diff --git "a/wandb/run-20220120_151428-zjqubvsf/files/diff.patch" "b/wandb/run-20220120_151428-zjqubvsf/files/diff.patch" new file mode 100644--- /dev/null +++ "b/wandb/run-20220120_151428-zjqubvsf/files/diff.patch" @@ -0,0 +1,1455 @@ +diff --git a/wandb/debug-internal.log b/wandb/debug-internal.log +index 06184a5..ac7431b 120000 +--- a/wandb/debug-internal.log ++++ b/wandb/debug-internal.log +@@ -1 +1 @@ +-run-20220120_135649-r5vgtpr6/logs/debug-internal.log +\ No newline at end of file ++run-20220120_151428-zjqubvsf/logs/debug-internal.log +\ No newline at end of file +diff --git a/wandb/debug.log b/wandb/debug.log +index 64abce4..b6e46ed 120000 +--- a/wandb/debug.log ++++ b/wandb/debug.log +@@ -1 +1 @@ +-run-20220120_135649-r5vgtpr6/logs/debug.log +\ No newline at end of file ++run-20220120_151428-zjqubvsf/logs/debug.log +\ No newline at end of file +diff --git a/wandb/latest-run b/wandb/latest-run +index a564754..b2cd69d 120000 +--- a/wandb/latest-run ++++ b/wandb/latest-run +@@ -1 +1 @@ +-run-20220120_135649-r5vgtpr6 +\ No newline at end of file ++run-20220120_151428-zjqubvsf +\ No newline at end of file +diff --git a/wandb/run-20220120_135649-r5vgtpr6/files/config.yaml b/wandb/run-20220120_135649-r5vgtpr6/files/config.yaml +index a5f2b1a..6d03e90 100644 +--- a/wandb/run-20220120_135649-r5vgtpr6/files/config.yaml ++++ b/wandb/run-20220120_135649-r5vgtpr6/files/config.yaml +@@ -15,6 +15,9 @@ _wandb: + 1: + - 11 + - 12 ++ 2: ++ - 11 ++ - 12 + 4: 3.8.10 + 5: 0.12.9 + 6: 4.16.0.dev0 +diff --git a/wandb/run-20220120_135649-r5vgtpr6/files/output.log b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +index 0363ada..b9210d3 100644 +--- a/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++++ b/wandb/run-20220120_135649-r5vgtpr6/files/output.log +@@ -1691,3 +1691,510 @@ Training...: 4%|████████▎ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++Step... (2000 | Loss: 1.330615520477295, Acc: 0.7090045213699341): 0%| | 0/3 [1:15:55 ++ main() ++ File "run_mlm_flax.py", line 718, in main ++ samples = [tokenized_datasets["train"][int(idx)] for idx in batch_idx] ++ File "run_mlm_flax.py", line 718, in ++ samples = [tokenized_datasets["train"][int(idx)] for idx in batch_idx] ++ File "/data/flax/lib/python3.8/site-packages/jax/_src/device_array.py", line 247, in ++ return (sl for chunk in self._chunk_iter(100) for sl in chunk._unstack()) ++ File "/data/flax/lib/python3.8/site-packages/jax/_src/numpy/lax_numpy.py", line 6792, in _chunk_iter ++ for i in range(num_chunks): ++ File "/usr/lib/python3.8/functools.py", line 399, in _method ++ return self.func(cls_or_self, *self.args, *args, **keywords) ++ File "/data/flax/lib/python3.8/site-packages/jax/_src/device_array.py", line 41, in _forward_method ++ return fun(getattr(self, attrname), *args) ++KeyboardInterrupt +\ No newline at end of file +diff --git a/wandb/run-20220120_135649-r5vgtpr6/files/wandb-summary.json b/wandb/run-20220120_135649-r5vgtpr6/files/wandb-summary.json +index 9e26dfe..493f68b 100644 +--- a/wandb/run-20220120_135649-r5vgtpr6/files/wandb-summary.json ++++ b/wandb/run-20220120_135649-r5vgtpr6/files/wandb-summary.json +@@ -1 +1 @@ +-{} +\ No newline at end of file ++{"_wandb": {"runtime": 4565}} +\ No newline at end of file +diff --git a/wandb/run-20220120_135649-r5vgtpr6/logs/debug-internal.log b/wandb/run-20220120_135649-r5vgtpr6/logs/debug-internal.log +index 1e33f0d..e0c3a96 100644 +--- a/wandb/run-20220120_135649-r5vgtpr6/logs/debug-internal.log ++++ b/wandb/run-20220120_135649-r5vgtpr6/logs/debug-internal.log +@@ -2322,3 +2322,766 @@ + 2022-01-20 14:56:14,433 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status + 2022-01-20 14:56:14,433 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status + 2022-01-20 14:56:27,163 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 14:56:30,010 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 14:56:30,011 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 14:56:32,139 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:56:34,140 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:56:36,141 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:56:38,141 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:56:40,142 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:56:42,143 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:56:44,144 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:56:45,144 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 14:56:45,145 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 14:56:46,146 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:56:48,146 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:56:50,147 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:56:52,148 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:56:54,149 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:56:56,150 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:56:57,236 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 14:56:58,151 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:00,152 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:00,362 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 14:57:00,362 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 14:57:02,153 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:04,153 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:06,154 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:08,155 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:10,156 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:12,157 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:14,158 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:15,501 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 14:57:15,502 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 14:57:16,158 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:18,159 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:20,161 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:22,162 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:24,163 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:26,163 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:27,309 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 14:57:28,164 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:30,165 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:30,645 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 14:57:30,645 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 14:57:32,167 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:34,168 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:36,169 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:38,170 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:40,171 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:42,172 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:44,173 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:45,782 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 14:57:45,782 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 14:57:46,174 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:48,175 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:50,176 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:52,177 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:54,178 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:56,179 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:57:57,383 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 14:57:58,180 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:00,181 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:00,947 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 14:58:00,948 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 14:58:02,182 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:04,183 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:06,184 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:08,185 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:10,186 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:12,187 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:14,188 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:16,091 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 14:58:16,092 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 14:58:16,189 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:18,190 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:20,191 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:22,191 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:24,192 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:26,193 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:27,455 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 14:58:28,194 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:30,195 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:31,240 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 14:58:31,241 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 14:58:32,198 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:35,200 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:37,200 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:39,201 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:41,202 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:43,203 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:45,204 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:46,521 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 14:58:46,521 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 14:58:47,205 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:49,206 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:51,207 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:53,208 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:55,209 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:57,210 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:58:57,530 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 14:58:59,211 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:01,212 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:01,664 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 14:59:01,664 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 14:59:03,213 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:05,214 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:07,215 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:09,215 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:11,216 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:13,217 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:15,218 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:16,800 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 14:59:16,800 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 14:59:17,220 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:19,221 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:21,221 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:23,222 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:25,223 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:27,224 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:27,604 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 14:59:29,225 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:31,225 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:31,940 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 14:59:31,941 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 14:59:33,226 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:35,227 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:37,228 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:39,229 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:41,230 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:43,231 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:45,232 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:47,087 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 14:59:47,087 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 14:59:47,233 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:49,234 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:51,235 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:53,236 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:55,237 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:57,238 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 14:59:57,679 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 14:59:59,239 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:01,240 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:02,226 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:00:02,226 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:00:03,241 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:05,242 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:07,243 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:09,244 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:11,245 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:13,246 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:15,247 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:17,247 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:17,365 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:00:17,365 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:00:19,248 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:21,249 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:23,251 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:25,252 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:27,253 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:27,751 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:00:29,254 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:31,255 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:32,504 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:00:32,504 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:00:33,256 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:35,257 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:37,258 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:39,258 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:41,259 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:43,260 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:45,261 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:47,262 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:47,639 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:00:47,639 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:00:49,263 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:51,264 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:53,265 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:55,266 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:57,267 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:00:57,821 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:00:59,268 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:01,269 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:02,777 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:01:02,777 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:01:03,270 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:05,271 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:07,272 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:09,273 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:11,274 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:13,275 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:15,276 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:17,277 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:17,911 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:01:17,911 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:01:19,278 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:21,278 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:23,279 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:25,280 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:27,281 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:27,890 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:01:29,282 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:31,283 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:33,048 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:01:33,049 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:01:33,284 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:35,285 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:37,286 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:39,287 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:41,288 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:43,289 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:45,290 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:47,291 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:48,189 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:01:48,190 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:01:49,292 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:51,293 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:53,294 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:55,295 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:57,296 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:01:57,962 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:01:59,297 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:01,297 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:03,298 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:03,331 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:02:03,331 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:02:05,299 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:07,300 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:09,301 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:11,302 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:13,303 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:15,304 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:17,304 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:18,497 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:02:18,498 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:02:19,305 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:21,306 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:23,307 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:25,308 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:27,309 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:28,036 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:02:29,310 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:31,311 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:33,312 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:33,637 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:02:33,637 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:02:35,313 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:37,314 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:39,315 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:41,316 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:43,317 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:45,317 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:47,318 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:48,772 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:02:48,773 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:02:49,319 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:51,320 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:53,321 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:55,322 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:57,322 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:02:58,112 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:02:59,323 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:01,324 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:03,325 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:03,908 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:03:03,909 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:03:05,326 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:07,327 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:09,327 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:11,328 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:13,329 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:15,330 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:17,331 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:19,257 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:03:19,258 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:03:19,333 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:21,334 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:23,334 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:25,335 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:27,336 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:28,188 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:03:29,337 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:31,338 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:33,339 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:34,408 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:03:34,408 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:03:35,340 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:37,341 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:39,342 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:41,343 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:43,344 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:45,345 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:47,346 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:49,347 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:49,548 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:03:49,549 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:03:51,348 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:53,349 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:55,350 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:57,351 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:03:58,262 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:03:59,352 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:01,353 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:03,354 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:04,686 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:04:04,686 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:04:05,355 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:07,356 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:09,357 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:11,358 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:13,359 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:15,360 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:17,361 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:19,362 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:19,826 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:04:19,826 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:04:21,363 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:23,364 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:25,365 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:27,366 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:28,330 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:04:29,367 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:31,368 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:33,369 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:34,962 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:04:34,962 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:04:35,370 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:37,371 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:39,372 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:41,373 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:43,374 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:45,374 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:47,375 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:49,377 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:50,096 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:04:50,097 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:04:51,378 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:53,378 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:55,379 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:57,380 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:04:58,400 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:04:59,381 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:01,382 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:03,383 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:05,240 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:05:05,240 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:05:05,384 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:07,385 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:09,386 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:11,387 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:13,388 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:15,389 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:17,390 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:19,391 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:20,378 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:05:20,379 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:05:21,392 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:23,393 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:25,394 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:27,395 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:28,471 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:05:29,396 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:31,397 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:33,398 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:35,399 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:35,510 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:05:35,510 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:05:37,400 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:39,400 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:41,401 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:43,402 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:45,403 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:47,404 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:49,405 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:50,660 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:05:50,661 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:05:51,406 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:53,407 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:55,408 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:57,409 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:05:58,543 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:05:59,410 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:01,410 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:03,411 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:05,412 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:05,799 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:06:05,799 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:06:07,413 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:09,414 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:11,415 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:13,416 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:15,417 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:17,418 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:19,419 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:20,940 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:06:20,941 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:06:21,420 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:23,421 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:25,422 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:27,423 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:28,607 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:06:29,424 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:31,425 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:33,426 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:35,427 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:36,087 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:06:36,087 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:06:37,428 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:39,429 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:41,429 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:43,431 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:45,432 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:47,433 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:50,434 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:51,301 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:06:51,302 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:06:52,435 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:54,437 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:56,438 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:58,439 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:06:58,671 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:07:00,440 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:02,441 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:04,442 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:06,442 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:07:06,443 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:07:06,444 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:08,444 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:10,445 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:12,447 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:14,448 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:16,449 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:18,450 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:20,451 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:21,584 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:07:21,584 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:07:22,452 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:24,453 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:26,454 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:28,455 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:28,736 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:07:30,456 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:32,457 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:34,458 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:36,459 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:36,778 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:07:36,779 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:07:38,460 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:40,461 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:42,462 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:44,463 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:46,464 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:48,465 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:50,466 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:51,916 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:07:51,916 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:07:52,467 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:54,469 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:56,470 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:58,471 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:07:58,803 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:08:00,472 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:02,473 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:04,474 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:06,475 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:07,056 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:08:07,057 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:08:08,476 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:10,477 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:12,477 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:14,478 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:16,479 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:18,480 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:20,481 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:22,206 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:08:22,207 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:08:22,482 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:24,483 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:26,484 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:28,485 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:28,868 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:08:30,486 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:32,487 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:34,488 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:36,489 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:37,346 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:08:37,346 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:08:38,490 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:40,491 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:42,492 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:44,493 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:46,494 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:48,495 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:50,496 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:52,487 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:08:52,487 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:08:52,497 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:54,498 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:56,498 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:58,499 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:08:58,935 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:09:00,500 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:02,501 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:04,502 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:06,503 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:07,627 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:09:07,627 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:09:08,504 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:10,505 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:12,506 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:14,507 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:16,508 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:18,509 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:20,510 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:22,511 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:23,088 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:09:23,088 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:09:24,512 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:26,513 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:28,514 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:29,008 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:09:30,515 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:32,516 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:34,517 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:36,518 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:38,235 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:09:38,236 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:09:38,519 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:40,520 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:42,521 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:44,522 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:46,523 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:48,524 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:50,525 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:52,526 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:53,400 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:09:53,400 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:09:54,527 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:56,528 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:58,529 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:09:59,085 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:10:00,530 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:02,531 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:04,532 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:06,533 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:08,534 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:08,536 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:10:08,537 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:10:10,535 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:12,536 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:14,537 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:16,538 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:18,539 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:20,540 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:22,541 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:23,715 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:10:23,715 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:10:24,542 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:26,543 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:28,544 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:29,159 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:10:30,545 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:32,546 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:34,547 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:36,548 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:38,549 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:39,078 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:10:39,078 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:10:40,550 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:42,551 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:44,552 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:46,553 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:48,554 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:50,555 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:52,556 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:54,235 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:10:54,235 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:10:54,557 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:56,558 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:58,559 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:10:59,233 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:11:00,560 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:02,561 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:04,562 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:06,563 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:08,564 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:09,375 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:11:09,375 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:11:10,565 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:12,567 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:14,568 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:16,569 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:18,570 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:20,570 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:22,572 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:24,563 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:11:24,563 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:11:24,572 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:26,574 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:28,575 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:29,308 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:11:30,576 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:32,577 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:34,578 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:36,579 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:38,580 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:39,697 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:11:39,697 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:11:40,581 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:42,582 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:44,583 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:46,584 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:48,585 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:50,586 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:52,587 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:54,588 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:54,866 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:11:54,867 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:11:56,588 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:58,589 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:11:59,383 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:12:00,590 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:02,591 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:04,591 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:06,592 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:08,593 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:10,005 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:12:10,006 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:12:10,594 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:12,595 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:14,596 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:16,597 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:18,597 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:20,598 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:22,599 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:24,600 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:25,144 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:12:25,144 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:12:26,601 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:28,602 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:29,458 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:12:30,603 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:32,603 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:34,604 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:36,605 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:38,606 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:40,316 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: stop_status ++2022-01-20 15:12:40,316 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: stop_status ++2022-01-20 15:12:40,606 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:42,607 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:44,608 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:46,609 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:48,610 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:50,611 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:52,612 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:53,208 WARNING MainThread:2513451 [internal.py:wandb_internal():146] Internal process interrupt: 1 ++2022-01-20 15:12:54,613 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:55,902 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit ++2022-01-20 15:12:55,903 DEBUG SenderThread:2513451 [sender.py:send():234] send: telemetry ++2022-01-20 15:12:55,903 DEBUG SenderThread:2513451 [sender.py:send():234] send: exit ++2022-01-20 15:12:55,903 INFO SenderThread:2513451 [sender.py:send_exit():366] handling exit code: 255 ++2022-01-20 15:12:55,903 INFO SenderThread:2513451 [sender.py:send_exit():368] handling runtime: 4565 ++2022-01-20 15:12:55,904 INFO SenderThread:2513451 [sender.py:_save_file():939] saving file wandb-summary.json with policy end ++2022-01-20 15:12:55,904 INFO SenderThread:2513451 [sender.py:send_exit():374] send defer ++2022-01-20 15:12:55,904 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit ++2022-01-20 15:12:55,905 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: defer ++2022-01-20 15:12:55,905 INFO HandlerThread:2513451 [handler.py:handle_request_defer():147] handle defer: 0 ++2022-01-20 15:12:55,905 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: defer ++2022-01-20 15:12:55,905 INFO SenderThread:2513451 [sender.py:send_request_defer():383] handle sender defer: 0 ++2022-01-20 15:12:55,905 INFO SenderThread:2513451 [sender.py:transition_state():387] send defer: 1 ++2022-01-20 15:12:55,905 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: defer ++2022-01-20 15:12:55,905 INFO HandlerThread:2513451 [handler.py:handle_request_defer():147] handle defer: 1 ++2022-01-20 15:12:55,924 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: defer ++2022-01-20 15:12:55,924 INFO SenderThread:2513451 [sender.py:send_request_defer():383] handle sender defer: 1 ++2022-01-20 15:12:55,924 INFO SenderThread:2513451 [sender.py:transition_state():387] send defer: 2 ++2022-01-20 15:12:55,924 DEBUG SenderThread:2513451 [sender.py:send():234] send: stats ++2022-01-20 15:12:55,925 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: defer ++2022-01-20 15:12:55,925 INFO HandlerThread:2513451 [handler.py:handle_request_defer():147] handle defer: 2 ++2022-01-20 15:12:55,925 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: defer ++2022-01-20 15:12:55,925 INFO SenderThread:2513451 [sender.py:send_request_defer():383] handle sender defer: 2 ++2022-01-20 15:12:55,925 INFO SenderThread:2513451 [sender.py:transition_state():387] send defer: 3 ++2022-01-20 15:12:55,926 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: defer ++2022-01-20 15:12:55,926 INFO HandlerThread:2513451 [handler.py:handle_request_defer():147] handle defer: 3 ++2022-01-20 15:12:55,926 DEBUG SenderThread:2513451 [sender.py:send():234] send: summary ++2022-01-20 15:12:55,926 INFO SenderThread:2513451 [sender.py:_save_file():939] saving file wandb-summary.json with policy end ++2022-01-20 15:12:55,926 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: defer ++2022-01-20 15:12:55,926 INFO SenderThread:2513451 [sender.py:send_request_defer():383] handle sender defer: 3 ++2022-01-20 15:12:55,926 INFO SenderThread:2513451 [sender.py:transition_state():387] send defer: 4 ++2022-01-20 15:12:55,927 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: defer ++2022-01-20 15:12:55,927 INFO HandlerThread:2513451 [handler.py:handle_request_defer():147] handle defer: 4 ++2022-01-20 15:12:55,927 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: defer ++2022-01-20 15:12:55,927 INFO SenderThread:2513451 [sender.py:send_request_defer():383] handle sender defer: 4 ++2022-01-20 15:12:56,006 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit ++2022-01-20 15:12:56,102 INFO SenderThread:2513451 [sender.py:transition_state():387] send defer: 5 ++2022-01-20 15:12:56,102 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit ++2022-01-20 15:12:56,103 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: defer ++2022-01-20 15:12:56,103 INFO HandlerThread:2513451 [handler.py:handle_request_defer():147] handle defer: 5 ++2022-01-20 15:12:56,103 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: defer ++2022-01-20 15:12:56,103 INFO SenderThread:2513451 [sender.py:send_request_defer():383] handle sender defer: 5 ++2022-01-20 15:12:56,103 INFO SenderThread:2513451 [dir_watcher.py:finish():283] shutting down directory watcher ++2022-01-20 15:12:56,204 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit ++2022-01-20 15:12:56,614 INFO Thread-8 :2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/config.yaml ++2022-01-20 15:12:56,615 INFO SenderThread:2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:56,615 INFO SenderThread:2513451 [dir_watcher.py:_on_file_modified():230] file/dir modified: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/wandb-summary.json ++2022-01-20 15:12:56,616 INFO SenderThread:2513451 [dir_watcher.py:finish():313] scan: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files ++2022-01-20 15:12:56,616 INFO SenderThread:2513451 [dir_watcher.py:finish():327] scan save: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/config.yaml config.yaml ++2022-01-20 15:12:56,616 INFO SenderThread:2513451 [dir_watcher.py:finish():327] scan save: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/diff.patch diff.patch ++2022-01-20 15:12:56,616 INFO SenderThread:2513451 [dir_watcher.py:finish():327] scan save: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/requirements.txt requirements.txt ++2022-01-20 15:12:56,616 INFO SenderThread:2513451 [dir_watcher.py:finish():327] scan save: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log output.log ++2022-01-20 15:12:56,616 INFO SenderThread:2513451 [dir_watcher.py:finish():327] scan save: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/wandb-summary.json wandb-summary.json ++2022-01-20 15:12:56,616 INFO SenderThread:2513451 [dir_watcher.py:finish():327] scan save: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/wandb-metadata.json wandb-metadata.json ++2022-01-20 15:12:56,617 INFO SenderThread:2513451 [dir_watcher.py:finish():327] scan save: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/code/run_mlm_flax.py code/run_mlm_flax.py ++2022-01-20 15:12:56,617 INFO SenderThread:2513451 [sender.py:transition_state():387] send defer: 6 ++2022-01-20 15:12:56,618 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit ++2022-01-20 15:12:56,622 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: defer ++2022-01-20 15:12:56,622 INFO HandlerThread:2513451 [handler.py:handle_request_defer():147] handle defer: 6 ++2022-01-20 15:12:56,622 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: defer ++2022-01-20 15:12:56,622 INFO SenderThread:2513451 [sender.py:send_request_defer():383] handle sender defer: 6 ++2022-01-20 15:12:56,622 INFO SenderThread:2513451 [file_pusher.py:finish():177] shutting down file pusher ++2022-01-20 15:12:56,720 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit ++2022-01-20 15:12:56,720 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit ++2022-01-20 15:12:56,822 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit ++2022-01-20 15:12:56,822 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit ++2022-01-20 15:12:56,924 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit ++2022-01-20 15:12:56,924 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit ++2022-01-20 15:12:57,026 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit ++2022-01-20 15:12:57,026 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit ++2022-01-20 15:12:57,061 INFO Thread-15 :2513451 [upload_job.py:push():137] Uploaded file /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/requirements.txt ++2022-01-20 15:12:57,074 INFO Thread-16 :2513451 [upload_job.py:push():137] Uploaded file /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/output.log ++2022-01-20 15:12:57,078 INFO Thread-14 :2513451 [upload_job.py:push():137] Uploaded file /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/config.yaml ++2022-01-20 15:12:57,084 INFO Thread-17 :2513451 [upload_job.py:push():137] Uploaded file /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/files/wandb-summary.json ++2022-01-20 15:12:57,128 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit ++2022-01-20 15:12:57,128 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit ++2022-01-20 15:12:57,230 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit ++2022-01-20 15:12:57,230 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit ++2022-01-20 15:12:57,285 INFO Thread-7 :2513451 [sender.py:transition_state():387] send defer: 7 ++2022-01-20 15:12:57,286 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: defer ++2022-01-20 15:12:57,286 INFO HandlerThread:2513451 [handler.py:handle_request_defer():147] handle defer: 7 ++2022-01-20 15:12:57,286 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: defer ++2022-01-20 15:12:57,286 INFO SenderThread:2513451 [sender.py:send_request_defer():383] handle sender defer: 7 ++2022-01-20 15:12:57,332 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit ++2022-01-20 15:12:57,712 INFO SenderThread:2513451 [sender.py:transition_state():387] send defer: 8 ++2022-01-20 15:12:57,713 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit ++2022-01-20 15:12:57,713 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: defer ++2022-01-20 15:12:57,713 INFO HandlerThread:2513451 [handler.py:handle_request_defer():147] handle defer: 8 ++2022-01-20 15:12:57,714 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: defer ++2022-01-20 15:12:57,714 INFO SenderThread:2513451 [sender.py:send_request_defer():383] handle sender defer: 8 ++2022-01-20 15:12:57,714 INFO SenderThread:2513451 [sender.py:transition_state():387] send defer: 9 ++2022-01-20 15:12:57,714 DEBUG SenderThread:2513451 [sender.py:send():234] send: final ++2022-01-20 15:12:57,714 DEBUG SenderThread:2513451 [sender.py:send():234] send: footer ++2022-01-20 15:12:57,714 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: defer ++2022-01-20 15:12:57,715 INFO HandlerThread:2513451 [handler.py:handle_request_defer():147] handle defer: 9 ++2022-01-20 15:12:57,715 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: defer ++2022-01-20 15:12:57,715 INFO SenderThread:2513451 [sender.py:send_request_defer():383] handle sender defer: 9 ++2022-01-20 15:12:57,814 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: poll_exit ++2022-01-20 15:12:57,815 DEBUG SenderThread:2513451 [sender.py:send_request():248] send_request: poll_exit ++2022-01-20 15:12:57,815 INFO SenderThread:2513451 [file_pusher.py:join():182] waiting for file pusher ++2022-01-20 15:12:58,081 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: get_summary ++2022-01-20 15:12:58,081 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: sampled_history ++2022-01-20 15:12:58,082 DEBUG HandlerThread:2513451 [handler.py:handle_request():130] handle_request: shutdown ++2022-01-20 15:12:58,082 INFO HandlerThread:2513451 [handler.py:finish():731] shutting down handler ++2022-01-20 15:12:58,715 INFO WriterThread:2513451 [datastore.py:close():281] close: /data/roberta-base-ncc-512b/wandb/run-20220120_135649-r5vgtpr6/run-r5vgtpr6.wandb ++2022-01-20 15:12:59,079 INFO SenderThread:2513451 [sender.py:finish():1070] shutting down sender ++2022-01-20 15:12:59,080 INFO SenderThread:2513451 [file_pusher.py:finish():177] shutting down file pusher ++2022-01-20 15:12:59,080 INFO SenderThread:2513451 [file_pusher.py:join():182] waiting for file pusher ++2022-01-20 15:12:59,082 INFO MainThread:2513451 [internal.py:handle_exit():77] Internal process exited +diff --git a/wandb/run-20220120_135649-r5vgtpr6/logs/debug.log b/wandb/run-20220120_135649-r5vgtpr6/logs/debug.log +index 4500fe3..e0fb1de 100644 +--- a/wandb/run-20220120_135649-r5vgtpr6/logs/debug.log ++++ b/wandb/run-20220120_135649-r5vgtpr6/logs/debug.log +@@ -23,3 +23,110 @@ config: {} + 2022-01-20 13:56:50,690 INFO MainThread:2512121 [wandb_run.py:_config_callback():956] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'per_device_train_batch_size': 46, 'per_device_eval_batch_size': 46, 'learning_rate': 0.00015, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.98, 'adam_epsilon': 1e-06, 'adafactor': False, 'num_train_epochs': 3.0, 'warmup_steps': 1000, 'logging_steps': 1000, 'save_steps': 1000, 'eval_steps': 1000, 'seed': 42, 'push_to_hub': True, 'hub_model_id': None, 'hub_token': None} + 2022-01-20 13:56:50,691 INFO MainThread:2512121 [wandb_run.py:_config_callback():956] config_cb None None {'model_name_or_path': 'versae/roberta-base-ncc', 'model_type': 'roberta', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'bfloat16'} + 2022-01-20 13:56:50,691 INFO MainThread:2512121 [wandb_run.py:_config_callback():956] config_cb None None {'dataset_name': 'NbAiLab/NCC', 'dataset_config_name': None, 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 512, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': True, 'line_by_line': False} ++2022-01-20 15:12:53,717 INFO MainThread:2512121 [wandb_run.py:_atexit_cleanup():1780] got exitcode: 255 ++2022-01-20 15:12:53,718 INFO MainThread:2512121 [wandb_run.py:_restore():1752] restore ++2022-01-20 15:12:55,905 INFO MainThread:2512121 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { ++ wandb_count: 2 ++ other_count: 1 ++} ++pusher_stats { ++ uploaded_bytes: 37926 ++ total_bytes: 37926 ++} ++ ++2022-01-20 15:12:56,103 INFO MainThread:2512121 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { ++ wandb_count: 2 ++ other_count: 1 ++} ++pusher_stats { ++ uploaded_bytes: 37926 ++ total_bytes: 37926 ++} ++ ++2022-01-20 15:12:56,618 INFO MainThread:2512121 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { ++ wandb_count: 6 ++ other_count: 1 ++} ++pusher_stats { ++ uploaded_bytes: 37926 ++ total_bytes: 47361 ++} ++ ++2022-01-20 15:12:56,721 INFO MainThread:2512121 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { ++ wandb_count: 6 ++ other_count: 1 ++} ++pusher_stats { ++ uploaded_bytes: 37926 ++ total_bytes: 47361 ++} ++ ++2022-01-20 15:12:56,823 INFO MainThread:2512121 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { ++ wandb_count: 6 ++ other_count: 1 ++} ++pusher_stats { ++ uploaded_bytes: 47361 ++ total_bytes: 47361 ++} ++ ++2022-01-20 15:12:56,925 INFO MainThread:2512121 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { ++ wandb_count: 6 ++ other_count: 1 ++} ++pusher_stats { ++ uploaded_bytes: 47361 ++ total_bytes: 47361 ++} ++ ++2022-01-20 15:12:57,027 INFO MainThread:2512121 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { ++ wandb_count: 6 ++ other_count: 1 ++} ++pusher_stats { ++ uploaded_bytes: 47361 ++ total_bytes: 47361 ++} ++ ++2022-01-20 15:12:57,129 INFO MainThread:2512121 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { ++ wandb_count: 6 ++ other_count: 1 ++} ++pusher_stats { ++ uploaded_bytes: 47361 ++ total_bytes: 47361 ++} ++ ++2022-01-20 15:12:57,231 INFO MainThread:2512121 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { ++ wandb_count: 6 ++ other_count: 1 ++} ++pusher_stats { ++ uploaded_bytes: 47361 ++ total_bytes: 47361 ++} ++ ++2022-01-20 15:12:57,714 INFO MainThread:2512121 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts { ++ wandb_count: 6 ++ other_count: 1 ++} ++pusher_stats { ++ uploaded_bytes: 47361 ++ total_bytes: 47361 ++} ++ ++2022-01-20 15:12:58,080 INFO MainThread:2512121 [wandb_run.py:_wait_for_finish():1912] got exit ret: done: true ++exit_result { ++} ++file_counts { ++ wandb_count: 6 ++ other_count: 1 ++} ++pusher_stats { ++ uploaded_bytes: 47361 ++ total_bytes: 47361 ++} ++local_info { ++} ++ ++2022-01-20 15:12:59,238 INFO MainThread:2512121 [wandb_run.py:_append_files():2180] logging synced files +diff --git a/wandb/run-20220120_135649-r5vgtpr6/run-r5vgtpr6.wandb b/wandb/run-20220120_135649-r5vgtpr6/run-r5vgtpr6.wandb +index 7d7df3a..76a4b0b 100644 +Binary files a/wandb/run-20220120_135649-r5vgtpr6/run-r5vgtpr6.wandb and b/wandb/run-20220120_135649-r5vgtpr6/run-r5vgtpr6.wandb differ