cyrusyc commited on
Commit
f0fdb88
1 Parent(s): df0fa6f

fix typo; add diatomic, combustion notebooks

Browse files
mlip_arena/tasks/combustion/water.ipynb ADDED
@@ -0,0 +1,763 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": 1,
6
+ "metadata": {},
7
+ "outputs": [],
8
+ "source": [
9
+ "from pathlib import Path\n",
10
+ "\n",
11
+ "from ase import units, Atoms\n",
12
+ "from ase.build import molecule\n",
13
+ "from ase.io import read, write\n",
14
+ "from dask.distributed import Client\n",
15
+ "from dask_jobqueue import SLURMCluster\n",
16
+ "from prefect import flow\n",
17
+ "from prefect_dask import DaskTaskRunner\n",
18
+ "from pymatgen.core import Molecule\n",
19
+ "from pymatgen.io.packmol import PackmolBoxGen\n",
20
+ "\n",
21
+ "from mlip_arena.models.utils import REGISTRY, MLIPEnum\n",
22
+ "from mlip_arena.tasks import MD"
23
+ ]
24
+ },
25
+ {
26
+ "cell_type": "code",
27
+ "execution_count": null,
28
+ "metadata": {},
29
+ "outputs": [],
30
+ "source": [
31
+ "h2 = molecule(\"H2\")\n",
32
+ "o2 = molecule(\"O2\")\n",
33
+ "h2o = molecule(\"H2O\")\n",
34
+ "\n",
35
+ "write(\"h2.xyz\", h2)\n",
36
+ "write(\"o2.xyz\", o2)\n",
37
+ "write(\"h2o.xyz\", h2o)"
38
+ ]
39
+ },
40
+ {
41
+ "cell_type": "code",
42
+ "execution_count": null,
43
+ "metadata": {},
44
+ "outputs": [],
45
+ "source": [
46
+ "h2 = Molecule.from_file(\"h2.xyz\")\n",
47
+ "o2 = Molecule.from_file(\"o2.xyz\")\n",
48
+ "h2o = Molecule.from_file(\"h2o.xyz\")"
49
+ ]
50
+ },
51
+ {
52
+ "cell_type": "code",
53
+ "execution_count": null,
54
+ "metadata": {},
55
+ "outputs": [],
56
+ "source": [
57
+ "molecules = []\n",
58
+ "\n",
59
+ "for m, number in zip([h2, o2], [128, 64]):\n",
60
+ " molecules.append(\n",
61
+ " {\n",
62
+ " \"name\": m.composition.to_pretty_string(),\n",
63
+ " \"number\": number,\n",
64
+ " \"coords\": m,\n",
65
+ " }\n",
66
+ " )"
67
+ ]
68
+ },
69
+ {
70
+ "cell_type": "code",
71
+ "execution_count": null,
72
+ "metadata": {},
73
+ "outputs": [],
74
+ "source": [
75
+ "tolerance = 2.0\n",
76
+ "input_gen = PackmolBoxGen(\n",
77
+ " tolerance=tolerance,\n",
78
+ " seed=1,\n",
79
+ ")\n",
80
+ "margin = 0.5 * tolerance\n",
81
+ "\n",
82
+ "a = 30\n",
83
+ "\n",
84
+ "packmol_set = input_gen.get_input_set(\n",
85
+ " molecules=molecules,\n",
86
+ " box=[margin, margin, margin, a - margin, a - margin, a - margin],\n",
87
+ ")\n",
88
+ "packmol_set.write_input(\".\")\n",
89
+ "packmol_set.run(\".\")\n",
90
+ "\n",
91
+ "atoms = read(\"packmol_out.xyz\")\n",
92
+ "atoms.cell = [a, a, a]\n",
93
+ "atoms.pbc = True\n",
94
+ "\n",
95
+ "print(atoms)\n",
96
+ "\n",
97
+ "write(f'{atoms.get_chemical_formula()}.extxyz', atoms)"
98
+ ]
99
+ },
100
+ {
101
+ "cell_type": "code",
102
+ "execution_count": 2,
103
+ "metadata": {
104
+ "tags": []
105
+ },
106
+ "outputs": [
107
+ {
108
+ "name": "stdout",
109
+ "output_type": "stream",
110
+ "text": [
111
+ "Atoms(symbols='H256O128', pbc=True, cell=[30.0, 30.0, 30.0])\n"
112
+ ]
113
+ }
114
+ ],
115
+ "source": [
116
+ "atoms = read(\"H256O128.extxyz\")\n",
117
+ "print(atoms)"
118
+ ]
119
+ },
120
+ {
121
+ "cell_type": "code",
122
+ "execution_count": 3,
123
+ "metadata": {},
124
+ "outputs": [
125
+ {
126
+ "name": "stdout",
127
+ "output_type": "stream",
128
+ "text": [
129
+ "#!/bin/bash\n",
130
+ "\n",
131
+ "#SBATCH -A matgen\n",
132
+ "#SBATCH --mem=0\n",
133
+ "#SBATCH -t 01:50:00\n",
134
+ "#SBATCH -J combustion-water\n",
135
+ "#SBATCH -q regular\n",
136
+ "#SBATCH -N 1\n",
137
+ "#SBATCH -C gpu\n",
138
+ "#SBATCH -G 4\n",
139
+ "source ~/.bashrc\n",
140
+ "module load python\n",
141
+ "source activate /pscratch/sd/c/cyrusyc/.conda/mlip-arena\n",
142
+ "/pscratch/sd/c/cyrusyc/.conda/mlip-arena/bin/python -m distributed.cli.dask_worker tcp://128.55.64.49:34761 --name dummy-name --nthreads 1 --memory-limit 59.60GiB --nanny --death-timeout 86400\n",
143
+ "\n"
144
+ ]
145
+ },
146
+ {
147
+ "name": "stderr",
148
+ "output_type": "stream",
149
+ "text": [
150
+ "/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/distributed/node.py:182: UserWarning: Port 8787 is already in use.\n",
151
+ "Perhaps you already have a cluster running?\n",
152
+ "Hosting the HTTP server on port 33279 instead\n",
153
+ " warnings.warn(\n",
154
+ "2024-08-29 07:01:08,914 - distributed.scheduler - ERROR - Task md-7-bdc25b05525842028653d9bf609a5611-1 marked as failed because 4 workers died while trying to run it\n",
155
+ "2024-08-29 07:01:09,223 - distributed.scheduler - ERROR - Task md-1-0dcbdd94b5ca46f7bdd90131bae6f396-1 marked as failed because 4 workers died while trying to run it\n"
156
+ ]
157
+ }
158
+ ],
159
+ "source": [
160
+ "nodes_per_alloc = 1\n",
161
+ "gpus_per_alloc = 4\n",
162
+ "ntasks = 1\n",
163
+ "\n",
164
+ "cluster_kwargs = dict(\n",
165
+ " cores=1,\n",
166
+ " memory=\"64 GB\",\n",
167
+ " shebang=\"#!/bin/bash\",\n",
168
+ " account=\"matgen\",\n",
169
+ " walltime=\"01:50:00\",\n",
170
+ " job_mem=\"0\",\n",
171
+ " job_script_prologue=[\n",
172
+ " \"source ~/.bashrc\",\n",
173
+ " \"module load python\",\n",
174
+ " \"source activate /pscratch/sd/c/cyrusyc/.conda/mlip-arena\",\n",
175
+ " ],\n",
176
+ " job_directives_skip=[\"-n\", \"--cpus-per-task\", \"-J\"],\n",
177
+ " job_extra_directives=[\n",
178
+ " \"-J combustion-water\",\n",
179
+ " \"-q regular\",\n",
180
+ " f\"-N {nodes_per_alloc}\",\n",
181
+ " \"-C gpu\",\n",
182
+ " f\"-G {gpus_per_alloc}\",\n",
183
+ " # f\"--exclusive\",\n",
184
+ " # \"--time-min=00:30:00\",\n",
185
+ " # \"--comment=1-00:00:00\",\n",
186
+ " # \"--signal=B:USR1@60\",\n",
187
+ " # \"--requeue\",\n",
188
+ " # \"--open-mode=append\"\n",
189
+ " ],\n",
190
+ " death_timeout=86400\n",
191
+ ")\n",
192
+ "\n",
193
+ "\n",
194
+ "cluster = SLURMCluster(**cluster_kwargs)\n",
195
+ "print(cluster.job_script())\n",
196
+ "cluster.adapt(minimum_jobs=5, maximum_jobs=50)\n",
197
+ "client = Client(cluster)"
198
+ ]
199
+ },
200
+ {
201
+ "cell_type": "code",
202
+ "execution_count": 4,
203
+ "metadata": {},
204
+ "outputs": [],
205
+ "source": [
206
+ "@flow(task_runner=DaskTaskRunner(address=client.scheduler.address), log_prints=True)\n",
207
+ "def combustion(atoms: Atoms):\n",
208
+ " futures = []\n",
209
+ "\n",
210
+ " for model in MLIPEnum:\n",
211
+ " future = MD.submit(\n",
212
+ " atoms=atoms,\n",
213
+ " calculator_name=model,\n",
214
+ " calculator_kwargs=None,\n",
215
+ " ensemble=\"nvt\",\n",
216
+ " dynamics=\"nose-hoover\",\n",
217
+ " time_step=None,\n",
218
+ " ase_md_kwargs=dict(ttime=25 * units.fs, pfactor=None),\n",
219
+ " total_time=1000_000,\n",
220
+ " temperature=[300, 3000, 3000, 300],\n",
221
+ " pressure=None,\n",
222
+ " mb_velocity_seed=0,\n",
223
+ " traj_file=Path(REGISTRY[model.name][\"family\"])\n",
224
+ " / f\"{atoms.get_chemical_formula()}.traj\",\n",
225
+ " traj_interval=1000,\n",
226
+ " restart=True,\n",
227
+ " )\n",
228
+ "\n",
229
+ " futures.append(future)\n",
230
+ " \n",
231
+ " return [future.result() for future in futures]"
232
+ ]
233
+ },
234
+ {
235
+ "cell_type": "code",
236
+ "execution_count": 5,
237
+ "metadata": {
238
+ "tags": []
239
+ },
240
+ "outputs": [
241
+ {
242
+ "data": {
243
+ "text/html": [
244
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:55.749 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | prefect.engine - Created flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> for flow<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> 'combustion'</span>\n",
245
+ "</pre>\n"
246
+ ],
247
+ "text/plain": [
248
+ "00:42:55.749 | \u001b[36mINFO\u001b[0m | prefect.engine - Created flow run\u001b[35m 'sly-yak'\u001b[0m for flow\u001b[1;35m 'combustion'\u001b[0m\n"
249
+ ]
250
+ },
251
+ "metadata": {},
252
+ "output_type": "display_data"
253
+ },
254
+ {
255
+ "data": {
256
+ "text/html": [
257
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:55.759 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - View at <span style=\"color: #0000ff; text-decoration-color: #0000ff\">https://app.prefect.cloud/account/f7d40474-9362-4bfa-8950-ee6a43ec00f3/workspace/d4bb0913-5f5e-49f7-bfc5-06509088baeb/flow-runs/flow-run/248fe963-31c9-4459-9c68-79fbd7418003</span>\n",
258
+ "</pre>\n"
259
+ ],
260
+ "text/plain": [
261
+ "00:42:55.759 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - View at \u001b[94mhttps://app.prefect.cloud/account/f7d40474-9362-4bfa-8950-ee6a43ec00f3/workspace/d4bb0913-5f5e-49f7-bfc5-06509088baeb/flow-runs/flow-run/248fe963-31c9-4459-9c68-79fbd7418003\u001b[0m\n"
262
+ ]
263
+ },
264
+ "metadata": {},
265
+ "output_type": "display_data"
266
+ },
267
+ {
268
+ "data": {
269
+ "text/html": [
270
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:55.761 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | prefect.task_runner.dask - Connecting to an existing Dask cluster at tcp://128.55.64.49:34761\n",
271
+ "</pre>\n"
272
+ ],
273
+ "text/plain": [
274
+ "00:42:55.761 | \u001b[36mINFO\u001b[0m | prefect.task_runner.dask - Connecting to an existing Dask cluster at tcp://128.55.64.49:34761\n"
275
+ ]
276
+ },
277
+ "metadata": {},
278
+ "output_type": "display_data"
279
+ },
280
+ {
281
+ "data": {
282
+ "text/html": [
283
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:55.774 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | prefect.task_runner.dask - The Dask dashboard is available at <span style=\"color: #0000ff; text-decoration-color: #0000ff\">http://128.55.64.49:33279/status</span>\n",
284
+ "</pre>\n"
285
+ ],
286
+ "text/plain": [
287
+ "00:42:55.774 | \u001b[36mINFO\u001b[0m | prefect.task_runner.dask - The Dask dashboard is available at \u001b[94mhttp://128.55.64.49:33279/status\u001b[0m\n"
288
+ ]
289
+ },
290
+ "metadata": {},
291
+ "output_type": "display_data"
292
+ },
293
+ {
294
+ "data": {
295
+ "text/html": [
296
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:56.291 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - Created task run 'md-0' for task 'md'\n",
297
+ "</pre>\n"
298
+ ],
299
+ "text/plain": [
300
+ "00:42:56.291 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - Created task run 'md-0' for task 'md'\n"
301
+ ]
302
+ },
303
+ "metadata": {},
304
+ "output_type": "display_data"
305
+ },
306
+ {
307
+ "data": {
308
+ "text/html": [
309
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:56.506 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - Created task run 'md-1' for task 'md'\n",
310
+ "</pre>\n"
311
+ ],
312
+ "text/plain": [
313
+ "00:42:56.506 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - Created task run 'md-1' for task 'md'\n"
314
+ ]
315
+ },
316
+ "metadata": {},
317
+ "output_type": "display_data"
318
+ },
319
+ {
320
+ "data": {
321
+ "text/html": [
322
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:56.544 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - Created task run 'md-2' for task 'md'\n",
323
+ "</pre>\n"
324
+ ],
325
+ "text/plain": [
326
+ "00:42:56.544 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - Created task run 'md-2' for task 'md'\n"
327
+ ]
328
+ },
329
+ "metadata": {},
330
+ "output_type": "display_data"
331
+ },
332
+ {
333
+ "data": {
334
+ "text/html": [
335
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:56.556 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - Created task run 'md-3' for task 'md'\n",
336
+ "</pre>\n"
337
+ ],
338
+ "text/plain": [
339
+ "00:42:56.556 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - Created task run 'md-3' for task 'md'\n"
340
+ ]
341
+ },
342
+ "metadata": {},
343
+ "output_type": "display_data"
344
+ },
345
+ {
346
+ "data": {
347
+ "text/html": [
348
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:56.570 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - Created task run 'md-6' for task 'md'\n",
349
+ "</pre>\n"
350
+ ],
351
+ "text/plain": [
352
+ "00:42:56.570 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - Created task run 'md-6' for task 'md'\n"
353
+ ]
354
+ },
355
+ "metadata": {},
356
+ "output_type": "display_data"
357
+ },
358
+ {
359
+ "data": {
360
+ "text/html": [
361
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:56.596 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - Created task run 'md-7' for task 'md'\n",
362
+ "</pre>\n"
363
+ ],
364
+ "text/plain": [
365
+ "00:42:56.596 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - Created task run 'md-7' for task 'md'\n"
366
+ ]
367
+ },
368
+ "metadata": {},
369
+ "output_type": "display_data"
370
+ },
371
+ {
372
+ "data": {
373
+ "text/html": [
374
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:56.604 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - Created task run 'md-8' for task 'md'\n",
375
+ "</pre>\n"
376
+ ],
377
+ "text/plain": [
378
+ "00:42:56.604 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - Created task run 'md-8' for task 'md'\n"
379
+ ]
380
+ },
381
+ "metadata": {},
382
+ "output_type": "display_data"
383
+ },
384
+ {
385
+ "data": {
386
+ "text/html": [
387
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:56.617 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - Created task run 'md-5' for task 'md'\n",
388
+ "</pre>\n"
389
+ ],
390
+ "text/plain": [
391
+ "00:42:56.617 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - Created task run 'md-5' for task 'md'\n"
392
+ ]
393
+ },
394
+ "metadata": {},
395
+ "output_type": "display_data"
396
+ },
397
+ {
398
+ "data": {
399
+ "text/html": [
400
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:56.622 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - Created task run 'md-4' for task 'md'\n",
401
+ "</pre>\n"
402
+ ],
403
+ "text/plain": [
404
+ "00:42:56.622 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - Created task run 'md-4' for task 'md'\n"
405
+ ]
406
+ },
407
+ "metadata": {},
408
+ "output_type": "display_data"
409
+ },
410
+ {
411
+ "data": {
412
+ "text/html": [
413
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:57.147 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - Submitted task run 'md-0' for execution.\n",
414
+ "</pre>\n"
415
+ ],
416
+ "text/plain": [
417
+ "00:42:57.147 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - Submitted task run 'md-0' for execution.\n"
418
+ ]
419
+ },
420
+ "metadata": {},
421
+ "output_type": "display_data"
422
+ },
423
+ {
424
+ "data": {
425
+ "text/html": [
426
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:57.160 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - Submitted task run 'md-5' for execution.\n",
427
+ "</pre>\n"
428
+ ],
429
+ "text/plain": [
430
+ "00:42:57.160 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - Submitted task run 'md-5' for execution.\n"
431
+ ]
432
+ },
433
+ "metadata": {},
434
+ "output_type": "display_data"
435
+ },
436
+ {
437
+ "data": {
438
+ "text/html": [
439
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:57.176 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - Submitted task run 'md-3' for execution.\n",
440
+ "</pre>\n"
441
+ ],
442
+ "text/plain": [
443
+ "00:42:57.176 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - Submitted task run 'md-3' for execution.\n"
444
+ ]
445
+ },
446
+ "metadata": {},
447
+ "output_type": "display_data"
448
+ },
449
+ {
450
+ "data": {
451
+ "text/html": [
452
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:57.196 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - Submitted task run 'md-2' for execution.\n",
453
+ "</pre>\n"
454
+ ],
455
+ "text/plain": [
456
+ "00:42:57.196 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - Submitted task run 'md-2' for execution.\n"
457
+ ]
458
+ },
459
+ "metadata": {},
460
+ "output_type": "display_data"
461
+ },
462
+ {
463
+ "data": {
464
+ "text/html": [
465
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:57.213 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - Submitted task run 'md-6' for execution.\n",
466
+ "</pre>\n"
467
+ ],
468
+ "text/plain": [
469
+ "00:42:57.213 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - Submitted task run 'md-6' for execution.\n"
470
+ ]
471
+ },
472
+ "metadata": {},
473
+ "output_type": "display_data"
474
+ },
475
+ {
476
+ "data": {
477
+ "text/html": [
478
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:57.230 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - Submitted task run 'md-8' for execution.\n",
479
+ "</pre>\n"
480
+ ],
481
+ "text/plain": [
482
+ "00:42:57.230 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - Submitted task run 'md-8' for execution.\n"
483
+ ]
484
+ },
485
+ "metadata": {},
486
+ "output_type": "display_data"
487
+ },
488
+ {
489
+ "data": {
490
+ "text/html": [
491
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:57.245 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - Submitted task run 'md-1' for execution.\n",
492
+ "</pre>\n"
493
+ ],
494
+ "text/plain": [
495
+ "00:42:57.245 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - Submitted task run 'md-1' for execution.\n"
496
+ ]
497
+ },
498
+ "metadata": {},
499
+ "output_type": "display_data"
500
+ },
501
+ {
502
+ "data": {
503
+ "text/html": [
504
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:57.256 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - Submitted task run 'md-4' for execution.\n",
505
+ "</pre>\n"
506
+ ],
507
+ "text/plain": [
508
+ "00:42:57.256 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - Submitted task run 'md-4' for execution.\n"
509
+ ]
510
+ },
511
+ "metadata": {},
512
+ "output_type": "display_data"
513
+ },
514
+ {
515
+ "data": {
516
+ "text/html": [
517
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">00:42:57.607 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - Submitted task run 'md-7' for execution.\n",
518
+ "</pre>\n"
519
+ ],
520
+ "text/plain": [
521
+ "00:42:57.607 | \u001b[36mINFO\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - Submitted task run 'md-7' for execution.\n"
522
+ ]
523
+ },
524
+ "metadata": {},
525
+ "output_type": "display_data"
526
+ },
527
+ {
528
+ "data": {
529
+ "text/html": [
530
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">07:01:09.228 | <span style=\"color: #d70000; text-decoration-color: #d70000\">ERROR</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - Encountered exception during execution:\n",
531
+ "Traceback (most recent call last):\n",
532
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/engine.py\", line 867, in orchestrate_flow_run\n",
533
+ " result = await flow_call.aresult()\n",
534
+ " ^^^^^^^^^^^^^^^^^^^^^^^^^\n",
535
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py\", line 327, in aresult\n",
536
+ " return await asyncio.wrap_future(self.future)\n",
537
+ " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
538
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py\", line 352, in _run_sync\n",
539
+ " result = self.fn(*self.args, **self.kwargs)\n",
540
+ " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
541
+ " File \"/tmp/ipykernel_699788/2249827672.py\", line 26, in combustion\n",
542
+ " return [future.result() for future in futures]\n",
543
+ " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
544
+ " File \"/tmp/ipykernel_699788/2249827672.py\", line 26, in &lt;listcomp&gt;\n",
545
+ " return [future.result() for future in futures]\n",
546
+ " ^^^^^^^^^^^^^^^\n",
547
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/futures.py\", line 228, in result\n",
548
+ " return from_sync.call_soon_in_loop_thread(result).result()\n",
549
+ " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
550
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py\", line 318, in result\n",
551
+ " return self.future.result(timeout=timeout)\n",
552
+ " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
553
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py\", line 188, in result\n",
554
+ " return self.__get_result()\n",
555
+ " ^^^^^^^^^^^^^^^^^^^\n",
556
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/concurrent/futures/_base.py\", line 401, in __get_result\n",
557
+ " raise self._exception\n",
558
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py\", line 389, in _run_async\n",
559
+ " result = await coro\n",
560
+ " ^^^^^^^^^^\n",
561
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/futures.py\", line 237, in _result\n",
562
+ " return await final_state.result(raise_on_failure=raise_on_failure, fetch=True)\n",
563
+ " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
564
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/states.py\", line 91, in _get_state_result\n",
565
+ " raise await get_state_exception(state)\n",
566
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect_dask/task_runners.py\", line 311, in wait\n",
567
+ " return await future.result(timeout=timeout)\n",
568
+ " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
569
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/distributed/client.py\", line 336, in _result\n",
570
+ " raise exc.with_traceback(tb)\n",
571
+ "distributed.scheduler.KilledWorker: Attempted to run task 'md-1-0dcbdd94b5ca46f7bdd90131bae6f396-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.69.212:43073. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see <span style=\"color: #0000ff; text-decoration-color: #0000ff\">https://distributed.dask.org/en/stable/killed.html.</span>\n",
572
+ "</pre>\n"
573
+ ],
574
+ "text/plain": [
575
+ "07:01:09.228 | \u001b[38;5;160mERROR\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - Encountered exception during execution:\n",
576
+ "Traceback (most recent call last):\n",
577
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/engine.py\", line 867, in orchestrate_flow_run\n",
578
+ " result = await flow_call.aresult()\n",
579
+ " ^^^^^^^^^^^^^^^^^^^^^^^^^\n",
580
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py\", line 327, in aresult\n",
581
+ " return await asyncio.wrap_future(self.future)\n",
582
+ " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
583
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py\", line 352, in _run_sync\n",
584
+ " result = self.fn(*self.args, **self.kwargs)\n",
585
+ " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
586
+ " File \"/tmp/ipykernel_699788/2249827672.py\", line 26, in combustion\n",
587
+ " return [future.result() for future in futures]\n",
588
+ " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
589
+ " File \"/tmp/ipykernel_699788/2249827672.py\", line 26, in <listcomp>\n",
590
+ " return [future.result() for future in futures]\n",
591
+ " ^^^^^^^^^^^^^^^\n",
592
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/futures.py\", line 228, in result\n",
593
+ " return from_sync.call_soon_in_loop_thread(result).result()\n",
594
+ " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
595
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py\", line 318, in result\n",
596
+ " return self.future.result(timeout=timeout)\n",
597
+ " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
598
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py\", line 188, in result\n",
599
+ " return self.__get_result()\n",
600
+ " ^^^^^^^^^^^^^^^^^^^\n",
601
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/concurrent/futures/_base.py\", line 401, in __get_result\n",
602
+ " raise self._exception\n",
603
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py\", line 389, in _run_async\n",
604
+ " result = await coro\n",
605
+ " ^^^^^^^^^^\n",
606
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/futures.py\", line 237, in _result\n",
607
+ " return await final_state.result(raise_on_failure=raise_on_failure, fetch=True)\n",
608
+ " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
609
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/states.py\", line 91, in _get_state_result\n",
610
+ " raise await get_state_exception(state)\n",
611
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect_dask/task_runners.py\", line 311, in wait\n",
612
+ " return await future.result(timeout=timeout)\n",
613
+ " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
614
+ " File \"/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/distributed/client.py\", line 336, in _result\n",
615
+ " raise exc.with_traceback(tb)\n",
616
+ "distributed.scheduler.KilledWorker: Attempted to run task 'md-1-0dcbdd94b5ca46f7bdd90131bae6f396-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.69.212:43073. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see \u001b[94mhttps://distributed.dask.org/en/stable/killed.html.\u001b[0m\n"
617
+ ]
618
+ },
619
+ "metadata": {},
620
+ "output_type": "display_data"
621
+ },
622
+ {
623
+ "data": {
624
+ "text/html": [
625
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">07:01:21.168 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Task run 'md-1' - Crash detected! Execution was interrupted by an unexpected exception: KilledWorker: Attempted to run task 'md-1-0dcbdd94b5ca46f7bdd90131bae6f396-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.69.212:43073. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see <span style=\"color: #0000ff; text-decoration-color: #0000ff\">https://distributed.dask.org/en/stable/killed.html.</span>\n",
626
+ "</pre>\n"
627
+ ],
628
+ "text/plain": [
629
+ "07:01:21.168 | \u001b[36mINFO\u001b[0m | Task run 'md-1' - Crash detected! Execution was interrupted by an unexpected exception: KilledWorker: Attempted to run task 'md-1-0dcbdd94b5ca46f7bdd90131bae6f396-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.69.212:43073. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see \u001b[94mhttps://distributed.dask.org/en/stable/killed.html.\u001b[0m\n"
630
+ ]
631
+ },
632
+ "metadata": {},
633
+ "output_type": "display_data"
634
+ },
635
+ {
636
+ "data": {
637
+ "text/html": [
638
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">07:01:21.625 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Task run 'md-2' - Crash detected! Execution was interrupted by an unexpected exception: KilledWorker: Attempted to run task 'md-2-1d496defb0564717b104fdbe3301ba7f-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.82.214:37797. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see <span style=\"color: #0000ff; text-decoration-color: #0000ff\">https://distributed.dask.org/en/stable/killed.html.</span>\n",
639
+ "</pre>\n"
640
+ ],
641
+ "text/plain": [
642
+ "07:01:21.625 | \u001b[36mINFO\u001b[0m | Task run 'md-2' - Crash detected! Execution was interrupted by an unexpected exception: KilledWorker: Attempted to run task 'md-2-1d496defb0564717b104fdbe3301ba7f-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.82.214:37797. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see \u001b[94mhttps://distributed.dask.org/en/stable/killed.html.\u001b[0m\n"
643
+ ]
644
+ },
645
+ "metadata": {},
646
+ "output_type": "display_data"
647
+ },
648
+ {
649
+ "data": {
650
+ "text/html": [
651
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">07:01:22.243 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Task run 'md-5' - Crash detected! Execution was interrupted by an unexpected exception: KilledWorker: Attempted to run task 'md-5-957565e01bfc458f8802ff6ed33304d4-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.82.215:36457. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see <span style=\"color: #0000ff; text-decoration-color: #0000ff\">https://distributed.dask.org/en/stable/killed.html.</span>\n",
652
+ "</pre>\n"
653
+ ],
654
+ "text/plain": [
655
+ "07:01:22.243 | \u001b[36mINFO\u001b[0m | Task run 'md-5' - Crash detected! Execution was interrupted by an unexpected exception: KilledWorker: Attempted to run task 'md-5-957565e01bfc458f8802ff6ed33304d4-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.82.215:36457. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see \u001b[94mhttps://distributed.dask.org/en/stable/killed.html.\u001b[0m\n"
656
+ ]
657
+ },
658
+ "metadata": {},
659
+ "output_type": "display_data"
660
+ },
661
+ {
662
+ "data": {
663
+ "text/html": [
664
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">07:01:23.569 | <span style=\"color: #008080; text-decoration-color: #008080\">INFO</span> | Task run 'md-7' - Crash detected! Execution was interrupted by an unexpected exception: KilledWorker: Attempted to run task 'md-7-bdc25b05525842028653d9bf609a5611-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.65.191:43365. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see <span style=\"color: #0000ff; text-decoration-color: #0000ff\">https://distributed.dask.org/en/stable/killed.html.</span>\n",
665
+ "</pre>\n"
666
+ ],
667
+ "text/plain": [
668
+ "07:01:23.569 | \u001b[36mINFO\u001b[0m | Task run 'md-7' - Crash detected! Execution was interrupted by an unexpected exception: KilledWorker: Attempted to run task 'md-7-bdc25b05525842028653d9bf609a5611-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.65.191:43365. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see \u001b[94mhttps://distributed.dask.org/en/stable/killed.html.\u001b[0m\n"
669
+ ]
670
+ },
671
+ "metadata": {},
672
+ "output_type": "display_data"
673
+ },
674
+ {
675
+ "data": {
676
+ "text/html": [
677
+ "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">07:01:24.000 | <span style=\"color: #d70000; text-decoration-color: #d70000\">ERROR</span> | Flow run<span style=\"color: #800080; text-decoration-color: #800080\"> 'sly-yak'</span> - Finished in state <span style=\"color: #d70000; text-decoration-color: #d70000\">Failed</span>(\"Flow run encountered an exception. KilledWorker: Attempted to run task 'md-1-0dcbdd94b5ca46f7bdd90131bae6f396-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.69.212:43073. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see <span style=\"color: #0000ff; text-decoration-color: #0000ff\">https://distributed.dask.org/en/stable/killed.html.</span>\")\n",
678
+ "</pre>\n"
679
+ ],
680
+ "text/plain": [
681
+ "07:01:24.000 | \u001b[38;5;160mERROR\u001b[0m | Flow run\u001b[35m 'sly-yak'\u001b[0m - Finished in state \u001b[38;5;160mFailed\u001b[0m(\"Flow run encountered an exception. KilledWorker: Attempted to run task 'md-1-0dcbdd94b5ca46f7bdd90131bae6f396-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.69.212:43073. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see \u001b[94mhttps://distributed.dask.org/en/stable/killed.html.\u001b[0m\")\n"
682
+ ]
683
+ },
684
+ "metadata": {},
685
+ "output_type": "display_data"
686
+ },
687
+ {
688
+ "ename": "KilledWorker",
689
+ "evalue": "Attempted to run task 'md-1-0dcbdd94b5ca46f7bdd90131bae6f396-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.69.212:43073. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see https://distributed.dask.org/en/stable/killed.html.",
690
+ "output_type": "error",
691
+ "traceback": [
692
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
693
+ "\u001b[0;31mKilledWorker\u001b[0m Traceback (most recent call last)",
694
+ "Cell \u001b[0;32mIn[5], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[43mcombustion\u001b[49m\u001b[43m(\u001b[49m\u001b[43matoms\u001b[49m\u001b[43m)\u001b[49m\n",
695
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/flows.py:1224\u001b[0m, in \u001b[0;36mFlow.__call__\u001b[0;34m(self, return_state, wait_for, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1219\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m task_viz_tracker:\n\u001b[1;32m 1220\u001b[0m \u001b[38;5;66;03m# this is a subflow, for now return a single task and do not go further\u001b[39;00m\n\u001b[1;32m 1221\u001b[0m \u001b[38;5;66;03m# we can add support for exploring subflows for tasks in the future.\u001b[39;00m\n\u001b[1;32m 1222\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m track_viz_task(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39misasync, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mname, parameters)\n\u001b[0;32m-> 1224\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43menter_flow_run_engine_from_flow_call\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1225\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1226\u001b[0m \u001b[43m \u001b[49m\u001b[43mparameters\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1227\u001b[0m \u001b[43m \u001b[49m\u001b[43mwait_for\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mwait_for\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1228\u001b[0m \u001b[43m \u001b[49m\u001b[43mreturn_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_type\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1229\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
696
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/engine.py:297\u001b[0m, in \u001b[0;36menter_flow_run_engine_from_flow_call\u001b[0;34m(flow, parameters, wait_for, return_type)\u001b[0m\n\u001b[1;32m 290\u001b[0m retval \u001b[38;5;241m=\u001b[39m from_async\u001b[38;5;241m.\u001b[39mwait_for_call_in_loop_thread(\n\u001b[1;32m 291\u001b[0m begin_run,\n\u001b[1;32m 292\u001b[0m done_callbacks\u001b[38;5;241m=\u001b[39mdone_callbacks,\n\u001b[1;32m 293\u001b[0m contexts\u001b[38;5;241m=\u001b[39mcontexts,\n\u001b[1;32m 294\u001b[0m )\n\u001b[1;32m 296\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 297\u001b[0m retval \u001b[38;5;241m=\u001b[39m \u001b[43mfrom_sync\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait_for_call_in_loop_thread\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 298\u001b[0m \u001b[43m \u001b[49m\u001b[43mbegin_run\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 299\u001b[0m \u001b[43m \u001b[49m\u001b[43mdone_callbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdone_callbacks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 300\u001b[0m \u001b[43m \u001b[49m\u001b[43mcontexts\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontexts\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 301\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 303\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m retval\n",
697
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/api.py:243\u001b[0m, in \u001b[0;36mfrom_sync.wait_for_call_in_loop_thread\u001b[0;34m(_from_sync__call, timeout, done_callbacks, contexts)\u001b[0m\n\u001b[1;32m 241\u001b[0m stack\u001b[38;5;241m.\u001b[39menter_context(context)\n\u001b[1;32m 242\u001b[0m waiter\u001b[38;5;241m.\u001b[39mwait()\n\u001b[0;32m--> 243\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mcall\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
698
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py:318\u001b[0m, in \u001b[0;36mCall.result\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 312\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mresult\u001b[39m(\u001b[38;5;28mself\u001b[39m, timeout: Optional[\u001b[38;5;28mfloat\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m T:\n\u001b[1;32m 313\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 314\u001b[0m \u001b[38;5;124;03m Wait for the result of the call.\u001b[39;00m\n\u001b[1;32m 315\u001b[0m \n\u001b[1;32m 316\u001b[0m \u001b[38;5;124;03m Not safe for use from asynchronous contexts.\u001b[39;00m\n\u001b[1;32m 317\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 318\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfuture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n",
699
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py:179\u001b[0m, in \u001b[0;36mFuture.result\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 177\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m CancelledError()\n\u001b[1;32m 178\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_state \u001b[38;5;241m==\u001b[39m FINISHED:\n\u001b[0;32m--> 179\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m__get_result\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 181\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_condition\u001b[38;5;241m.\u001b[39mwait(timeout)\n\u001b[1;32m 183\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_state \u001b[38;5;129;01min\u001b[39;00m [CANCELLED, CANCELLED_AND_NOTIFIED]:\n\u001b[1;32m 184\u001b[0m \u001b[38;5;66;03m# Raise Prefect cancelled error instead of\u001b[39;00m\n\u001b[1;32m 185\u001b[0m \u001b[38;5;66;03m# `concurrent.futures._base.CancelledError`\u001b[39;00m\n",
700
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/concurrent/futures/_base.py:401\u001b[0m, in \u001b[0;36mFuture.__get_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 399\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_exception:\n\u001b[1;32m 400\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 401\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_exception\n\u001b[1;32m 402\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 403\u001b[0m \u001b[38;5;66;03m# Break a reference cycle with the exception in self._exception\u001b[39;00m\n\u001b[1;32m 404\u001b[0m \u001b[38;5;28mself\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
701
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py:389\u001b[0m, in \u001b[0;36mCall._run_async\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m 387\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfuture\u001b[38;5;241m.\u001b[39menforce_async_deadline() \u001b[38;5;28;01mas\u001b[39;00m cancel_scope:\n\u001b[1;32m 388\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 389\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m coro\n\u001b[1;32m 390\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 391\u001b[0m \u001b[38;5;66;03m# Forget this call's arguments in order to free up any memory\u001b[39;00m\n\u001b[1;32m 392\u001b[0m \u001b[38;5;66;03m# that may be referenced by them; after a call has happened,\u001b[39;00m\n\u001b[1;32m 393\u001b[0m \u001b[38;5;66;03m# there's no need to keep a reference to them\u001b[39;00m\n\u001b[1;32m 394\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
702
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/client/utilities.py:78\u001b[0m, in \u001b[0;36minject_client.<locals>.with_injected_client\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 76\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m context \u001b[38;5;28;01mas\u001b[39;00m new_client:\n\u001b[1;32m 77\u001b[0m kwargs\u001b[38;5;241m.\u001b[39msetdefault(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mclient\u001b[39m\u001b[38;5;124m\"\u001b[39m, new_client \u001b[38;5;129;01mor\u001b[39;00m client)\n\u001b[0;32m---> 78\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m fn(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
703
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/engine.py:400\u001b[0m, in \u001b[0;36mcreate_then_begin_flow_run\u001b[0;34m(flow, parameters, wait_for, return_type, client, user_thread)\u001b[0m\n\u001b[1;32m 398\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state\n\u001b[1;32m 399\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m return_type \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mresult\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m--> 400\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m state\u001b[38;5;241m.\u001b[39mresult(fetch\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 401\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 402\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInvalid return type for flow engine \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mreturn_type\u001b[38;5;132;01m!r}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
704
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/states.py:91\u001b[0m, in \u001b[0;36m_get_state_result\u001b[0;34m(state, raise_on_failure)\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m UnfinishedRun(\n\u001b[1;32m 85\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRun is in \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mstate\u001b[38;5;241m.\u001b[39mtype\u001b[38;5;241m.\u001b[39mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m state, its result is not available.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 86\u001b[0m )\n\u001b[1;32m 88\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m raise_on_failure \u001b[38;5;129;01mand\u001b[39;00m (\n\u001b[1;32m 89\u001b[0m state\u001b[38;5;241m.\u001b[39mis_crashed() \u001b[38;5;129;01mor\u001b[39;00m state\u001b[38;5;241m.\u001b[39mis_failed() \u001b[38;5;129;01mor\u001b[39;00m state\u001b[38;5;241m.\u001b[39mis_cancelled()\n\u001b[1;32m 90\u001b[0m ):\n\u001b[0;32m---> 91\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m get_state_exception(state)\n\u001b[1;32m 93\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(state\u001b[38;5;241m.\u001b[39mdata, DataDocument):\n\u001b[1;32m 94\u001b[0m result \u001b[38;5;241m=\u001b[39m result_from_state_with_data_document(\n\u001b[1;32m 95\u001b[0m state, raise_on_failure\u001b[38;5;241m=\u001b[39mraise_on_failure\n\u001b[1;32m 96\u001b[0m )\n",
705
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/engine.py:867\u001b[0m, in \u001b[0;36morchestrate_flow_run\u001b[0;34m(flow, flow_run, parameters, wait_for, interruptible, client, partial_flow_run_context, user_thread)\u001b[0m\n\u001b[1;32m 862\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 863\u001b[0m from_async\u001b[38;5;241m.\u001b[39mcall_soon_in_new_thread(\n\u001b[1;32m 864\u001b[0m flow_call, timeout\u001b[38;5;241m=\u001b[39mflow\u001b[38;5;241m.\u001b[39mtimeout_seconds\n\u001b[1;32m 865\u001b[0m )\n\u001b[0;32m--> 867\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m flow_call\u001b[38;5;241m.\u001b[39maresult()\n\u001b[1;32m 869\u001b[0m waited_for_task_runs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m wait_for_task_runs_and_report_crashes(\n\u001b[1;32m 870\u001b[0m flow_run_context\u001b[38;5;241m.\u001b[39mtask_run_futures, client\u001b[38;5;241m=\u001b[39mclient\n\u001b[1;32m 871\u001b[0m )\n\u001b[1;32m 872\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m PausedRun \u001b[38;5;28;01mas\u001b[39;00m exc:\n\u001b[1;32m 873\u001b[0m \u001b[38;5;66;03m# could get raised either via utility or by returning Paused from a task run\u001b[39;00m\n\u001b[1;32m 874\u001b[0m \u001b[38;5;66;03m# if a task run pauses, we set its state as the flow's state\u001b[39;00m\n\u001b[1;32m 875\u001b[0m \u001b[38;5;66;03m# to preserve reschedule and timeout behavior\u001b[39;00m\n",
706
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py:327\u001b[0m, in \u001b[0;36mCall.aresult\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 321\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 322\u001b[0m \u001b[38;5;124;03mWait for the result of the call.\u001b[39;00m\n\u001b[1;32m 323\u001b[0m \n\u001b[1;32m 324\u001b[0m \u001b[38;5;124;03mFor use from asynchronous contexts.\u001b[39;00m\n\u001b[1;32m 325\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 326\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 327\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mwrap_future(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfuture)\n\u001b[1;32m 328\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m asyncio\u001b[38;5;241m.\u001b[39mCancelledError \u001b[38;5;28;01mas\u001b[39;00m exc:\n\u001b[1;32m 329\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m CancelledError() \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mexc\u001b[39;00m\n",
707
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py:352\u001b[0m, in \u001b[0;36mCall._run_sync\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m 350\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfuture\u001b[38;5;241m.\u001b[39menforce_sync_deadline() \u001b[38;5;28;01mas\u001b[39;00m cancel_scope:\n\u001b[1;32m 351\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 352\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 353\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 354\u001b[0m \u001b[38;5;66;03m# Forget this call's arguments in order to free up any memory\u001b[39;00m\n\u001b[1;32m 355\u001b[0m \u001b[38;5;66;03m# that may be referenced by them; after a call has happened,\u001b[39;00m\n\u001b[1;32m 356\u001b[0m \u001b[38;5;66;03m# there's no need to keep a reference to them\u001b[39;00m\n\u001b[1;32m 357\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
708
+ "Cell \u001b[0;32mIn[4], line 26\u001b[0m, in \u001b[0;36mcombustion\u001b[0;34m(atoms)\u001b[0m\n\u001b[1;32m 6\u001b[0m future \u001b[38;5;241m=\u001b[39m MD\u001b[38;5;241m.\u001b[39msubmit(\n\u001b[1;32m 7\u001b[0m atoms\u001b[38;5;241m=\u001b[39matoms,\n\u001b[1;32m 8\u001b[0m calculator_name\u001b[38;5;241m=\u001b[39mmodel,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 21\u001b[0m restart\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 22\u001b[0m )\n\u001b[1;32m 24\u001b[0m futures\u001b[38;5;241m.\u001b[39mappend(future)\n\u001b[0;32m---> 26\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m[\u001b[49m\u001b[43mfuture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mfuture\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mfutures\u001b[49m\u001b[43m]\u001b[49m\n",
709
+ "Cell \u001b[0;32mIn[4], line 26\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 6\u001b[0m future \u001b[38;5;241m=\u001b[39m MD\u001b[38;5;241m.\u001b[39msubmit(\n\u001b[1;32m 7\u001b[0m atoms\u001b[38;5;241m=\u001b[39matoms,\n\u001b[1;32m 8\u001b[0m calculator_name\u001b[38;5;241m=\u001b[39mmodel,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 21\u001b[0m restart\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 22\u001b[0m )\n\u001b[1;32m 24\u001b[0m futures\u001b[38;5;241m.\u001b[39mappend(future)\n\u001b[0;32m---> 26\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m [\u001b[43mfuture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m future \u001b[38;5;129;01min\u001b[39;00m futures]\n",
710
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/futures.py:228\u001b[0m, in \u001b[0;36mPrefectFuture.result\u001b[0;34m(self, timeout, raise_on_failure)\u001b[0m\n\u001b[1;32m 226\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m from_async\u001b[38;5;241m.\u001b[39mcall_soon_in_loop_thread(result)\u001b[38;5;241m.\u001b[39maresult()\n\u001b[1;32m 227\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 228\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfrom_sync\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcall_soon_in_loop_thread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresult\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
711
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py:318\u001b[0m, in \u001b[0;36mCall.result\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 312\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mresult\u001b[39m(\u001b[38;5;28mself\u001b[39m, timeout: Optional[\u001b[38;5;28mfloat\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m T:\n\u001b[1;32m 313\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 314\u001b[0m \u001b[38;5;124;03m Wait for the result of the call.\u001b[39;00m\n\u001b[1;32m 315\u001b[0m \n\u001b[1;32m 316\u001b[0m \u001b[38;5;124;03m Not safe for use from asynchronous contexts.\u001b[39;00m\n\u001b[1;32m 317\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 318\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfuture\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n",
712
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py:188\u001b[0m, in \u001b[0;36mFuture.result\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 186\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m CancelledError()\n\u001b[1;32m 187\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_state \u001b[38;5;241m==\u001b[39m FINISHED:\n\u001b[0;32m--> 188\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m__get_result\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 189\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 190\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTimeoutError\u001b[39;00m()\n",
713
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/concurrent/futures/_base.py:401\u001b[0m, in \u001b[0;36mFuture.__get_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 399\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_exception:\n\u001b[1;32m 400\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 401\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_exception\n\u001b[1;32m 402\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 403\u001b[0m \u001b[38;5;66;03m# Break a reference cycle with the exception in self._exception\u001b[39;00m\n\u001b[1;32m 404\u001b[0m \u001b[38;5;28mself\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
714
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/_internal/concurrency/calls.py:389\u001b[0m, in \u001b[0;36mCall._run_async\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m 387\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfuture\u001b[38;5;241m.\u001b[39menforce_async_deadline() \u001b[38;5;28;01mas\u001b[39;00m cancel_scope:\n\u001b[1;32m 388\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 389\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m coro\n\u001b[1;32m 390\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 391\u001b[0m \u001b[38;5;66;03m# Forget this call's arguments in order to free up any memory\u001b[39;00m\n\u001b[1;32m 392\u001b[0m \u001b[38;5;66;03m# that may be referenced by them; after a call has happened,\u001b[39;00m\n\u001b[1;32m 393\u001b[0m \u001b[38;5;66;03m# there's no need to keep a reference to them\u001b[39;00m\n\u001b[1;32m 394\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
715
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/futures.py:237\u001b[0m, in \u001b[0;36mPrefectFuture._result\u001b[0;34m(self, timeout, raise_on_failure)\u001b[0m\n\u001b[1;32m 235\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m final_state:\n\u001b[1;32m 236\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTimeoutError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCall timed out before task finished.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 237\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m final_state\u001b[38;5;241m.\u001b[39mresult(raise_on_failure\u001b[38;5;241m=\u001b[39mraise_on_failure, fetch\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
716
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect/states.py:91\u001b[0m, in \u001b[0;36m_get_state_result\u001b[0;34m(state, raise_on_failure)\u001b[0m\n\u001b[1;32m 84\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m UnfinishedRun(\n\u001b[1;32m 85\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRun is in \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mstate\u001b[38;5;241m.\u001b[39mtype\u001b[38;5;241m.\u001b[39mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m state, its result is not available.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 86\u001b[0m )\n\u001b[1;32m 88\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m raise_on_failure \u001b[38;5;129;01mand\u001b[39;00m (\n\u001b[1;32m 89\u001b[0m state\u001b[38;5;241m.\u001b[39mis_crashed() \u001b[38;5;129;01mor\u001b[39;00m state\u001b[38;5;241m.\u001b[39mis_failed() \u001b[38;5;129;01mor\u001b[39;00m state\u001b[38;5;241m.\u001b[39mis_cancelled()\n\u001b[1;32m 90\u001b[0m ):\n\u001b[0;32m---> 91\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m get_state_exception(state)\n\u001b[1;32m 93\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(state\u001b[38;5;241m.\u001b[39mdata, DataDocument):\n\u001b[1;32m 94\u001b[0m result \u001b[38;5;241m=\u001b[39m result_from_state_with_data_document(\n\u001b[1;32m 95\u001b[0m state, raise_on_failure\u001b[38;5;241m=\u001b[39mraise_on_failure\n\u001b[1;32m 96\u001b[0m )\n",
717
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/prefect_dask/task_runners.py:311\u001b[0m, in \u001b[0;36mDaskTaskRunner.wait\u001b[0;34m(self, key, timeout)\u001b[0m\n\u001b[1;32m 309\u001b[0m future \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_dask_future(key)\n\u001b[1;32m 310\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 311\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m future\u001b[38;5;241m.\u001b[39mresult(timeout\u001b[38;5;241m=\u001b[39mtimeout)\n\u001b[1;32m 312\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m distributed\u001b[38;5;241m.\u001b[39mTimeoutError:\n\u001b[1;32m 313\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n",
718
+ "File \u001b[0;32m/pscratch/sd/c/cyrusyc/.conda/mlip-arena/lib/python3.11/site-packages/distributed/client.py:336\u001b[0m, in \u001b[0;36mFuture._result\u001b[0;34m(self, raiseit)\u001b[0m\n\u001b[1;32m 334\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m raiseit:\n\u001b[1;32m 335\u001b[0m typ, exc, tb \u001b[38;5;241m=\u001b[39m exc\n\u001b[0;32m--> 336\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m exc\u001b[38;5;241m.\u001b[39mwith_traceback(tb)\n\u001b[1;32m 337\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 338\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m exc\n",
719
+ "\u001b[0;31mKilledWorker\u001b[0m: Attempted to run task 'md-1-0dcbdd94b5ca46f7bdd90131bae6f396-1' on 4 different workers, but all those workers died while running it. The last worker that attempt to run the task was tcp://128.55.69.212:43073. Inspecting worker logs is often a good next step to diagnose what went wrong. For more information see https://distributed.dask.org/en/stable/killed.html."
720
+ ]
721
+ }
722
+ ],
723
+ "source": [
724
+ "results = combustion(atoms)"
725
+ ]
726
+ },
727
+ {
728
+ "cell_type": "code",
729
+ "execution_count": null,
730
+ "metadata": {},
731
+ "outputs": [],
732
+ "source": []
733
+ }
734
+ ],
735
+ "metadata": {
736
+ "kernelspec": {
737
+ "display_name": "mlip-arena",
738
+ "language": "python",
739
+ "name": "mlip-arena"
740
+ },
741
+ "language_info": {
742
+ "codemirror_mode": {
743
+ "name": "ipython",
744
+ "version": 3
745
+ },
746
+ "file_extension": ".py",
747
+ "mimetype": "text/x-python",
748
+ "name": "python",
749
+ "nbconvert_exporter": "python",
750
+ "pygments_lexer": "ipython3",
751
+ "version": "3.11.8"
752
+ },
753
+ "widgets": {
754
+ "application/vnd.jupyter.widget-state+json": {
755
+ "state": {},
756
+ "version_major": 2,
757
+ "version_minor": 0
758
+ }
759
+ }
760
+ },
761
+ "nbformat": 4,
762
+ "nbformat_minor": 4
763
+ }
mlip_arena/tasks/diatomics/run.ipynb ADDED
The diff for this file is too large to render. See raw diff
 
serve/tasks/combustion.py CHANGED
@@ -74,7 +74,7 @@ for method in df["method"].unique():
74
  )
75
 
76
  fig.update_layout(
77
- title="Hydrogen Combusiton (2H2 + O2 -> 2H2O, 64 units)",
78
  xaxis_title="Timesteps",
79
  yaxis_title="Number of water molecules",
80
  )
@@ -113,7 +113,7 @@ fig.add_trace(
113
  )
114
 
115
  fig.update_layout(
116
- title="Hydrogen Combusiton (2H2 + O2 -> 2H2O, 64 units)",
117
  xaxis_title="Timesteps",
118
  yaxis_title="Temperatures",
119
  yaxis2=dict(
 
74
  )
75
 
76
  fig.update_layout(
77
+ title="Hydrogen Combustion (2H2 + O2 -> 2H2O, 64 units)",
78
  xaxis_title="Timesteps",
79
  yaxis_title="Number of water molecules",
80
  )
 
113
  )
114
 
115
  fig.update_layout(
116
+ title="Hydrogen Combustion (2H2 + O2 -> 2H2O, 64 units)",
117
  xaxis_title="Timesteps",
118
  yaxis_title="Temperatures",
119
  yaxis2=dict(