From f5c0a14ad98c2f206867b01f1d1085d6ac1a60cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Nov 2021 13:58:41 +0000 Subject: [PATCH 01/23] Bump matplotlib from 3.4.3 to 3.5.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 17516db4..63398be4 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ "pytest==6.2.5", "pytest-cov==3.0.0", "FireWorks==1.9.7", - "matplotlib==3.4.3", + "matplotlib==3.5.0", "pydot==1.4.2", ], "dev": ["pre-commit>=2.12.1"], From fb36c1a85b10164dbd1b371c16cf829b5ec7bee8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Nov 2021 13:59:03 +0000 Subject: [PATCH 02/23] Bump furo from 2021.11.15 to 2021.11.16 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 63398be4..8a0f4f61 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ extras_require={ "docs": [ "sphinx==4.3.0", - "furo==2021.11.15", + "furo==2021.11.16", "m2r2==0.3.1", "ipython==7.29.0", "nbsphinx==0.8.7", From 5a4f804e5aa872cdff16bd9acb48272dfc4adb10 Mon Sep 17 00:00:00 2001 From: David Waroquiers Date: Thu, 18 Nov 2021 18:30:53 +0100 Subject: [PATCH 03/23] Added first implementation of a myqueue conversion scheme for jobflow Flows. This would allow jobflow Flows to be run within Myqueue. --- src/jobflow/managers/myqueue.py | 110 ++++++++++++++++++ .../managers/myqueue_scripts/workflow.py | 26 +++++ 2 files changed, 136 insertions(+) create mode 100644 src/jobflow/managers/myqueue.py create mode 100644 src/jobflow/managers/myqueue_scripts/workflow.py diff --git a/src/jobflow/managers/myqueue.py b/src/jobflow/managers/myqueue.py new file mode 100644 index 00000000..8448fb0c --- /dev/null +++ b/src/jobflow/managers/myqueue.py @@ -0,0 +1,110 @@ +"""Tools for running :obj:`Flow` and :obj:`Job` objects using the Myqueue package. + +Notes +----- +Myqueue heavily relies on the file system. To submit a workflow, one has to run: +mq workflow workflow.py DIRECTORY_PATTERNS +where workflow.py is a python script defining one workflow. For jobflow Flows, the +workflow.py file in myqueue_scripts has to be used. +""" + +from __future__ import annotations + +import json +import os +import typing +from datetime import datetime +from pathlib import Path +from random import randint + +from monty.json import MontyDecoder, MontyEncoder +from monty.os import cd + +if typing.TYPE_CHECKING: + from typing import List, Union + + import jobflow + +from maggma.stores import JSONStore + +from jobflow import JobStore + +__all__ = ["flow_to_myqueue", "run_myqueue_task"] + +FLOW_JSON = "flow.json" +JOB_STORE_JSON = "job_store.json" + + +def flow_to_myqueue( + flow: Union[jobflow.Flow, jobflow.Job, List[jobflow.Job]], +): + """ + Convert a jobflow Flow to myqueue. + + This is basically just dumping the jobflow Flow to a flow.json file. + The flow.json file is then read again when the user wants to submit + the workflow using myqueue. + + Parameters + ---------- + flow + A flow or job. + + """ + from jobflow.core.flow import get_flow + + flow = get_flow(flow) + with open(FLOW_JSON, "w") as f: + json.dump(flow, f, cls=MontyEncoder, indent=2) + + +def run_myqueue_task(uuid): + """ + Run a job in myqueue. + + Parameters + ---------- + uuid + Unique identifier of the job that needs to be executed. + """ + root_dir = Path.cwd() + # First get the jobflow Flow from the flow.json file + with open("flow.json", "r") as f: + flow = json.load(f, cls=MontyDecoder) + + # Get the jobflow Job corresponding to the uuid + job = _get_job(flow, uuid) + job_dir = _get_job_dir(root_dir=root_dir) + + # Initialize the store for output references + job_store_json_path = os.path.join(root_dir, JOB_STORE_JSON) + if not os.path.exists(job_store_json_path): + with open(job_store_json_path, "w") as f: + json.dump([], f) + store = JobStore(JSONStore(job_store_json_path, writable=True)) + store.connect() + + # Run the job + with cd(job_dir): + job.run(store=store) + + +def _get_job(flow, uuid): + myjob = None + for job, _ in flow.iterflow(): + if job.uuid == uuid: + if myjob is not None: + raise RuntimeError(f"Multiple jobs with uuid {uuid}") + myjob = job + return myjob + + +def _get_job_dir(root_dir): + time_now = datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S-%f") + job_dir = root_dir / f"job_{time_now}-{randint(10000, 99999)}" + job_dir.mkdir() + return job_dir + + +if __name__ == "__main__": + pass diff --git a/src/jobflow/managers/myqueue_scripts/workflow.py b/src/jobflow/managers/myqueue_scripts/workflow.py new file mode 100644 index 00000000..206c4154 --- /dev/null +++ b/src/jobflow/managers/myqueue_scripts/workflow.py @@ -0,0 +1,26 @@ +"""Template workflow.py file for running jobflow workflows in myqueue.""" + + +import json + +from monty.json import MontyDecoder +from myqueue.task import task + + +def create_tasks(): + """Create tasks for myqueue.""" + # First reconstruct the jobflow Flow object + with open("flow.json", "r") as f: + flow = json.load(f, cls=MontyDecoder) + + tasks = [] + uuid2task = {} + for job, parents in flow.iterflow(): + deps = [uuid2task[parent_uuid] for parent_uuid in parents] + t = task( + "jobflow.managers.myqueue@run_myqueue_task", args=[job.uuid], deps=deps + ) + uuid2task[job.uuid] = t + tasks.append(t) + + return tasks From 5278c62cce33eba5e5b3ce10c1070b0ee92d91cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 18:20:41 +0000 Subject: [PATCH 04/23] Bump furo from 2021.11.16 to 2021.11.23 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8a0f4f61..bf12d7ed 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ extras_require={ "docs": [ "sphinx==4.3.0", - "furo==2021.11.16", + "furo==2021.11.23", "m2r2==0.3.1", "ipython==7.29.0", "nbsphinx==0.8.7", From c6e1d4b5ddd86c976c7539a8415be8e8596f953f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 18:21:09 +0000 Subject: [PATCH 05/23] Bump maggma from 0.32.1 to 0.32.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 23bb4f4e..fe39b83d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ monty==2021.8.17 networkx==2.6.3 pydash==5.1.0 -maggma==0.32.1 +maggma==0.32.3 pydantic==1.8.2 PyYAML==6.0 From f31d7573f38d8d5446275e314f456087b56a37c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 18:21:48 +0000 Subject: [PATCH 06/23] Bump ipython from 7.29.0 to 7.30.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bf12d7ed..1b2216c5 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ "sphinx==4.3.0", "furo==2021.11.23", "m2r2==0.3.1", - "ipython==7.29.0", + "ipython==7.30.0", "nbsphinx==0.8.7", "nbsphinx-link==1.3.0", "FireWorks==1.9.7", From 96b5d6a4c31c21bdfab945f2e4a33b011286c567 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 18:29:04 +0000 Subject: [PATCH 07/23] Bump sphinx from 4.3.0 to 4.3.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1b2216c5..a0fdce86 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ ], extras_require={ "docs": [ - "sphinx==4.3.0", + "sphinx==4.3.1", "furo==2021.11.23", "m2r2==0.3.1", "ipython==7.30.0", From 2178d9c0f925276db82cf936ad8c247e44577597 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Dec 2021 18:21:10 +0000 Subject: [PATCH 08/23] Bump fireworks from 1.9.7 to 1.9.8 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a0fdce86..d2804f00 100644 --- a/setup.py +++ b/setup.py @@ -39,13 +39,13 @@ "ipython==7.30.0", "nbsphinx==0.8.7", "nbsphinx-link==1.3.0", - "FireWorks==1.9.7", + "FireWorks==1.9.8", "autodoc_pydantic==1.5.1", ], "tests": [ "pytest==6.2.5", "pytest-cov==3.0.0", - "FireWorks==1.9.7", + "FireWorks==1.9.8", "matplotlib==3.5.0", "pydot==1.4.2", ], From cd0c260753f1aba41e11c479a91d51712ebe7b7c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Dec 2021 18:25:46 +0000 Subject: [PATCH 09/23] Bump ipython from 7.30.0 to 7.30.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d2804f00..0156a7b6 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ "sphinx==4.3.1", "furo==2021.11.23", "m2r2==0.3.1", - "ipython==7.30.0", + "ipython==7.30.1", "nbsphinx==0.8.7", "nbsphinx-link==1.3.0", "FireWorks==1.9.8", From aa2be5b39aa957bb060804047dd963fa8df3d7cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Dec 2021 18:26:25 +0000 Subject: [PATCH 10/23] Bump matplotlib from 3.5.0 to 3.5.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0156a7b6..bbe3cb8f 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ "pytest==6.2.5", "pytest-cov==3.0.0", "FireWorks==1.9.8", - "matplotlib==3.5.0", + "matplotlib==3.5.1", "pydot==1.4.2", ], "dev": ["pre-commit>=2.12.1"], From 312b86b5592576d59c7619a3450fb58f25587c1d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Dec 2021 18:30:16 +0000 Subject: [PATCH 11/23] Bump m2r2 from 0.3.1 to 0.3.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bbe3cb8f..e5880ae1 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ "docs": [ "sphinx==4.3.1", "furo==2021.11.23", - "m2r2==0.3.1", + "m2r2==0.3.2", "ipython==7.30.1", "nbsphinx==0.8.7", "nbsphinx-link==1.3.0", From 9ed307fa5f044e16c0c1fbe1223750beb7fa4b1f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Dec 2021 18:23:18 +0000 Subject: [PATCH 12/23] Bump sphinx from 4.3.1 to 4.3.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e5880ae1..219fa50e 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ ], extras_require={ "docs": [ - "sphinx==4.3.1", + "sphinx==4.3.2", "furo==2021.11.23", "m2r2==0.3.2", "ipython==7.30.1", From 02102a908c2e20b5eaec81518659915e9ea28d58 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jan 2022 18:18:10 +0000 Subject: [PATCH 13/23] Bump nbsphinx from 0.8.7 to 0.8.8 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 219fa50e..b2b8971f 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ "furo==2021.11.23", "m2r2==0.3.2", "ipython==7.30.1", - "nbsphinx==0.8.7", + "nbsphinx==0.8.8", "nbsphinx-link==1.3.0", "FireWorks==1.9.8", "autodoc_pydantic==1.5.1", From b0350acc611f5f8a2a354abca735c83512aec279 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jan 2022 18:18:12 +0000 Subject: [PATCH 14/23] Bump pydantic from 1.8.2 to 1.9.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fe39b83d..79af6a42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,5 @@ monty==2021.8.17 networkx==2.6.3 pydash==5.1.0 maggma==0.32.3 -pydantic==1.8.2 +pydantic==1.9.0 PyYAML==6.0 From d8747aa0be3f53ac7a0d831fe84875cb061a60ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jan 2022 18:18:23 +0000 Subject: [PATCH 15/23] Bump furo from 2021.11.23 to 2022.1.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b2b8971f..b1899c70 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ extras_require={ "docs": [ "sphinx==4.3.2", - "furo==2021.11.23", + "furo==2022.1.2", "m2r2==0.3.2", "ipython==7.30.1", "nbsphinx==0.8.8", From 13813fa97d7b2358bf9462d4a4b8ebe02d82183b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jan 2022 18:25:04 +0000 Subject: [PATCH 16/23] Bump monty from 2021.8.17 to 2021.12.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 79af6a42..1efb1a2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -monty==2021.8.17 +monty==2021.12.1 networkx==2.6.3 pydash==5.1.0 maggma==0.32.3 From 52b073ac90a98655af3e7e816c9a8d12b715e231 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 18:24:26 +0000 Subject: [PATCH 17/23] Bump autodoc-pydantic from 1.5.1 to 1.6.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b1899c70..2ca0e34d 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ "nbsphinx==0.8.8", "nbsphinx-link==1.3.0", "FireWorks==1.9.8", - "autodoc_pydantic==1.5.1", + "autodoc_pydantic==1.6.0", ], "tests": [ "pytest==6.2.5", From 461589ac6d666b7fa0273b513441b1fedab5af2a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 18:24:32 +0000 Subject: [PATCH 18/23] Bump ipython from 7.30.1 to 7.31.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2ca0e34d..db4e3202 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ "sphinx==4.3.2", "furo==2022.1.2", "m2r2==0.3.2", - "ipython==7.30.1", + "ipython==7.31.0", "nbsphinx==0.8.8", "nbsphinx-link==1.3.0", "FireWorks==1.9.8", From fc0880c72bdadd958d1950b8566c45aea97af3d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jan 2022 18:14:02 +0000 Subject: [PATCH 19/23] Bump sphinx from 4.3.2 to 4.4.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index db4e3202..1a828183 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ ], extras_require={ "docs": [ - "sphinx==4.3.2", + "sphinx==4.4.0", "furo==2022.1.2", "m2r2==0.3.2", "ipython==7.31.0", From 80e17a0c5191b38f68fa346faa770d6a15af7613 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jan 2022 18:14:10 +0000 Subject: [PATCH 20/23] Bump ipython from 7.31.0 to 8.0.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1a828183..cfa9d717 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ "sphinx==4.4.0", "furo==2022.1.2", "m2r2==0.3.2", - "ipython==7.31.0", + "ipython==8.0.0", "nbsphinx==0.8.8", "nbsphinx-link==1.3.0", "FireWorks==1.9.8", From 3eaf8bd8d283b230fbecdd418871fc3cf34af9d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jan 2022 18:14:24 +0000 Subject: [PATCH 21/23] Bump monty from 2021.12.1 to 2022.1.12.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1efb1a2f..551ab9be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -monty==2021.12.1 +monty==2022.1.12.1 networkx==2.6.3 pydash==5.1.0 maggma==0.32.3 From 23baaf342074395fd7573789cfc796a902f9b4cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jan 2022 18:15:22 +0000 Subject: [PATCH 22/23] Bump ipython from 8.0.0 to 8.0.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cfa9d717..78f45982 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ "sphinx==4.4.0", "furo==2022.1.2", "m2r2==0.3.2", - "ipython==8.0.0", + "ipython==8.0.1", "nbsphinx==0.8.8", "nbsphinx-link==1.3.0", "FireWorks==1.9.8", From 467584369b378f5e071823ddd76ba534c79a5222 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Jan 2022 18:15:25 +0000 Subject: [PATCH 23/23] Bump monty from 2022.1.12.1 to 2022.1.19 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 551ab9be..de0f4bc5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -monty==2022.1.12.1 +monty==2022.1.19 networkx==2.6.3 pydash==5.1.0 maggma==0.32.3