From 9e60e6525edc3c26d182d2b49e3d898c2ae30e15 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 23 Mar 2026 08:53:44 +0000 Subject: [PATCH 1/5] Take picker id out of murfey recipes --- recipes/murfey/em-spa-bfactor.json | 1 - recipes/murfey/em-spa-class2d.json | 1 - recipes/murfey/em-spa-class3d.json | 1 - recipes/murfey/em-spa-refine.json | 2 -- src/cryoemservices/services/postprocess.py | 2 +- src/cryoemservices/wrappers/class2d_wrapper.py | 8 ++++---- src/cryoemservices/wrappers/class3d_wrapper.py | 12 ++++++------ 7 files changed, 11 insertions(+), 16 deletions(-) diff --git a/recipes/murfey/em-spa-bfactor.json b/recipes/murfey/em-spa-bfactor.json index 88859f33..f4ba4b34 100644 --- a/recipes/murfey/em-spa-bfactor.json +++ b/recipes/murfey/em-spa-bfactor.json @@ -41,7 +41,6 @@ "node_creator": 2 }, "parameters": { - "picker_id": "{picker_id}", "refined_class_uuid": "{refined_class_uuid}", "refined_grp_uuid": "{refined_grp_uuid}" }, diff --git a/recipes/murfey/em-spa-class2d.json b/recipes/murfey/em-spa-class2d.json index 8c69c892..adde5629 100644 --- a/recipes/murfey/em-spa-class2d.json +++ b/recipes/murfey/em-spa-class2d.json @@ -20,7 +20,6 @@ "mpi_run_command": "srun -n 5", "particle_diameter": "{particle_diameter}", "particles_file": "{particles_file}", - "picker_id": "{picker_id}", "relion_options": {}, "threads": 8 }, diff --git a/recipes/murfey/em-spa-class3d.json b/recipes/murfey/em-spa-class3d.json index c6e3ee9e..15fbf8e7 100644 --- a/recipes/murfey/em-spa-class3d.json +++ b/recipes/murfey/em-spa-class3d.json @@ -19,7 +19,6 @@ "mpi_run_command": "srun -n 9", "particle_diameter": "{particle_diameter}", "particles_file": "{particles_file}", - "picker_id": "{picker_id}", "relion_options": {}, "symmetry": "{symmetry}", "threads": 4 diff --git a/recipes/murfey/em-spa-refine.json b/recipes/murfey/em-spa-refine.json index 9eae817d..148c850d 100644 --- a/recipes/murfey/em-spa-refine.json +++ b/recipes/murfey/em-spa-refine.json @@ -41,7 +41,6 @@ "refine_wrapper": 7 }, "parameters": { - "picker_id": "{picker_id}", "refined_class_uuid": "{refined_class_uuid}", "refined_grp_uuid": "{refined_grp_uuid}" }, @@ -83,7 +82,6 @@ "node_creator": 2 }, "parameters": { - "picker_id": "{picker_id}", "refined_class_uuid": "{symmetry_refined_class_uuid}", "refined_grp_uuid": "{symmetry_refined_grp_uuid}" }, diff --git a/src/cryoemservices/services/postprocess.py b/src/cryoemservices/services/postprocess.py index c52f3668..d53c0dbd 100644 --- a/src/cryoemservices/services/postprocess.py +++ b/src/cryoemservices/services/postprocess.py @@ -35,7 +35,7 @@ class PostProcessParameters(BaseModel): postprocess_lowres: float = 10 symmetry: str = "C1" particles_file: str = "" - picker_id: int + picker_id: int | None = None refined_grp_uuid: int refined_class_uuid: int relion_options: RelionServiceOptions diff --git a/src/cryoemservices/wrappers/class2d_wrapper.py b/src/cryoemservices/wrappers/class2d_wrapper.py index 4ac0013d..45141123 100644 --- a/src/cryoemservices/wrappers/class2d_wrapper.py +++ b/src/cryoemservices/wrappers/class2d_wrapper.py @@ -6,7 +6,7 @@ import re import subprocess from pathlib import Path -from typing import Callable, Optional +from typing import Callable import numpy as np from gemmi import cif @@ -36,7 +36,7 @@ class Class2DParameters(BaseModel): vdam_final_fraction: float = 0.1 dont_combine_weights_via_disc: bool = True preread_images: bool = True - scratch_dir: Optional[str] = None + scratch_dir: str | None = None nr_pool: int = 100 pad: int = 2 skip_gridding: bool = False @@ -47,7 +47,7 @@ class Class2DParameters(BaseModel): class2d_nr_classes: int = 50 flatten_solvent: bool = True do_zero_mask: bool = True - highres_limit: Optional[float] = None + highres_limit: float | None = None centre_classes: bool = True oversampling: int = 1 skip_align: bool = False @@ -60,7 +60,7 @@ class Class2DParameters(BaseModel): mpi_run_command: str = "srun -n 5" threads: int = 8 gpus: str = "0" - picker_id: int + picker_id: int | None = None class2d_grp_uuid: int class_uuids: str do_icebreaker_jobs: bool = True diff --git a/src/cryoemservices/wrappers/class3d_wrapper.py b/src/cryoemservices/wrappers/class3d_wrapper.py index 53dc97e3..bd3b3be1 100644 --- a/src/cryoemservices/wrappers/class3d_wrapper.py +++ b/src/cryoemservices/wrappers/class3d_wrapper.py @@ -6,7 +6,7 @@ import re import subprocess from pathlib import Path -from typing import Callable, Optional, Tuple +from typing import Callable, Tuple import numpy as np from gemmi import cif @@ -50,14 +50,14 @@ class Class3DParameters(BaseModel): particle_diameter: float = 0 mask_diameter: float = 190 do_initial_model: bool = False - initial_model_file: Optional[str] = None + initial_model_file: str | None = None initial_model_iterations: int = 200 initial_model_offset_range: float = 6 initial_model_offset_step: float = 2 start_initial_model_C1: bool = True dont_combine_weights_via_disc: bool = True preread_images: bool = True - scratch_dir: Optional[str] = None + scratch_dir: str | None = None nr_pool: int = 10 pad: int = 2 skip_gridding: bool = False @@ -71,8 +71,8 @@ class Class3DParameters(BaseModel): class3d_nr_classes: int = 4 flatten_solvent: bool = True do_zero_mask: bool = True - highres_limit: Optional[float] = None - fn_mask: Optional[str] = None + highres_limit: float | None = None + fn_mask: str | None = None oversampling: int = 1 skip_align: bool = False healpix_order: int = 2 @@ -86,7 +86,7 @@ class Class3DParameters(BaseModel): threads: int = 8 gpus: str = "0:1:2:3" initial_model_gpus: str = "0,1,2,3" - picker_id: int + picker_id: int | None = None class3d_grp_uuid: int class_uuids: str relion_options: RelionServiceOptions From 2336c3b24172fb17bad48b0c689da27bf3fc812a Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 23 Mar 2026 09:03:54 +0000 Subject: [PATCH 2/5] Change default mpi command --- recipes/murfey/em-spa-class2d.json | 1 - recipes/murfey/em-spa-class3d.json | 1 - src/cryoemservices/wrappers/class2d_wrapper.py | 2 +- src/cryoemservices/wrappers/class3d_wrapper.py | 2 +- src/cryoemservices/wrappers/refine3d_wrapper.py | 2 +- 5 files changed, 3 insertions(+), 5 deletions(-) diff --git a/recipes/murfey/em-spa-class2d.json b/recipes/murfey/em-spa-class2d.json index adde5629..ecfa38b5 100644 --- a/recipes/murfey/em-spa-class2d.json +++ b/recipes/murfey/em-spa-class2d.json @@ -17,7 +17,6 @@ "class_uuids": "{class_uuids}", "do_icebreaker_jobs": "False", "mask_diameter": "{mask_diameter}", - "mpi_run_command": "srun -n 5", "particle_diameter": "{particle_diameter}", "particles_file": "{particles_file}", "relion_options": {}, diff --git a/recipes/murfey/em-spa-class3d.json b/recipes/murfey/em-spa-class3d.json index 15fbf8e7..77467cbe 100644 --- a/recipes/murfey/em-spa-class3d.json +++ b/recipes/murfey/em-spa-class3d.json @@ -16,7 +16,6 @@ "initial_model_file": "{initial_model_file}", "initial_model_iterations": "{initial_model_iterations}", "mask_diameter": "{mask_diameter}", - "mpi_run_command": "srun -n 9", "particle_diameter": "{particle_diameter}", "particles_file": "{particles_file}", "relion_options": {}, diff --git a/src/cryoemservices/wrappers/class2d_wrapper.py b/src/cryoemservices/wrappers/class2d_wrapper.py index 45141123..14ae1572 100644 --- a/src/cryoemservices/wrappers/class2d_wrapper.py +++ b/src/cryoemservices/wrappers/class2d_wrapper.py @@ -57,7 +57,7 @@ class Class2DParameters(BaseModel): allow_coarser: bool = False do_norm: bool = True do_scale: bool = True - mpi_run_command: str = "srun -n 5" + mpi_run_command: str = "mpirun -n 5" threads: int = 8 gpus: str = "0" picker_id: int | None = None diff --git a/src/cryoemservices/wrappers/class3d_wrapper.py b/src/cryoemservices/wrappers/class3d_wrapper.py index bd3b3be1..b5d5062d 100644 --- a/src/cryoemservices/wrappers/class3d_wrapper.py +++ b/src/cryoemservices/wrappers/class3d_wrapper.py @@ -82,7 +82,7 @@ class Class3DParameters(BaseModel): symmetry: str = "C1" do_norm: bool = True do_scale: bool = True - mpi_run_command: str = "srun -n 5" + mpi_run_command: str = "mpirun -n 5" threads: int = 8 gpus: str = "0:1:2:3" initial_model_gpus: str = "0,1,2,3" diff --git a/src/cryoemservices/wrappers/refine3d_wrapper.py b/src/cryoemservices/wrappers/refine3d_wrapper.py index 78bc9eab..8f86b034 100644 --- a/src/cryoemservices/wrappers/refine3d_wrapper.py +++ b/src/cryoemservices/wrappers/refine3d_wrapper.py @@ -68,7 +68,7 @@ class RefineParameters(BaseModel): mask_lowpass: float = 15 mask_extend: int = 3 mask_soft_edge: int = 3 - mpi_run_command: str = "srun -n 5" + mpi_run_command: str = "mpirun -n 5" dont_correct_greyscale: bool = True initial_lowpass: float = 20.0 dont_combine_weights_via_disc: bool = True From 06341fe4efe2ebdfd9f8efc607a396c24e8158c1 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 23 Mar 2026 09:42:18 +0000 Subject: [PATCH 3/5] Update the tests --- tests/services/test_class2d.py | 2 +- tests/services/test_class3d.py | 3 ++- tests/wrappers/test_class2d_wrapper.py | 2 +- tests/wrappers/test_class3d_wrapper.py | 2 +- tests/wrappers/test_refine3d_wrapper.py | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/services/test_class2d.py b/tests/services/test_class2d.py index c6d766f6..06207108 100644 --- a/tests/services/test_class2d.py +++ b/tests/services/test_class2d.py @@ -330,7 +330,7 @@ def test_class2d_service_complete_batch(mock_subprocess, offline_transport, tmp_ # Check the expected command was run class2d_command = [ - "srun", + "mpirun", "-n", "5", "relion_refine_mpi", diff --git a/tests/services/test_class3d.py b/tests/services/test_class3d.py index 544592ab..cefc0d35 100644 --- a/tests/services/test_class3d.py +++ b/tests/services/test_class3d.py @@ -103,7 +103,7 @@ def test_class3d_service_has_initial_model( # Check the expected 3D classifcation command was run assert mock_subprocess.call_count == 4 class3d_command = [ - "srun", + "mpirun", "-n", "5", "relion_refine_mpi", @@ -282,6 +282,7 @@ def test_class3d_service_rerun( "do_initial_model": False, "initial_model_file": f"{tmp_path}/initial_model.mrc", "mask_diameter": "190.0", + "mpi_run_command": "srun -n 5", "particle_diameter": "180", "particles_file": f"{tmp_path}/Select/job013/particles_100000.star", "picker_id": "6", diff --git a/tests/wrappers/test_class2d_wrapper.py b/tests/wrappers/test_class2d_wrapper.py index 6e7dd703..399f5c16 100644 --- a/tests/wrappers/test_class2d_wrapper.py +++ b/tests/wrappers/test_class2d_wrapper.py @@ -352,7 +352,7 @@ def test_class2d_wrapper_complete_batch( # Check the expected command was run class2d_command = [ - "srun", + "mpirun", "-n", "5", "relion_refine_mpi", diff --git a/tests/wrappers/test_class3d_wrapper.py b/tests/wrappers/test_class3d_wrapper.py index 53789a05..593831f8 100644 --- a/tests/wrappers/test_class3d_wrapper.py +++ b/tests/wrappers/test_class3d_wrapper.py @@ -499,7 +499,7 @@ def test_class3d_wrapper_has_initial_model( # Check the expected 3D classifcation command was run assert mock_subprocess.call_count == 4 class3d_command = [ - "srun", + "mpirun", "-n", "5", "relion_refine_mpi", diff --git a/tests/wrappers/test_refine3d_wrapper.py b/tests/wrappers/test_refine3d_wrapper.py index cd13ed3a..084176d5 100644 --- a/tests/wrappers/test_refine3d_wrapper.py +++ b/tests/wrappers/test_refine3d_wrapper.py @@ -350,7 +350,7 @@ def test_refine3d_wrapper_no_mask( ) refine3d_command = [ - "srun", + "mpirun", "-n", "5", "relion_refine_mpi", From ef3878ed12e62bde7e324941d75407996c800de9 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 15 Apr 2026 10:37:08 +0100 Subject: [PATCH 4/5] Wipe picker id and update mpi number --- recipes/ispyb/em-spa-bfactor-wrapper.json | 1 - recipes/ispyb/em-spa-class2d-wrapper.json | 1 - recipes/ispyb/em-spa-class3d-wrapper.json | 1 - recipes/ispyb/em-spa-refine-wrapper.json | 4 ++-- src/cryoemservices/wrappers/class3d_wrapper.py | 4 ++-- 5 files changed, 4 insertions(+), 7 deletions(-) diff --git a/recipes/ispyb/em-spa-bfactor-wrapper.json b/recipes/ispyb/em-spa-bfactor-wrapper.json index 701c05ce..f5144f9e 100644 --- a/recipes/ispyb/em-spa-bfactor-wrapper.json +++ b/recipes/ispyb/em-spa-bfactor-wrapper.json @@ -65,7 +65,6 @@ "node_creator": 2 }, "parameters": { - "picker_id": "{picker_id}", "refined_class_uuid": "{refined_class_uuid}", "refined_grp_uuid": "{refined_grp_uuid}" }, diff --git a/recipes/ispyb/em-spa-class2d-wrapper.json b/recipes/ispyb/em-spa-class2d-wrapper.json index 5a761ea0..53044c7f 100644 --- a/recipes/ispyb/em-spa-class2d-wrapper.json +++ b/recipes/ispyb/em-spa-class2d-wrapper.json @@ -13,7 +13,6 @@ "mpi_run_command": "srun -n 5", "particle_diameter": "{particle_diameter}", "particles_file": "{particles_file}", - "picker_id": "{picker_id}", "relion_options": {}, "threads": 8 }, diff --git a/recipes/ispyb/em-spa-class3d-wrapper.json b/recipes/ispyb/em-spa-class3d-wrapper.json index a4fdbf41..36838a46 100644 --- a/recipes/ispyb/em-spa-class3d-wrapper.json +++ b/recipes/ispyb/em-spa-class3d-wrapper.json @@ -14,7 +14,6 @@ "mpi_run_command": "srun -n 9", "particle_diameter": "{particle_diameter}", "particles_file": "{particles_file}", - "picker_id": "{picker_id}", "relion_options": {}, "symmetry": "{symmetry}", "threads": 4 diff --git a/recipes/ispyb/em-spa-refine-wrapper.json b/recipes/ispyb/em-spa-refine-wrapper.json index f48b6021..e4b88afd 100644 --- a/recipes/ispyb/em-spa-refine-wrapper.json +++ b/recipes/ispyb/em-spa-refine-wrapper.json @@ -45,6 +45,7 @@ "gpus": 4, "job_name": "Refine3D-EM", "memory_per_node": 96000, + "mpi_run_command": "srun -n 5", "nodes": 1, "partition": "{partition}", "prefer": "{partition_preference}", @@ -65,7 +66,6 @@ "refine_wrapper": 7 }, "parameters": { - "picker_id": "{picker_id}", "refined_class_uuid": "{refined_class_uuid}", "refined_grp_uuid": "{refined_grp_uuid}" }, @@ -113,6 +113,7 @@ "gpus": 4, "job_name": "Symmetry3D-EM", "memory_per_node": 96000, + "mpi_run_command": "srun -n 5", "nodes": 1, "partition": "{partition}", "prefer": "{partition_preference}", @@ -134,7 +135,6 @@ "node_creator": 2 }, "parameters": { - "picker_id": "{picker_id}", "refined_class_uuid": "{symmetry_refined_class_uuid}", "refined_grp_uuid": "{symmetry_refined_grp_uuid}" }, diff --git a/src/cryoemservices/wrappers/class3d_wrapper.py b/src/cryoemservices/wrappers/class3d_wrapper.py index b5d5062d..87828502 100644 --- a/src/cryoemservices/wrappers/class3d_wrapper.py +++ b/src/cryoemservices/wrappers/class3d_wrapper.py @@ -82,8 +82,8 @@ class Class3DParameters(BaseModel): symmetry: str = "C1" do_norm: bool = True do_scale: bool = True - mpi_run_command: str = "mpirun -n 5" - threads: int = 8 + mpi_run_command: str = "mpirun -n 9" + threads: int = 4 gpus: str = "0:1:2:3" initial_model_gpus: str = "0,1,2,3" picker_id: int | None = None From eb9881024c1b46c465aff3e13835b83432b4f78e Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 15 Apr 2026 10:48:05 +0100 Subject: [PATCH 5/5] Update tests and mpirun recipe location --- recipes/ispyb/em-spa-refine-wrapper.json | 4 ++-- tests/services/test_class3d.py | 6 +++--- tests/wrappers/test_class3d_wrapper.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/recipes/ispyb/em-spa-refine-wrapper.json b/recipes/ispyb/em-spa-refine-wrapper.json index e4b88afd..c5ae8b3b 100644 --- a/recipes/ispyb/em-spa-refine-wrapper.json +++ b/recipes/ispyb/em-spa-refine-wrapper.json @@ -22,6 +22,7 @@ "3": { "job_parameters": { "mask_diameter": "{mask_diameter}", + "mpi_run_command": "srun -n 5", "particle_diameter": "{particle_diameter}", "relion_options": {}, "symmetry": "{symmetry}" @@ -45,7 +46,6 @@ "gpus": 4, "job_name": "Refine3D-EM", "memory_per_node": 96000, - "mpi_run_command": "srun -n 5", "nodes": 1, "partition": "{partition}", "prefer": "{partition_preference}", @@ -92,6 +92,7 @@ "7": { "job_parameters": { "mask_diameter": "{mask_diameter}", + "mpi_run_command": "srun -n 5", "particle_diameter": "{particle_diameter}" }, "output": { @@ -113,7 +114,6 @@ "gpus": 4, "job_name": "Symmetry3D-EM", "memory_per_node": 96000, - "mpi_run_command": "srun -n 5", "nodes": 1, "partition": "{partition}", "prefer": "{partition_preference}", diff --git a/tests/services/test_class3d.py b/tests/services/test_class3d.py index cefc0d35..e9917554 100644 --- a/tests/services/test_class3d.py +++ b/tests/services/test_class3d.py @@ -105,7 +105,7 @@ def test_class3d_service_has_initial_model( class3d_command = [ "mpirun", "-n", - "5", + "9", "relion_refine_mpi", "--i", "Select/job013/particles_100000.star", @@ -146,7 +146,7 @@ def test_class3d_service_has_initial_model( "--norm", "--scale", "--j", - "8", + "4", "--gpu", "0:1:2:3", "--pipeline_control", @@ -377,7 +377,7 @@ def test_class3d_service_rerun( "--norm", "--scale", "--j", - "8", + "4", "--gpu", "0:1:2:3", "--pipeline_control", diff --git a/tests/wrappers/test_class3d_wrapper.py b/tests/wrappers/test_class3d_wrapper.py index 593831f8..7eb3430f 100644 --- a/tests/wrappers/test_class3d_wrapper.py +++ b/tests/wrappers/test_class3d_wrapper.py @@ -501,7 +501,7 @@ def test_class3d_wrapper_has_initial_model( class3d_command = [ "mpirun", "-n", - "5", + "9", "relion_refine_mpi", "--i", "Select/job013/particles_100000.star", @@ -542,7 +542,7 @@ def test_class3d_wrapper_has_initial_model( "--norm", "--scale", "--j", - "8", + "4", "--gpu", "0:1:2:3", "--pipeline_control",