From 4c91101389e6aeb25997cb9e57f2fd23343229b9 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 16 Oct 2025 10:10:09 +0100 Subject: [PATCH 01/13] Way to run parallel denoising for tilts --- src/cryoemservices/services/tomo_align.py | 38 +++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/cryoemservices/services/tomo_align.py b/src/cryoemservices/services/tomo_align.py index dc8ef203..68f7a09e 100644 --- a/src/cryoemservices/services/tomo_align.py +++ b/src/cryoemservices/services/tomo_align.py @@ -25,6 +25,25 @@ from cryoemservices.util.tomo_output_files import _get_tilt_number_v5_12 +def run_tilt_denoising(tilt: str) -> str | None: + denoised_tilt = "/".join("tmp" if p == "processed" else p for p in Path(tilt).parts) + denoised_tilt = str(Path(denoised_tilt).with_suffix("_denoised.mrc")) + denoise_result = subprocess.run( + [ + "python", + "run_denoiser.py", + "--nimage", + str(tilt), + "--dimage", + str(denoised_tilt), + ], + capture_output=True, + ) + if denoise_result.returncode: + return None + return denoised_tilt + + class TomoParameters(BaseModel): stack_file: str = Field(..., min_length=1) pixel_size: float @@ -52,6 +71,7 @@ class TomoParameters(BaseModel): out_imod_xf: Optional[int] = None dark_tol: Optional[float] = None manual_tilt_offset: Optional[float] = None + denoise_tilts: bool = False relion_options: RelionServiceOptions @model_validator(mode="before") @@ -304,6 +324,24 @@ def _tilt(file_list_for_tilts): for index in sorted(tilts_to_remove, reverse=True): self.input_file_list_of_lists.remove(self.input_file_list_of_lists[index]) + # Decide whether to denoise + if not tomo_params.denoise_tilts: + rw.send_to("tomo_align", {"denoise": True}) + else: + new_input_list_of_lists = [] + for tname, tangle in self.input_file_list_of_lists: + denoised_tilt = run_tilt_denoising(tname) + if not denoised_tilt: + self.log.error(f"Failed to denoise {tname}") + rw.transport.nack(header) + return + new_input_list_of_lists.append([denoised_tilt, tangle]) + self.input_file_list_of_lists = new_input_list_of_lists + tomo_params.stack_file = "/".join( + "tmp" if p == "processed" else p + for p in Path(tomo_params.stack_file).parts + ) + # Find the input image dimensions with mrcfile.open(self.input_file_list_of_lists[0][0]) as mrc: mrc_header = mrc.header From b2124819cc9e95978fcb63c62dc17c66ad3d9717 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 16 Oct 2025 10:25:18 +0100 Subject: [PATCH 02/13] Put denoising and quality into recipe --- recipes/em-tomo-align.json | 21 ++++++++++++++++++++- src/cryoemservices/services/tomo_align.py | 12 ++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/recipes/em-tomo-align.json b/recipes/em-tomo-align.json index d413ef34..75730d13 100644 --- a/recipes/em-tomo-align.json +++ b/recipes/em-tomo-align.json @@ -23,9 +23,11 @@ "node_creator": 11, "projxy": 6, "projxz": 6, - "success": 7 + "success": 7, + "tomo_align_denoise": 12 }, "parameters": { + "denoise_tilts": 1, "dose_per_frame": "{dose_per_frame}", "frame_count": "{frame_count}", "input_file_list": "{input_file_list}", @@ -118,5 +120,22 @@ "queue": "node_creator", "service": "NodeCreator" }, + "12": { + "parameters": { + "denoise_tilts": 2, + "dose_per_frame": "{dose_per_frame}", + "frame_count": "{frame_count}", + "input_file_list": "{input_file_list}", + "kv": "{kv}", + "manual_tilt_offset": "{manual_tilt_offset}", + "path_pattern": "{path_pattern}", + "pixel_size": "{pixel_size}", + "relion_options": {}, + "stack_file": "{stack_file}", + "tilt_axis": "{tilt_axis}" + }, + "queue": "tomo_align", + "service": "TomoAlign" + }, "start": [[1, []]] } diff --git a/src/cryoemservices/services/tomo_align.py b/src/cryoemservices/services/tomo_align.py index 68f7a09e..5f5859ba 100644 --- a/src/cryoemservices/services/tomo_align.py +++ b/src/cryoemservices/services/tomo_align.py @@ -71,7 +71,7 @@ class TomoParameters(BaseModel): out_imod_xf: Optional[int] = None dark_tol: Optional[float] = None manual_tilt_offset: Optional[float] = None - denoise_tilts: bool = False + denoise_tilts: int = 0 relion_options: RelionServiceOptions @model_validator(mode="before") @@ -325,9 +325,9 @@ def _tilt(file_list_for_tilts): self.input_file_list_of_lists.remove(self.input_file_list_of_lists[index]) # Decide whether to denoise - if not tomo_params.denoise_tilts: - rw.send_to("tomo_align", {"denoise": True}) - else: + if tomo_params.denoise_tilts == 1: + rw.send_to("tomo_align_denoise", {"denoise_tilts": 2}) + elif tomo_params.denoise_tilts == 2: new_input_list_of_lists = [] for tname, tangle in self.input_file_list_of_lists: denoised_tilt = run_tilt_denoising(tname) @@ -540,6 +540,10 @@ def _tilt(file_list_for_tilts): } ] + # Write the score somewhere + with open(aretomo_output_path.with_suffix(".com"), "a") as comfile: + comfile.write(f"\n\nAlignment quality {self.alignment_quality}") + # Find the indexes of the dark images removed by AreTomo missing_indices = [] dark_images_file = Path(stack_name + "_DarkImgs.txt") From bcd7e27526d0f06ec3acac88d3413fe0d9c4d4c6 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 16 Oct 2025 10:38:49 +0100 Subject: [PATCH 03/13] Test that denoiser gets called --- src/cryoemservices/services/tomo_align.py | 12 ++++++++---- tests/services/test_tomo_align.py | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/cryoemservices/services/tomo_align.py b/src/cryoemservices/services/tomo_align.py index 5f5859ba..0fe92980 100644 --- a/src/cryoemservices/services/tomo_align.py +++ b/src/cryoemservices/services/tomo_align.py @@ -26,8 +26,12 @@ def run_tilt_denoising(tilt: str) -> str | None: - denoised_tilt = "/".join("tmp" if p == "processed" else p for p in Path(tilt).parts) - denoised_tilt = str(Path(denoised_tilt).with_suffix("_denoised.mrc")) + denoised_tilt = "/" + "/".join( + "tmp" if p == "processed" else p for p in Path(tilt).parts[1:] + ) + denoised_tilt = str( + Path(denoised_tilt).parent / (Path(denoised_tilt).stem + "_denoised.mrc") + ) denoise_result = subprocess.run( [ "python", @@ -337,9 +341,9 @@ def _tilt(file_list_for_tilts): return new_input_list_of_lists.append([denoised_tilt, tangle]) self.input_file_list_of_lists = new_input_list_of_lists - tomo_params.stack_file = "/".join( + tomo_params.stack_file = "/" + "/".join( "tmp" if p == "processed" else p - for p in Path(tomo_params.stack_file).parts + for p in Path(tomo_params.stack_file).parts[1:] ) # Find the input image dimensions diff --git a/tests/services/test_tomo_align.py b/tests/services/test_tomo_align.py index 182fce66..8af7d6a1 100644 --- a/tests/services/test_tomo_align.py +++ b/tests/services/test_tomo_align.py @@ -1456,3 +1456,22 @@ def test_parse_tomo_align_output(offline_transport): assert service.rot_centre_z_list == ["300.0", "350.0"] assert service.tilt_offset == 1.0 assert service.alignment_quality == 0.07568 + + +@mock.patch("cryoemservices.services.tomo_align.subprocess.run") +def test_run_tilt_denoising(mock_subprocess, tmp_path): + mock_subprocess().returncode = 0 + + tilt_in = f"{tmp_path}/processed/relion_murfey/MotionCorr/job002/Movies/tilt.mrc" + tilt_out = ( + f"{tmp_path}/tmp/relion_murfey/MotionCorr/job002/Movies/tilt_denoised.mrc" + ) + + denoised_tilt = tomo_align.run_tilt_denoising(tilt_in) + + assert denoised_tilt == tilt_out + + mock_subprocess.assert_called_with( + ["python", "run_denoiser.py", "--nimage", tilt_in, "--dimage", tilt_out], + capture_output=True, + ) From 425dc2b13f2a4ad628747a8a006f51ddace55142 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 17 Oct 2025 09:58:33 +0100 Subject: [PATCH 04/13] Put in spool and ensure exists --- src/cryoemservices/services/tomo_align.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/cryoemservices/services/tomo_align.py b/src/cryoemservices/services/tomo_align.py index 0fe92980..8512b1ce 100644 --- a/src/cryoemservices/services/tomo_align.py +++ b/src/cryoemservices/services/tomo_align.py @@ -27,11 +27,12 @@ def run_tilt_denoising(tilt: str) -> str | None: denoised_tilt = "/" + "/".join( - "tmp" if p == "processed" else p for p in Path(tilt).parts[1:] + "spool" if p == "processed" else p for p in Path(tilt).parts[1:] ) denoised_tilt = str( Path(denoised_tilt).parent / (Path(denoised_tilt).stem + "_denoised.mrc") ) + Path(denoised_tilt).parent.mkdir(parents=True, exist_ok=True) denoise_result = subprocess.run( [ "python", @@ -342,9 +343,10 @@ def _tilt(file_list_for_tilts): new_input_list_of_lists.append([denoised_tilt, tangle]) self.input_file_list_of_lists = new_input_list_of_lists tomo_params.stack_file = "/" + "/".join( - "tmp" if p == "processed" else p + "spool" if p == "processed" else p for p in Path(tomo_params.stack_file).parts[1:] ) + Path(tomo_params.stack_file).parent.mkdir(parents=True, exist_ok=True) # Find the input image dimensions with mrcfile.open(self.input_file_list_of_lists[0][0]) as mrc: From ba95bf00d0011a7725d7211c7b040ee5005664e2 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 17 Oct 2025 13:23:00 +0100 Subject: [PATCH 05/13] Add some logs --- src/cryoemservices/services/tomo_align.py | 2 ++ tests/services/test_tomo_align.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cryoemservices/services/tomo_align.py b/src/cryoemservices/services/tomo_align.py index 8512b1ce..12320e7a 100644 --- a/src/cryoemservices/services/tomo_align.py +++ b/src/cryoemservices/services/tomo_align.py @@ -331,8 +331,10 @@ def _tilt(file_list_for_tilts): # Decide whether to denoise if tomo_params.denoise_tilts == 1: + self.log.info("Sending to tilt denoising and alignment re-run") rw.send_to("tomo_align_denoise", {"denoise_tilts": 2}) elif tomo_params.denoise_tilts == 2: + self.log.info("Running tilt denoising") new_input_list_of_lists = [] for tname, tangle in self.input_file_list_of_lists: denoised_tilt = run_tilt_denoising(tname) diff --git a/tests/services/test_tomo_align.py b/tests/services/test_tomo_align.py index 8af7d6a1..87dfec23 100644 --- a/tests/services/test_tomo_align.py +++ b/tests/services/test_tomo_align.py @@ -1464,7 +1464,7 @@ def test_run_tilt_denoising(mock_subprocess, tmp_path): tilt_in = f"{tmp_path}/processed/relion_murfey/MotionCorr/job002/Movies/tilt.mrc" tilt_out = ( - f"{tmp_path}/tmp/relion_murfey/MotionCorr/job002/Movies/tilt_denoised.mrc" + f"{tmp_path}/spool/relion_murfey/MotionCorr/job002/Movies/tilt_denoised.mrc" ) denoised_tilt = tomo_align.run_tilt_denoising(tilt_in) From 1b5dd5250fa177f992f3e326fe2f5b1b624c0fe9 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 28 Oct 2025 12:09:47 +0000 Subject: [PATCH 06/13] Method for denoising using IRIS --- .../templates/deployment.yaml | 2 + src/cryoemservices/services/tomo_align.py | 65 ++++++++++-------- .../services/tomo_align_slurm.py | 66 ++++++++++++++++++- src/cryoemservices/util/slurm_submission.py | 3 + 4 files changed, 106 insertions(+), 30 deletions(-) diff --git a/Helm/charts/tomo_align_slurm/templates/deployment.yaml b/Helm/charts/tomo_align_slurm/templates/deployment.yaml index 1639be16..94c8c79a 100644 --- a/Helm/charts/tomo_align_slurm/templates/deployment.yaml +++ b/Helm/charts/tomo_align_slurm/templates/deployment.yaml @@ -32,6 +32,8 @@ spec: - >- {{ .Values.command }} env: + - name: TILT_DENOISING_SIF + value: "{{ .Values.tiltDenoisingSIF }}" - name: ARETOMO2_EXECUTABLE value: "{{ .Values.aretomoExecutable }}" - name: EXTRA_LIBRARIES diff --git a/src/cryoemservices/services/tomo_align.py b/src/cryoemservices/services/tomo_align.py index 1e12d1bb..0610b731 100644 --- a/src/cryoemservices/services/tomo_align.py +++ b/src/cryoemservices/services/tomo_align.py @@ -25,30 +25,6 @@ from cryoemservices.util.tomo_output_files import _get_tilt_number_v5_12 -def run_tilt_denoising(tilt: str) -> str | None: - denoised_tilt = "/" + "/".join( - "spool" if p == "processed" else p for p in Path(tilt).parts[1:] - ) - denoised_tilt = str( - Path(denoised_tilt).parent / (Path(denoised_tilt).stem + "_denoised.mrc") - ) - Path(denoised_tilt).parent.mkdir(parents=True, exist_ok=True) - denoise_result = subprocess.run( - [ - "python", - "run_denoiser.py", - "--nimage", - str(tilt), - "--dimage", - str(denoised_tilt), - ], - capture_output=True, - ) - if denoise_result.returncode: - return None - return denoised_tilt - - class TomoParameters(BaseModel): stack_file: str = Field(..., min_length=1) pixel_size: float @@ -162,6 +138,34 @@ def parse_tomo_output(self, tomo_stdout: str): if line.startswith("Best tilt axis"): self.alignment_quality = float(line.split()[5]) + def get_denoised_tilt_name(self, tilt: str) -> str: + denoised_tilt = "/" + "/".join( + "spool" if p == "processed" else p for p in Path(tilt).parts[1:] + ) + denoised_tilt = str( + Path(denoised_tilt).parent / (Path(denoised_tilt).stem + "_denoised.mrc") + ) + Path(denoised_tilt).parent.mkdir(parents=True, exist_ok=True) + return denoised_tilt + + def run_tilt_denoising(self, tilt_list: list[str]) -> bool: + for tilt in tilt_list: + denoised_tilt = self.get_denoised_tilt_name(tilt) + denoise_result = subprocess.run( + [ + "python", + "run_denoiser.py", + "--nimage", + str(tilt), + "--dimage", + str(denoised_tilt), + ], + capture_output=True, + ) + if denoise_result.returncode: + return False + return True + def extract_from_aln(self, tomo_parameters, alignment_output_dir, plot_path): tomo_aln_file = None self.rot = None @@ -336,13 +340,16 @@ def _tilt(file_list_for_tilts): elif tomo_params.denoise_tilts == 2: self.log.info("Running tilt denoising") new_input_list_of_lists = [] + tilts_to_denoise = [] for tname, tangle in self.input_file_list_of_lists: - denoised_tilt = run_tilt_denoising(tname) - if not denoised_tilt: - self.log.error(f"Failed to denoise {tname}") - rw.transport.nack(header) - return + denoised_tilt = self.get_denoised_tilt_name(tname) + tilts_to_denoise.append(tname) new_input_list_of_lists.append([denoised_tilt, tangle]) + denoise_success = self.run_tilt_denoising(tilts_to_denoise) + if not denoise_success: + self.log.error("Failed to denoise tilts") + rw.transport.nack(header) + return self.input_file_list_of_lists = new_input_list_of_lists tomo_params.stack_file = "/" + "/".join( "spool" if p == "processed" else p diff --git a/src/cryoemservices/services/tomo_align_slurm.py b/src/cryoemservices/services/tomo_align_slurm.py index 1596756d..bd937b47 100644 --- a/src/cryoemservices/services/tomo_align_slurm.py +++ b/src/cryoemservices/services/tomo_align_slurm.py @@ -7,7 +7,10 @@ from typing import List from cryoemservices.services.tomo_align import TomoAlign, TomoParameters -from cryoemservices.util.slurm_submission import slurm_submission_for_services +from cryoemservices.util.slurm_submission import ( + slurm_submission_for_services, + wait_for_job_completion, +) def retrieve_files( @@ -80,6 +83,67 @@ def parse_tomo_output(self, tomo_output_file): self.alignment_quality = float(line.split()[5]) tomo_file.close() + def run_tilt_denoising(self, tilt_list: list[str]) -> bool: + transfer_status = transfer_files([Path(tilt) for tilt in tilt_list]) + if len(transfer_status) != len(tilt_list): + self.log.error( + f"Unable to transfer files: desired {tilt_list}, done {transfer_status}" + ) + return False + self.log.info("All files transferred") + + job_ids = [] + final_tilts = [] + for tilt in tilt_list: + denoised_tilt = self.get_denoised_tilt_name(tilt) + command = [ + "python", + "run_denoiser.py", + "--nimage", + str(tilt), + "--dimage", + str(denoised_tilt), + ] + tilt_job_id = slurm_submission_for_services( + log=self.log, + service_config_file=self._environment["config"], + slurm_cluster=self._environment["slurm_cluster"], + job_name="TiltDenoise", + command=command, + project_dir=Path(denoised_tilt).parent, + output_file=Path(denoised_tilt), + cpus=1, + use_gpu=True, + use_singularity=True, + cif_name=os.environ["TILT_DENOISING_SIF"], + external_filesystem=True, + wait_for_completion=False, + ) + job_ids.append(tilt_job_id.returncode) + final_tilts.append(denoised_tilt) + + for tid, job_id in enumerate(job_ids): + wait_for_job_completion( + job_id=job_id, + logger=self.log, + service_config=self._environment["config"], + cluster_name=self._environment["slurm_cluster"], + ) + + # Get back any output files and clean up + self.log.info("Retrieving output files...") + retrieve_files( + job_directory=Path(final_tilts[tid]).parent, + files_to_skip=[Path(tilt_list[tid])], + basepath=str(Path(tilt_list[tid]).stem), + ) + self.log.info("All denoising jobs finished and output files retrieved") + + for out_tilt in final_tilts: + if not Path(out_tilt).is_file(): + return False + return True + def aretomo( self, tomo_parameters: TomoParameters, diff --git a/src/cryoemservices/util/slurm_submission.py b/src/cryoemservices/util/slurm_submission.py index 4ddcd003..f0a5c65b 100644 --- a/src/cryoemservices/util/slurm_submission.py +++ b/src/cryoemservices/util/slurm_submission.py @@ -274,6 +274,7 @@ def slurm_submission_for_services( memory_request: int = 12000, external_filesystem: bool = False, extra_singularity_directories: Optional[list[str]] = None, + wait_for_completion: bool = True, ) -> subprocess.CompletedProcess: """Submit jobs to a slurm cluster via the RestAPI""" # Load the service config with slurm credentials @@ -371,6 +372,8 @@ def slurm_submission_for_services( # Get the status of the submitted job from the restAPI log.info(f"Submitted job {job_id} for {job_name} to slurm. Waiting...") + if not wait_for_completion: + return subprocess.CompletedProcess(args="", returncode=job_id) slurm_job_state = wait_for_job_completion( job_id=job_id, logger=log, From 3e4aae17de8b1ea4ce0c3a0771c5c04bc1c6373f Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 28 Oct 2025 16:17:48 +0000 Subject: [PATCH 07/13] Need to run in singularity with full path --- src/cryoemservices/services/tomo_align_slurm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cryoemservices/services/tomo_align_slurm.py b/src/cryoemservices/services/tomo_align_slurm.py index bd937b47..b615d683 100644 --- a/src/cryoemservices/services/tomo_align_slurm.py +++ b/src/cryoemservices/services/tomo_align_slurm.py @@ -98,7 +98,7 @@ def run_tilt_denoising(self, tilt_list: list[str]) -> bool: denoised_tilt = self.get_denoised_tilt_name(tilt) command = [ "python", - "run_denoiser.py", + "/install/denoiser/run_denoiser.py", "--nimage", str(tilt), "--dimage", From 3619abbc6f3aea281183851d8d014e401d2ca0b6 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 29 Oct 2025 10:51:21 +0000 Subject: [PATCH 08/13] Need to set the home --- Helm/charts/tomo_align/templates/deployment.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Helm/charts/tomo_align/templates/deployment.yaml b/Helm/charts/tomo_align/templates/deployment.yaml index 88d9028d..55ab951d 100644 --- a/Helm/charts/tomo_align/templates/deployment.yaml +++ b/Helm/charts/tomo_align/templates/deployment.yaml @@ -33,6 +33,9 @@ spec: - -c - >- {{ .Values.command }} + env: + - name: HOME + value: "/tmp" volumeMounts: - name: config-file mountPath: /cryoemservices/config From fb40a8085dabacb7ad2669a40afac0811691e2c3 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Wed, 29 Oct 2025 13:52:58 +0000 Subject: [PATCH 09/13] Need proper config not file name --- src/cryoemservices/services/tomo_align_slurm.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cryoemservices/services/tomo_align_slurm.py b/src/cryoemservices/services/tomo_align_slurm.py index b615d683..d7c424e8 100644 --- a/src/cryoemservices/services/tomo_align_slurm.py +++ b/src/cryoemservices/services/tomo_align_slurm.py @@ -8,6 +8,7 @@ from cryoemservices.services.tomo_align import TomoAlign, TomoParameters from cryoemservices.util.slurm_submission import ( + config_from_file, slurm_submission_for_services, wait_for_job_completion, ) @@ -122,16 +123,15 @@ def run_tilt_denoising(self, tilt_list: list[str]) -> bool: job_ids.append(tilt_job_id.returncode) final_tilts.append(denoised_tilt) + service_config = config_from_file(self._environment["config"]) + self.log.info("Waiting for completion and retrieval of output files...") for tid, job_id in enumerate(job_ids): wait_for_job_completion( job_id=job_id, logger=self.log, - service_config=self._environment["config"], + service_config=service_config, cluster_name=self._environment["slurm_cluster"], ) - - # Get back any output files and clean up - self.log.info("Retrieving output files...") retrieve_files( job_directory=Path(final_tilts[tid]).parent, files_to_skip=[Path(tilt_list[tid])], From eb489f85a59776e30bfa7c3272282bf5452c305f Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 4 Nov 2025 17:18:34 +0000 Subject: [PATCH 10/13] Fix output file names --- src/cryoemservices/services/tomo_align_slurm.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/cryoemservices/services/tomo_align_slurm.py b/src/cryoemservices/services/tomo_align_slurm.py index d7c424e8..d78a9f42 100644 --- a/src/cryoemservices/services/tomo_align_slurm.py +++ b/src/cryoemservices/services/tomo_align_slurm.py @@ -72,7 +72,7 @@ class TomoAlignSlurm(TomoAlign): # Logger name _logger_name = "cryoemservices.services.tomo_align_slurm" - def parse_tomo_output(self, tomo_output_file): + def parse_tomo_output_file(self, tomo_output_file: Path): tomo_file = open(tomo_output_file, "r") lines = tomo_file.readlines() for line in lines: @@ -221,10 +221,10 @@ def aretomo( command, ) - slurm_output_file = f"{aretomo_output_path}.out" - slurm_error_file = f"{aretomo_output_path}.out" - if tomo_parameters.tilt_cor and Path(slurm_output_file).is_file(): - self.parse_tomo_output(slurm_output_file) + slurm_output_file = aretomo_output_path.with_suffix(".out") + slurm_error_file = aretomo_output_path.with_suffix(".err") + if tomo_parameters.tilt_cor and slurm_output_file.is_file(): + self.parse_tomo_output_file(slurm_output_file) try: with open(slurm_output_file, "r") as slurm_stdout: From 4a48ee762e482d5bbaa5576f0ee24e7d8756d5fa Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 14 Nov 2025 11:17:16 +0000 Subject: [PATCH 11/13] More logs to assess job status --- src/cryoemservices/services/tomo_align.py | 4 ++++ src/cryoemservices/services/tomo_align_slurm.py | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/cryoemservices/services/tomo_align.py b/src/cryoemservices/services/tomo_align.py index 0610b731..a2a40e5c 100644 --- a/src/cryoemservices/services/tomo_align.py +++ b/src/cryoemservices/services/tomo_align.py @@ -163,6 +163,10 @@ def run_tilt_denoising(self, tilt_list: list[str]) -> bool: capture_output=True, ) if denoise_result.returncode: + self.log.error(f"Failed to denoise tilt {tilt}") + self.log.error( + f"Denoise reason: {denoise_result.stdout.decode('utf8')} {denoise_result.stderr.decode('utf8')}" + ) return False return True diff --git a/src/cryoemservices/services/tomo_align_slurm.py b/src/cryoemservices/services/tomo_align_slurm.py index d78a9f42..e8da1e48 100644 --- a/src/cryoemservices/services/tomo_align_slurm.py +++ b/src/cryoemservices/services/tomo_align_slurm.py @@ -126,7 +126,7 @@ def run_tilt_denoising(self, tilt_list: list[str]) -> bool: service_config = config_from_file(self._environment["config"]) self.log.info("Waiting for completion and retrieval of output files...") for tid, job_id in enumerate(job_ids): - wait_for_job_completion( + job_state = wait_for_job_completion( job_id=job_id, logger=self.log, service_config=service_config, @@ -137,10 +137,17 @@ def run_tilt_denoising(self, tilt_list: list[str]) -> bool: files_to_skip=[Path(tilt_list[tid])], basepath=str(Path(tilt_list[tid]).stem), ) + if job_state != "COMPLETED": + self.log.error(f"Job {job_id} failed with {job_state}") self.log.info("All denoising jobs finished and output files retrieved") for out_tilt in final_tilts: if not Path(out_tilt).is_file(): + self.log.info(f"Tilt denoising failed for {out_tilt}") + if Path(out_tilt).with_suffix(".err").is_file(): + with open(Path(out_tilt).with_suffix(".err"), "r") as slurm_stderr: + stderr = slurm_stderr.read() + self.log.error(stderr) return False return True From d61d3e85858b9046bfc25648739021f806b55eab Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 26 Jan 2026 09:28:09 +0000 Subject: [PATCH 12/13] Fix numbering in recipe --- recipes/em-tomo-align.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/em-tomo-align.json b/recipes/em-tomo-align.json index 72b4ba96..4f8a8019 100644 --- a/recipes/em-tomo-align.json +++ b/recipes/em-tomo-align.json @@ -24,7 +24,7 @@ "projxy": 6, "projxz": 6, "success": 7, - "tomo_align_denoise": 12 + "tomo_align_denoise": 13 }, "parameters": { "denoise_tilts": 1, @@ -128,7 +128,7 @@ "queue": "node_creator", "service": "NodeCreator" }, - "12": { + "13": { "parameters": { "denoise_tilts": 2, "dose_per_frame": "{dose_per_frame}", From ae03e50a7e3101236dff018160929e662ef1c101 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 30 Apr 2026 10:53:48 +0100 Subject: [PATCH 13/13] Skip denoising on iris --- src/cryoemservices/services/tomo_align_slurm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cryoemservices/services/tomo_align_slurm.py b/src/cryoemservices/services/tomo_align_slurm.py index d82e45aa..dd7c0d4e 100644 --- a/src/cryoemservices/services/tomo_align_slurm.py +++ b/src/cryoemservices/services/tomo_align_slurm.py @@ -108,6 +108,8 @@ def check_visit(tomo_params: TomoParameters): visit_search = re.search( "/[a-z]{2}[0-9]{5}-[0-9]{1,3}/", tomo_params.stack_file ) + if tomo_params.denoise_tilts == 2: + return False if visit_search: visit_name = visit_search[0][1:-1] visit_code = visit_name[:2]