From 3410fa552542e1fdca082e50a6932079b95f5264 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 17 Oct 2025 13:36:37 +0100 Subject: [PATCH 01/12] Add recipe for B24 tomogram alignment --- recipes/b24-tomo-align.json | 103 ++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 recipes/b24-tomo-align.json diff --git a/recipes/b24-tomo-align.json b/recipes/b24-tomo-align.json new file mode 100644 index 00000000..65c4b250 --- /dev/null +++ b/recipes/b24-tomo-align.json @@ -0,0 +1,103 @@ +{ + "1": { + "output": 2, + "parameters": { + "dcid": "{dcid}", + "ispyb_command": "insert_tomogram", + "program_id": "{appid}", + "store_result": "ispyb_tomogram_id" + }, + "queue": "ispyb_connector", + "service": "EMISPyB" + }, + "2": { + "output": { + "denoise": 5, + "failure": 3, + "images": 6, + "ispyb_connector": 4, + "movie": 6, + "projxy": 6, + "projxz": 6, + "success": 7 + }, + "parameters": { + "align_z": 200, + "dose_per_frame": "{dose_per_frame}", + "frame_count": "{frame_count}", + "input_file_list": "{input_file_list}", + "manual_tilt_offset": "{manual_tilt_offset}", + "path_pattern": "{path_pattern}", + "pixel_size": "{pixel_size}", + "relion_options": {}, + "stack_file": "{stack_file}", + "tilt_axis": "{tilt_axis}", + "vol_z": 700, + "wbp": 1 + }, + "queue": "tomo_align", + "service": "TomoAlign" + }, + "3": { + "parameters": { + "ispyb_command": "update_processing_status", + "status_message": "processing failure", + "program_id": "{appid}", + "status": "failure" + }, + "queue": "ispyb_connector", + "service": "EMISPyB" + }, + "4": { + "parameters": { + "dcid": "{dcid}", + "ispyb_command": "multipart_message", + "program_id": "{appid}", + "tomogram_id": "$ispyb_tomogram_id" + }, + "queue": "ispyb_connector", + "service": "EMISPyB" + }, + "5": { + "output": { + "images": 6, + "ispyb_connector": 9, + "movie": 6, + "segmentation": 8 + }, + "queue": "denoise", + "service": "Denoise" + }, + "6": { + "queue": "images", + "service": "Images" + }, + "7": { + "parameters": { + "ispyb_command": "update_processing_status", + "status_message": "processing successful", + "program_id": "{appid}", + "status": "success" + }, + "queue": "ispyb_connector", + "service": "EMISPyB" + }, + "8": { + "output": { + "images": 6, + "ispyb_connector": 9, + "movie": 6 + }, + "queue": "segmentation", + "service": "MembrainSeg" + }, + "9": { + "parameters": { + "ispyb_command": "insert_processed_tomogram", + "tomogram_id": "$ispyb_tomogram_id" + }, + "queue": "ispyb_connector", + "service": "EMISPyB" + }, + "start": [[1, []]] +} From c4565b79e6464108459e1ec36c4b398e13d2b924 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 9 Dec 2025 12:09:43 +0000 Subject: [PATCH 02/12] Route to opening and reading txrm files --- pyproject.toml | 1 + recipes/b24-tomo-align.json | 15 ++- src/cryoemservices/services/tomo_align.py | 119 +++++++++++++++------- 3 files changed, 91 insertions(+), 44 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f6378d39..25707d8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ dependencies = [ "starfile", "stomp-py>8.1.1", "tifffile", # CLEM workflow + "txrm2tiff", "workflows>=3", ] [project.optional-dependencies] diff --git a/recipes/b24-tomo-align.json b/recipes/b24-tomo-align.json index 65c4b250..d4736a0c 100644 --- a/recipes/b24-tomo-align.json +++ b/recipes/b24-tomo-align.json @@ -22,17 +22,14 @@ "success": 7 }, "parameters": { - "align_z": 200, - "dose_per_frame": "{dose_per_frame}", - "frame_count": "{frame_count}", - "input_file_list": "{input_file_list}", - "manual_tilt_offset": "{manual_tilt_offset}", - "path_pattern": "{path_pattern}", - "pixel_size": "{pixel_size}", + "dark_tol": 0, + "manual_tilt_offset": 0, + "out_bin": 1, + "pixel_size": 100, "relion_options": {}, "stack_file": "{stack_file}", - "tilt_axis": "{tilt_axis}", - "vol_z": 700, + "tilt_axis": 0, + "txrm_file": "{txrm_file}", "wbp": 1 }, "queue": "tomo_align", diff --git a/src/cryoemservices/services/tomo_align.py b/src/cryoemservices/services/tomo_align.py index 81561f31..5cbc836f 100644 --- a/src/cryoemservices/services/tomo_align.py +++ b/src/cryoemservices/services/tomo_align.py @@ -12,6 +12,7 @@ import mrcfile import numpy as np import plotly.express as px +import tifffile from gemmi import cif from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator from workflows.recipe import wrap_subscribe @@ -48,6 +49,7 @@ class TomoParameters(BaseModel): aretomo_version: Literal[2, 3] = 3 stack_file: str = Field(..., min_length=1) pixel_size: float + txrm_file: Optional[str] = None path_pattern: Optional[str] = None input_file_list: Optional[str] = None vol_z: Optional[int] = None @@ -86,12 +88,15 @@ class TomoParameters(BaseModel): def check_only_one_is_provided(cls, values): input_file_list = values.get("input_file_list") path_pattern = values.get("path_pattern") - if not input_file_list and not path_pattern: - raise ValueError("input_file_list or path_pattern must be provided") - if input_file_list and path_pattern: + txrm_file = values.get("txrm_file") + if not input_file_list and not path_pattern and not txrm_file: raise ValueError( - "Message must only include one of 'path_pattern' and 'input_file_list'." - " Both are set or one has been set by the recipe." + "input_file_list or path_pattern or txrm_file must be provided" + ) + if input_file_list and path_pattern and txrm_file: + raise ValueError( + "Message must only include one of " + "'path_pattern', 'input_file_list' or 'txrm_file'." ) return values @@ -127,6 +132,7 @@ class TomoAlign(CommonService): # Values to extract for ISPyB input_file_list_of_lists: List[Any] + tilt_angles: dict refined_tilts: List[float] x_shift: List[float] y_shift: List[float] @@ -139,6 +145,7 @@ class TomoAlign(CommonService): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.tilt_angles = {} self.refined_tilts = [] self.rot_centre_z_list = [] @@ -254,15 +261,8 @@ def _tilt(file_list_for_tilts): # If no volume provided, set it to the maximum we allow tomo_params.vol_z = tomo_params.max_vol - # Update the relion options - tomo_params.relion_options = update_relion_options( - tomo_params.relion_options, dict(tomo_params) - ) - tomo_params.relion_options.pixel_size_downscaled = ( - tomo_params.pixel_size * tomo_params.out_bin - ) - # Convert a path pattern into a file list + self.tilt_angles = {} if tomo_params.path_pattern: directory = Path(tomo_params.path_pattern).parent @@ -276,10 +276,23 @@ def _tilt(file_list_for_tilts): elif tomo_params.input_file_list: file_list = ast.literal_eval(tomo_params.input_file_list) self.input_file_list_of_lists = file_list + else: + tomo_params.pixel_size = self.convert_txrm_to_stack(tomo_params.txrm_file) + self.input_file_list_of_lists = [] - self.log.info(f"Input list {self.input_file_list_of_lists}") + self.log.info( + f"Input list {self.input_file_list_of_lists or tomo_params.txrm_file}" + ) self.input_file_list_of_lists.sort(key=_tilt) + # Update the relion options + tomo_params.relion_options = update_relion_options( + tomo_params.relion_options, dict(tomo_params) + ) + tomo_params.relion_options.pixel_size_downscaled = ( + tomo_params.pixel_size * tomo_params.out_bin + ) + # Find all the tilt angles and remove duplicates tilt_dict: dict = {} for tilt in self.input_file_list_of_lists: @@ -333,11 +346,18 @@ def _tilt(file_list_for_tilts): self.input_file_list_of_lists.remove(self.input_file_list_of_lists[index]) # Find the input image dimensions - with mrcfile.open(self.input_file_list_of_lists[0][0]) as mrc: - mrc_header = mrc.header - # x and y get flipped on tomogram creation - tomo_params.relion_options.tomo_size_x = int(mrc_header["nx"]) - tomo_params.relion_options.tomo_size_y = int(mrc_header["ny"]) + if self.input_file_list_of_lists: + with mrcfile.open(self.input_file_list_of_lists[0][0]) as mrc: + mrc_header = mrc.header + tomo_params.relion_options.tomo_size_x = int(mrc_header["nx"]) + tomo_params.relion_options.tomo_size_y = int(mrc_header["ny"]) + else: + # Read shape of the tiff stack if using txrm + data_shape = tifffile.imread(tomo_params.stack_file).shape + tomo_params.relion_options.tomo_size_x = data_shape[2] + tomo_params.relion_options.tomo_size_x = data_shape[1] + + # Scale size of output scaled_x_size = tomo_params.relion_options.tomo_size_x / int( tomo_params.out_bin ) @@ -360,38 +380,38 @@ def _tilt(file_list_for_tilts): rw.transport.nack(header) return - # Stack the tilts with newstack - newstack_path = alignment_output_dir / f"{stack_name}_newstack.txt" - newstack_result = self.newstack(tomo_params, newstack_path) - if newstack_result.returncode: - self.log.error( - f"Newstack failed with exitcode {newstack_result.returncode}:\n" - + newstack_result.stderr.decode("utf8", "replace") - ) - rw.transport.nack(header) - return + if self.input_file_list_of_lists: + # Stack the tilts with newstack + newstack_path = alignment_output_dir / f"{stack_name}_newstack.txt" + newstack_result = self.newstack(tomo_params, newstack_path) + if newstack_result.returncode: + self.log.error( + f"Newstack failed with exitcode {newstack_result.returncode}:\n" + + newstack_result.stderr.decode("utf8", "replace") + ) + rw.transport.nack(header) + return # Set up the angle file needed for dose weighting angle_file = ( Path(tomo_params.stack_file).parent / f"{Path(tomo_params.stack_file).stem}_TLT.txt" ) - tilt_angles = {} for i in range(len(self.input_file_list_of_lists)): tilt_index = _get_tilt_number_v5_12( Path(self.input_file_list_of_lists[i][0]) ) tilt_index -= len(np.where(removed_tilt_numbers < tilt_index)[0]) - tilt_angles[tilt_index] = float(self.input_file_list_of_lists[i][1]) + self.tilt_angles[tilt_index] = float(self.input_file_list_of_lists[i][1]) with open(angle_file, "w") as angfile: - for tilt_id in tilt_angles.keys(): + for tilt_id in self.tilt_angles.keys(): if tomo_params.aretomo_version == 3 and tomo_params.manual_tilt_offset: # AreTomo3 performs better with pre-shifted angles angfile.write( - f"{tilt_angles[tilt_id] + tomo_params.manual_tilt_offset:.2f} {int(tilt_id)}\n" + f"{self.tilt_angles[tilt_id] + tomo_params.manual_tilt_offset:.2f} {int(tilt_id)}\n" ) else: - angfile.write(f"{tilt_angles[tilt_id]:.2f} {int(tilt_id)}\n") + angfile.write(f"{self.tilt_angles[tilt_id]:.2f} {int(tilt_id)}\n") # Do alignment with AreTomo aretomo_output_path = alignment_output_dir / f"{stack_name}_Vol.mrc" @@ -436,7 +456,9 @@ def _tilt(file_list_for_tilts): node_creator_parameters = { "experiment_type": "tomography", "job_type": self.job_type, - "input_file": self.input_file_list_of_lists[0][0], + "input_file": self.input_file_list_of_lists[0][0] + if self.input_file_list_of_lists + else tomo_params.txrm_file, "output_file": str(aretomo_output_path), "relion_options": dict(tomo_params.relion_options), "command": " ".join(aretomo_command), @@ -775,6 +797,33 @@ def _tilt(file_list_for_tilts): self.log.info(f"Done tomogram alignment for {tomo_params.stack_file}") rw.transport.ack(header) + def convert_txrm_to_stack(self, txrm_file) -> float: + from txrm2tiff.inspector import Inspector + from txrm2tiff.txrm import open_txrm + from txrm2tiff.txrm_functions.general import read_stream + from txrm2tiff.xradia_properties.enums import XrmDataTypes + + with open_txrm( + txrm_file, load_images=False, load_reference=False, strict=False + ) as txrm: + inspector = Inspector(txrm) + angles = read_stream( + inspector.txrm.ole, + "ImageInfo/Angles", + XrmDataTypes.XRM_FLOAT, + strict=True, + ) + pixel_size_microns = read_stream( + inspector.txrm.ole, + "ImageInfo/PixelSize", + XrmDataTypes.XRM_FLOAT, + strict=True, + ) + self.tilt_angles = dict(enumerate(angles)) + if pixel_size_microns: + return pixel_size_microns[0] * 1000 + return 100 + def newstack(self, tomo_parameters: TomoParameters, newstack_path: Path): """ Construct file containing a list of files From d939f69266dbcb1e726df4fa7045373cae554354 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 9 Dec 2025 14:58:17 +0000 Subject: [PATCH 03/12] Fixes and tests for txrm tomo align --- src/cryoemservices/services/tomo_align.py | 43 +-- tests/services/test_tomo_align.py | 304 ++++++++++++++++++++++ 2 files changed, 331 insertions(+), 16 deletions(-) diff --git a/src/cryoemservices/services/tomo_align.py b/src/cryoemservices/services/tomo_align.py index 5cbc836f..274cd522 100644 --- a/src/cryoemservices/services/tomo_align.py +++ b/src/cryoemservices/services/tomo_align.py @@ -276,9 +276,15 @@ def _tilt(file_list_for_tilts): elif tomo_params.input_file_list: file_list = ast.literal_eval(tomo_params.input_file_list) self.input_file_list_of_lists = file_list - else: - tomo_params.pixel_size = self.convert_txrm_to_stack(tomo_params.txrm_file) + elif tomo_params.txrm_file and Path(tomo_params.txrm_file).is_file(): + tomo_params.pixel_size = self.convert_txrm_to_stack( + tomo_params.txrm_file, tomo_params.stack_file + ) self.input_file_list_of_lists = [] + else: + self.log.warning(f"Invalid input or {tomo_params.txrm_file} is not a file") + rw.transport.nack(header, requeue=True) + return self.log.info( f"Input list {self.input_file_list_of_lists or tomo_params.txrm_file}" @@ -355,7 +361,7 @@ def _tilt(file_list_for_tilts): # Read shape of the tiff stack if using txrm data_shape = tifffile.imread(tomo_params.stack_file).shape tomo_params.relion_options.tomo_size_x = data_shape[2] - tomo_params.relion_options.tomo_size_x = data_shape[1] + tomo_params.relion_options.tomo_size_y = data_shape[1] # Scale size of output scaled_x_size = tomo_params.relion_options.tomo_size_x / int( @@ -599,18 +605,19 @@ def _tilt(file_list_for_tilts): im_diff = 0 # TiltImageAlignment (one per movie) node_creator_params_list = [] - (project_dir / f"ExcludeTiltImages/job{job_number - 2:03}").mkdir( - parents=True, exist_ok=True - ) - if not ( - project_dir / f"ExcludeTiltImages/job{job_number - 2:03}/tilts" - ).is_symlink(): - ( + if self.input_file_list_of_lists: + (project_dir / f"ExcludeTiltImages/job{job_number - 2:03}").mkdir( + parents=True, exist_ok=True + ) + if not ( project_dir / f"ExcludeTiltImages/job{job_number - 2:03}/tilts" - ).symlink_to(project_dir / "MotionCorr/job002/Movies") - (project_dir / f"AlignTiltSeries/job{job_number - 1:03}").mkdir( - parents=True, exist_ok=True - ) + ).is_symlink(): + ( + project_dir / f"ExcludeTiltImages/job{job_number - 2:03}/tilts" + ).symlink_to(project_dir / "MotionCorr/job002/Movies") + (project_dir / f"AlignTiltSeries/job{job_number - 1:03}").mkdir( + parents=True, exist_ok=True + ) if self.rot: tomo_params.relion_options.tilt_axis_angle = self.rot for im, movie in enumerate(self.input_file_list_of_lists): @@ -763,7 +770,6 @@ def _tilt(file_list_for_tilts): self.thickness_pixels * tomo_params.pixel_size ) rw.send_to("images", images_call_params) - print(images_call_params) # Forward results to denoise service self.log.info(f"Sending to denoise service {aretomo_output_path}") @@ -797,12 +803,14 @@ def _tilt(file_list_for_tilts): self.log.info(f"Done tomogram alignment for {tomo_params.stack_file}") rw.transport.ack(header) - def convert_txrm_to_stack(self, txrm_file) -> float: + def convert_txrm_to_stack(self, txrm_file, stack_file) -> float: from txrm2tiff.inspector import Inspector + from txrm2tiff.main import convert_and_save from txrm2tiff.txrm import open_txrm from txrm2tiff.txrm_functions.general import read_stream from txrm2tiff.xradia_properties.enums import XrmDataTypes + # Read the tilt angles and pixel size from the txrm with open_txrm( txrm_file, load_images=False, load_reference=False, strict=False ) as txrm: @@ -820,6 +828,9 @@ def convert_txrm_to_stack(self, txrm_file) -> float: strict=True, ) self.tilt_angles = dict(enumerate(angles)) + + # Convert the txrm to a tiff stack + convert_and_save(txrm_file, stack_file, custom_reference=None) if pixel_size_microns: return pixel_size_microns[0] * 1000 return 100 diff --git a/tests/services/test_tomo_align.py b/tests/services/test_tomo_align.py index 51a9aafc..c7a9b44b 100644 --- a/tests/services/test_tomo_align.py +++ b/tests/services/test_tomo_align.py @@ -8,6 +8,7 @@ import mrcfile import numpy as np import pytest +from txrm2tiff.xradia_properties.enums import XrmDataTypes from workflows.transport.offline_transport import OfflineTransport from cryoemservices.services import tomo_align @@ -2020,6 +2021,309 @@ def write_aretomo_outputs(command, capture_output): offline_transport.send.assert_any_call("failure", {}) +@mock.patch("cryoemservices.services.tomo_align.subprocess.run") +@mock.patch("cryoemservices.services.tomo_align.tifffile") +@mock.patch("txrm2tiff.main.convert_and_save") +@mock.patch("txrm2tiff.txrm.open_txrm") +@mock.patch("txrm2tiff.inspector.Inspector") +@mock.patch("txrm2tiff.txrm_functions.general.read_stream") +@mock.patch("cryoemservices.services.tomo_align.resize_tomogram") +def test_tomo_align_service_txrm( + mock_resize, + mock_read_stream, + mock_inspector, + mock_open_txrm, + mock_convert_and_save, + mock_tifffile, + mock_subprocess, + offline_transport, + tmp_path, +): + """ + Send a test message to TomoAlign (AreTomo3) for a txrm file + """ + mock_tifffile.imread().shape = (600, 3000, 4000) + mock_read_stream.return_value = [0.1, 0.3, 0.5] + + header = { + "message-id": mock.sentinel, + "subscription": mock.sentinel, + } + tomo_align_test_message = { + "stack_file": f"{tmp_path}/Tomograms/job001/stack.mrc", + "txrm_file": f"{tmp_path}/tilt_stack.txrm", + "pixel_size": 100, + "relion_options": {}, + "dark_tol": 0, + "manual_tilt_offset": 0, + "out_bin": 1, + "tilt_axis": 0, + "wbp": 1, + } + output_relion_options = dict(RelionServiceOptions()) + output_relion_options["pixel_size"] = 100 + output_relion_options["pixel_size_downscaled"] = 100 + output_relion_options["tomo_size_x"] = 4000 + output_relion_options["tomo_size_y"] = 3000 + output_relion_options["vol_z"] = 530 + + # Touch the expected input files + (tmp_path / "tilt_stack.txrm").touch() + + # Set up the mock service + service = tomo_align.TomoAlign( + environment={"queue": ""}, transport=offline_transport + ) + service.initializing() + + def write_aretomo_outputs(command, capture_output): + if command[0] != "AreTomo3": + return CompletedProcess("", returncode=0) + # Set up outputs: stack_Imod file like AreTomo3, no exclusions but with space + (tmp_path / "Tomograms/job001/stack_Imod").mkdir(parents=True) + (tmp_path / "Tomograms/job001/stack_Vol.mrc").touch() + with open(tmp_path / "Tomograms/job001/stack_Imod/tilt.com", "w") as dark_file: + dark_file.write("EXCLUDELIST ") + with open(tmp_path / "Tomograms/job001/stack.aln", "w") as aln_file: + aln_file.write("# Thickness = 130\ndummy 0.0 1000 1.2 2.3 5 6 7 8 4.5") + return CompletedProcess( + "", + returncode=0, + stdout=( + "Rot center Z 100.0 200.0 3.1\n" + "Rot center Z 150.0 250.0 2.1\n" + "Tilt offset 1.1, CC: 0.5\n" + "Best tilt axis: 57, Score: 0.5\n" + ).encode("ascii"), + stderr="stderr".encode("ascii"), + ) + + mock_subprocess.side_effect = write_aretomo_outputs + + # Send a message to the service + service.tomo_align(None, header=header, message=tomo_align_test_message) + + aretomo_command = [ + "AreTomo3", + "-Cmd", + "1", + "-InPrefix", + tomo_align_test_message["stack_file"], + "-OutDir", + f"{tmp_path}/Tomograms/job001", + "-TiltCor", + "1", + "0.0", + "-TiltAxis", + "0.0", + "1", + "-AtBin", + "1", + "-PixSize", + "100.0", + "-VolZ", + "1800", + "-ExtZ", + "400", + "-FlipVol", + "0", + "-Wbp", + "1", + "-OutImod", + "1", + "-DarkTol", + "0.0", + ] + + # Check txrm file reading + mock_open_txrm.assert_called_once_with( + tomo_align_test_message["txrm_file"], + load_images=False, + load_reference=False, + strict=False, + ) + mock_open_txrm().__enter__.assert_called_once() + mock_inspector.assert_called_once() + mock_read_stream.assert_any_call( + mock.ANY, + "ImageInfo/Angles", + XrmDataTypes.XRM_FLOAT, + strict=True, + ) + mock_read_stream.assert_any_call( + mock.ANY, + "ImageInfo/PixelSize", + XrmDataTypes.XRM_FLOAT, + strict=True, + ) + mock_convert_and_save.assert_called_once_with( + tomo_align_test_message["txrm_file"], + tomo_align_test_message["stack_file"], + custom_reference=None, + ) + + # Check the expected calls were made + assert mock_subprocess.call_count == 2 + mock_subprocess.assert_any_call( + aretomo_command, + capture_output=True, + ) + mock_subprocess.assert_any_call( + [ + "rotatevol", + "-i", + f"{tmp_path}/Tomograms/job001/stack_Vol.mrc", + "-ou", + f"{tmp_path}/Tomograms/job001/stack_Vol.mrc", + "-size", + "4000,3000,530", + "-a", + "0,0,-90", + ], + capture_output=True, + ) + + # Check resizing + mock_resize.assert_called_once_with( + tmp_path / "Tomograms/job001/stack_Vol.mrc", 530 + ) + + # Check the angle file + assert (tmp_path / "Tomograms/job001/stack_TLT.txt").is_file() + with open(tmp_path / "Tomograms/job001/stack_TLT.txt", "r") as angfile: + angles_data = angfile.read() + assert angles_data == "0.10 0\n0.30 1\n0.50 2\n" + + # Check the shift plot + with open(tmp_path / "Tomograms/job001/stack_xy_shift_plot.json") as shift_plot: + shift_data = json.load(shift_plot) + assert shift_data["data"][0]["x"] == [1.2] + assert shift_data["data"][0]["y"] == [2.3] + + # Check that the correct messages were sent + assert offline_transport.send.call_count == 11 + offline_transport.send.assert_any_call( + "node_creator", + { + "experiment_type": "tomography", + "job_type": "relion.reconstructtomograms", + "input_file": f"{tmp_path}/tilt_stack.txrm", + "output_file": f"{tmp_path}/Tomograms/job001/stack_Vol.mrc", + "relion_options": output_relion_options, + "command": " ".join(aretomo_command), + "stdout": ( + "Rot center Z 100.0 200.0 3.1\n" + "Rot center Z 150.0 250.0 2.1\n" + "Tilt offset 1.1, CC: 0.5\n" + "Best tilt axis: 57, Score: 0.5\n" + ), + "stderr": "stderr", + "success": True, + }, + ) + offline_transport.send.assert_any_call( + "ispyb_connector", + { + "ispyb_command": "multipart_message", + "ispyb_command_list": [ + { + "ispyb_command": "insert_tomogram", + "volume_file": "stack_Vol.mrc", + "stack_file": tomo_align_test_message["stack_file"], + "size_x": 4000.0, + "size_y": 3000.0, + "size_z": 530, + "pixel_spacing": "100.0", + "tilt_angle_offset": "1.1", + "z_shift": "2.1", + "file_directory": f"{tmp_path}/Tomograms/job001", + "central_slice_image": "stack_Vol_thumbnail.jpeg", + "tomogram_movie": "stack_Vol_movie.png", + "xy_shift_plot": "stack_xy_shift_plot.json", + "proj_xy": "stack_Vol_projXY.jpeg", + "proj_xz": "stack_Vol_projXZ.jpeg", + "alignment_quality": "0.5", + }, + { + "ispyb_command": "insert_processed_tomogram", + "file_path": f"{tmp_path}/Tomograms/job001/stack.mrc", + "processing_type": "Stack", + }, + { + "ispyb_command": "insert_processed_tomogram", + "file_path": f"{tmp_path}/Tomograms/job001/stack_alignment.jpeg", + "processing_type": "Alignment", + }, + ], + }, + ) + offline_transport.send.assert_any_call( + "images", + { + "image_command": "tilt_series_alignment", + "file": tomo_align_test_message["stack_file"], + "aln_file": f"{tmp_path}/Tomograms/job001/stack.aln", + "pixel_size": tomo_align_test_message["pixel_size"], + }, + ) + offline_transport.send.assert_any_call( + "images", + { + "image_command": "mrc_central_slice", + "file": tomo_align_test_message["stack_file"], + }, + ) + offline_transport.send.assert_any_call( + "images", + { + "image_command": "mrc_to_apng", + "file": tomo_align_test_message["stack_file"], + }, + ) + offline_transport.send.assert_any_call( + "images", + { + "image_command": "mrc_central_slice", + "file": f"{tmp_path}/Tomograms/job001/stack_Vol.mrc", + }, + ) + offline_transport.send.assert_any_call( + "images", + { + "image_command": "mrc_to_apng", + "file": f"{tmp_path}/Tomograms/job001/stack_Vol.mrc", + }, + ) + offline_transport.send.assert_any_call( + "images", + { + "image_command": "mrc_projection", + "file": f"{tmp_path}/Tomograms/job001/stack_Vol.mrc", + "projection": "XY", + "pixel_spacing": 100.0, + }, + ) + offline_transport.send.assert_any_call( + "images", + { + "image_command": "mrc_projection", + "file": f"{tmp_path}/Tomograms/job001/stack_Vol.mrc", + "projection": "YZ", + "pixel_spacing": 100.0, + "thickness_ang": 13000.0, + }, + ) + offline_transport.send.assert_any_call( + "denoise", + { + "volume": f"{tmp_path}/Tomograms/job001/stack_Vol.mrc", + "output_dir": f"{tmp_path}/Denoise/job002/tomograms", + "relion_options": output_relion_options, + }, + ) + offline_transport.send.assert_any_call("success", {}) + + def test_parse_tomo_align_output(offline_transport): """ Send test lines to the output parser From 5f45b2b9bdbc5bf3f50147e50b58d855ce45e984 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 18 Dec 2025 11:21:19 +0000 Subject: [PATCH 04/12] Stack file must be mrc for aretomo3 --- src/cryoemservices/services/tomo_align.py | 46 +++++++++++++---------- tests/services/test_tomo_align.py | 20 +++++++--- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/src/cryoemservices/services/tomo_align.py b/src/cryoemservices/services/tomo_align.py index 274cd522..b395ab6e 100644 --- a/src/cryoemservices/services/tomo_align.py +++ b/src/cryoemservices/services/tomo_align.py @@ -12,7 +12,6 @@ import mrcfile import numpy as np import plotly.express as px -import tifffile from gemmi import cif from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator from workflows.recipe import wrap_subscribe @@ -261,6 +260,11 @@ def _tilt(file_list_for_tilts): # If no volume provided, set it to the maximum we allow tomo_params.vol_z = tomo_params.max_vol + # Get the names of the output files expected + alignment_output_dir = Path(tomo_params.stack_file).parent + Path(tomo_params.stack_file).parent.mkdir(parents=True, exist_ok=True) + stack_name = str(Path(tomo_params.stack_file).stem) + # Convert a path pattern into a file list self.tilt_angles = {} if tomo_params.path_pattern: @@ -281,9 +285,14 @@ def _tilt(file_list_for_tilts): tomo_params.txrm_file, tomo_params.stack_file ) self.input_file_list_of_lists = [] + # Check we now have the expected stack file + if not Path(tomo_params.stack_file).is_file(): + self.log.warning("Stack file generation failed") + rw.transport.nack(header) + return else: self.log.warning(f"Invalid input or {tomo_params.txrm_file} is not a file") - rw.transport.nack(header, requeue=True) + rw.transport.nack(header) return self.log.info( @@ -353,15 +362,16 @@ def _tilt(file_list_for_tilts): # Find the input image dimensions if self.input_file_list_of_lists: + # Read the first image if a list is given with mrcfile.open(self.input_file_list_of_lists[0][0]) as mrc: mrc_header = mrc.header - tomo_params.relion_options.tomo_size_x = int(mrc_header["nx"]) - tomo_params.relion_options.tomo_size_y = int(mrc_header["ny"]) + else: - # Read shape of the tiff stack if using txrm - data_shape = tifffile.imread(tomo_params.stack_file).shape - tomo_params.relion_options.tomo_size_x = data_shape[2] - tomo_params.relion_options.tomo_size_y = data_shape[1] + # Read shape of the stack if using txrm + with mrcfile.open(tomo_params.stack_file) as mrc: + mrc_header = mrc.header + tomo_params.relion_options.tomo_size_x = int(mrc_header["nx"]) + tomo_params.relion_options.tomo_size_y = int(mrc_header["ny"]) # Scale size of output scaled_x_size = tomo_params.relion_options.tomo_size_x / int( @@ -371,11 +381,6 @@ def _tilt(file_list_for_tilts): tomo_params.out_bin ) - # Get the names of the output files expected - alignment_output_dir = Path(tomo_params.stack_file).parent - Path(tomo_params.stack_file).parent.mkdir(parents=True, exist_ok=True) - stack_name = str(Path(tomo_params.stack_file).stem) - project_dir_search = re.search(".+/job[0-9]+/", tomo_params.stack_file) job_num_search = re.search("/job[0-9]+", tomo_params.stack_file) if project_dir_search and job_num_search: @@ -399,10 +404,7 @@ def _tilt(file_list_for_tilts): return # Set up the angle file needed for dose weighting - angle_file = ( - Path(tomo_params.stack_file).parent - / f"{Path(tomo_params.stack_file).stem}_TLT.txt" - ) + angle_file = Path(tomo_params.stack_file).parent / f"{stack_name}_TLT.txt" for i in range(len(self.input_file_list_of_lists)): tilt_index = _get_tilt_number_v5_12( Path(self.input_file_list_of_lists[i][0]) @@ -803,7 +805,7 @@ def _tilt(file_list_for_tilts): self.log.info(f"Done tomogram alignment for {tomo_params.stack_file}") rw.transport.ack(header) - def convert_txrm_to_stack(self, txrm_file, stack_file) -> float: + def convert_txrm_to_stack(self, txrm_file: str, stack_file: str) -> float: from txrm2tiff.inspector import Inspector from txrm2tiff.main import convert_and_save from txrm2tiff.txrm import open_txrm @@ -829,8 +831,12 @@ def convert_txrm_to_stack(self, txrm_file, stack_file) -> float: ) self.tilt_angles = dict(enumerate(angles)) - # Convert the txrm to a tiff stack - convert_and_save(txrm_file, stack_file, custom_reference=None) + # Convert the txrm to a tiff stack, then convert to mrc for aretomo + tifftomo = Path(stack_file).with_suffix(".tiff") + convert_and_save(txrm_file, str(tifftomo), custom_reference=None) + # Let this run, and check later if the output file exists + subprocess.run(["tif2mrc", str(tifftomo), stack_file]) + tifftomo.unlink(missing_ok=True) if pixel_size_microns: return pixel_size_microns[0] * 1000 return 100 diff --git a/tests/services/test_tomo_align.py b/tests/services/test_tomo_align.py index c7a9b44b..3dc5119e 100644 --- a/tests/services/test_tomo_align.py +++ b/tests/services/test_tomo_align.py @@ -2022,7 +2022,7 @@ def write_aretomo_outputs(command, capture_output): @mock.patch("cryoemservices.services.tomo_align.subprocess.run") -@mock.patch("cryoemservices.services.tomo_align.tifffile") +@mock.patch("cryoemservices.services.tomo_align.mrcfile") @mock.patch("txrm2tiff.main.convert_and_save") @mock.patch("txrm2tiff.txrm.open_txrm") @mock.patch("txrm2tiff.inspector.Inspector") @@ -2034,7 +2034,7 @@ def test_tomo_align_service_txrm( mock_inspector, mock_open_txrm, mock_convert_and_save, - mock_tifffile, + mock_mrcfile, mock_subprocess, offline_transport, tmp_path, @@ -2042,7 +2042,7 @@ def test_tomo_align_service_txrm( """ Send a test message to TomoAlign (AreTomo3) for a txrm file """ - mock_tifffile.imread().shape = (600, 3000, 4000) + mock_mrcfile.open().__enter__().header = {"nx": 4000, "ny": 3000, "nz": 600} mock_read_stream.return_value = [0.1, 0.3, 0.5] header = { @@ -2076,8 +2076,9 @@ def test_tomo_align_service_txrm( ) service.initializing() - def write_aretomo_outputs(command, capture_output): + def write_aretomo_outputs(command, capture_output: bool = False): if command[0] != "AreTomo3": + (tmp_path / "Tomograms/job001/stack.mrc").touch(exist_ok=True) return CompletedProcess("", returncode=0) # Set up outputs: stack_Imod file like AreTomo3, no exclusions but with space (tmp_path / "Tomograms/job001/stack_Imod").mkdir(parents=True) @@ -2158,12 +2159,19 @@ def write_aretomo_outputs(command, capture_output): ) mock_convert_and_save.assert_called_once_with( tomo_align_test_message["txrm_file"], - tomo_align_test_message["stack_file"], + f"{tmp_path}/Tomograms/job001/stack.tiff", custom_reference=None, ) # Check the expected calls were made - assert mock_subprocess.call_count == 2 + assert mock_subprocess.call_count == 3 + mock_subprocess.assert_any_call( + [ + "tif2mrc", + f"{tmp_path}/Tomograms/job001/stack.tiff", + f"{tmp_path}/Tomograms/job001/stack.mrc", + ] + ) mock_subprocess.assert_any_call( aretomo_command, capture_output=True, From 11758291df28c078c16c732ea15ca6bc46751745 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 31 Mar 2026 16:58:53 +0100 Subject: [PATCH 05/12] Allow non-project txrm files --- src/cryoemservices/services/tomo_align.py | 18 +++-- tests/services/test_tomo_align.py | 88 ++++++++++------------- 2 files changed, 50 insertions(+), 56 deletions(-) diff --git a/src/cryoemservices/services/tomo_align.py b/src/cryoemservices/services/tomo_align.py index 7ff2b535..f412f757 100644 --- a/src/cryoemservices/services/tomo_align.py +++ b/src/cryoemservices/services/tomo_align.py @@ -105,7 +105,7 @@ class TomoParameters(BaseModel): input_file_list: Optional[str] = None vol_z: Optional[int] = None max_vol: int = 1800 - min_vol: int = 800 + min_vol: int = 600 extra_vol: int = 400 out_bin: int = 4 second_bin: Optional[int] = None @@ -435,11 +435,15 @@ def _tilt(file_list_for_tilts): tomo_params.out_bin ) - project_dir_search = re.search(".+/job[0-9]+/", tomo_params.stack_file) + project_dir_search = re.search(".+/Tomograms/", tomo_params.stack_file) job_num_search = re.search("/job[0-9]+", tomo_params.stack_file) if project_dir_search and job_num_search: - project_dir = Path(project_dir_search[0]).parent.parent + project_dir = Path(project_dir_search[0]).parent job_number = int(job_num_search[0][4:]) + elif project_dir_search and tomo_params.txrm_file: + # Allow non-Relion projects for txrm + project_dir = Path(project_dir_search[0]).parent + job_number = 0 else: self.log.warning(f"Invalid project directory in {tomo_params.stack_file}") rw.transport.nack(header) @@ -813,13 +817,15 @@ def _tilt(file_list_for_tilts): # Forward results to denoise service self.log.info(f"Sending to denoise service {aretomo_output_path}") + if job_number: + denoise_dir = project_dir / f"Denoise/job{job_number + 1:03}/tomograms" + else: + denoise_dir = project_dir / "Denoise" rw.send_to( "denoise", { "volume": str(aretomo_output_path), - "output_dir": str( - project_dir / f"Denoise/job{job_number + 1:03}/tomograms" - ), + "output_dir": str(denoise_dir), "relion_options": dict(tomo_params.relion_options), }, ) diff --git a/tests/services/test_tomo_align.py b/tests/services/test_tomo_align.py index 7723c323..0c2f1af7 100644 --- a/tests/services/test_tomo_align.py +++ b/tests/services/test_tomo_align.py @@ -17,9 +17,10 @@ class MrcFileHeader: - def __init__(self, nx, ny): + def __init__(self, nx, ny, nz=1): self.nx = nx self.ny = ny + self.nz = nz @pytest.fixture @@ -1411,7 +1412,7 @@ def test_tomo_align_service_path_pattern( output_relion_options["frame_count"] = 6 output_relion_options["dose_per_frame"] = 0.2 # Volume default to minimum - output_relion_options["vol_z"] = 800 + output_relion_options["vol_z"] = 600 # Set up the mock service service = tomo_align.TomoAlign( @@ -1511,10 +1512,10 @@ def write_aretomo_outputs(command, capture_output): # Check resizing and rotating assert mock_resize.call_count == 2 mock_resize.assert_any_call( - tmp_path / "Tomograms/job006/tomograms/test_stack_Vol.mrc", int(800 / 4) + tmp_path / "Tomograms/job006/tomograms/test_stack_Vol.mrc", int(600 / 4) ) mock_resize.assert_any_call( - tmp_path / "Tomograms/job006/tomograms/test_stack_2ND_Vol.mrc", int(800 / 2) + tmp_path / "Tomograms/job006/tomograms/test_stack_2ND_Vol.mrc", int(600 / 2) ) mock_rotate.assert_not_called() @@ -2087,7 +2088,9 @@ def write_aretomo_outputs(command, capture_output): @mock.patch("txrm2tiff.inspector.Inspector") @mock.patch("txrm2tiff.txrm_functions.general.read_stream") @mock.patch("cryoemservices.services.tomo_align.resize_tomogram") +@mock.patch("cryoemservices.services.tomo_align.rotate_tomogram") def test_tomo_align_service_txrm( + mock_rotate, mock_resize, mock_read_stream, mock_inspector, @@ -2101,7 +2104,7 @@ def test_tomo_align_service_txrm( """ Send a test message to TomoAlign (AreTomo3) for a txrm file """ - mock_mrcfile.open().__enter__().header = {"nx": 4000, "ny": 3000, "nz": 600} + mock_mrcfile.open().__enter__().header = MrcFileHeader(nx=4000, ny=3000, nz=600) mock_read_stream.return_value = [0.1, 0.3, 0.5] header = { @@ -2109,7 +2112,7 @@ def test_tomo_align_service_txrm( "subscription": mock.sentinel, } tomo_align_test_message = { - "stack_file": f"{tmp_path}/Tomograms/job001/stack.mrc", + "stack_file": f"{tmp_path}/Tomograms/stack.mrc", "txrm_file": f"{tmp_path}/tilt_stack.txrm", "pixel_size": 100, "relion_options": {}, @@ -2124,7 +2127,7 @@ def test_tomo_align_service_txrm( output_relion_options["pixel_size_downscaled"] = 100 output_relion_options["tomo_size_x"] = 4000 output_relion_options["tomo_size_y"] = 3000 - output_relion_options["vol_z"] = 530 + output_relion_options["vol_z"] = 600 # Touch the expected input files (tmp_path / "tilt_stack.txrm").touch() @@ -2137,14 +2140,14 @@ def test_tomo_align_service_txrm( def write_aretomo_outputs(command, capture_output: bool = False): if command[0] != "AreTomo3": - (tmp_path / "Tomograms/job001/stack.mrc").touch(exist_ok=True) + (tmp_path / "Tomograms/stack.mrc").touch(exist_ok=True) return CompletedProcess("", returncode=0) # Set up outputs: stack_Imod file like AreTomo3, no exclusions but with space - (tmp_path / "Tomograms/job001/stack_Imod").mkdir(parents=True) - (tmp_path / "Tomograms/job001/stack_Vol.mrc").touch() - with open(tmp_path / "Tomograms/job001/stack_Imod/tilt.com", "w") as dark_file: + (tmp_path / "Tomograms/stack_Imod").mkdir(parents=True) + (tmp_path / "Tomograms/stack_Vol.mrc").touch() + with open(tmp_path / "Tomograms/stack_Imod/tilt.com", "w") as dark_file: dark_file.write("EXCLUDELIST ") - with open(tmp_path / "Tomograms/job001/stack.aln", "w") as aln_file: + with open(tmp_path / "Tomograms/stack.aln", "w") as aln_file: aln_file.write("# Thickness = 130\ndummy 0.0 1000 1.2 2.3 5 6 7 8 4.5") return CompletedProcess( "", @@ -2170,7 +2173,7 @@ def write_aretomo_outputs(command, capture_output: bool = False): "-InPrefix", tomo_align_test_message["stack_file"], "-OutDir", - f"{tmp_path}/Tomograms/job001", + f"{tmp_path}/Tomograms", "-TiltCor", "1", "0.0", @@ -2218,51 +2221,36 @@ def write_aretomo_outputs(command, capture_output: bool = False): ) mock_convert_and_save.assert_called_once_with( tomo_align_test_message["txrm_file"], - f"{tmp_path}/Tomograms/job001/stack.tiff", + f"{tmp_path}/Tomograms/stack.tiff", custom_reference=None, ) # Check the expected calls were made - assert mock_subprocess.call_count == 3 + assert mock_subprocess.call_count == 2 mock_subprocess.assert_any_call( [ "tif2mrc", - f"{tmp_path}/Tomograms/job001/stack.tiff", - f"{tmp_path}/Tomograms/job001/stack.mrc", + f"{tmp_path}/Tomograms/stack.tiff", + f"{tmp_path}/Tomograms/stack.mrc", ] ) mock_subprocess.assert_any_call( aretomo_command, capture_output=True, ) - mock_subprocess.assert_any_call( - [ - "rotatevol", - "-i", - f"{tmp_path}/Tomograms/job001/stack_Vol.mrc", - "-ou", - f"{tmp_path}/Tomograms/job001/stack_Vol.mrc", - "-size", - "4000,3000,530", - "-a", - "0,0,-90", - ], - capture_output=True, - ) - # Check resizing - mock_resize.assert_called_once_with( - tmp_path / "Tomograms/job001/stack_Vol.mrc", 530 - ) + # Check resizing and rotating + mock_resize.assert_called_once_with(tmp_path / "Tomograms/stack_Vol.mrc", 600) + mock_rotate.assert_called_once_with(tmp_path / "Tomograms/stack_Vol.mrc", 0) # Check the angle file - assert (tmp_path / "Tomograms/job001/stack_TLT.txt").is_file() - with open(tmp_path / "Tomograms/job001/stack_TLT.txt", "r") as angfile: + assert (tmp_path / "Tomograms/stack_TLT.txt").is_file() + with open(tmp_path / "Tomograms/stack_TLT.txt", "r") as angfile: angles_data = angfile.read() assert angles_data == "0.10 0\n0.30 1\n0.50 2\n" # Check the shift plot - with open(tmp_path / "Tomograms/job001/stack_xy_shift_plot.json") as shift_plot: + with open(tmp_path / "Tomograms/stack_xy_shift_plot.json") as shift_plot: shift_data = json.load(shift_plot) assert shift_data["data"][0]["x"] == [1.2] assert shift_data["data"][0]["y"] == [2.3] @@ -2275,7 +2263,7 @@ def write_aretomo_outputs(command, capture_output: bool = False): "experiment_type": "tomography", "job_type": "relion.reconstructtomograms", "input_file": f"{tmp_path}/tilt_stack.txrm", - "output_file": f"{tmp_path}/Tomograms/job001/stack_Vol.mrc", + "output_file": f"{tmp_path}/Tomograms/stack_Vol.mrc", "relion_options": output_relion_options, "command": " ".join(aretomo_command), "stdout": ( @@ -2299,11 +2287,11 @@ def write_aretomo_outputs(command, capture_output: bool = False): "stack_file": tomo_align_test_message["stack_file"], "size_x": 4000.0, "size_y": 3000.0, - "size_z": 530, + "size_z": 600, "pixel_spacing": "100.0", "tilt_angle_offset": "1.1", "z_shift": "2.1", - "file_directory": f"{tmp_path}/Tomograms/job001", + "file_directory": f"{tmp_path}/Tomograms", "central_slice_image": "stack_Vol_thumbnail.jpeg", "tomogram_movie": "stack_Vol_movie.png", "xy_shift_plot": "stack_xy_shift_plot.json", @@ -2313,12 +2301,12 @@ def write_aretomo_outputs(command, capture_output: bool = False): }, { "ispyb_command": "insert_processed_tomogram", - "file_path": f"{tmp_path}/Tomograms/job001/stack.mrc", + "file_path": f"{tmp_path}/Tomograms/stack.mrc", "processing_type": "Stack", }, { "ispyb_command": "insert_processed_tomogram", - "file_path": f"{tmp_path}/Tomograms/job001/stack_alignment.jpeg", + "file_path": f"{tmp_path}/Tomograms/stack_alignment.jpeg", "processing_type": "Alignment", }, ], @@ -2329,7 +2317,7 @@ def write_aretomo_outputs(command, capture_output: bool = False): { "image_command": "tilt_series_alignment", "file": tomo_align_test_message["stack_file"], - "aln_file": f"{tmp_path}/Tomograms/job001/stack.aln", + "aln_file": f"{tmp_path}/Tomograms/stack.aln", "pixel_size": tomo_align_test_message["pixel_size"], }, ) @@ -2351,21 +2339,21 @@ def write_aretomo_outputs(command, capture_output: bool = False): "images", { "image_command": "mrc_central_slice", - "file": f"{tmp_path}/Tomograms/job001/stack_Vol.mrc", + "file": f"{tmp_path}/Tomograms/stack_Vol.mrc", }, ) offline_transport.send.assert_any_call( "images", { "image_command": "mrc_to_apng", - "file": f"{tmp_path}/Tomograms/job001/stack_Vol.mrc", + "file": f"{tmp_path}/Tomograms/stack_Vol.mrc", }, ) offline_transport.send.assert_any_call( "images", { "image_command": "mrc_projection", - "file": f"{tmp_path}/Tomograms/job001/stack_Vol.mrc", + "file": f"{tmp_path}/Tomograms/stack_Vol.mrc", "projection": "XY", "pixel_spacing": 100.0, }, @@ -2374,7 +2362,7 @@ def write_aretomo_outputs(command, capture_output: bool = False): "images", { "image_command": "mrc_projection", - "file": f"{tmp_path}/Tomograms/job001/stack_Vol.mrc", + "file": f"{tmp_path}/Tomograms/stack_Vol.mrc", "projection": "YZ", "pixel_spacing": 100.0, "thickness_ang": 13000.0, @@ -2383,8 +2371,8 @@ def write_aretomo_outputs(command, capture_output: bool = False): offline_transport.send.assert_any_call( "denoise", { - "volume": f"{tmp_path}/Tomograms/job001/stack_Vol.mrc", - "output_dir": f"{tmp_path}/Denoise/job002/tomograms", + "volume": f"{tmp_path}/Tomograms/stack_Vol.mrc", + "output_dir": f"{tmp_path}/Denoise", "relion_options": output_relion_options, }, ) From 49771c8c5d601756c9d7cdd9cfb69d0ad20ec56d Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 31 Mar 2026 17:05:58 +0100 Subject: [PATCH 06/12] Update broken test --- tests/services/test_tomo_align_slurm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/services/test_tomo_align_slurm.py b/tests/services/test_tomo_align_slurm.py index 11330320..68485bb9 100644 --- a/tests/services/test_tomo_align_slurm.py +++ b/tests/services/test_tomo_align_slurm.py @@ -253,7 +253,7 @@ def test_tomo_align_slurm_service_aretomo3( # Check resizing and rotating at min vol mock_resize.assert_called_once_with( tmp_path / "cm12345-6/Tomograms/job006/tomograms/test_stack_Vol.mrc", - int(800 / 4), + int(600 / 4), ) mock_rotate.assert_not_called() From ce880f7130c838a432bdb9d7d8e9f45e3ca573fe Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 7 Apr 2026 13:35:21 +0100 Subject: [PATCH 07/12] Shift import out of function --- src/cryoemservices/services/tomo_align.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/cryoemservices/services/tomo_align.py b/src/cryoemservices/services/tomo_align.py index f412f757..ee5c47da 100644 --- a/src/cryoemservices/services/tomo_align.py +++ b/src/cryoemservices/services/tomo_align.py @@ -14,6 +14,11 @@ import plotly.express as px from gemmi import cif from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator +from txrm2tiff.inspector import Inspector +from txrm2tiff.main import convert_and_save +from txrm2tiff.txrm import open_txrm +from txrm2tiff.txrm_functions.general import read_stream +from txrm2tiff.xradia_properties.enums import XrmDataTypes from workflows.recipe import wrap_subscribe from cryoemservices.services.common_service import CommonService @@ -850,12 +855,6 @@ def _tilt(file_list_for_tilts): rw.transport.ack(header) def convert_txrm_to_stack(self, txrm_file: str, stack_file: str) -> float: - from txrm2tiff.inspector import Inspector - from txrm2tiff.main import convert_and_save - from txrm2tiff.txrm import open_txrm - from txrm2tiff.txrm_functions.general import read_stream - from txrm2tiff.xradia_properties.enums import XrmDataTypes - # Read the tilt angles and pixel size from the txrm with open_txrm( txrm_file, load_images=False, load_reference=False, strict=False From 30f8fd346cb247883219c4c45241e9899de1dc07 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 7 Apr 2026 13:50:30 +0100 Subject: [PATCH 08/12] Move recipe and reset pixel size --- .../{b24-tomo-align.json => ispyb/sxt-aretomo.json} | 0 src/cryoemservices/services/tomo_align.py | 9 ++++++--- tests/services/test_tomo_align.py | 11 +++++++---- 3 files changed, 13 insertions(+), 7 deletions(-) rename recipes/{b24-tomo-align.json => ispyb/sxt-aretomo.json} (100%) diff --git a/recipes/b24-tomo-align.json b/recipes/ispyb/sxt-aretomo.json similarity index 100% rename from recipes/b24-tomo-align.json rename to recipes/ispyb/sxt-aretomo.json diff --git a/src/cryoemservices/services/tomo_align.py b/src/cryoemservices/services/tomo_align.py index ee5c47da..1f886693 100644 --- a/src/cryoemservices/services/tomo_align.py +++ b/src/cryoemservices/services/tomo_align.py @@ -499,7 +499,6 @@ def _tilt(file_list_for_tilts): plot_path = alignment_output_dir / plot_file # Extract results - pixel_spacing: str = str(tomo_params.pixel_size * tomo_params.out_bin) aln_file = self.extract_from_aln(tomo_params, alignment_output_dir, plot_path) if not aretomo_result.returncode and not aln_file: self.log.error("Failed to read alignment file") @@ -614,7 +613,7 @@ def _tilt(file_list_for_tilts): "size_x": scaled_x_size, # volume image size, pix "size_y": scaled_y_size, "size_z": scaled_z_size, - "pixel_spacing": pixel_spacing, + "pixel_spacing": str(tomo_params.relion_options.pixel_size_downscaled), "tilt_angle_offset": str( self.tilt_offset or tomo_params.manual_tilt_offset ), @@ -812,7 +811,7 @@ def _tilt(file_list_for_tilts): "image_command": "mrc_projection", "file": str(aretomo_output_path), "projection": projection_type, - "pixel_spacing": float(pixel_spacing), + "pixel_spacing": tomo_params.relion_options.pixel_size_downscaled, } if projection_type in ["XZ", "YZ"] and self.thickness_pixels: images_call_params["thickness_ang"] = ( @@ -825,6 +824,10 @@ def _tilt(file_list_for_tilts): if job_number: denoise_dir = project_dir / f"Denoise/job{job_number + 1:03}/tomograms" else: + # This is used for SXT + # Resets the pixel size to prevent segmentation rescaling + tomo_params.relion_options.pixel_size = 10 + tomo_params.relion_options.pixel_size_downscaled = 10 denoise_dir = project_dir / "Denoise" rw.send_to( "denoise", diff --git a/tests/services/test_tomo_align.py b/tests/services/test_tomo_align.py index 0c2f1af7..c8a3bca6 100644 --- a/tests/services/test_tomo_align.py +++ b/tests/services/test_tomo_align.py @@ -2083,10 +2083,10 @@ def write_aretomo_outputs(command, capture_output): @mock.patch("cryoemservices.services.tomo_align.subprocess.run") @mock.patch("cryoemservices.services.tomo_align.mrcfile") -@mock.patch("txrm2tiff.main.convert_and_save") -@mock.patch("txrm2tiff.txrm.open_txrm") -@mock.patch("txrm2tiff.inspector.Inspector") -@mock.patch("txrm2tiff.txrm_functions.general.read_stream") +@mock.patch("cryoemservices.services.tomo_align.convert_and_save") +@mock.patch("cryoemservices.services.tomo_align.open_txrm") +@mock.patch("cryoemservices.services.tomo_align.Inspector") +@mock.patch("cryoemservices.services.tomo_align.read_stream") @mock.patch("cryoemservices.services.tomo_align.resize_tomogram") @mock.patch("cryoemservices.services.tomo_align.rotate_tomogram") def test_tomo_align_service_txrm( @@ -2368,6 +2368,9 @@ def write_aretomo_outputs(command, capture_output: bool = False): "thickness_ang": 13000.0, }, ) + # Denoise gets sent with reset pixel size + output_relion_options["pixel_size"] = 10 + output_relion_options["pixel_size_downscaled"] = 10 offline_transport.send.assert_any_call( "denoise", { From 894989026a7390fb06c8eb1d18dae15c7d4d06ce Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 9 Apr 2026 10:30:21 +0100 Subject: [PATCH 09/12] Skip node creator if no job numbers --- src/cryoemservices/services/tomo_align.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/cryoemservices/services/tomo_align.py b/src/cryoemservices/services/tomo_align.py index 1f886693..959538bf 100644 --- a/src/cryoemservices/services/tomo_align.py +++ b/src/cryoemservices/services/tomo_align.py @@ -445,8 +445,8 @@ def _tilt(file_list_for_tilts): if project_dir_search and job_num_search: project_dir = Path(project_dir_search[0]).parent job_number = int(job_num_search[0][4:]) - elif project_dir_search and tomo_params.txrm_file: - # Allow non-Relion projects for txrm + elif project_dir_search: + # Allow processing without job numbers, but then skip node creator sends project_dir = Path(project_dir_search[0]).parent job_number = 0 else: @@ -522,7 +522,7 @@ def _tilt(file_list_for_tilts): tomo_params.vol_z = tomo_params.min_vol tomo_params.relion_options.vol_z = tomo_params.vol_z - if not job_is_rerun: + if job_number and not job_is_rerun: # Send to node creator if this is the first time this tomogram is made self.log.info("Sending tomo align to node creator") node_creator_parameters = { @@ -653,7 +653,7 @@ def _tilt(file_list_for_tilts): im_diff = 0 # TiltImageAlignment (one per movie) node_creator_params_list = [] - if self.input_file_list_of_lists: + if self.input_file_list_of_lists and job_number: (project_dir / f"ExcludeTiltImages/job{job_number - 2:03}").mkdir( parents=True, exist_ok=True ) @@ -741,8 +741,9 @@ def _tilt(file_list_for_tilts): rw.transport.nack(header) return - for tilt_params in node_creator_params_list: - rw.send_to("node_creator", tilt_params) + if job_number: + for tilt_params in node_creator_params_list: + rw.send_to("node_creator", tilt_params) ispyb_command_list.append( { From 8153e779713ff0d023477c4127d55bd74bef13fe Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Thu, 9 Apr 2026 10:36:39 +0100 Subject: [PATCH 10/12] Shorter header reading logic --- src/cryoemservices/services/tomo_align.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/cryoemservices/services/tomo_align.py b/src/cryoemservices/services/tomo_align.py index 959538bf..92d66e50 100644 --- a/src/cryoemservices/services/tomo_align.py +++ b/src/cryoemservices/services/tomo_align.py @@ -420,14 +420,12 @@ def _tilt(file_list_for_tilts): # Find the input image dimensions if self.input_file_list_of_lists: - # Read the first image if a list is given - with mrcfile.open(self.input_file_list_of_lists[0][0]) as mrc: - mrc_header = mrc.header - + input_image = self.input_file_list_of_lists[0][0] else: - # Read shape of the stack if using txrm - with mrcfile.open(tomo_params.stack_file) as mrc: - mrc_header = mrc.header + input_image = tomo_params.stack_file + # Read the first image if a list is given or stack if using txrm + with mrcfile.open(input_image) as mrc: + mrc_header = mrc.header # x and y get flipped on tomogram creation tomo_params.relion_options.tomo_size_x = int(mrc_header.nx) tomo_params.relion_options.tomo_size_y = int(mrc_header.ny) From 59226bc181676f522dc532dd6764b522d9372294 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Fri, 10 Apr 2026 11:57:17 +0100 Subject: [PATCH 11/12] Update test as no node creator for txrm --- tests/services/test_tomo_align.py | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/tests/services/test_tomo_align.py b/tests/services/test_tomo_align.py index c8a3bca6..bc6536a7 100644 --- a/tests/services/test_tomo_align.py +++ b/tests/services/test_tomo_align.py @@ -2256,26 +2256,7 @@ def write_aretomo_outputs(command, capture_output: bool = False): assert shift_data["data"][0]["y"] == [2.3] # Check that the correct messages were sent - assert offline_transport.send.call_count == 11 - offline_transport.send.assert_any_call( - "node_creator", - { - "experiment_type": "tomography", - "job_type": "relion.reconstructtomograms", - "input_file": f"{tmp_path}/tilt_stack.txrm", - "output_file": f"{tmp_path}/Tomograms/stack_Vol.mrc", - "relion_options": output_relion_options, - "command": " ".join(aretomo_command), - "stdout": ( - "Rot center Z 100.0 200.0 3.1\n" - "Rot center Z 150.0 250.0 2.1\n" - "Tilt offset 1.1, CC: 0.5\n" - "Best tilt axis: 57, Score: 0.5\n" - ), - "stderr": "stderr", - "success": True, - }, - ) + assert offline_transport.send.call_count == 10 offline_transport.send.assert_any_call( "ispyb_connector", { From ca7c879165f157f698e610b4ee2a106b26232379 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 13 Apr 2026 10:33:28 +0100 Subject: [PATCH 12/12] Safer pixel size scaling only when large --- src/cryoemservices/services/tomo_align.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cryoemservices/services/tomo_align.py b/src/cryoemservices/services/tomo_align.py index 92d66e50..40ccaa19 100644 --- a/src/cryoemservices/services/tomo_align.py +++ b/src/cryoemservices/services/tomo_align.py @@ -823,11 +823,12 @@ def _tilt(file_list_for_tilts): if job_number: denoise_dir = project_dir / f"Denoise/job{job_number + 1:03}/tomograms" else: + denoise_dir = project_dir / "Denoise" + if tomo_params.relion_options.pixel_size_downscaled > 20: # This is used for SXT # Resets the pixel size to prevent segmentation rescaling tomo_params.relion_options.pixel_size = 10 tomo_params.relion_options.pixel_size_downscaled = 10 - denoise_dir = project_dir / "Denoise" rw.send_to( "denoise", {