From 92b7ddfe6077c01bacc37846810b76416f0a389a Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 30 Apr 2026 15:00:12 +0100 Subject: [PATCH 01/19] Rearranged stag position fields based on their location in the 'ProjectData.dat' file --- src/murfey/util/models.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/murfey/util/models.py b/src/murfey/util/models.py index 11d1b851..9b05fca8 100644 --- a/src/murfey/util/models.py +++ b/src/murfey/util/models.py @@ -128,20 +128,22 @@ class StagePositionInfo(BaseModel): "ChunkCoincidenceStagePosition" currently correspond to. """ + # Top-level values preparation: StagePositionValues | None = ( None # PreparationSiteLocation/StagePosition/StagePosition ) - chunk_coincidence: StagePositionValues | None = ( - None # Parameters/ChunkCoincidenceStagePosition/StagePosition - ) chunk: StagePositionValues | None = ( None # ChunkSiteLocation/StagePosition/StagePosition ) thinning_1: StagePositionValues | None = ( - None # Parameters/ThinningStagePosition/StagePosition + None # ThinningSiteLocation/StagePosition/StagePosition + ) + # Stored under Parameters + chunk_coincidence: StagePositionValues | None = ( + None # Parameters/ChunkCoincidenceStagePosition/StagePosition ) thinning_2: StagePositionValues | None = ( - None # ThinningSiteLocation/StagePosition/StagePosition + None # Parameters/ThinningStagePosition/StagePosition ) From 8ad14f0506a3989f785d07a14673c889e9896747 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 30 Apr 2026 15:33:42 +0100 Subject: [PATCH 02/19] Moved the 'make_gif' function over to the backend server's FIB workflow module and optimised input parameters required --- src/murfey/server/api/workflow.py | 74 --------------------------- src/murfey/server/api/workflow_fib.py | 71 ++++++++++++++++++++++++- src/murfey/util/route_manifest.yaml | 18 +++---- 3 files changed, 77 insertions(+), 86 deletions(-) diff --git a/src/murfey/server/api/workflow.py b/src/murfey/server/api/workflow.py index ce63b6e8..4eb7906b 100644 --- a/src/murfey/server/api/workflow.py +++ b/src/murfey/server/api/workflow.py @@ -5,7 +5,6 @@ from pathlib import Path from typing import Any, Dict, List, Optional -import numpy as np import sqlalchemy from fastapi import APIRouter, Depends from ispyb.sqlalchemy import ( @@ -21,11 +20,6 @@ from sqlmodel import col, select from werkzeug.utils import secure_filename -try: - from PIL import Image -except ImportError: - Image = None - try: from smartem_backend.api_client import SmartEMAPIClient from smartem_common.schemas import ( @@ -1194,71 +1188,3 @@ def register_sample_image( if _transport_object: return _transport_object.do_insert_sample_image(record) return {"success": False} - - -class MillingParameters(BaseModel): - lamella_number: int - images: List[str] - raw_directory: str - - -@correlative_router.post( - "/year/{year}/visits/{visit_name}/sessions/{session_id}/make_milling_gif" -) -async def make_gif( - year: int, - visit_name: str, - session_id: int, - gif_params: MillingParameters, - db=murfey_db, -): - instrument_name = ( - db.exec(select(Session).where(Session.id == session_id)).one().instrument_name - ) - machine_config = get_machine_config(instrument_name=instrument_name)[ - instrument_name - ] - output_dir = ( - (machine_config.rsync_basepath or Path("")).resolve() - / secure_filename(str(year)) - / secure_filename(visit_name) - / "processed" - ) - output_dir.mkdir(exist_ok=True) - os.chmod(output_dir, mode=machine_config.mkdir_chmod) - output_dir = output_dir / secure_filename(gif_params.raw_directory) - output_dir.mkdir(exist_ok=True) - os.chmod(output_dir, mode=machine_config.mkdir_chmod) - output_path = output_dir / f"lamella_{gif_params.lamella_number}_milling.gif" - - if Image is not None: - images = [Image.open(f) for f in gif_params.images] - else: - images = [] - for im in images: - im.thumbnail((512, 512)) - - # Normalize and convert individual frames to 8-bit - arr: list[np.ndarray] = [] - for im in images: - frame = np.array(im).astype(np.float32) - vmin, vmax = np.percentile(frame, (0.5, 99.5)) - scale = 255 / ((vmax - vmin) or 1) - np.clip(frame, a_min=vmin, a_max=vmax, out=frame) - np.subtract(frame, vmin, out=frame) - np.multiply(frame, scale, out=frame) - arr.append(frame.astype(np.uint8)) - arr = np.array(arr).astype(np.uint8) - - # Convert back to Image objects and save as GIF - converted = [Image.fromarray(arr[f], mode="L") for f in range(len(images))] - converted[0].save( - output_path, - format="GIF", - append_images=converted[1:], - save_all=True, - duration=30, - loop=0, - ) - - return {"output_gif": str(output_path)} diff --git a/src/murfey/server/api/workflow_fib.py b/src/murfey/server/api/workflow_fib.py index 44406757..269899f5 100644 --- a/src/murfey/server/api/workflow_fib.py +++ b/src/murfey/server/api/workflow_fib.py @@ -1,14 +1,19 @@ import json import logging +import os from importlib.metadata import entry_points from pathlib import Path +import numpy as np +import PIL.Image from fastapi import APIRouter, Depends from pydantic import BaseModel -from sqlmodel import Session +from sqlmodel import Session, select +import murfey.util.db as MurfeyDB from murfey.server.api.auth import validate_instrument_token from murfey.server.murfey_db import murfey_db +from murfey.util.config import get_machine_config from murfey.util.models import LamellaSiteInfo logger = logging.getLogger("murfey.server.api.workflow_fib") @@ -57,3 +62,67 @@ def register_fib_milling_progress( "Received the following FIB metadata for registration:\n" f"{json.dumps(site_info.model_dump(exclude_none=True), indent=2, default=str)}" ) + + +class FIBGIFParameters(BaseModel): + lamella_number: int + images: list[Path] + output_file: Path + + +@router.post("/sessions/{session_id}/make_gif") +async def make_gif( + session_id: int, + gif_params: FIBGIFParameters, + db=murfey_db, +): + # Load machine config and session info + session_entry = db.exec( + select(MurfeyDB.Session).where(MurfeyDB.Session.id == session_id) + ).one() + instrument_name = session_entry.instrument_name + visit_name = session_entry.visit + machine_config = get_machine_config(instrument_name=instrument_name)[ + instrument_name + ] + + # Create the directory structure + if not (output_dir := gif_params.output_file.parent).exists(): + output_dir.mkdir(parents=True) + logger.debug(f"Created output directory {output_dir}") + visit_index = output_dir.parts.index(visit_name) + # Change permissions for folders in the visit directory and onwards + for current_path in list(reversed(output_dir.parents))[visit_index + 1 :]: + os.chmod(current_path, mode=machine_config.mkdir_chmod) + + if PIL.Image is not None: + images = [PIL.Image.open(f) for f in gif_params.images] + else: + images = [] + for im in images: + im.thumbnail((512, 512)) + + # Normalize and convert individual frames to 8-bit + arr: list[np.ndarray] = [] + for im in images: + frame = np.array(im).astype(np.float32) + vmin, vmax = np.percentile(frame, (0.5, 99.5)) + scale = 255 / ((vmax - vmin) or 1) + np.clip(frame, a_min=vmin, a_max=vmax, out=frame) + np.subtract(frame, vmin, out=frame) + np.multiply(frame, scale, out=frame) + arr.append(frame.astype(np.uint8)) + arr = np.array(arr).astype(np.uint8) + + # Convert back to PIL.Image objects and save as GIF + converted = [PIL.Image.fromarray(arr[f], mode="L") for f in range(len(images))] + converted[0].save( + gif_params.output_file, + format="GIF", + append_images=converted[1:], + save_all=True, + duration=30, + loop=0, + ) + + return {"output_gif": str(gif_params.output_file)} diff --git a/src/murfey/util/route_manifest.yaml b/src/murfey/util/route_manifest.yaml index dcbc9ece..65dad8b9 100644 --- a/src/murfey/util/route_manifest.yaml +++ b/src/murfey/util/route_manifest.yaml @@ -1292,17 +1292,6 @@ murfey.server.api.workflow.correlative_router: type: str methods: - POST - - path: /workflow/correlative/year/{year}/visits/{visit_name}/sessions/{session_id}/make_milling_gif - function: make_gif - path_params: - - name: year - type: int - - name: visit_name - type: str - - name: session_id - type: int - methods: - - POST murfey.server.api.workflow.router: - path: /workflow/visits/{visit_name}/sessions/{session_id}/register_data_collection_group function: register_dc_group @@ -1447,3 +1436,10 @@ murfey.server.api.workflow_fib.router: type: int methods: - POST + - path: /workflow/fib/sessions/{session_id}/make_gif + function: make_gif + path_params: + - name: session_id + type: int + methods: + - POST From b100c56a7b3a6c4478573cd827964e4ff5f8a438 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 30 Apr 2026 16:22:27 +0100 Subject: [PATCH 03/19] Updated the logic used to post requests to make GIFs from the drift correction images --- src/murfey/client/contexts/fib.py | 272 ++++++++++++++++++++---------- 1 file changed, 187 insertions(+), 85 deletions(-) diff --git a/src/murfey/client/contexts/fib.py b/src/murfey/client/contexts/fib.py index fff544e2..beaa30fd 100644 --- a/src/murfey/client/contexts/fib.py +++ b/src/murfey/client/contexts/fib.py @@ -5,7 +5,6 @@ import threading import xml.etree.ElementTree as ET from dataclasses import dataclass -from datetime import datetime from pathlib import Path from typing import Callable, Type, TypeVar @@ -25,12 +24,6 @@ lock = threading.Lock() -@dataclass -class MillingImage: - file: Path - timestamp: float - - def _number_from_name(name: str) -> int: """ In the AutoTEM and Maps workflows for the FIB, the sites and images are @@ -170,10 +163,10 @@ def _parse_boolean(text: str): # Map class attribute to element name # Paths are relative to the "Site" node "preparation": "PreparationSiteLocation/StagePosition/StagePosition", - "chunk_coincidence": "Parameters/ChunkCoincidenceStagePosition/StagePosition", "chunk": "ChunkSiteLocation/StagePosition/StagePosition", - "thinning_1": "Parameters/ThinningStagePosition/StagePosition", - "thinning_2": "ThinningSiteLocation/StagePosition/StagePosition", + "thinning_1": "ThinningSiteLocation/StagePosition/StagePosition", + "chunk_coincidence": "Parameters/ChunkCoincidenceStagePosition/StagePosition", + "thinning_2": "Parameters/ThinningStagePosition/StagePosition", } @@ -233,6 +226,13 @@ def _file_transferred_to( return destination +@dataclass +class FIBImage: + images: list[Path] = [] + output_file: Path | None = None + is_submitted: bool = False + + class FIBContext(Context): def __init__( self, @@ -245,7 +245,7 @@ def __init__( self._basepath = basepath self._machine_config = machine_config self._site_info: dict[int, LamellaSiteInfo] = {} - self._drift_correction_images: dict[int, list[MillingImage]] = {} + self._drift_correction_images: dict[int, FIBImage] = {} def post_transfer( self, @@ -262,7 +262,6 @@ def post_transfer( # AutoTEM # ----------------------------------------------------------------------------- if self._acquisition_software == "autotem": - parts = transferred_file.parts if transferred_file.name == "ProjectData.dat": logger.info(f"Found metadata file {transferred_file} for parsing") @@ -289,82 +288,32 @@ def post_transfer( # Update existing dict self._site_info[site_num] = site_info_new logger.info(f"Updating metadata for site {site_num}") - return None - elif "DCImages" in parts and transferred_file.suffix == ".png": - lamella_name = parts[parts.index("Sites") + 1] - lamella_number = _number_from_name(lamella_name) - time_from_name = transferred_file.name.split("-")[:6] - timestamp = datetime.timestamp( - datetime( - year=int(time_from_name[0]), - month=int(time_from_name[1]), - day=int(time_from_name[2]), - hour=int(time_from_name[3]), - minute=int(time_from_name[4]), - second=int(time_from_name[5]), - ) - ) - if not (source := _get_source(transferred_file, environment)): - logger.warning(f"No source found for file {transferred_file}") - return - if not ( - destination_file := _file_transferred_to( - environment=environment, - source=source, - file_path=transferred_file, - rsync_basepath=Path( - self._machine_config.get("rsync_basepath", "") - ), - ) - ): - logger.warning( - f"File {transferred_file.name!r} not found on storage system" - ) - return - if not self._drift_correction_images.get(lamella_number): - self._drift_correction_images[lamella_number] = [ - MillingImage( - timestamp=timestamp, - file=destination_file, - ) - ] - else: - self._drift_correction_images[lamella_number].append( - MillingImage( - timestamp=timestamp, - file=destination_file, - ) - ) - gif_list = [ - l.file - for l in sorted( - self._drift_correction_images[lamella_number], - key=lambda x: x.timestamp, - ) - ] - raw_directory = Path( - environment.default_destinations[self._basepath] - ).name - # Submit job to backend to construct a GIF - capture_post( - base_url=str(environment.url.geturl()), - router_name="workflow.correlative_router", - function_name="make_gif", - token=self._token, - instrument_name=environment.instrument_name, - data={ - "lamella_number": lamella_number, - "images": [str(file) for file in gif_list], - "raw_directory": raw_directory, - }, - # Endpoint kwargs - year=datetime.now().year, - visit_name=environment.visit, - session_id=environment.murfey_session, - ) + # Post drift correction GIF request if it hasn't already been done + if ( + (fib_image := self._drift_correction_images.get(site_num, None)) + is not None + and not fib_image.is_submitted + and fib_image.output_file is not None + ): + if self._make_gif( + environment=environment, + lamella_number=site_num, + images=fib_image.images, + output_file=fib_image.output_file, + ): + with lock: + self._drift_correction_images[ + site_num + ].is_submitted = True return None + elif ( + "DCImages" in transferred_file.parts + and transferred_file.suffix == ".png" + ): + self._make_drift_correction_gif(transferred_file, environment) + # ----------------------------------------------------------------------------- # Maps # ----------------------------------------------------------------------------- @@ -506,6 +455,159 @@ def _parse_autotem_metadata(self, file: Path): logger.info(f"Successfully extracted AutoTEM metadata from file {file}") return all_site_info + def _make_drift_correction_gif( + self, + file: Path, + environment: MurfeyInstanceEnvironment, + ): + """ + Helper function to create GIFs using the drift correction images seen by the + FIBContext class. The function uses the metadata returned + """ + parts = file.parts + try: + lamella_name = parts[parts.index("Sites") + 1] + lamella_number = _number_from_name(lamella_name) + except Exception: + logger.warning( + f"Could not extract metadata from file {file}", exc_info=True + ) + return None + if not (source := _get_source(file, environment)): + logger.warning(f"No source found for file {file}") + return + if not ( + destination_file := _file_transferred_to( + environment=environment, + source=source, + file_path=file, + rsync_basepath=Path(self._machine_config.get("rsync_basepath", "")), + ) + ): + logger.warning(f"File {file.name!r} not found on storage system") + return + + # Create FIBImage instance for this lamella site, or update existing one + if not self._drift_correction_images.get(lamella_number): + with lock: + self._drift_correction_images[lamella_number] = FIBImage( + images=[destination_file] + ) + else: + with lock: + self._drift_correction_images[lamella_number].images.append( + destination_file + ) + self._drift_correction_images[lamella_number].is_submitted = False + + # Determine the output directory to save the milling image to + output_file = self._drift_correction_images[lamella_number].output_file + if output_file is None: + # Early exits if data for creating output image path is absent + # No site info + if (site_info := self._site_info.get(lamella_number)) is None: + logger.debug(f"No metadata found for site {lamella_number} yet") + return None + # No project name + if (project_name := site_info.project_name) is None: + logger.warning(f"No project name associated with site {lamella_number}") + return None + # No stage position information + if all( + getattr(site_info.stage_info, stage_name, None) is None + for stage_name in STAGE_POSITION_NAMES.keys() + ): + logger.warning( + f"No stage position information associated with site {lamella_number}" + ) + return None + # Determine the slot number + slot_number: int | None = None + for stage_name in reversed(STAGE_POSITION_NAMES.keys()): + if ( + stage_info := getattr(site_info.stage_info, stage_name, None) + ) is None: + continue + if stage_info.slot_number is None: + continue + else: + slot_number = stage_info.slot_number + break + # Early exit if no slot number + if slot_number is None: + logger.warning( + f"Could not determine slot number of site {lamella_number}" + ) + return None + # Determine the path to save the GIF to + raw_dir = Path(environment.default_destinations[self._basepath]) + try: + visit_index = raw_dir.parts.index(environment.visit) + visit_dir = list(reversed(raw_dir.parents))[visit_index] + output_file = ( + visit_dir + / "processed" + / project_name + / f"grid_{slot_number}" + / "drift_correction" + / f"lamella_{lamella_number}.gif" + ) + with lock: + self._drift_correction_images[ + lamella_number + ].output_file = output_file + except Exception: + logger.error( + f"Could not construct drift correction GIF output path for site {lamella_number}" + ) + return None + + # Submit job to backend to construct a GIF + if self._make_gif( + environment=environment, + lamella_number=lamella_number, + images=sorted(self._drift_correction_images[lamella_number].images), + output_file=output_file, + ): + # Mark this dataset as having been submitted + with lock: + self._drift_correction_images[lamella_number].is_submitted = True + logger.info( + f"Submitted request to create drift correction GIF for site {lamella_number}" + ) + return None + + def _make_gif( + self, + environment: MurfeyInstanceEnvironment, + lamella_number: int, + images: list[Path], + output_file: Path, + ): + """ + Submits a POST request to the backend server to create a GIF using the + JSON payload provided. The payload will contain + """ + try: + capture_post( + base_url=str(environment.url.geturl()), + router_name="workflow_fib.router", + function_name="make_gif", + token=self._token, + instrument_name=environment.instrument_name, + data={ + "lamella_number": lamella_number, + "images": [str(file) for file in images], + "output_file": str(output_file), + }, + # Endpoint kwargs + session_id=environment.murfey_session, + ) + return True + except Exception: + logger.error(f"Could not submit GIF for site {lamella_number}") + return False + def _register_atlas(self, file: Path, environment: MurfeyInstanceEnvironment): """ Constructs the URL and dictionary to be posted to the server, which then triggers From b7f740edbdd011813f961628b53ac86ee46f1a37 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 30 Apr 2026 16:45:52 +0100 Subject: [PATCH 04/19] Need 'default_factory' for list --- src/murfey/client/contexts/fib.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/murfey/client/contexts/fib.py b/src/murfey/client/contexts/fib.py index beaa30fd..4cc537c1 100644 --- a/src/murfey/client/contexts/fib.py +++ b/src/murfey/client/contexts/fib.py @@ -4,7 +4,7 @@ import re import threading import xml.etree.ElementTree as ET -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Callable, Type, TypeVar @@ -228,7 +228,7 @@ def _file_transferred_to( @dataclass class FIBImage: - images: list[Path] = [] + images: list[Path] = field(default_factory=list) output_file: Path | None = None is_submitted: bool = False @@ -440,9 +440,9 @@ def _parse_autotem_metadata(self, file: Path): ) # Iteratively update fields in the MillingSteps model it's not None - for field, path, func in ACTIVITY_FIELD_MAP: + for field_name, path, func in ACTIVITY_FIELD_MAP: if (value := _parse_xml_text(activity, path, func)) is not None: - step_info.__setattr__(field, value) + step_info.__setattr__(field_name, value) # Add info for current step to the site info model site_info.steps.__setattr__( From 20f3fd15b4ade612aaf0a973c3a3e5203344e354 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 30 Apr 2026 17:10:19 +0100 Subject: [PATCH 05/19] Use current file's destination path to determine GIF file path --- src/murfey/client/contexts/fib.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/murfey/client/contexts/fib.py b/src/murfey/client/contexts/fib.py index 4cc537c1..02ed649d 100644 --- a/src/murfey/client/contexts/fib.py +++ b/src/murfey/client/contexts/fib.py @@ -540,10 +540,9 @@ def _make_drift_correction_gif( ) return None # Determine the path to save the GIF to - raw_dir = Path(environment.default_destinations[self._basepath]) try: - visit_index = raw_dir.parts.index(environment.visit) - visit_dir = list(reversed(raw_dir.parents))[visit_index] + visit_index = destination_file.parts.index(environment.visit) + visit_dir = list(reversed(destination_file.parents))[visit_index] output_file = ( visit_dir / "processed" From 654561e0b80d715f0a5bd492b59152646dbba47e Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 30 Apr 2026 17:30:17 +0100 Subject: [PATCH 06/19] One more log to indicate successful creation of GIF file --- src/murfey/server/api/workflow_fib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/server/api/workflow_fib.py b/src/murfey/server/api/workflow_fib.py index 269899f5..7dcae98c 100644 --- a/src/murfey/server/api/workflow_fib.py +++ b/src/murfey/server/api/workflow_fib.py @@ -124,5 +124,5 @@ async def make_gif( duration=30, loop=0, ) - + logger.info(f"Created GIF file {gif_params.output_file}") return {"output_gif": str(gif_params.output_file)} From 04b73583240d3975186f1c1ab892fb1a8df33c5d Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 30 Apr 2026 17:43:12 +0100 Subject: [PATCH 07/19] try-except the 'chmod' function --- src/murfey/server/api/workflow_fib.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/murfey/server/api/workflow_fib.py b/src/murfey/server/api/workflow_fib.py index 7dcae98c..ed961aa0 100644 --- a/src/murfey/server/api/workflow_fib.py +++ b/src/murfey/server/api/workflow_fib.py @@ -93,7 +93,13 @@ async def make_gif( visit_index = output_dir.parts.index(visit_name) # Change permissions for folders in the visit directory and onwards for current_path in list(reversed(output_dir.parents))[visit_index + 1 :]: - os.chmod(current_path, mode=machine_config.mkdir_chmod) + try: + os.chmod(current_path, mode=machine_config.mkdir_chmod) + except PermissionError: + logger.warning( + f"Insufficient permissions to modify directory {current_path}" + ) + continue if PIL.Image is not None: images = [PIL.Image.open(f) for f in gif_params.images] From 2c113040b92f5a0b8a6b2f9c3f167144992b3c68 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 30 Apr 2026 17:57:29 +0100 Subject: [PATCH 08/19] Migrated and fixed test for 'make_gif' function --- tests/server/api/test_workflow.py | 80 -------------------------- tests/server/api/test_workflow_fib.py | 81 ++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 81 deletions(-) diff --git a/tests/server/api/test_workflow.py b/tests/server/api/test_workflow.py index 1f6ee0c5..a1ec8336 100644 --- a/tests/server/api/test_workflow.py +++ b/tests/server/api/test_workflow.py @@ -1,17 +1,9 @@ -from pathlib import Path from unittest import mock -from unittest.mock import MagicMock -import numpy as np -import PIL.Image -import pytest -from pytest_mock import MockerFixture from sqlmodel import Session, select from murfey.server.api.workflow import ( DCGroupParameters, - MillingParameters, - make_gif, register_dc_group, ) from murfey.util.db import DataCollectionGroup, SearchMap @@ -438,75 +430,3 @@ def test_register_dc_group_new_atlas_with_searchmaps( murfey_db_session, close_db=False, ) - - -@pytest.mark.asyncio -async def test_make_gif( - mocker: MockerFixture, - tmp_path: Path, -): - # Set up test variables - session_id = 10 - instrument_name = "test_instrument" - rsync_basepath = tmp_path / "data" - visit_name = "cm12345-6" - year = 2020 - visit_dir = rsync_basepath / str(year) / visit_name - lamella_num = 12 - lamella_folder = "Lamella" - if lamella_num > 1: - lamella_folder += f" ({lamella_num})" - raw_directory = "autotem" - - # Create a list of test image file paths - raw_images = [ - visit_dir - / "autotem" - / visit_name - / "Sites" - / lamella_folder - / "DCImages/DCM_asdfjkl/asdfjkl-Polishing-dc_rescan-image-.png" - ] * 5 - # Mock the output of PIL.Image.open to always return a NumPY array - mocker.patch( - "murfey.server.api.workflow.Image.open", - return_value=PIL.Image.fromarray(np.ones((512, 512), dtype=np.uint16)), - ) - - # Create the Pydantic model - params = MillingParameters( - lamella_number=lamella_num, - images=[str(f) for f in raw_images], - raw_directory=raw_directory, - ) - - # Mock the database query - mock_db = MagicMock() - mock_db.exec.return_value.one.return_value.instrument_name = instrument_name - - # Mock the machine config and 'get_machine_config' - mock_machine_config = MagicMock() - mock_machine_config.rsync_basepath = rsync_basepath - mock_machine_config.mkdir_chmod = 0o775 - mocker.patch( - "murfey.server.api.workflow.get_machine_config", - return_value={ - instrument_name: mock_machine_config, - }, - ) - - # Create the save directory directory - save_dir = visit_dir / "processed" / raw_directory - save_dir.mkdir(parents=True, exist_ok=True) - - # Run the function and check that the expected outputs are there - result = await make_gif( - year=year, - visit_name=visit_name, - session_id=session_id, - gif_params=params, - db=mock_db, - ) - image_path = save_dir / f"lamella_{lamella_num}_milling.gif" - assert image_path.exists() - assert result.get("output_gif") == str(image_path) diff --git a/tests/server/api/test_workflow_fib.py b/tests/server/api/test_workflow_fib.py index e72deb57..6baadcd0 100644 --- a/tests/server/api/test_workflow_fib.py +++ b/tests/server/api/test_workflow_fib.py @@ -1,10 +1,17 @@ from pathlib import Path from unittest.mock import MagicMock +import numpy as np +import PIL.Image import pytest from pytest_mock import MockerFixture -from murfey.server.api.workflow_fib import FIBAtlasInfo, register_fib_atlas +from murfey.server.api.workflow_fib import ( + FIBAtlasInfo, + FIBGIFParameters, + make_gif, + register_fib_atlas, +) def test_register_fib_atlas( @@ -52,3 +59,75 @@ def test_register_fib_atlas_no_entry_point( fib_atlas_info=fib_atlas_info, db=mock_db, ) + + +@pytest.mark.asyncio +async def test_make_gif( + mocker: MockerFixture, + tmp_path: Path, +): + # Set up test variables + session_id = 10 + instrument_name = "test_instrument" + rsync_basepath = tmp_path / "data" + visit_name = "cm12345-6" + year = 2020 + visit_dir = rsync_basepath / str(year) / visit_name + lamella_num = 12 + lamella_folder = "Lamella" + if lamella_num > 1: + lamella_folder += f" ({lamella_num})" + output_file = ( + visit_dir + / "processed" + / "project_name" + / "grid_1" + / "drift_correction" + / f"lamella_{lamella_num}.gif" + ) + + # Create a list of test image file paths + raw_images = [ + visit_dir + / "autotem" + / visit_name + / "Sites" + / lamella_folder + / "DCImages/DCM_asdfjkl/asdfjkl-Polishing-dc_rescan-image-.png" + ] * 5 + # Mock the output of PIL.Image.open to always return a NumPY array + mocker.patch( + "murfey.server.api.workflow_fib.PIL.Image.open", + return_value=PIL.Image.fromarray(np.ones((512, 512), dtype=np.uint16)), + ) + + # Create the Pydantic model + params = FIBGIFParameters( + lamella_number=lamella_num, + images=[str(f) for f in raw_images], + output_file=output_file, + ) + + # Mock the database query + mock_db = MagicMock() + mock_db.exec.return_value.one.return_value.instrument_name = instrument_name + mock_db.exec.return_value.one.return_value.visit = visit_name + + # Mock the machine config and 'get_machine_config' + mock_machine_config = MagicMock() + mock_machine_config.mkdir_chmod = 0o775 + mocker.patch( + "murfey.server.api.workflow_fib.get_machine_config", + return_value={ + instrument_name: mock_machine_config, + }, + ) + + # Run the function and check that the expected outputs are there + result = await make_gif( + session_id=session_id, + gif_params=params, + db=mock_db, + ) + assert output_file.exists() + assert result.get("output_gif") == str(output_file) From 85186905ef6b0b91936adee4552a8eba91a44526 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 30 Apr 2026 18:47:53 +0100 Subject: [PATCH 09/19] Forgot to sort images --- src/murfey/client/contexts/fib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/client/contexts/fib.py b/src/murfey/client/contexts/fib.py index 02ed649d..68474183 100644 --- a/src/murfey/client/contexts/fib.py +++ b/src/murfey/client/contexts/fib.py @@ -299,7 +299,7 @@ def post_transfer( if self._make_gif( environment=environment, lamella_number=site_num, - images=fib_image.images, + images=sorted(fib_image.images), output_file=fib_image.output_file, ): with lock: From b8d2b4f7836dbab90f793553da0c1e05280a347b Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 30 Apr 2026 18:49:45 +0100 Subject: [PATCH 10/19] Updated test for FIBContext to handle new drift correction FIB image logic --- tests/client/contexts/test_fib.py | 97 ++++++++++++++++++++++++++++--- 1 file changed, 89 insertions(+), 8 deletions(-) diff --git a/tests/client/contexts/test_fib.py b/tests/client/contexts/test_fib.py index 91b9b766..58161bbb 100644 --- a/tests/client/contexts/test_fib.py +++ b/tests/client/contexts/test_fib.py @@ -16,6 +16,7 @@ _number_from_name, _parse_boolean, ) +from murfey.util.models import LamellaSiteInfo # Mock session values num_lamellae = 5 @@ -595,27 +596,49 @@ def test_fib_autotem_context_projectdata( @pytest.mark.parametrize( "test_params", - ( # Use environment? | Find source? | Find destination? - (True, True, True), - (False, True, True), - (True, False, True), - (True, True, False), + ( + # Early exits + # No MurfeyInstanceEnvironment + (False, True, True, True, True, True, True), + # No source + (True, False, True, True, True, True, True), + # No destination + (True, True, False, True, True, True, True), + # No site info + (True, True, True, False, True, True, True), + # No project name + (True, True, True, True, False, True, True), + # No stage position + (True, True, True, True, True, False, True), + # No stage position values + (True, True, True, True, True, True, False), + # Successful case + (True, True, True, True, True, True, True), ), ) def test_fib_autotem_context_drift_correction_images( mocker: MockerFixture, - test_params: tuple[bool, bool, bool], + test_params: tuple[bool, bool, bool, bool, bool, bool, bool], tmp_path: Path, visit_dir: Path, fib_autotem_dc_images: list[Path], ): # Unpack test params - use_env, find_source, find_dst = test_params + ( + use_env, + find_source, + find_dst, + has_site_info, + has_project_name, + has_stage_position, + has_stage_values, + ) = test_params # Mock the environment mock_environment = None if use_env: mock_environment = MagicMock() + mock_environment.visit = visit_name # Mock the logger to check if specific logs are triggered mock_logger = mocker.patch("murfey.client.contexts.fib.logger") @@ -649,6 +672,23 @@ def test_fib_autotem_context_drift_correction_images( token="", ) + # Create the Pydantic model for each site and add metadata + for i in range(num_lamellae): + lamella_num = i + 1 + metadata_dict = { + "site_name": f"Lamella ({lamella_num})", + "site_number": lamella_num, + } + if has_project_name: + metadata_dict["project_name"] = project_name + if has_stage_position: + stage_dict: dict[str, dict] = {"preparation": {}} + if has_stage_values: + stage_dict["preparation"] = {"x": 0.003} + metadata_dict["stage_info"] = stage_dict + if has_site_info: + context._site_info[lamella_num] = LamellaSiteInfo(**metadata_dict) + # Parse images one-by-one and check that expected calls were made for file in fib_autotem_dc_images: context.post_transfer(file, environment=mock_environment) @@ -660,6 +700,22 @@ def test_fib_autotem_context_drift_correction_images( mock_logger.warning.assert_called_with( f"File {file.name!r} not found on storage system" ) + elif not has_site_info: + mock_logger.debug.assert_called_with( + f"No metadata found for site {lamella_num} yet" + ) + elif not has_project_name: + mock_logger.warning.assert_called_with( + f"No project name associated with site {lamella_num}" + ) + elif not has_stage_position: + mock_logger.warning.assert_called_with( + f"No stage position information associated with site {lamella_num}" + ) + elif not has_stage_values: + mock_logger.warning.assert_called_with( + f"Could not determine slot number of site {lamella_num}" + ) else: mock_get_source.assert_called_with(file, mock_environment) mock_file_transferred_to.assert_called_with( @@ -668,9 +724,34 @@ def test_fib_autotem_context_drift_correction_images( file_path=file, rsync_basepath=Path(""), ) - assert mock_capture_post.call_count == len(fib_autotem_dc_images) assert len(context._drift_correction_images) == num_lamellae + for i in range(num_lamellae): + lamella_num = i + 1 + # The '_site_info' attribute should now be populated + assert ( + context._site_info[lamella_num].stage_info.preparation.slot_number == 2 + ) + + # The output file should point to 'grid_2' for a positive x stage position + output_file = ( + tmp_path + / "fib" + / "data" + / "current_year" + / visit_name + / "processed" + / project_name + / "grid_2" + / "drift_correction" + / f"lamella_{lamella_num}.gif" + ) + assert ( + context._drift_correction_images[lamella_num].output_file == output_file + ) + # 'capture_post' should be called for every image + assert mock_capture_post.call_count == len(destination_files) + def test_fib_maps_context( mocker: MockerFixture, From d9db5317639d9f1ba8342def359a002a3a52009c Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 5 May 2026 10:22:08 +0100 Subject: [PATCH 11/19] Sanitise and verify FIB output file path --- src/murfey/server/api/workflow_fib.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/murfey/server/api/workflow_fib.py b/src/murfey/server/api/workflow_fib.py index ed961aa0..ee8b41cb 100644 --- a/src/murfey/server/api/workflow_fib.py +++ b/src/murfey/server/api/workflow_fib.py @@ -13,6 +13,7 @@ import murfey.util.db as MurfeyDB from murfey.server.api.auth import validate_instrument_token from murfey.server.murfey_db import murfey_db +from murfey.util import sanitise_path from murfey.util.config import get_machine_config from murfey.util.models import LamellaSiteInfo @@ -85,6 +86,12 @@ async def make_gif( machine_config = get_machine_config(instrument_name=instrument_name)[ instrument_name ] + rsync_basepath = machine_config.rsync_basepath or Path(".").resolve() + + # Sanitise and verify that the output directory is relative to rsync basepath + output_dir = sanitise_path(gif_params.output_file.parent) + if not output_dir.is_relative_to(rsync_basepath): + logger.error("Output directory path is not permitted") # Create the directory structure if not (output_dir := gif_params.output_file.parent).exists(): From cc44ea02e0ddbe6de67db63c78fbdf7545df50a6 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 5 May 2026 10:27:48 +0100 Subject: [PATCH 12/19] Change sanitisation logic --- src/murfey/server/api/workflow_fib.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/murfey/server/api/workflow_fib.py b/src/murfey/server/api/workflow_fib.py index ee8b41cb..a0f77bf9 100644 --- a/src/murfey/server/api/workflow_fib.py +++ b/src/murfey/server/api/workflow_fib.py @@ -89,12 +89,12 @@ async def make_gif( rsync_basepath = machine_config.rsync_basepath or Path(".").resolve() # Sanitise and verify that the output directory is relative to rsync basepath - output_dir = sanitise_path(gif_params.output_file.parent) - if not output_dir.is_relative_to(rsync_basepath): - logger.error("Output directory path is not permitted") + output_file = sanitise_path(gif_params.output_file) + if not output_file.is_relative_to(rsync_basepath): + logger.error("Output file path is not permitted") # Create the directory structure - if not (output_dir := gif_params.output_file.parent).exists(): + if not (output_dir := output_file.parent).exists(): output_dir.mkdir(parents=True) logger.debug(f"Created output directory {output_dir}") visit_index = output_dir.parts.index(visit_name) @@ -130,12 +130,12 @@ async def make_gif( # Convert back to PIL.Image objects and save as GIF converted = [PIL.Image.fromarray(arr[f], mode="L") for f in range(len(images))] converted[0].save( - gif_params.output_file, + output_file, format="GIF", append_images=converted[1:], save_all=True, duration=30, loop=0, ) - logger.info(f"Created GIF file {gif_params.output_file}") - return {"output_gif": str(gif_params.output_file)} + logger.info(f"Created GIF file {output_file}") + return {"output_gif": str(output_file)} From 316d837b14a8de22da5285a533fb1c153977ed88 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 5 May 2026 10:54:07 +0100 Subject: [PATCH 13/19] PIL.Image conditional no longer needed --- src/murfey/server/api/workflow_fib.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/murfey/server/api/workflow_fib.py b/src/murfey/server/api/workflow_fib.py index a0f77bf9..9a632b4e 100644 --- a/src/murfey/server/api/workflow_fib.py +++ b/src/murfey/server/api/workflow_fib.py @@ -108,10 +108,8 @@ async def make_gif( ) continue - if PIL.Image is not None: - images = [PIL.Image.open(f) for f in gif_params.images] - else: - images = [] + # Load the images as PIL Image objects + images = [PIL.Image.open(f) for f in gif_params.images] for im in images: im.thumbnail((512, 512)) From 849d597269e674f8c2f34267fba799679ce340fd Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 5 May 2026 11:14:28 +0100 Subject: [PATCH 14/19] Add logic to check that 'make_gif' commands were being sent correctly after parsing FIB AutoTEM metadata --- tests/client/contexts/test_fib.py | 64 ++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/tests/client/contexts/test_fib.py b/tests/client/contexts/test_fib.py index 58161bbb..e2014bd0 100644 --- a/tests/client/contexts/test_fib.py +++ b/tests/client/contexts/test_fib.py @@ -11,6 +11,7 @@ STAGE_POSITION_NAMES, STAGE_POSITION_VALUES, FIBContext, + FIBImage, _file_transferred_to, _get_source, _number_from_name, @@ -469,23 +470,24 @@ def test_file_transferred_to( @pytest.mark.parametrize( "test_params", - ( # Has environment | Has Project name | Has Site | Has Site name | Has Recipe | Has Recipe name | Has Activity | Has Activity name - # Pass case - (True, True, True, True, True, True, True, True), - # Only one of these should be False at a given time - (True, True, True, True, True, True, True, False), - (True, True, True, True, True, True, False, True), - (True, True, True, True, True, False, True, True), - (True, True, True, True, False, True, True, True), - (True, True, True, False, True, True, True, True), - (True, True, False, True, True, True, True, True), - (True, False, True, True, True, True, True, True), - (False, True, True, True, True, True, True, True), + ( + # Pass cases + (True, True, True, True, True, True, True, True, True), # DC images + (True, True, True, True, True, True, True, True, False), # No DC images + # Only one of these, and the last one, should be False at a given time + (True, True, True, True, True, True, True, False, False), # No activity name + (True, True, True, True, True, True, False, True, False), # No activity + (True, True, True, True, True, False, True, True, False), # No recipe name + (True, True, True, True, False, True, True, True, False), # No recipe + (True, True, True, False, True, True, True, True, False), # No site name + (True, True, False, True, True, True, True, True, False), # No sites + (True, False, True, True, True, True, True, True, False), # No project name + (False, True, True, True, True, True, True, True, False), # No environment ), ) def test_fib_autotem_context_projectdata( mocker: MockerFixture, - test_params: tuple[bool, bool, bool, bool, bool, bool, bool, bool], + test_params: tuple[bool, bool, bool, bool, bool, bool, bool, bool, bool], tmp_path: Path, visit_dir: Path, ): @@ -499,6 +501,7 @@ def test_fib_autotem_context_projectdata( has_recipe_name, has_activities, has_activity_name, + has_drift_correction_images, ) = test_params # Mock the environment @@ -532,6 +535,14 @@ def test_fib_autotem_context_projectdata( machine_config={}, token="", ) + if has_drift_correction_images: + for i in range(num_lamellae): + context._drift_correction_images[i + 1] = FIBImage( + images=[tmp_path / "dummy.png"], + output_file=tmp_path / "dc_image.gif", + is_submitted=False, + ) + context.post_transfer(mock_projectdata, environment=mock_environment) # Check the success case @@ -547,10 +558,35 @@ def test_fib_autotem_context_projectdata( has_activity_name, ) ): - assert mock_capture_post.call_count == num_lamellae + assert mock_capture_post.call_count == num_lamellae * ( + 2 if has_drift_correction_images else 1 + ) assert len(context._site_info) == num_lamellae for i in range(num_lamellae): + mock_capture_post.assert_any_call( + base_url=mock.ANY, + router_name="workflow_fib.router", + function_name="register_fib_milling_progress", + token=mock.ANY, + instrument_name=mock.ANY, + data=mock.ANY, + session_id=mock.ANY, + ) mock_logger.info.assert_any_call(f"Updating metadata for site {i + 1}") + if has_drift_correction_images: + mock_capture_post.assert_any_call( + base_url=mock.ANY, + router_name="workflow_fib.router", + function_name="make_gif", + token=mock.ANY, + instrument_name=mock.ANY, + data={ + "lamella_number": i + 1, + "images": [str(tmp_path / "dummy.png")], + "output_file": str(tmp_path / "dc_image.gif"), + }, + session_id=mock.ANY, + ) # These fail cases will return an empty dict and not call "post_transfer" if not has_environment: mock_logger.warning.assert_called_with("No environment passed in") From cf722194f1f41bc92ba64ffa8b2c7368bca61b0c Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 5 May 2026 16:11:46 +0100 Subject: [PATCH 15/19] Create a class function to determine the output directory to save processed files to for each lamella site; updated output file determination logic when creating FIB GIFs to reflect this --- src/murfey/client/contexts/fib.py | 187 +++++++++++++++++++----------- 1 file changed, 118 insertions(+), 69 deletions(-) diff --git a/src/murfey/client/contexts/fib.py b/src/murfey/client/contexts/fib.py index 68474183..fcc0c1f1 100644 --- a/src/murfey/client/contexts/fib.py +++ b/src/murfey/client/contexts/fib.py @@ -291,16 +291,59 @@ def post_transfer( # Post drift correction GIF request if it hasn't already been done if ( - (fib_image := self._drift_correction_images.get(site_num, None)) - is not None - and not fib_image.is_submitted - and fib_image.output_file is not None - ): + fib_image := self._drift_correction_images.get(site_num, None) + ) is not None and not fib_image.is_submitted: + # Construct the output file name if it doesn't already exist + if (output_file := fib_image.output_file) is None: + if not ( + source := _get_source(transferred_file, environment) + ): + logger.warning( + f"No source found for file {transferred_file}" + ) + continue + if not ( + destination_file := _file_transferred_to( + environment=environment, + source=source, + file_path=transferred_file, + rsync_basepath=Path( + self._machine_config.get("rsync_basepath", "") + ), + ) + ): + logger.warning( + f"Could not find destination file path for {transferred_file.name!r}" + ) + continue + if ( + output_dir := self._determine_output_dir( + site_num, + destination_file, + environment, + ) + ) is None: + logger.warning( + f"Could not determine output directory for lamella {site_num}" + ) + continue + with lock: + output_file = ( + output_dir + / "drift_correction" + / f"lamella_{site_num}.gif" + ) + self._drift_correction_images[ + site_num + ].output_file = output_file + # Reload the new object + fib_image = self._drift_correction_images[site_num] + if self._make_gif( environment=environment, lamella_number=site_num, images=sorted(fib_image.images), - output_file=fib_image.output_file, + output_file=output_file, ): with lock: self._drift_correction_images[ @@ -337,7 +380,7 @@ def post_transfer( ) ): logger.warning( - f"File {transferred_file.name!r} not found on storage system" + f"Could not find destination file path for {transferred_file.name!r}" ) return None @@ -455,6 +498,59 @@ def _parse_autotem_metadata(self, file: Path): logger.info(f"Successfully extracted AutoTEM metadata from file {file}") return all_site_info + def _determine_output_dir( + self, + lamella_number: int, + destination_file: Path, + environment: MurfeyInstanceEnvironment, + ): + """ + Helper function to determine the output directory for the current lamella site + on the server side. + """ + # Early exits if data for creating output path is absent + # No site info + if (site_info := self._site_info.get(lamella_number)) is None: + logger.debug(f"No metadata found for site {lamella_number} yet") + return None + # No project name + if (project_name := site_info.project_name) is None: + logger.warning(f"No project name associated with site {lamella_number}") + return None + # No stage position information + if all( + getattr(site_info.stage_info, stage_name, None) is None + for stage_name in STAGE_POSITION_NAMES.keys() + ): + logger.warning( + f"No stage position information associated with site {lamella_number}" + ) + return None + # Determine the slot number + slot_number: int | None = None + for stage_name in reversed(STAGE_POSITION_NAMES.keys()): + if (stage_info := getattr(site_info.stage_info, stage_name, None)) is None: + continue + if stage_info.slot_number is None: + continue + else: + slot_number = stage_info.slot_number + break + # Early exit if no slot number + if slot_number is None: + logger.warning(f"Could not determine slot number of site {lamella_number}") + return None + # Determine the path to save the GIF to + try: + visit_index = destination_file.parts.index(environment.visit) + visit_dir = list(reversed(destination_file.parents))[visit_index] + return visit_dir / "processed" / project_name / f"grid_{slot_number}" + except Exception: + logger.error( + f"Could not construct output directory path for site {lamella_number}" + ) + return None + def _make_drift_correction_gif( self, file: Path, @@ -484,7 +580,7 @@ def _make_drift_correction_gif( rsync_basepath=Path(self._machine_config.get("rsync_basepath", "")), ) ): - logger.warning(f"File {file.name!r} not found on storage system") + logger.warning(f"Could not find destination file path for {file.name!r}") return # Create FIBImage instance for this lamella site, or update existing one @@ -499,67 +595,20 @@ def _make_drift_correction_gif( destination_file ) self._drift_correction_images[lamella_number].is_submitted = False - - # Determine the output directory to save the milling image to - output_file = self._drift_correction_images[lamella_number].output_file - if output_file is None: - # Early exits if data for creating output image path is absent - # No site info - if (site_info := self._site_info.get(lamella_number)) is None: - logger.debug(f"No metadata found for site {lamella_number} yet") - return None - # No project name - if (project_name := site_info.project_name) is None: - logger.warning(f"No project name associated with site {lamella_number}") - return None - # No stage position information - if all( - getattr(site_info.stage_info, stage_name, None) is None - for stage_name in STAGE_POSITION_NAMES.keys() - ): - logger.warning( - f"No stage position information associated with site {lamella_number}" - ) - return None - # Determine the slot number - slot_number: int | None = None - for stage_name in reversed(STAGE_POSITION_NAMES.keys()): - if ( - stage_info := getattr(site_info.stage_info, stage_name, None) - ) is None: - continue - if stage_info.slot_number is None: - continue - else: - slot_number = stage_info.slot_number - break - # Early exit if no slot number - if slot_number is None: - logger.warning( - f"Could not determine slot number of site {lamella_number}" - ) - return None - # Determine the path to save the GIF to - try: - visit_index = destination_file.parts.index(environment.visit) - visit_dir = list(reversed(destination_file.parents))[visit_index] - output_file = ( - visit_dir - / "processed" - / project_name - / f"grid_{slot_number}" - / "drift_correction" - / f"lamella_{lamella_number}.gif" - ) - with lock: - self._drift_correction_images[ - lamella_number - ].output_file = output_file - except Exception: - logger.error( - f"Could not construct drift correction GIF output path for site {lamella_number}" - ) - return None + if ( + output_dir := self._determine_output_dir( + lamella_number, + destination_file, + environment, + ) + ) is None: + logger.warning( + f"Could not determine output directory for lamella {lamella_number}" + ) + return None + output_file = output_dir / "drift_correction" / f"lamella_{lamella_number}.gif" + with lock: + self._drift_correction_images[lamella_number].output_file = output_file # Submit job to backend to construct a GIF if self._make_gif( From 016f3c6f1d45da028df6276c0307ee8a1269a3fd Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 5 May 2026 16:17:41 +0100 Subject: [PATCH 16/19] Updated tests --- tests/client/contexts/test_fib.py | 66 +++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 11 deletions(-) diff --git a/tests/client/contexts/test_fib.py b/tests/client/contexts/test_fib.py index e2014bd0..0eb93083 100644 --- a/tests/client/contexts/test_fib.py +++ b/tests/client/contexts/test_fib.py @@ -334,7 +334,7 @@ def create_fib_autotem_project_data( autotem_node.append(project_node) # Save the mock XML file - file = visit_dir / "autotem/visit/ProjectData.dat" + file = visit_dir / f"autotem/{project_name}/ProjectData.dat" file.parent.mkdir(parents=True, exist_ok=True) tree = ET.ElementTree(autotem_node) ET.indent(tree, space=" ") @@ -379,7 +379,7 @@ def fib_maps_images(visit_dir: Path): name = "Electron Snapshot" if i > 0: name += f" ({i + 1})" - file = visit_dir / "maps/visit/LayersData/Layer" / f"{name}.tiff" + file = visit_dir / f"maps/{project_name}/LayersData/Layer" / f"{name}.tiff" if not file.exists(): file.parent.mkdir(parents=True, exist_ok=True) file.touch() @@ -508,10 +508,41 @@ def test_fib_autotem_context_projectdata( mock_environment = None if has_environment: mock_environment = MagicMock() + mock_environment.visit = visit_name # Mock the logger to check that specific logs are called mock_logger = mocker.patch("murfey.client.contexts.fib.logger") + # Mock '_get_source' + mock_get_source = mocker.patch("murfey.client.contexts.fib._get_source") + mock_get_source.return_value = tmp_path + + # Mock '_file_transferred_to' + mock_file_transferred_to = mocker.patch( + "murfey.client.contexts.fib._file_transferred_to" + ) + mock_file_transferred_to.return_value = ( + tmp_path + / "fib" + / "data" + / "current_year" + / visit_name + / "autotem" + / project_name + / "ProjectData.dat" + ) + # Set the expected output directory to be derived from metadata + output_dir = ( + tmp_path + / "fib" + / "data" + / "current_year" + / visit_name + / "processed" + / project_name + / "grid_2" + ) + # Mock the functions used in 'post_transfer' mock_capture_post = mocker.patch("murfey.client.contexts.fib.capture_post") @@ -536,13 +567,15 @@ def test_fib_autotem_context_projectdata( token="", ) if has_drift_correction_images: + # Add drift correction images for i in range(num_lamellae): context._drift_correction_images[i + 1] = FIBImage( images=[tmp_path / "dummy.png"], - output_file=tmp_path / "dc_image.gif", + output_file=None, is_submitted=False, ) + # Run 'post_transfer' and check for expected calls and outputs context.post_transfer(mock_projectdata, environment=mock_environment) # Check the success case @@ -558,11 +591,15 @@ def test_fib_autotem_context_projectdata( has_activity_name, ) ): + # 'capture_post' should be called once when registering the site + # and again if registering a drift correction image assert mock_capture_post.call_count == num_lamellae * ( 2 if has_drift_correction_images else 1 ) + # There should be one dictionary entry for each lamella now assert len(context._site_info) == num_lamellae for i in range(num_lamellae): + lamella_number = i + 1 mock_capture_post.assert_any_call( base_url=mock.ANY, router_name="workflow_fib.router", @@ -572,7 +609,10 @@ def test_fib_autotem_context_projectdata( data=mock.ANY, session_id=mock.ANY, ) - mock_logger.info.assert_any_call(f"Updating metadata for site {i + 1}") + mock_logger.info.assert_any_call( + f"Updating metadata for site {lamella_number}" + ) + if has_drift_correction_images: mock_capture_post.assert_any_call( base_url=mock.ANY, @@ -581,9 +621,13 @@ def test_fib_autotem_context_projectdata( token=mock.ANY, instrument_name=mock.ANY, data={ - "lamella_number": i + 1, + "lamella_number": lamella_number, "images": [str(tmp_path / "dummy.png")], - "output_file": str(tmp_path / "dc_image.gif"), + "output_file": str( + output_dir + / "drift_correction" + / f"lamella_{lamella_number}.gif" + ), }, session_id=mock.ANY, ) @@ -612,7 +656,7 @@ def test_fib_autotem_context_projectdata( ) mock_capture_post.assert_not_called() # These fail cases will produce LamellaSiteInfo dicts with default values - # "post_transfer" will still be called + # "capture_post" will still be called elif not has_recipe_name: mock_logger.warning.assert_any_call("Recipe doesn't have a name, skipping") assert mock_capture_post.call_count == num_lamellae @@ -734,22 +778,22 @@ def test_fib_autotem_context_drift_correction_images( mock_logger.warning.assert_called_with(f"No source found for file {file}") elif not find_dst: mock_logger.warning.assert_called_with( - f"File {file.name!r} not found on storage system" + f"Could not find destination file path for {file.name!r}" ) elif not has_site_info: mock_logger.debug.assert_called_with( f"No metadata found for site {lamella_num} yet" ) elif not has_project_name: - mock_logger.warning.assert_called_with( + mock_logger.warning.assert_any_call( f"No project name associated with site {lamella_num}" ) elif not has_stage_position: - mock_logger.warning.assert_called_with( + mock_logger.warning.assert_any_call( f"No stage position information associated with site {lamella_num}" ) elif not has_stage_values: - mock_logger.warning.assert_called_with( + mock_logger.warning.assert_any_call( f"Could not determine slot number of site {lamella_num}" ) else: From c9293b17eec3a8e1aa5728ce04d97d199984343c Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 5 May 2026 16:30:34 +0100 Subject: [PATCH 17/19] Forgot to include most nested directory in iterative 'os.chmod' run --- src/murfey/server/api/workflow_fib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/server/api/workflow_fib.py b/src/murfey/server/api/workflow_fib.py index 9a632b4e..2cc68cb4 100644 --- a/src/murfey/server/api/workflow_fib.py +++ b/src/murfey/server/api/workflow_fib.py @@ -99,7 +99,7 @@ async def make_gif( logger.debug(f"Created output directory {output_dir}") visit_index = output_dir.parts.index(visit_name) # Change permissions for folders in the visit directory and onwards - for current_path in list(reversed(output_dir.parents))[visit_index + 1 :]: + for current_path in list(reversed(output_file.parents))[visit_index + 1 :]: try: os.chmod(current_path, mode=machine_config.mkdir_chmod) except PermissionError: From 94ecd0b494e96c36f325550b0aa5e914f4539f5a Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 5 May 2026 17:52:16 +0100 Subject: [PATCH 18/19] Adjust 'mkdir' logic --- src/murfey/server/api/workflow_fib.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/murfey/server/api/workflow_fib.py b/src/murfey/server/api/workflow_fib.py index 2cc68cb4..f40629fa 100644 --- a/src/murfey/server/api/workflow_fib.py +++ b/src/murfey/server/api/workflow_fib.py @@ -92,14 +92,14 @@ async def make_gif( output_file = sanitise_path(gif_params.output_file) if not output_file.is_relative_to(rsync_basepath): logger.error("Output file path is not permitted") - - # Create the directory structure - if not (output_dir := output_file.parent).exists(): - output_dir.mkdir(parents=True) - logger.debug(f"Created output directory {output_dir}") - visit_index = output_dir.parts.index(visit_name) - # Change permissions for folders in the visit directory and onwards - for current_path in list(reversed(output_file.parents))[visit_index + 1 :]: + raise ValueError + + # Create folders in the visit directory and onwards and change permissions + visit_index = output_file.parts.index(visit_name) + for current_path in list(reversed(output_file.parents))[visit_index + 1 :]: + if not current_path.exists(): + current_path.mkdir(parents=True) + logger.debug(f"Created output directory {current_path}") try: os.chmod(current_path, mode=machine_config.mkdir_chmod) except PermissionError: From b27e198d03cf159e2626c4bcfeeaac610de1e687 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 5 May 2026 18:41:26 +0100 Subject: [PATCH 19/19] Fixed broken test --- tests/server/api/test_workflow_fib.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/server/api/test_workflow_fib.py b/tests/server/api/test_workflow_fib.py index 6baadcd0..3afdb935 100644 --- a/tests/server/api/test_workflow_fib.py +++ b/tests/server/api/test_workflow_fib.py @@ -115,7 +115,8 @@ async def test_make_gif( # Mock the machine config and 'get_machine_config' mock_machine_config = MagicMock() - mock_machine_config.mkdir_chmod = 0o775 + mock_machine_config.mkdir_chmod = 0o2775 + mock_machine_config.rsync_basepath = rsync_basepath mocker.patch( "murfey.server.api.workflow_fib.get_machine_config", return_value={