From 3e949614d3ff28249778fedf9f7a59680282e6e9 Mon Sep 17 00:00:00 2001 From: Hamdah Shafqat Abbasi Date: Thu, 7 Nov 2024 12:52:49 -0500 Subject: [PATCH 01/17] fix_worker_issue --- formats/ome-converter-tool/.bumpversion.cfg | 2 +- formats/ome-converter-tool/Dockerfile | 2 +- formats/ome-converter-tool/README.md | 4 ++-- formats/ome-converter-tool/VERSION | 2 +- formats/ome-converter-tool/ict.yaml | 4 ++-- formats/ome-converter-tool/omeconverter.cwl | 2 +- formats/ome-converter-tool/plugin.json | 4 ++-- formats/ome-converter-tool/pyproject.toml | 4 ++-- .../images/formats/ome_converter/__init__.py | 2 +- .../formats/ome_converter/image_converter.py | 18 +++++++++++++----- 10 files changed, 26 insertions(+), 18 deletions(-) diff --git a/formats/ome-converter-tool/.bumpversion.cfg b/formats/ome-converter-tool/.bumpversion.cfg index d277ae70a..cdc0fcc4c 100644 --- a/formats/ome-converter-tool/.bumpversion.cfg +++ b/formats/ome-converter-tool/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.3 +current_version = 0.3.3-dev4 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? diff --git a/formats/ome-converter-tool/Dockerfile b/formats/ome-converter-tool/Dockerfile index 53a844554..6c8cb479b 100644 --- a/formats/ome-converter-tool/Dockerfile +++ b/formats/ome-converter-tool/Dockerfile @@ -1,4 +1,4 @@ -FROM polusai/bfio:2.4.4 +FROM polusai/bfio:2.4.5 # environment variables defined in polusai/bfio ENV EXEC_DIR="/opt/executables" diff --git a/formats/ome-converter-tool/README.md b/formats/ome-converter-tool/README.md index f1cf158cc..6aea1fac0 100644 --- a/formats/ome-converter-tool/README.md +++ b/formats/ome-converter-tool/README.md @@ -1,4 +1,4 @@ -# OME Converter (v0.3.3) +# OME Converter (v0.3.3-dev4) This WIPP plugin converts BioFormats supported data types to the OME Zarr or OME TIF file format. This is not a complete implementation, rather it implements a file @@ -35,4 +35,4 @@ This plugin takes 2 input arguments and 1 output argument: ## Docker Command ```bash -docker run -e POLUS_IMG_EXT=".ome.zarr" -v /Users/username/:/Users/username/ polusai/ome-converter-tool:0.3.3 --inpDir=/Users/path/to/Images/ --filePattern=".*.tif" --outDir=/Users/path/to/outputs +docker run -e POLUS_IMG_EXT=".ome.zarr" -v /Users/username/:/Users/username/ polusai/ome-converter-tool:0.3.3-dev4 --inpDir=/Users/path/to/Images/ --filePattern=".*.tif" --outDir=/Users/path/to/outputs diff --git a/formats/ome-converter-tool/VERSION b/formats/ome-converter-tool/VERSION index 1c09c74e2..764989e54 100644 --- a/formats/ome-converter-tool/VERSION +++ b/formats/ome-converter-tool/VERSION @@ -1 +1 @@ -0.3.3 +0.3.3-dev4 diff --git a/formats/ome-converter-tool/ict.yaml b/formats/ome-converter-tool/ict.yaml index a522e96a9..a6ff5cbdc 100644 --- a/formats/ome-converter-tool/ict.yaml +++ b/formats/ome-converter-tool/ict.yaml @@ -2,7 +2,7 @@ author: - Nick Schaub - Hamdah Shafqat contact: nick.schaub@nih.gov -container: polusai/ome-converter-tool:0.3.3 +container: polusai/ome-converter-tool:0.3.3-dev4 description: Convert Bioformats supported format to OME Zarr or OME TIF entrypoint: python3 -m polus.images.formats.ome_converter inputs: @@ -38,4 +38,4 @@ ui: key: inputs.filePattern title: Filepattern type: text -version: 0.3.3 +version: 0.3.3-dev4 diff --git a/formats/ome-converter-tool/omeconverter.cwl b/formats/ome-converter-tool/omeconverter.cwl index 47ab1a4ef..31afb4eac 100644 --- a/formats/ome-converter-tool/omeconverter.cwl +++ b/formats/ome-converter-tool/omeconverter.cwl @@ -20,7 +20,7 @@ outputs: type: Directory requirements: DockerRequirement: - dockerPull: polusai/ome-converter-tool:0.3.3 + dockerPull: polusai/ome-converter-tool:0.3.3-dev4 InitialWorkDirRequirement: listing: - entry: $(inputs.outDir) diff --git a/formats/ome-converter-tool/plugin.json b/formats/ome-converter-tool/plugin.json index 6ebed8f30..a74037b35 100644 --- a/formats/ome-converter-tool/plugin.json +++ b/formats/ome-converter-tool/plugin.json @@ -1,6 +1,6 @@ { "name": "OME Converter", - "version": "0.3.3", + "version": "0.3.3-dev4", "title": "OME Converter", "description": "Convert Bioformats supported format to OME Zarr or OME TIF", "author": "Nick Schaub (nick.schaub@nih.gov), Hamdah Shafqat Abbasi (hamdahshafqat.abbasi@nih.gov)", @@ -8,7 +8,7 @@ "repository": "https://github.com/PolusAI/image-tools", "website": "https://ncats.nih.gov/preclinical/core/informatics", "citation": "", - "containerId": "polusai/ome-converter-tool:0.3.3", + "containerId": "polusai/ome-converter-tool:0.3.3-dev4", "baseCommand": [ "python3", "-m", diff --git a/formats/ome-converter-tool/pyproject.toml b/formats/ome-converter-tool/pyproject.toml index d37c5ae4b..bee0f7885 100644 --- a/formats/ome-converter-tool/pyproject.toml +++ b/formats/ome-converter-tool/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "polus-images-formats-ome-converter" -version = "0.3.3" +version = "0.3.3-dev4" description = "Convert BioFormats datatypes to ome.tif or ome.zarr file format" authors = [ "Nick Schaub ", @@ -12,7 +12,7 @@ packages = [{include = "polus", from = "src"}] [tool.poetry.dependencies] python = ">=3.9,<3.12" -bfio = {version = "^2.4.4", extras = ["all"]} +bfio = {version = "^2.4.5", extras = ["all"]} filepattern = "^2.0.4" typer = "^0.7.0" tqdm = "^4.64.1" diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py index b0b13b640..820c6a067 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py @@ -1,6 +1,6 @@ """Ome Converter.""" -__version__ = "0.3.3" +__version__ = "0.3.3-dev4" from .image_converter import batch_convert from .image_converter import convert_image diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py index 825df42b9..7dbe9d3ef 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py @@ -3,9 +3,9 @@ import logging import os import pathlib +import platform from concurrent.futures import as_completed from itertools import product -from sys import platform from typing import Optional import filepattern as fp @@ -20,11 +20,19 @@ TILE_SIZE = 2**13 - -if platform.startswith("linux"): - NUM_THREADS = len(os.sched_getaffinity(0)) // 2 # type: ignore +# Determine the number of worker threads based on the platform and environment +if platform.system().lower() == "linux": + try: + # Use the number of CPUs available to the current process + NUM_THREADS = len(os.sched_getaffinity(0)) // 2 # type: ignore + except AttributeError: + # Fallback if os.sched_getaffinity is not available + NUM_THREADS = os.cpu_count() // 2 # type: ignore else: - NUM_THREADS = os.cpu_count() # type: ignore + # Use os.cpu_count() for non-Linux platforms + NUM_THREADS = os.cpu_count() // 2 # type: ignore + +NUM_THREADS = max(1, NUM_THREADS) def write_image( From 67934bad108f0a1a5fef763e1bcedf051616020b Mon Sep 17 00:00:00 2001 From: Hamdah Shafqat Abbasi Date: Wed, 4 Dec 2024 11:02:39 -0500 Subject: [PATCH 02/17] fix logic for channelnames --- .../polus/images/formats/ome_converter/image_converter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py index 7dbe9d3ef..dcab3054e 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py @@ -125,7 +125,7 @@ def convert_image( ) # Initialize the complete_image if not done yet - final_image = np.zeros((br.Y, br.X, 1, 1, 1), dtype=br.dtype) + final_image = np.zeros((br.Y, br.X), dtype=br.dtype) # Process each tile in the image using itertools.product for y, x in product(range(0, br.Y, TILE_SIZE), range(0, br.X, TILE_SIZE)): @@ -139,8 +139,9 @@ def convert_image( c, t, ] + # Place the tile into the correct position in the complete image - final_image[y:y_max, x:x_max, z, c, t] = image + final_image[y:y_max, x:x_max] = image write_image( br=br, From e8877ad2da1fee32f1985050500392541a216c7b Mon Sep 17 00:00:00 2001 From: Hamdah Shafqat Abbasi Date: Mon, 10 Mar 2025 13:16:10 -0400 Subject: [PATCH 03/17] func to handle path and list of files --- formats/ome-converter-tool/.bumpversion.cfg | 2 +- formats/ome-converter-tool/Dockerfile | 2 +- formats/ome-converter-tool/README.md | 4 ++-- formats/ome-converter-tool/VERSION | 2 +- formats/ome-converter-tool/ict.yaml | 6 +++--- formats/ome-converter-tool/omeconverter.cwl | 4 ++-- formats/ome-converter-tool/plugin.json | 6 +++--- formats/ome-converter-tool/pyproject.toml | 4 ++-- .../images/formats/ome_converter/__init__.py | 2 +- .../images/formats/ome_converter/__main__.py | 20 +++++++++++++------ .../formats/ome_converter/image_converter.py | 4 ++-- 11 files changed, 32 insertions(+), 24 deletions(-) diff --git a/formats/ome-converter-tool/.bumpversion.cfg b/formats/ome-converter-tool/.bumpversion.cfg index cdc0fcc4c..06d60b4a8 100644 --- a/formats/ome-converter-tool/.bumpversion.cfg +++ b/formats/ome-converter-tool/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.3-dev4 +current_version = 0.3.3-dev5 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? diff --git a/formats/ome-converter-tool/Dockerfile b/formats/ome-converter-tool/Dockerfile index 6c8cb479b..6af9d5553 100644 --- a/formats/ome-converter-tool/Dockerfile +++ b/formats/ome-converter-tool/Dockerfile @@ -1,4 +1,4 @@ -FROM polusai/bfio:2.4.5 +FROM sameeul/image-processor:latest # environment variables defined in polusai/bfio ENV EXEC_DIR="/opt/executables" diff --git a/formats/ome-converter-tool/README.md b/formats/ome-converter-tool/README.md index 6aea1fac0..b8f9c274b 100644 --- a/formats/ome-converter-tool/README.md +++ b/formats/ome-converter-tool/README.md @@ -1,4 +1,4 @@ -# OME Converter (v0.3.3-dev4) +# OME Converter (v0.3.3-dev5) This WIPP plugin converts BioFormats supported data types to the OME Zarr or OME TIF file format. This is not a complete implementation, rather it implements a file @@ -35,4 +35,4 @@ This plugin takes 2 input arguments and 1 output argument: ## Docker Command ```bash -docker run -e POLUS_IMG_EXT=".ome.zarr" -v /Users/username/:/Users/username/ polusai/ome-converter-tool:0.3.3-dev4 --inpDir=/Users/path/to/Images/ --filePattern=".*.tif" --outDir=/Users/path/to/outputs +docker run -e POLUS_IMG_EXT=".ome.zarr" -v /Users/username/:/Users/username/ polusai/ome-converter-tool:0.3.3-dev5 --inpDir=/Users/path/to/Images/ --filePattern=".*.tif" --outDir=/Users/path/to/outputs diff --git a/formats/ome-converter-tool/VERSION b/formats/ome-converter-tool/VERSION index 764989e54..4f5d11fcc 100644 --- a/formats/ome-converter-tool/VERSION +++ b/formats/ome-converter-tool/VERSION @@ -1 +1 @@ -0.3.3-dev4 +0.3.3-dev5 diff --git a/formats/ome-converter-tool/ict.yaml b/formats/ome-converter-tool/ict.yaml index a6ff5cbdc..bcb1271db 100644 --- a/formats/ome-converter-tool/ict.yaml +++ b/formats/ome-converter-tool/ict.yaml @@ -2,13 +2,13 @@ author: - Nick Schaub - Hamdah Shafqat contact: nick.schaub@nih.gov -container: polusai/ome-converter-tool:0.3.3-dev4 +container: polusai/ome-converter-tool:0.3.3-dev5 description: Convert Bioformats supported format to OME Zarr or OME TIF entrypoint: python3 -m polus.images.formats.ome_converter inputs: - description: Input generic data collection to be processed by this plugin format: - - genericData + - string name: inpDir required: true type: path @@ -38,4 +38,4 @@ ui: key: inputs.filePattern title: Filepattern type: text -version: 0.3.3-dev4 +version: 0.3.3-dev5 diff --git a/formats/ome-converter-tool/omeconverter.cwl b/formats/ome-converter-tool/omeconverter.cwl index 31afb4eac..070e2da58 100644 --- a/formats/ome-converter-tool/omeconverter.cwl +++ b/formats/ome-converter-tool/omeconverter.cwl @@ -8,7 +8,7 @@ inputs: inpDir: inputBinding: prefix: --inpDir - type: Directory + type: string outDir: inputBinding: prefix: --outDir @@ -20,7 +20,7 @@ outputs: type: Directory requirements: DockerRequirement: - dockerPull: polusai/ome-converter-tool:0.3.3-dev4 + dockerPull: polusai/ome-converter-tool:0.3.3-dev5 InitialWorkDirRequirement: listing: - entry: $(inputs.outDir) diff --git a/formats/ome-converter-tool/plugin.json b/formats/ome-converter-tool/plugin.json index a74037b35..cdaffb014 100644 --- a/formats/ome-converter-tool/plugin.json +++ b/formats/ome-converter-tool/plugin.json @@ -1,6 +1,6 @@ { "name": "OME Converter", - "version": "0.3.3-dev4", + "version": "0.3.3-dev5", "title": "OME Converter", "description": "Convert Bioformats supported format to OME Zarr or OME TIF", "author": "Nick Schaub (nick.schaub@nih.gov), Hamdah Shafqat Abbasi (hamdahshafqat.abbasi@nih.gov)", @@ -8,7 +8,7 @@ "repository": "https://github.com/PolusAI/image-tools", "website": "https://ncats.nih.gov/preclinical/core/informatics", "citation": "", - "containerId": "polusai/ome-converter-tool:0.3.3-dev4", + "containerId": "polusai/ome-converter-tool:0.3.3-dev5", "baseCommand": [ "python3", "-m", @@ -17,7 +17,7 @@ "inputs": [ { "name": "inpDir", - "type": "genericData", + "type": "string", "description": "Input generic data collection to be processed by this plugin", "required": true }, diff --git a/formats/ome-converter-tool/pyproject.toml b/formats/ome-converter-tool/pyproject.toml index bee0f7885..e70ac8648 100644 --- a/formats/ome-converter-tool/pyproject.toml +++ b/formats/ome-converter-tool/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "polus-images-formats-ome-converter" -version = "0.3.3-dev4" +version = "0.3.3-dev5" description = "Convert BioFormats datatypes to ome.tif or ome.zarr file format" authors = [ "Nick Schaub ", @@ -12,7 +12,7 @@ packages = [{include = "polus", from = "src"}] [tool.poetry.dependencies] python = ">=3.9,<3.12" -bfio = {version = "^2.4.5", extras = ["all"]} +bfio = {version = "2.4.7", extras = ["all"]} filepattern = "^2.0.4" typer = "^0.7.0" tqdm = "^4.64.1" diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py index 820c6a067..f5ec8be08 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py @@ -1,6 +1,6 @@ """Ome Converter.""" -__version__ = "0.3.3-dev4" +__version__ = "0.3.3-dev5" from .image_converter import batch_convert from .image_converter import convert_image diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py index 969c803ff..40a80e3a5 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py @@ -6,6 +6,7 @@ from concurrent.futures import as_completed from typing import Any from typing import Optional +from typing import Union import filepattern as fp import preadator @@ -26,17 +27,24 @@ POLUS_IMG_EXT = os.environ.get("POLUS_IMG_EXT", ".ome.tif") +def parse_input_dir(value: str) -> Union[pathlib.Path, list[str]]: + """Parse inp_dir as either a directory path or a comma-separated list of files.""" + if "," in value: + return [f.strip() for f in value.split(",")] + path = pathlib.Path(value) + if not (path.exists() and path.is_dir()): + msg = f"Directory {path} does not exist" + raise typer.BadParameter(msg) + return path + + @app.command() def main( - inp_dir: pathlib.Path = typer.Option( + inp_dir: str = typer.Option( ..., "--inpDir", help="Input generic data collection to be processed by this plugin", - exists=True, - resolve_path=True, - readable=True, - file_okay=False, - dir_okay=True, + callback=parse_input_dir, ), pattern: str = typer.Option( ".+", diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py index dcab3054e..de446884e 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py @@ -125,7 +125,7 @@ def convert_image( ) # Initialize the complete_image if not done yet - final_image = np.zeros((br.Y, br.X), dtype=br.dtype) + final_image = np.zeros((br.Y, br.X, br.Z, br.C, br.T), dtype=br.dtype) # Process each tile in the image using itertools.product for y, x in product(range(0, br.Y, TILE_SIZE), range(0, br.X, TILE_SIZE)): @@ -141,7 +141,7 @@ def convert_image( ] # Place the tile into the correct position in the complete image - final_image[y:y_max, x:x_max] = image + final_image[y:y_max, x:x_max, z, c, t] = image write_image( br=br, From 5b4974332108816b7c61bb0be54fb9dd37cfba63 Mon Sep 17 00:00:00 2001 From: Hamdah Shafqat Abbasi Date: Tue, 25 Mar 2025 17:32:50 -0400 Subject: [PATCH 04/17] =?UTF-8?q?Bump=20version:=200.3.3-dev5=20=E2=86=92?= =?UTF-8?q?=200.3.3-dev6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- formats/ome-converter-tool/.bumpversion.cfg | 2 +- formats/ome-converter-tool/README.md | 4 ++-- formats/ome-converter-tool/VERSION | 2 +- formats/ome-converter-tool/ict.yaml | 4 ++-- formats/ome-converter-tool/omeconverter.cwl | 2 +- formats/ome-converter-tool/plugin.json | 4 ++-- formats/ome-converter-tool/pyproject.toml | 3 ++- .../src/polus/images/formats/ome_converter/__init__.py | 2 +- 8 files changed, 12 insertions(+), 11 deletions(-) diff --git a/formats/ome-converter-tool/.bumpversion.cfg b/formats/ome-converter-tool/.bumpversion.cfg index 06d60b4a8..9c5b3ea4c 100644 --- a/formats/ome-converter-tool/.bumpversion.cfg +++ b/formats/ome-converter-tool/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.3-dev5 +current_version = 0.3.3-dev6 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? diff --git a/formats/ome-converter-tool/README.md b/formats/ome-converter-tool/README.md index b8f9c274b..d3e93d08f 100644 --- a/formats/ome-converter-tool/README.md +++ b/formats/ome-converter-tool/README.md @@ -1,4 +1,4 @@ -# OME Converter (v0.3.3-dev5) +# OME Converter (v0.3.3-dev6) This WIPP plugin converts BioFormats supported data types to the OME Zarr or OME TIF file format. This is not a complete implementation, rather it implements a file @@ -35,4 +35,4 @@ This plugin takes 2 input arguments and 1 output argument: ## Docker Command ```bash -docker run -e POLUS_IMG_EXT=".ome.zarr" -v /Users/username/:/Users/username/ polusai/ome-converter-tool:0.3.3-dev5 --inpDir=/Users/path/to/Images/ --filePattern=".*.tif" --outDir=/Users/path/to/outputs +docker run -e POLUS_IMG_EXT=".ome.zarr" -v /Users/username/:/Users/username/ polusai/ome-converter-tool:0.3.3-dev6 --inpDir=/Users/path/to/Images/ --filePattern=".*.tif" --outDir=/Users/path/to/outputs diff --git a/formats/ome-converter-tool/VERSION b/formats/ome-converter-tool/VERSION index 4f5d11fcc..2d516df6e 100644 --- a/formats/ome-converter-tool/VERSION +++ b/formats/ome-converter-tool/VERSION @@ -1 +1 @@ -0.3.3-dev5 +0.3.3-dev6 diff --git a/formats/ome-converter-tool/ict.yaml b/formats/ome-converter-tool/ict.yaml index bcb1271db..b61e9be58 100644 --- a/formats/ome-converter-tool/ict.yaml +++ b/formats/ome-converter-tool/ict.yaml @@ -2,7 +2,7 @@ author: - Nick Schaub - Hamdah Shafqat contact: nick.schaub@nih.gov -container: polusai/ome-converter-tool:0.3.3-dev5 +container: polusai/ome-converter-tool:0.3.3-dev6 description: Convert Bioformats supported format to OME Zarr or OME TIF entrypoint: python3 -m polus.images.formats.ome_converter inputs: @@ -38,4 +38,4 @@ ui: key: inputs.filePattern title: Filepattern type: text -version: 0.3.3-dev5 +version: 0.3.3-dev6 diff --git a/formats/ome-converter-tool/omeconverter.cwl b/formats/ome-converter-tool/omeconverter.cwl index 070e2da58..ee59749a1 100644 --- a/formats/ome-converter-tool/omeconverter.cwl +++ b/formats/ome-converter-tool/omeconverter.cwl @@ -20,7 +20,7 @@ outputs: type: Directory requirements: DockerRequirement: - dockerPull: polusai/ome-converter-tool:0.3.3-dev5 + dockerPull: polusai/ome-converter-tool:0.3.3-dev6 InitialWorkDirRequirement: listing: - entry: $(inputs.outDir) diff --git a/formats/ome-converter-tool/plugin.json b/formats/ome-converter-tool/plugin.json index cdaffb014..807d4d6e9 100644 --- a/formats/ome-converter-tool/plugin.json +++ b/formats/ome-converter-tool/plugin.json @@ -1,6 +1,6 @@ { "name": "OME Converter", - "version": "0.3.3-dev5", + "version": "0.3.3-dev6", "title": "OME Converter", "description": "Convert Bioformats supported format to OME Zarr or OME TIF", "author": "Nick Schaub (nick.schaub@nih.gov), Hamdah Shafqat Abbasi (hamdahshafqat.abbasi@nih.gov)", @@ -8,7 +8,7 @@ "repository": "https://github.com/PolusAI/image-tools", "website": "https://ncats.nih.gov/preclinical/core/informatics", "citation": "", - "containerId": "polusai/ome-converter-tool:0.3.3-dev5", + "containerId": "polusai/ome-converter-tool:0.3.3-dev6", "baseCommand": [ "python3", "-m", diff --git a/formats/ome-converter-tool/pyproject.toml b/formats/ome-converter-tool/pyproject.toml index e70ac8648..0f78cc193 100644 --- a/formats/ome-converter-tool/pyproject.toml +++ b/formats/ome-converter-tool/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "polus-images-formats-ome-converter" -version = "0.3.3-dev5" +version = "0.3.3-dev6" description = "Convert BioFormats datatypes to ome.tif or ome.zarr file format" authors = [ "Nick Schaub ", @@ -26,6 +26,7 @@ pytest = "^7.2.1" ipykernel = "^6.21.2" requests = "^2.28.2" scikit-image = "^0.19.3" +matplotlib = "3.9.4" [build-system] requires = ["poetry-core"] diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py index f5ec8be08..ffa697c2e 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py @@ -1,6 +1,6 @@ """Ome Converter.""" -__version__ = "0.3.3-dev5" +__version__ = "0.3.3-dev6" from .image_converter import batch_convert from .image_converter import convert_image From 35901d813e33a07f209a27d6c1cbf4819943ad5b Mon Sep 17 00:00:00 2001 From: Hamdah Shafqat Abbasi Date: Wed, 2 Apr 2025 13:47:43 -0400 Subject: [PATCH 05/17] fix the bug for flex file --- formats/ome-converter-tool/Dockerfile | 1 + .../images/formats/ome_converter/__main__.py | 22 +- .../formats/ome_converter/image_converter.py | 262 ++++++++++++------ 3 files changed, 194 insertions(+), 91 deletions(-) diff --git a/formats/ome-converter-tool/Dockerfile b/formats/ome-converter-tool/Dockerfile index 6af9d5553..504961fce 100644 --- a/formats/ome-converter-tool/Dockerfile +++ b/formats/ome-converter-tool/Dockerfile @@ -5,6 +5,7 @@ ENV EXEC_DIR="/opt/executables" ENV POLUS_IMG_EXT=".ome.tif" ENV POLUS_TAB_EXT=".csv" ENV POLUS_LOG="INFO" +ENV NUM_THREADS="1" # Work directory defined in the base container WORKDIR ${EXEC_DIR} diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py index 40a80e3a5..8cce13059 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py @@ -3,15 +3,16 @@ import logging import os import pathlib +from concurrent.futures import ProcessPoolExecutor from concurrent.futures import as_completed from typing import Any from typing import Optional from typing import Union import filepattern as fp -import preadator import typer from polus.images.formats.ome_converter.image_converter import NUM_THREADS +from polus.images.formats.ome_converter.image_converter import POLUS_IMG_EXT from polus.images.formats.ome_converter.image_converter import convert_image from tqdm import tqdm @@ -24,7 +25,6 @@ ) logger = logging.getLogger("polus.images.formats.ome_converter") logger.setLevel(os.environ.get("POLUS_LOG", logging.INFO)) -POLUS_IMG_EXT = os.environ.get("POLUS_IMG_EXT", ".ome.tif") def parse_input_dir(value: str) -> Union[pathlib.Path, list[str]]: @@ -86,21 +86,17 @@ def main( json.dump(out_json, jfile, indent=2) return - with preadator.ProcessManager( - name="ome_converter", - num_processes=NUM_THREADS, - threads_per_process=2, - ) as executor: - threads = [] + # Use ProcessPoolExecutor for multiprocessing + with ProcessPoolExecutor(max_workers=NUM_THREADS) as executor: + futures = [] for files in fps(): file = files[1][0] - threads.append( - executor.submit_process(convert_image, file, POLUS_IMG_EXT, out_dir), - ) + futures.append(executor.submit(convert_image, file, POLUS_IMG_EXT, out_dir)) + # Process results with progress bar for f in tqdm( - as_completed(threads), - total=len(threads), + as_completed(futures), + total=len(futures), mininterval=5, desc=f"converting images to {POLUS_IMG_EXT}", initial=0, diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py index de446884e..1b3d467e9 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py @@ -1,16 +1,15 @@ """Ome Converter.""" - import logging import os import pathlib import platform +from concurrent.futures import ProcessPoolExecutor from concurrent.futures import as_completed from itertools import product from typing import Optional import filepattern as fp import numpy as np -import preadator from bfio import BioReader from bfio import BioWriter from tqdm import tqdm @@ -19,20 +18,23 @@ logger.setLevel(os.environ.get("POLUS_LOG", logging.INFO)) TILE_SIZE = 2**13 +POLUS_IMG_EXT = os.environ.get("POLUS_IMG_EXT", ".ome.tif") +NUM_THREADS_ENV = os.environ.get("NUM_THREADS") -# Determine the number of worker threads based on the platform and environment -if platform.system().lower() == "linux": - try: - # Use the number of CPUs available to the current process - NUM_THREADS = len(os.sched_getaffinity(0)) // 2 # type: ignore - except AttributeError: - # Fallback if os.sched_getaffinity is not available - NUM_THREADS = os.cpu_count() // 2 # type: ignore +if not NUM_THREADS_ENV or NUM_THREADS_ENV == "1": + if platform.system().lower() == "linux": + try: + NUM_THREADS = len(os.sched_getaffinity(0)) // 2 # type: ignore + except AttributeError: + cpu_count = os.cpu_count() + NUM_THREADS = cpu_count // 2 if cpu_count is not None else 1 + else: + cpu_count = os.cpu_count() + NUM_THREADS = cpu_count // 2 if cpu_count is not None else 1 else: - # Use os.cpu_count() for non-Linux platforms - NUM_THREADS = os.cpu_count() // 2 # type: ignore + NUM_THREADS = int(NUM_THREADS_ENV) # Convert str to int safely -NUM_THREADS = max(1, NUM_THREADS) +NUM_THREADS = max(1, NUM_THREADS) # Ensure at least 1 thread, type is int def write_image( @@ -75,73 +77,175 @@ def write_image( bw[:] = image +def get_num_series(inp_image: pathlib.Path) -> int: + """Get the number of series/levels in an individual image file. + + Args: + inp_image: Path to the input image file. + + Returns: + int: Number of series/levels in the image. + """ + try: + # Open the specific file with BioReader + br = BioReader(inp_image, max_workers=1) + + # Initialize series count + num_series = 1 + # For formats with metadata.images, verify it's for this file only + if hasattr(br.metadata, "images"): + # Check if images relate to this specific file + image_count = len(br.metadata.images) + num_series = image_count + logger.debug( + f"count:{num_series} for {inp_image.name}", + ) + + # Log and return result + logger.info(f"File {inp_image.name} has {num_series}") + + # Clean up + br.close() + del br + + return num_series + + except (OSError, ValueError) as e: + logger.error(f"Error determining number of series for {inp_image}: {e!s}") + return 1 + + def convert_image( inp_image: pathlib.Path, file_extension: str, out_dir: pathlib.Path, ) -> None: - """Convert bioformats supported datatypes to ome.tif or ome.zarr file format. + """Convert bioformats supported datatypes to ome.tif or ome.zarr file format.""" + # First, determine the number of series + num_series = get_num_series(inp_image) + logger.info(f"Detected {num_series} series/levels in image {inp_image.name}") - Args: - inp_image: Path of an input image. - file_extension: Type of data conversion. - out_dir: Path to output directory. - """ - # Loop through timepoints, channels and z-slices - with BioReader(inp_image, max_workers=NUM_THREADS) as br: - for t, c, z in product(range(br.T), range(br.C), range(br.Z)): - extension = "".join( - [ - suffix - for suffix in inp_image.suffixes[-2:] - if len(suffix) < 6 # noqa: PLR2004 - ], - ) + # Process each series independently + for idx in range(num_series): + logger.info(f"Processing levels {idx} of {inp_image.name}") - out_path = out_dir.joinpath( - inp_image.name.replace(extension, file_extension), - ) - if br.C > 1: - out_path = out_dir.joinpath( - out_path.name.replace( - file_extension, - f"_c{c}" + file_extension, - ), - ) - if br.T > 1: - out_path = out_dir.joinpath( - out_path.name.replace( - file_extension, - f"_t{t}" + file_extension, - ), + try: + # Explicitly set the series/level parameter when opening the file + with BioReader(inp_image, max_workers=NUM_THREADS, level=idx) as br: + logger.debug( + f"Level {idx}: T={br.T}, C={br.C}, Z={br.Z}, Y={br.Y}, X={br.X}", ) - if br.Z > 1: - out_path = out_dir.joinpath( - out_path.name.replace( - file_extension, - f"_z{z}" + file_extension, - ), + # Process each view (t, c, z) + for t, c, z in product(range(br.T), range(br.C), range(br.Z)): + # Build the output path + if inp_image.suffix: + extension_len = 6 + extension = "".join( + [ + suffix + for suffix in inp_image.suffixes[-2:] + if len(suffix) < extension_len + ], + ) + out_path = out_dir.joinpath( + inp_image.name.replace(extension, file_extension), + ) + else: + out_path = out_dir.joinpath(f"{inp_image.name}{file_extension}") + + # Build suffix components + suffix_parts = [] + if br.C > 1: + suffix_parts.append(f"_c{c}") + if br.T > 1: + suffix_parts.append(f"_t{t}") + if br.Z > 1: + suffix_parts.append(f"_z{z}") + if num_series > 1: + suffix_parts.append(f"_level_{idx}") + + # Apply combined suffix + if suffix_parts: + suffix = "".join(suffix_parts) + out_path = out_dir.joinpath( + out_path.name.replace( + file_extension, + f"{suffix}{file_extension}", + ), + ) + + # Try primary method first, then fall back to chunked approach + process_single_view(br, c, t, z, out_path) + + except (OSError, ValueError) as e: + logger.error( + f"Failed to process series {idx} of {inp_image.name}: {e!s}", + ) + + +def process_single_view( + br: "BioReader", + c: int, + t: int, + z: int, + out_path: pathlib.Path, +) -> None: + """Process a single view (t,c,z) from a BioReader.""" + try: + # Try direct read first + final_image = br[:, :, z, c, t] + + write_image( + br=br, + c=c, + image=final_image, + out_path=out_path, + max_workers=NUM_THREADS, + ) + logger.debug(f"Successfully processed {out_path.name} using direct read") + + except (OSError, ValueError) as e: + logger.warning( + f"Direct read failed for {out_path.name}, trying chunked approach: {e!s}", + ) + try: + # Fallback to chunked approach + actual_y, actual_x = br.Y, br.X + + if actual_y <= 0 or actual_x <= 0: + logger.error( + f"Invalid dimensions: {actual_y}x{actual_x} for {out_path.name}", ) + return + + final_image = np.zeros((actual_y, actual_x), dtype=br.dtype) + + # Use smaller chunks for problematic images + chunk_size = min(TILE_SIZE, min(actual_y, actual_x) // 2) - # Initialize the complete_image if not done yet - final_image = np.zeros((br.Y, br.X, br.Z, br.C, br.T), dtype=br.dtype) + # Track successful chunks for debugging + total_chunks = 0 + successful_chunks = 0 - # Process each tile in the image using itertools.product - for y, x in product(range(0, br.Y, TILE_SIZE), range(0, br.X, TILE_SIZE)): - y_max = min(br.Y, y + TILE_SIZE) - x_max = min(br.X, x + TILE_SIZE) + for y in range(0, actual_y, chunk_size): + for x in range(0, actual_x, chunk_size): + y_max = min(actual_y, y + chunk_size) + x_max = min(actual_x, x + chunk_size) + total_chunks += 1 - image = br[ - y:y_max, - x:x_max, - z, - c, - t, - ] + try: + chunk = br[y:y_max, x:x_max, z, c, t] + final_image[y:y_max, x:x_max] = chunk + successful_chunks += 1 + except (OSError, ValueError) as e: + logger.warning( + f"Failed to read chunk at ({y}:{y_max}, {x}:{x_max}): {e}", + ) - # Place the tile into the correct position in the complete image - final_image[y:y_max, x:x_max, z, c, t] = image + logger.info( + f"Chunked read: {successful_chunks}/{total_chunks} successfully", + ) write_image( br=br, @@ -150,6 +254,14 @@ def convert_image( out_path=out_path, max_workers=NUM_THREADS, ) + logger.debug( + f"Successfully processed {out_path.name} using chunked approach", + ) + + except (OSError, ValueError) as e: + logger.error( + f"Failed to process {out_path.name} with chunked approach: {e!s}", + ) def batch_convert( @@ -175,23 +287,17 @@ def batch_convert( fps = fp.FilePattern(inp_dir, file_pattern) - with preadator.ProcessManager( - name="ome_converter", - num_processes=NUM_THREADS, - threads_per_process=2, - ) as executor: - threads = [] + with ProcessPoolExecutor(max_workers=NUM_THREADS) as executor: + futures = [] for files in fps(): file = files[1][0] - threads.append( - executor.submit(convert_image, file, file_extension, out_dir), - ) + futures.append(executor.submit(convert_image, file, POLUS_IMG_EXT, out_dir)) for f in tqdm( - as_completed(threads), - total=len(threads), + as_completed(futures), + total=len(futures), mininterval=5, - desc=f"converting images to {file_extension}", + desc=f"converting images to {POLUS_IMG_EXT}", initial=0, unit_scale=True, colour="cyan", From 348ccfa3e78f78723e1da07a8c45c8adac95a20d Mon Sep 17 00:00:00 2001 From: Hamdah Shafqat Abbasi Date: Wed, 2 Apr 2025 14:43:26 -0400 Subject: [PATCH 06/17] =?UTF-8?q?Bump=20version:=200.3.3-dev6=20=E2=86=92?= =?UTF-8?q?=200.3.4-dev0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- formats/ome-converter-tool/.bumpversion.cfg | 2 +- formats/ome-converter-tool/README.md | 4 ++-- formats/ome-converter-tool/VERSION | 2 +- formats/ome-converter-tool/ict.yaml | 4 ++-- formats/ome-converter-tool/omeconverter.cwl | 2 +- formats/ome-converter-tool/plugin.json | 4 ++-- formats/ome-converter-tool/pyproject.toml | 2 +- .../src/polus/images/formats/ome_converter/__init__.py | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/formats/ome-converter-tool/.bumpversion.cfg b/formats/ome-converter-tool/.bumpversion.cfg index 9c5b3ea4c..c14abf22e 100644 --- a/formats/ome-converter-tool/.bumpversion.cfg +++ b/formats/ome-converter-tool/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.3-dev6 +current_version = 0.3.4-dev0 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? diff --git a/formats/ome-converter-tool/README.md b/formats/ome-converter-tool/README.md index d3e93d08f..8523fc3ac 100644 --- a/formats/ome-converter-tool/README.md +++ b/formats/ome-converter-tool/README.md @@ -1,4 +1,4 @@ -# OME Converter (v0.3.3-dev6) +# OME Converter (v0.3.4-dev0) This WIPP plugin converts BioFormats supported data types to the OME Zarr or OME TIF file format. This is not a complete implementation, rather it implements a file @@ -35,4 +35,4 @@ This plugin takes 2 input arguments and 1 output argument: ## Docker Command ```bash -docker run -e POLUS_IMG_EXT=".ome.zarr" -v /Users/username/:/Users/username/ polusai/ome-converter-tool:0.3.3-dev6 --inpDir=/Users/path/to/Images/ --filePattern=".*.tif" --outDir=/Users/path/to/outputs +docker run -e POLUS_IMG_EXT=".ome.zarr" -v /Users/username/:/Users/username/ polusai/ome-converter-tool:0.3.4-dev0 --inpDir=/Users/path/to/Images/ --filePattern=".*.tif" --outDir=/Users/path/to/outputs diff --git a/formats/ome-converter-tool/VERSION b/formats/ome-converter-tool/VERSION index 2d516df6e..424eb6c61 100644 --- a/formats/ome-converter-tool/VERSION +++ b/formats/ome-converter-tool/VERSION @@ -1 +1 @@ -0.3.3-dev6 +0.3.4-dev0 diff --git a/formats/ome-converter-tool/ict.yaml b/formats/ome-converter-tool/ict.yaml index b61e9be58..18a66a1aa 100644 --- a/formats/ome-converter-tool/ict.yaml +++ b/formats/ome-converter-tool/ict.yaml @@ -2,7 +2,7 @@ author: - Nick Schaub - Hamdah Shafqat contact: nick.schaub@nih.gov -container: polusai/ome-converter-tool:0.3.3-dev6 +container: polusai/ome-converter-tool:0.3.4-dev0 description: Convert Bioformats supported format to OME Zarr or OME TIF entrypoint: python3 -m polus.images.formats.ome_converter inputs: @@ -38,4 +38,4 @@ ui: key: inputs.filePattern title: Filepattern type: text -version: 0.3.3-dev6 +version: 0.3.4-dev0 diff --git a/formats/ome-converter-tool/omeconverter.cwl b/formats/ome-converter-tool/omeconverter.cwl index ee59749a1..048087a3b 100644 --- a/formats/ome-converter-tool/omeconverter.cwl +++ b/formats/ome-converter-tool/omeconverter.cwl @@ -20,7 +20,7 @@ outputs: type: Directory requirements: DockerRequirement: - dockerPull: polusai/ome-converter-tool:0.3.3-dev6 + dockerPull: polusai/ome-converter-tool:0.3.4-dev0 InitialWorkDirRequirement: listing: - entry: $(inputs.outDir) diff --git a/formats/ome-converter-tool/plugin.json b/formats/ome-converter-tool/plugin.json index 807d4d6e9..bf3603221 100644 --- a/formats/ome-converter-tool/plugin.json +++ b/formats/ome-converter-tool/plugin.json @@ -1,6 +1,6 @@ { "name": "OME Converter", - "version": "0.3.3-dev6", + "version": "0.3.4-dev0", "title": "OME Converter", "description": "Convert Bioformats supported format to OME Zarr or OME TIF", "author": "Nick Schaub (nick.schaub@nih.gov), Hamdah Shafqat Abbasi (hamdahshafqat.abbasi@nih.gov)", @@ -8,7 +8,7 @@ "repository": "https://github.com/PolusAI/image-tools", "website": "https://ncats.nih.gov/preclinical/core/informatics", "citation": "", - "containerId": "polusai/ome-converter-tool:0.3.3-dev6", + "containerId": "polusai/ome-converter-tool:0.3.4-dev0", "baseCommand": [ "python3", "-m", diff --git a/formats/ome-converter-tool/pyproject.toml b/formats/ome-converter-tool/pyproject.toml index 0f78cc193..86bf92ff7 100644 --- a/formats/ome-converter-tool/pyproject.toml +++ b/formats/ome-converter-tool/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "polus-images-formats-ome-converter" -version = "0.3.3-dev6" +version = "0.3.4-dev0" description = "Convert BioFormats datatypes to ome.tif or ome.zarr file format" authors = [ "Nick Schaub ", diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py index ffa697c2e..e6c65e454 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py @@ -1,6 +1,6 @@ """Ome Converter.""" -__version__ = "0.3.3-dev6" +__version__ = "0.3.4-dev0" from .image_converter import batch_convert from .image_converter import convert_image From 0a18d39558caa09f4896dd8549724e5ace8b8c48 Mon Sep 17 00:00:00 2001 From: Hamdah Shafqat Abbasi Date: Wed, 2 Apr 2025 14:46:22 -0400 Subject: [PATCH 07/17] fix the bug for flex file --- formats/ome-converter-tool/Dockerfile | 2 +- formats/ome-converter-tool/pyproject.toml | 3 +-- formats/ome-converter-tool/run-plugin.sh | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/formats/ome-converter-tool/Dockerfile b/formats/ome-converter-tool/Dockerfile index 504961fce..831abef6b 100644 --- a/formats/ome-converter-tool/Dockerfile +++ b/formats/ome-converter-tool/Dockerfile @@ -5,7 +5,7 @@ ENV EXEC_DIR="/opt/executables" ENV POLUS_IMG_EXT=".ome.tif" ENV POLUS_TAB_EXT=".csv" ENV POLUS_LOG="INFO" -ENV NUM_THREADS="1" +ENV NUM_THREADS=1 # Work directory defined in the base container WORKDIR ${EXEC_DIR} diff --git a/formats/ome-converter-tool/pyproject.toml b/formats/ome-converter-tool/pyproject.toml index 86bf92ff7..2cf753cf6 100644 --- a/formats/ome-converter-tool/pyproject.toml +++ b/formats/ome-converter-tool/pyproject.toml @@ -12,11 +12,10 @@ packages = [{include = "polus", from = "src"}] [tool.poetry.dependencies] python = ">=3.9,<3.12" -bfio = {version = "2.4.7", extras = ["all"]} +bfio = {version = "^2.4.7", extras = ["all"]} filepattern = "^2.0.4" typer = "^0.7.0" tqdm = "^4.64.1" -preadator = "0.4.0-dev2" numpy = "<2.0.0" [tool.poetry.group.dev.dependencies] diff --git a/formats/ome-converter-tool/run-plugin.sh b/formats/ome-converter-tool/run-plugin.sh index b579515f9..40a667371 100755 --- a/formats/ome-converter-tool/run-plugin.sh +++ b/formats/ome-converter-tool/run-plugin.sh @@ -15,7 +15,7 @@ outDir=/data/output # docker run polusai/ome-converter-plugin:${version} # Run the plugin -docker run -e POLUS_IMG_EXT=${fileExtension} --mount type=bind,source=${datapath},target=/data/ \ +docker run -e POLUS_IMG_EXT=${fileExtension} -e NUM_THREADS=4 --mount type=bind,source=${datapath},target=/data/ \ polusai/ome-converter-tool:${version} \ --inpDir ${inpDir} \ --filePattern ${filePattern} \ From 2df2ef19b33fe487528d5be9f62e895b6db74aac Mon Sep 17 00:00:00 2001 From: Hamdah Shafqat Abbasi Date: Wed, 2 Apr 2025 14:52:54 -0400 Subject: [PATCH 08/17] fix the bug for flex file --- formats/ome-converter-tool/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/formats/ome-converter-tool/README.md b/formats/ome-converter-tool/README.md index 8523fc3ac..25fb13af4 100644 --- a/formats/ome-converter-tool/README.md +++ b/formats/ome-converter-tool/README.md @@ -35,4 +35,4 @@ This plugin takes 2 input arguments and 1 output argument: ## Docker Command ```bash -docker run -e POLUS_IMG_EXT=".ome.zarr" -v /Users/username/:/Users/username/ polusai/ome-converter-tool:0.3.4-dev0 --inpDir=/Users/path/to/Images/ --filePattern=".*.tif" --outDir=/Users/path/to/outputs +docker run -e POLUS_IMG_EXT=".ome.zarr" --e NUM_THREADS=4 -v /Users/username/:/Users/username/ polusai/ome-converter-tool:0.3.4-dev0 --inpDir=/Users/path/to/Images/ --filePattern=".*.tif" --outDir=/Users/path/to/outputs From 745b2c242f72f701d8d5c1f2ed28aa0cb5a2ef07 Mon Sep 17 00:00:00 2001 From: Hamdah Shafqat Abbasi Date: Tue, 1 Jul 2025 12:07:35 -0400 Subject: [PATCH 09/17] =?UTF-8?q?Bump=20version:=200.3.4-dev0=20=E2=86=92?= =?UTF-8?q?=200.3.4-dev1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- formats/ome-converter-tool/.bumpversion.cfg | 2 +- formats/ome-converter-tool/README.md | 4 ++-- formats/ome-converter-tool/VERSION | 2 +- formats/ome-converter-tool/ict.yaml | 4 ++-- formats/ome-converter-tool/omeconverter.cwl | 2 +- formats/ome-converter-tool/plugin.json | 4 ++-- formats/ome-converter-tool/pyproject.toml | 2 +- .../src/polus/images/formats/ome_converter/__init__.py | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/formats/ome-converter-tool/.bumpversion.cfg b/formats/ome-converter-tool/.bumpversion.cfg index c14abf22e..a7d733ed9 100644 --- a/formats/ome-converter-tool/.bumpversion.cfg +++ b/formats/ome-converter-tool/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.4-dev0 +current_version = 0.3.4-dev1 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? diff --git a/formats/ome-converter-tool/README.md b/formats/ome-converter-tool/README.md index 25fb13af4..7c4f062d7 100644 --- a/formats/ome-converter-tool/README.md +++ b/formats/ome-converter-tool/README.md @@ -1,4 +1,4 @@ -# OME Converter (v0.3.4-dev0) +# OME Converter (v0.3.4-dev1) This WIPP plugin converts BioFormats supported data types to the OME Zarr or OME TIF file format. This is not a complete implementation, rather it implements a file @@ -35,4 +35,4 @@ This plugin takes 2 input arguments and 1 output argument: ## Docker Command ```bash -docker run -e POLUS_IMG_EXT=".ome.zarr" --e NUM_THREADS=4 -v /Users/username/:/Users/username/ polusai/ome-converter-tool:0.3.4-dev0 --inpDir=/Users/path/to/Images/ --filePattern=".*.tif" --outDir=/Users/path/to/outputs +docker run -e POLUS_IMG_EXT=".ome.zarr" --e NUM_THREADS=4 -v /Users/username/:/Users/username/ polusai/ome-converter-tool:0.3.4-dev1 --inpDir=/Users/path/to/Images/ --filePattern=".*.tif" --outDir=/Users/path/to/outputs diff --git a/formats/ome-converter-tool/VERSION b/formats/ome-converter-tool/VERSION index 424eb6c61..a2d952023 100644 --- a/formats/ome-converter-tool/VERSION +++ b/formats/ome-converter-tool/VERSION @@ -1 +1 @@ -0.3.4-dev0 +0.3.4-dev1 diff --git a/formats/ome-converter-tool/ict.yaml b/formats/ome-converter-tool/ict.yaml index 18a66a1aa..de4a6f079 100644 --- a/formats/ome-converter-tool/ict.yaml +++ b/formats/ome-converter-tool/ict.yaml @@ -2,7 +2,7 @@ author: - Nick Schaub - Hamdah Shafqat contact: nick.schaub@nih.gov -container: polusai/ome-converter-tool:0.3.4-dev0 +container: polusai/ome-converter-tool:0.3.4-dev1 description: Convert Bioformats supported format to OME Zarr or OME TIF entrypoint: python3 -m polus.images.formats.ome_converter inputs: @@ -38,4 +38,4 @@ ui: key: inputs.filePattern title: Filepattern type: text -version: 0.3.4-dev0 +version: 0.3.4-dev1 diff --git a/formats/ome-converter-tool/omeconverter.cwl b/formats/ome-converter-tool/omeconverter.cwl index 048087a3b..a13304179 100644 --- a/formats/ome-converter-tool/omeconverter.cwl +++ b/formats/ome-converter-tool/omeconverter.cwl @@ -20,7 +20,7 @@ outputs: type: Directory requirements: DockerRequirement: - dockerPull: polusai/ome-converter-tool:0.3.4-dev0 + dockerPull: polusai/ome-converter-tool:0.3.4-dev1 InitialWorkDirRequirement: listing: - entry: $(inputs.outDir) diff --git a/formats/ome-converter-tool/plugin.json b/formats/ome-converter-tool/plugin.json index bf3603221..6c3f40a04 100644 --- a/formats/ome-converter-tool/plugin.json +++ b/formats/ome-converter-tool/plugin.json @@ -1,6 +1,6 @@ { "name": "OME Converter", - "version": "0.3.4-dev0", + "version": "0.3.4-dev1", "title": "OME Converter", "description": "Convert Bioformats supported format to OME Zarr or OME TIF", "author": "Nick Schaub (nick.schaub@nih.gov), Hamdah Shafqat Abbasi (hamdahshafqat.abbasi@nih.gov)", @@ -8,7 +8,7 @@ "repository": "https://github.com/PolusAI/image-tools", "website": "https://ncats.nih.gov/preclinical/core/informatics", "citation": "", - "containerId": "polusai/ome-converter-tool:0.3.4-dev0", + "containerId": "polusai/ome-converter-tool:0.3.4-dev1", "baseCommand": [ "python3", "-m", diff --git a/formats/ome-converter-tool/pyproject.toml b/formats/ome-converter-tool/pyproject.toml index 2cf753cf6..9f221132d 100644 --- a/formats/ome-converter-tool/pyproject.toml +++ b/formats/ome-converter-tool/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "polus-images-formats-ome-converter" -version = "0.3.4-dev0" +version = "0.3.4-dev1" description = "Convert BioFormats datatypes to ome.tif or ome.zarr file format" authors = [ "Nick Schaub ", diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py index e6c65e454..260a520dc 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py @@ -1,6 +1,6 @@ """Ome Converter.""" -__version__ = "0.3.4-dev0" +__version__ = "0.3.4-dev1" from .image_converter import batch_convert from .image_converter import convert_image From 3bceb17f7a64154594d6f01414ee1fa4c01fb4af Mon Sep 17 00:00:00 2001 From: hamshkhawar Date: Tue, 3 Mar 2026 11:23:42 -0500 Subject: [PATCH 10/17] =?UTF-8?q?Bump=20version:=200.3.4-dev1=20=E2=86=92?= =?UTF-8?q?=200.3.4-dev2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- formats/ome-converter-tool/.bumpversion.cfg | 2 +- formats/ome-converter-tool/Dockerfile | 5 +- formats/ome-converter-tool/README.md | 4 +- formats/ome-converter-tool/VERSION | 2 +- formats/ome-converter-tool/ict.yaml | 4 +- formats/ome-converter-tool/omeconverter.cwl | 2 +- formats/ome-converter-tool/plugin.json | 4 +- formats/ome-converter-tool/pyproject.toml | 15 +- .../images/formats/ome_converter/__init__.py | 2 +- .../images/formats/ome_converter/__main__.py | 98 ++++---- .../formats/ome_converter/image_converter.py | 216 ++++++------------ 11 files changed, 146 insertions(+), 208 deletions(-) diff --git a/formats/ome-converter-tool/.bumpversion.cfg b/formats/ome-converter-tool/.bumpversion.cfg index a7d733ed9..5486ac653 100644 --- a/formats/ome-converter-tool/.bumpversion.cfg +++ b/formats/ome-converter-tool/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.4-dev1 +current_version = 0.3.4-dev2 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? diff --git a/formats/ome-converter-tool/Dockerfile b/formats/ome-converter-tool/Dockerfile index 831abef6b..087bb74e4 100644 --- a/formats/ome-converter-tool/Dockerfile +++ b/formats/ome-converter-tool/Dockerfile @@ -1,11 +1,12 @@ -FROM sameeul/image-processor:latest +FROM polusai/bfio:2.5.0 # environment variables defined in polusai/bfio ENV EXEC_DIR="/opt/executables" ENV POLUS_IMG_EXT=".ome.tif" ENV POLUS_TAB_EXT=".csv" ENV POLUS_LOG="INFO" -ENV NUM_THREADS=1 +ENV NUM_THREADS=2 +ENV NUM_WORKERS=4 # Work directory defined in the base container WORKDIR ${EXEC_DIR} diff --git a/formats/ome-converter-tool/README.md b/formats/ome-converter-tool/README.md index 7c4f062d7..40103c67f 100644 --- a/formats/ome-converter-tool/README.md +++ b/formats/ome-converter-tool/README.md @@ -1,4 +1,4 @@ -# OME Converter (v0.3.4-dev1) +# OME Converter (v0.3.4-dev2) This WIPP plugin converts BioFormats supported data types to the OME Zarr or OME TIF file format. This is not a complete implementation, rather it implements a file @@ -35,4 +35,4 @@ This plugin takes 2 input arguments and 1 output argument: ## Docker Command ```bash -docker run -e POLUS_IMG_EXT=".ome.zarr" --e NUM_THREADS=4 -v /Users/username/:/Users/username/ polusai/ome-converter-tool:0.3.4-dev1 --inpDir=/Users/path/to/Images/ --filePattern=".*.tif" --outDir=/Users/path/to/outputs +docker run -e POLUS_IMG_EXT=".ome.zarr" --e NUM_THREADS=4 -v /Users/username/:/Users/username/ polusai/ome-converter-tool:0.3.4-dev2 --inpDir=/Users/path/to/Images/ --filePattern=".*.tif" --outDir=/Users/path/to/outputs diff --git a/formats/ome-converter-tool/VERSION b/formats/ome-converter-tool/VERSION index a2d952023..ee14bbbd0 100644 --- a/formats/ome-converter-tool/VERSION +++ b/formats/ome-converter-tool/VERSION @@ -1 +1 @@ -0.3.4-dev1 +0.3.4-dev2 diff --git a/formats/ome-converter-tool/ict.yaml b/formats/ome-converter-tool/ict.yaml index de4a6f079..a0899413a 100644 --- a/formats/ome-converter-tool/ict.yaml +++ b/formats/ome-converter-tool/ict.yaml @@ -2,7 +2,7 @@ author: - Nick Schaub - Hamdah Shafqat contact: nick.schaub@nih.gov -container: polusai/ome-converter-tool:0.3.4-dev1 +container: polusai/ome-converter-tool:0.3.4-dev2 description: Convert Bioformats supported format to OME Zarr or OME TIF entrypoint: python3 -m polus.images.formats.ome_converter inputs: @@ -38,4 +38,4 @@ ui: key: inputs.filePattern title: Filepattern type: text -version: 0.3.4-dev1 +version: 0.3.4-dev2 diff --git a/formats/ome-converter-tool/omeconverter.cwl b/formats/ome-converter-tool/omeconverter.cwl index a13304179..c6f38b610 100644 --- a/formats/ome-converter-tool/omeconverter.cwl +++ b/formats/ome-converter-tool/omeconverter.cwl @@ -20,7 +20,7 @@ outputs: type: Directory requirements: DockerRequirement: - dockerPull: polusai/ome-converter-tool:0.3.4-dev1 + dockerPull: polusai/ome-converter-tool:0.3.4-dev2 InitialWorkDirRequirement: listing: - entry: $(inputs.outDir) diff --git a/formats/ome-converter-tool/plugin.json b/formats/ome-converter-tool/plugin.json index 6c3f40a04..60edecfc6 100644 --- a/formats/ome-converter-tool/plugin.json +++ b/formats/ome-converter-tool/plugin.json @@ -1,6 +1,6 @@ { "name": "OME Converter", - "version": "0.3.4-dev1", + "version": "0.3.4-dev2", "title": "OME Converter", "description": "Convert Bioformats supported format to OME Zarr or OME TIF", "author": "Nick Schaub (nick.schaub@nih.gov), Hamdah Shafqat Abbasi (hamdahshafqat.abbasi@nih.gov)", @@ -8,7 +8,7 @@ "repository": "https://github.com/PolusAI/image-tools", "website": "https://ncats.nih.gov/preclinical/core/informatics", "citation": "", - "containerId": "polusai/ome-converter-tool:0.3.4-dev1", + "containerId": "polusai/ome-converter-tool:0.3.4-dev2", "baseCommand": [ "python3", "-m", diff --git a/formats/ome-converter-tool/pyproject.toml b/formats/ome-converter-tool/pyproject.toml index 9f221132d..2b9b9bd6c 100644 --- a/formats/ome-converter-tool/pyproject.toml +++ b/formats/ome-converter-tool/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "polus-images-formats-ome-converter" -version = "0.3.4-dev1" +version = "0.3.4-dev2" description = "Convert BioFormats datatypes to ome.tif or ome.zarr file format" authors = [ "Nick Schaub ", @@ -11,12 +11,13 @@ readme = "README.md" packages = [{include = "polus", from = "src"}] [tool.poetry.dependencies] -python = ">=3.9,<3.12" -bfio = {version = "^2.4.7", extras = ["all"]} -filepattern = "^2.0.4" -typer = "^0.7.0" +python = ">=3.11,<3.13" +bfio = "2.5.0" +filepattern = "2.1.4" +typer = "0.24.1" tqdm = "^4.64.1" -numpy = "<2.0.0" +numpy = ">2.0.0" +scikit-image = "^0.26.0" [tool.poetry.group.dev.dependencies] bump2version = "^1.0.1" @@ -24,7 +25,7 @@ pre-commit = "^3.0.4" pytest = "^7.2.1" ipykernel = "^6.21.2" requests = "^2.28.2" -scikit-image = "^0.19.3" +scikit-image = ">=0.26.0" matplotlib = "3.9.4" [build-system] diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py index 260a520dc..439dcf7ad 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__init__.py @@ -1,6 +1,6 @@ """Ome Converter.""" -__version__ = "0.3.4-dev1" +__version__ = "0.3.4-dev2" from .image_converter import batch_convert from .image_converter import convert_image diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py index 8cce13059..c152d1430 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py @@ -3,18 +3,13 @@ import logging import os import pathlib -from concurrent.futures import ProcessPoolExecutor -from concurrent.futures import as_completed from typing import Any from typing import Optional -from typing import Union import filepattern as fp import typer -from polus.images.formats.ome_converter.image_converter import NUM_THREADS from polus.images.formats.ome_converter.image_converter import POLUS_IMG_EXT -from polus.images.formats.ome_converter.image_converter import convert_image -from tqdm import tqdm +from polus.images.formats.ome_converter.image_converter import batch_convert app = typer.Typer() @@ -27,17 +22,56 @@ logger.setLevel(os.environ.get("POLUS_LOG", logging.INFO)) -def parse_input_dir(value: str) -> Union[pathlib.Path, list[str]]: - """Parse inp_dir as either a directory path or a comma-separated list of files.""" +def parse_input_dir(value: str) -> pathlib.Path | list[pathlib.Path]: + """Parse input as either: + + - a directory path, which must exist + - a comma-separated list of file paths (strings). + """ if "," in value: - return [f.strip() for f in value.split(",")] - path = pathlib.Path(value) - if not (path.exists() and path.is_dir()): + files = [ + pathlib.Path(f.strip()).expanduser().resolve() + for f in value.split(",") + if f.strip() + ] + if not files: + msg = "Empty file list provided" + raise typer.BadParameter(msg) + return files + path = pathlib.Path(value).expanduser().resolve() + if not path.is_dir(): msg = f"Directory {path} does not exist" raise typer.BadParameter(msg) return path +def _collect_files( + inp: pathlib.Path | list[pathlib.Path], + pattern: str, +) -> list[pathlib.Path]: + """Normalize input into a list of files.""" + if isinstance(inp, pathlib.Path): + fps = fp.FilePattern(inp, pattern) + return [f[1][0] for f in fps()] + return list(inp) + + +def write_preview( + out_dir: pathlib.Path, + file_pattern: str, + files: list[pathlib.Path], +) -> None: + """Write a JSON preview of the files that would be converted.""" + preview: dict[str, Any] = { + "filePattern": file_pattern, + "outDir": [f.stem + POLUS_IMG_EXT for f in files], + } + preview_path = out_dir / "preview.json" + with preview_path.open("w") as f: + json.dump(preview, f, indent=2) + logger.info(f"Preview written to {preview_path}") + + @app.command() def main( inp_dir: str = typer.Option( @@ -46,7 +80,7 @@ def main( help="Input generic data collection to be processed by this plugin", callback=parse_input_dir, ), - pattern: str = typer.Option( + file_pattern: str = typer.Option( ".+", "--filePattern", help="A filepattern defining the images to be converted", @@ -70,40 +104,22 @@ def main( """Convert bioformat supported image datatypes conversion to ome.tif or ome.zarr.""" logger.info(f"inpDir = {inp_dir}") logger.info(f"outDir = {out_dir}") - logger.info(f"filePattern = {pattern}") + logger.info(f"filePattern = {file_pattern}") + logger.info(f"preview = {preview}") - fps = fp.FilePattern(inp_dir, pattern) + files = _collect_files(inp_dir, file_pattern) + logger.info(f"Found {len(files)} file(s) to process.") if preview: - with out_dir.joinpath("preview.json").open("w") as jfile: - out_json: dict[str, Any] = { - "filepattern": pattern, - "outDir": [], - } - for file in fps(): - out_name = str(file[1][0].name.split(".")[0]) + POLUS_IMG_EXT - out_json["outDir"].append(out_name) - json.dump(out_json, jfile, indent=2) + write_preview(out_dir, file_pattern, files) return - # Use ProcessPoolExecutor for multiprocessing - with ProcessPoolExecutor(max_workers=NUM_THREADS) as executor: - futures = [] - for files in fps(): - file = files[1][0] - futures.append(executor.submit(convert_image, file, POLUS_IMG_EXT, out_dir)) - - # Process results with progress bar - for f in tqdm( - as_completed(futures), - total=len(futures), - mininterval=5, - desc=f"converting images to {POLUS_IMG_EXT}", - initial=0, - unit_scale=True, - colour="cyan", - ): - f.result() + batch_convert( + inp_dir=inp_dir, + out_dir=out_dir, + file_pattern=file_pattern, + file_extension=POLUS_IMG_EXT, + ) if __name__ == "__main__": diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py index 1b3d467e9..8ebadc0e2 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py @@ -3,13 +3,12 @@ import os import pathlib import platform -from concurrent.futures import ProcessPoolExecutor from concurrent.futures import as_completed from itertools import product from typing import Optional import filepattern as fp -import numpy as np +import preadator from bfio import BioReader from bfio import BioWriter from tqdm import tqdm @@ -19,62 +18,21 @@ TILE_SIZE = 2**13 POLUS_IMG_EXT = os.environ.get("POLUS_IMG_EXT", ".ome.tif") -NUM_THREADS_ENV = os.environ.get("NUM_THREADS") -if not NUM_THREADS_ENV or NUM_THREADS_ENV == "1": - if platform.system().lower() == "linux": - try: - NUM_THREADS = len(os.sched_getaffinity(0)) // 2 # type: ignore - except AttributeError: - cpu_count = os.cpu_count() - NUM_THREADS = cpu_count // 2 if cpu_count is not None else 1 - else: - cpu_count = os.cpu_count() - NUM_THREADS = cpu_count // 2 if cpu_count is not None else 1 +# NUM_THREADS: default to half of CPU cores, min 1 +if "NUM_THREADS" in os.environ: + NUM_THREADS = max(1, int(os.environ["NUM_THREADS"])) else: - NUM_THREADS = int(NUM_THREADS_ENV) # Convert str to int safely - -NUM_THREADS = max(1, NUM_THREADS) # Ensure at least 1 thread, type is int - - -def write_image( - br: BioReader, - c: int, - image: np.ndarray, - out_path: pathlib.Path, - max_workers: int, -) -> None: - """Write an image to OME-TIFF or OME-ZARR format using BioFormats. + try: + if platform.system().lower() == "linux": + NUM_THREADS = max(1, len(os.sched_getaffinity(0)) // 2) + else: + NUM_THREADS = max(1, os.cpu_count() // 2) + except TypeError: + NUM_THREADS = 1 # fallback if cpu_count returns None - This function converts a given image to the OME-TIFF or OME-ZARR format, - ensuring that the data type is compatible and handling any necessary - byte order adjustments. It utilizes the BioWriter for writing the image - and manages channel names based on the provided BioReader. - - Args: - br: An instance of BioReader containing metadata for the image. - c: The index of the channel in the image. - image: The image data to be written. - out_path: Path to an output image. - max_workers: The maximum number of worker threads to use for writing. - """ - if image.dtype == ">u2": - image = image.byteswap().newbyteorder("<") - - with BioWriter( - out_path, - max_workers, - ) as bw: - # Handling of parsing channels when channels names are not provided. - if bw.channel_names != [None]: - bw.channel_names = [br.channel_names[c]] - bw.C = 1 - bw.T = 1 - bw.Z = 1 - bw.X = image.shape[1] - bw.Y = image.shape[0] - bw.dtype = image.dtype - bw[:] = image +# NUM_WORKERS: default to 1 process +NUM_WORKERS = max(1, int(os.environ.get("NUM_WORKERS", "1"))) def get_num_series(inp_image: pathlib.Path) -> int: @@ -88,7 +46,7 @@ def get_num_series(inp_image: pathlib.Path) -> int: """ try: # Open the specific file with BioReader - br = BioReader(inp_image, max_workers=1) + br = BioReader(inp_image, max_workers=NUM_THREADS) # Initialize series count num_series = 1 @@ -119,12 +77,14 @@ def convert_image( inp_image: pathlib.Path, file_extension: str, out_dir: pathlib.Path, -) -> None: +) -> None: # noqa: C901 """Convert bioformats supported datatypes to ome.tif or ome.zarr file format.""" # First, determine the number of series num_series = get_num_series(inp_image) + logger.info(f"Detected {num_series} series/levels in image {inp_image.name}") + files_written = 0 # Process each series independently for idx in range(num_series): logger.info(f"Processing levels {idx} of {inp_image.name}") @@ -163,7 +123,7 @@ def convert_image( if br.Z > 1: suffix_parts.append(f"_z{z}") if num_series > 1: - suffix_parts.append(f"_level_{idx}") + suffix_parts.append(f"_s{idx}") # Apply combined suffix if suffix_parts: @@ -175,8 +135,37 @@ def convert_image( ), ) - # Try primary method first, then fall back to chunked approach - process_single_view(br, c, t, z, out_path) + with BioWriter( + out_path, + max_workers=NUM_THREADS, + metadata=br.metadata, + ) as bw: + bw.C = 1 + bw.T = 1 + bw.Z = 1 + if bw.channel_names != [None]: + bw.channel_names = [br.channel_names[c]] + + for y in range(0, br.Y, TILE_SIZE): + y_max = min(br.Y, y + TILE_SIZE) + for x in range(0, br.X, TILE_SIZE): + x_max = min(br.X, x + TILE_SIZE) + bw[ + y:y_max, + x:x_max, + 0, + 0, + 0, + ] = br[ + y:y_max, + x:x_max, + z : z + 1, + c, + t, + ] + + files_written += 1 + logger.debug(f"Written: {out_path.name}") except (OSError, ValueError) as e: logger.error( @@ -184,88 +173,8 @@ def convert_image( ) -def process_single_view( - br: "BioReader", - c: int, - t: int, - z: int, - out_path: pathlib.Path, -) -> None: - """Process a single view (t,c,z) from a BioReader.""" - try: - # Try direct read first - final_image = br[:, :, z, c, t] - - write_image( - br=br, - c=c, - image=final_image, - out_path=out_path, - max_workers=NUM_THREADS, - ) - logger.debug(f"Successfully processed {out_path.name} using direct read") - - except (OSError, ValueError) as e: - logger.warning( - f"Direct read failed for {out_path.name}, trying chunked approach: {e!s}", - ) - try: - # Fallback to chunked approach - actual_y, actual_x = br.Y, br.X - - if actual_y <= 0 or actual_x <= 0: - logger.error( - f"Invalid dimensions: {actual_y}x{actual_x} for {out_path.name}", - ) - return - - final_image = np.zeros((actual_y, actual_x), dtype=br.dtype) - - # Use smaller chunks for problematic images - chunk_size = min(TILE_SIZE, min(actual_y, actual_x) // 2) - - # Track successful chunks for debugging - total_chunks = 0 - successful_chunks = 0 - - for y in range(0, actual_y, chunk_size): - for x in range(0, actual_x, chunk_size): - y_max = min(actual_y, y + chunk_size) - x_max = min(actual_x, x + chunk_size) - total_chunks += 1 - - try: - chunk = br[y:y_max, x:x_max, z, c, t] - final_image[y:y_max, x:x_max] = chunk - successful_chunks += 1 - except (OSError, ValueError) as e: - logger.warning( - f"Failed to read chunk at ({y}:{y_max}, {x}:{x_max}): {e}", - ) - - logger.info( - f"Chunked read: {successful_chunks}/{total_chunks} successfully", - ) - - write_image( - br=br, - c=c, - image=final_image, - out_path=out_path, - max_workers=NUM_THREADS, - ) - logger.debug( - f"Successfully processed {out_path.name} using chunked approach", - ) - - except (OSError, ValueError) as e: - logger.error( - f"Failed to process {out_path.name} with chunked approach: {e!s}", - ) - - def batch_convert( - inp_dir: pathlib.Path, + inp_dir: pathlib.Path | list[pathlib.Path], out_dir: pathlib.Path, file_pattern: Optional[str], file_extension: str, @@ -283,16 +192,27 @@ def batch_convert( logger.info(f"file_pattern = {file_pattern}") logger.info(f"file_extension = {file_extension}") - file_pattern = ".+" if file_pattern is None else file_pattern + if isinstance(inp_dir, pathlib.Path): + file_pattern = ".+" if file_pattern is None else file_pattern + fps = fp.FilePattern(inp_dir, file_pattern) + files = [files[1][0] for files in fps()] + else: + files = list(inp_dir) - fps = fp.FilePattern(inp_dir, file_pattern) + if not files: + logger.warning("No files found to process.") + return - with ProcessPoolExecutor(max_workers=NUM_THREADS) as executor: + with preadator.ProcessManager( + name="ome_converter", + num_processes=NUM_WORKERS, + threads_per_process=NUM_THREADS, + ) as executor: futures = [] - for files in fps(): - file = files[1][0] - futures.append(executor.submit(convert_image, file, POLUS_IMG_EXT, out_dir)) - + futures = [ + executor.submit(convert_image, file, POLUS_IMG_EXT, out_dir) + for file in files + ] for f in tqdm( as_completed(futures), total=len(futures), From fe6d3e61ddba26504bad15c8feab9d25f48535b7 Mon Sep 17 00:00:00 2001 From: hamshkhawar Date: Tue, 3 Mar 2026 11:36:02 -0500 Subject: [PATCH 11/17] addressed Nick comments --- .../formats/ome_converter/image_converter.py | 16 ++++++++-------- formats/ome-converter-tool/tests/conftest.py | 4 +--- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py index 8ebadc0e2..d690b375c 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py @@ -4,11 +4,11 @@ import pathlib import platform from concurrent.futures import as_completed +from concurrent.futures import ProcessPoolExecutor from itertools import product from typing import Optional import filepattern as fp -import preadator from bfio import BioReader from bfio import BioWriter from tqdm import tqdm @@ -77,7 +77,7 @@ def convert_image( inp_image: pathlib.Path, file_extension: str, out_dir: pathlib.Path, -) -> None: # noqa: C901 +) -> None: # noqa: C901 """Convert bioformats supported datatypes to ome.tif or ome.zarr file format.""" # First, determine the number of series num_series = get_num_series(inp_image) @@ -203,12 +203,9 @@ def batch_convert( logger.warning("No files found to process.") return - with preadator.ProcessManager( - name="ome_converter", - num_processes=NUM_WORKERS, - threads_per_process=NUM_THREADS, + with ProcessPoolExecutor( + max_workers=NUM_WORKERS, ) as executor: - futures = [] futures = [ executor.submit(convert_image, file, POLUS_IMG_EXT, out_dir) for file in files @@ -222,4 +219,7 @@ def batch_convert( unit_scale=True, colour="cyan", ): - f.result() + try: + f.result() + except Exception as e: + logger.error(f"Failed to convert file: {e}") \ No newline at end of file diff --git a/formats/ome-converter-tool/tests/conftest.py b/formats/ome-converter-tool/tests/conftest.py index b3dc8a7b9..9094aba73 100644 --- a/formats/ome-converter-tool/tests/conftest.py +++ b/formats/ome-converter-tool/tests/conftest.py @@ -5,12 +5,10 @@ import shutil import tempfile import typing - +import skimage import numpy import pytest import requests -import skimage.data -import skimage.measure def pytest_addoption(parser: pytest.Parser) -> None: From eb633cc8632113c809794341385eb08c07cf6c64 Mon Sep 17 00:00:00 2001 From: hamshkhawar Date: Tue, 3 Mar 2026 13:21:19 -0500 Subject: [PATCH 12/17] fix ruff mypy checks --- formats/ome-converter-tool/mypy.ini | 16 +++++++++ formats/ome-converter-tool/pyproject.toml | 1 + .../images/formats/ome_converter/__main__.py | 28 +++++++++------- .../formats/ome_converter/image_converter.py | 33 +++++++++++-------- formats/ome-converter-tool/tests/conftest.py | 9 ++--- formats/ome-converter-tool/tests/test_main.py | 4 +-- 6 files changed, 60 insertions(+), 31 deletions(-) create mode 100644 formats/ome-converter-tool/mypy.ini diff --git a/formats/ome-converter-tool/mypy.ini b/formats/ome-converter-tool/mypy.ini new file mode 100644 index 000000000..16e95711f --- /dev/null +++ b/formats/ome-converter-tool/mypy.ini @@ -0,0 +1,16 @@ +[mypy] +files = src/polus/images/formats/ome_converter, tests/ +exclude = \.git/ +disable_error_code = import-untyped + +[mypy-bfio] +ignore_missing_imports = True + +[mypy-filepattern] +ignore_missing_imports = True + +[mypy-tqdm] +ignore_missing_imports = True + +# [mypy-polus.images.formats.ome_converter.*] +# ignore_missing_imports = False \ No newline at end of file diff --git a/formats/ome-converter-tool/pyproject.toml b/formats/ome-converter-tool/pyproject.toml index 2b9b9bd6c..c2a009e24 100644 --- a/formats/ome-converter-tool/pyproject.toml +++ b/formats/ome-converter-tool/pyproject.toml @@ -28,6 +28,7 @@ requests = "^2.28.2" scikit-image = ">=0.26.0" matplotlib = "3.9.4" + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py index c152d1430..b3ea99c8d 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py @@ -23,11 +23,7 @@ def parse_input_dir(value: str) -> pathlib.Path | list[pathlib.Path]: - """Parse input as either: - - - a directory path, which must exist - - a comma-separated list of file paths (strings). - """ + """Parse a directory or comma-separated file paths.""" if "," in value: files = [ pathlib.Path(f.strip()).expanduser().resolve() @@ -46,14 +42,24 @@ def parse_input_dir(value: str) -> pathlib.Path | list[pathlib.Path]: def _collect_files( - inp: pathlib.Path | list[pathlib.Path], + inp: str | pathlib.Path | list[pathlib.Path], pattern: str, ) -> list[pathlib.Path]: """Normalize input into a list of files.""" - if isinstance(inp, pathlib.Path): - fps = fp.FilePattern(inp, pattern) - return [f[1][0] for f in fps()] - return list(inp) + if isinstance(inp, (str, pathlib.Path)): + dir_path = pathlib.Path(inp) if isinstance(inp, str) else inp + if not dir_path.is_dir(): + raise ValueError(f"Input path is not a directory: {dir_path}") + fps = fp.FilePattern(dir_path, pattern) + return [files[1][0] for files in fps()] + + elif isinstance(inp, list): + return [pathlib.Path(p) if isinstance(p, str) else p for p in inp] + else: + raise TypeError( + f"Unsupported input type: {type(inp).__name__}. " + "Expected str, Path, or list[Path/str]." + ) def write_preview( @@ -107,7 +113,7 @@ def main( logger.info(f"filePattern = {file_pattern}") logger.info(f"preview = {preview}") - files = _collect_files(inp_dir, file_pattern) + files = _collect_files(inp_dir, file_pattern) # type: ignore logger.info(f"Found {len(files)} file(s) to process.") if preview: diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py index d690b375c..cb686080c 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py @@ -3,8 +3,8 @@ import os import pathlib import platform -from concurrent.futures import as_completed from concurrent.futures import ProcessPoolExecutor +from concurrent.futures import as_completed from itertools import product from typing import Optional @@ -19,17 +19,22 @@ TILE_SIZE = 2**13 POLUS_IMG_EXT = os.environ.get("POLUS_IMG_EXT", ".ome.tif") -# NUM_THREADS: default to half of CPU cores, min 1 -if "NUM_THREADS" in os.environ: - NUM_THREADS = max(1, int(os.environ["NUM_THREADS"])) -else: + +def get_num_threads() -> int: + """Determine default number of threads (half CPU cores, min 1).""" + # Allow environment override + if "NUM_THREADS" in os.environ: + return max(1, int(os.environ["NUM_THREADS"])) try: - if platform.system().lower() == "linux": - NUM_THREADS = max(1, len(os.sched_getaffinity(0)) // 2) + if platform.system().lower() == "linux" and hasattr(os, "sched_getaffinity"): + cpu_count = len(os.sched_getaffinity(0)) else: - NUM_THREADS = max(1, os.cpu_count() // 2) - except TypeError: - NUM_THREADS = 1 # fallback if cpu_count returns None + cpu_count = os.cpu_count() or 1 # fallback if None + return max(1, cpu_count // 2) + except Exception: + return 1 + +NUM_THREADS: int = get_num_threads() # NUM_WORKERS: default to 1 process NUM_WORKERS = max(1, int(os.environ.get("NUM_WORKERS", "1"))) @@ -73,11 +78,11 @@ def get_num_series(inp_image: pathlib.Path) -> int: return 1 -def convert_image( +def convert_image( # noqa:C901,PLR0912 inp_image: pathlib.Path, file_extension: str, out_dir: pathlib.Path, -) -> None: # noqa: C901 +) -> None: """Convert bioformats supported datatypes to ome.tif or ome.zarr file format.""" # First, determine the number of series num_series = get_num_series(inp_image) @@ -221,5 +226,5 @@ def batch_convert( ): try: f.result() - except Exception as e: - logger.error(f"Failed to convert file: {e}") \ No newline at end of file + except (OSError, ValueError) as e: + logger.error(f"Failed to convert file: {e}") diff --git a/formats/ome-converter-tool/tests/conftest.py b/formats/ome-converter-tool/tests/conftest.py index 9094aba73..1da67a3b7 100644 --- a/formats/ome-converter-tool/tests/conftest.py +++ b/formats/ome-converter-tool/tests/conftest.py @@ -5,10 +5,11 @@ import shutil import tempfile import typing -import skimage + import numpy import pytest import requests +import skimage def pytest_addoption(parser: pytest.Parser) -> None: @@ -41,7 +42,7 @@ def get_params(request) -> tuple[int, str]: # noqa: ANN001 return request.param -@pytest.fixture() +@pytest.fixture def synthetic_images( get_params: tuple[int, str], ) -> typing.Generator[tuple[list[numpy.ndarray], pathlib.Path], None, None]: @@ -73,7 +74,7 @@ def synthetic_images( shutil.rmtree(syn_dir) -@pytest.fixture() +@pytest.fixture def output_directory() -> typing.Generator[pathlib.Path, None, None]: """Generate random synthetic images.""" out_dir = pathlib.Path(tempfile.mkdtemp(suffix="_out_dir")) @@ -81,7 +82,7 @@ def output_directory() -> typing.Generator[pathlib.Path, None, None]: shutil.rmtree(out_dir) -@pytest.fixture() +@pytest.fixture def download_images() -> typing.Generator[pathlib.Path, None, None]: """Download test.""" imagelist = { diff --git a/formats/ome-converter-tool/tests/test_main.py b/formats/ome-converter-tool/tests/test_main.py index 8e91d9480..745fd434e 100644 --- a/formats/ome-converter-tool/tests/test_main.py +++ b/formats/ome-converter-tool/tests/test_main.py @@ -67,11 +67,11 @@ def test_cli( app, [ "--inpDir", - inp_dir, + str(inp_dir), "--filePattern", ".+", "--outDir", - output_directory, + str(output_directory), ], ) From b512a9e56974a41c0c77c6c3234f78805d834ccf Mon Sep 17 00:00:00 2001 From: hamshkhawar Date: Tue, 3 Mar 2026 13:26:01 -0500 Subject: [PATCH 13/17] Apply black formatting --- formats/ome-converter-tool/tests/test_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/formats/ome-converter-tool/tests/test_main.py b/formats/ome-converter-tool/tests/test_main.py index 745fd434e..0bec2775c 100644 --- a/formats/ome-converter-tool/tests/test_main.py +++ b/formats/ome-converter-tool/tests/test_main.py @@ -71,7 +71,7 @@ def test_cli( "--filePattern", ".+", "--outDir", - str(output_directory), + str(output_directory), ], ) From 1230dafb939a6c1f007c0a8c0635e8875b6bbc28 Mon Sep 17 00:00:00 2001 From: hamshkhawar Date: Tue, 3 Mar 2026 13:44:19 -0500 Subject: [PATCH 14/17] fixing ruff mypy checks --- formats/ome-converter-tool/pyproject.toml | 8 ++++++++ .../polus/images/formats/ome_converter/__main__.py | 12 +++++------- .../images/formats/ome_converter/image_converter.py | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/formats/ome-converter-tool/pyproject.toml b/formats/ome-converter-tool/pyproject.toml index c2a009e24..f68ae78f7 100644 --- a/formats/ome-converter-tool/pyproject.toml +++ b/formats/ome-converter-tool/pyproject.toml @@ -28,6 +28,14 @@ requests = "^2.28.2" scikit-image = ">=0.26.0" matplotlib = "3.9.4" +[tool.mypy] +files = "." + +# Use strict defaults +strict = true +warn_unreachable = true +warn_no_return = true + [build-system] requires = ["poetry-core"] diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py index b3ea99c8d..a807b8362 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py @@ -49,17 +49,15 @@ def _collect_files( if isinstance(inp, (str, pathlib.Path)): dir_path = pathlib.Path(inp) if isinstance(inp, str) else inp if not dir_path.is_dir(): - raise ValueError(f"Input path is not a directory: {dir_path}") + msg= f"Input path is not a directory: {dir_path}" + raise ValueError(msg) fps = fp.FilePattern(dir_path, pattern) return [files[1][0] for files in fps()] - elif isinstance(inp, list): + if isinstance(inp, list): return [pathlib.Path(p) if isinstance(p, str) else p for p in inp] - else: - raise TypeError( - f"Unsupported input type: {type(inp).__name__}. " - "Expected str, Path, or list[Path/str]." - ) + msg = "Expected str, Path, or list[Path/str]" + raise TypeError(msg) def write_preview( diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py index cb686080c..d364e5a50 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py @@ -31,7 +31,7 @@ def get_num_threads() -> int: else: cpu_count = os.cpu_count() or 1 # fallback if None return max(1, cpu_count // 2) - except Exception: + except (AttributeError, OSError): return 1 NUM_THREADS: int = get_num_threads() From ca42e7c2b1a596aa02b1b3ea780ee0b73538e271 Mon Sep 17 00:00:00 2001 From: hamshkhawar Date: Tue, 3 Mar 2026 14:11:38 -0500 Subject: [PATCH 15/17] updating docs --- formats/ome-converter-tool/run-plugin.sh | 14 +++++++++----- .../polus/images/formats/ome_converter/__main__.py | 4 ++-- .../formats/ome_converter/image_converter.py | 3 ++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/formats/ome-converter-tool/run-plugin.sh b/formats/ome-converter-tool/run-plugin.sh index 40a667371..711222cf9 100755 --- a/formats/ome-converter-tool/run-plugin.sh +++ b/formats/ome-converter-tool/run-plugin.sh @@ -15,8 +15,12 @@ outDir=/data/output # docker run polusai/ome-converter-plugin:${version} # Run the plugin -docker run -e POLUS_IMG_EXT=${fileExtension} -e NUM_THREADS=4 --mount type=bind,source=${datapath},target=/data/ \ - polusai/ome-converter-tool:${version} \ - --inpDir ${inpDir} \ - --filePattern ${filePattern} \ - --outDir ${outDir} +docker run \ + -e POLUS_IMG_EXT=${fileExtension} \ + -e NUM_THREADS=2 \ + -e NUM_WORKERS=4 \ + --mount type=bind,source=${datapath},target=/data/ \ + polusai/ome-converter-tool:${version} \ + --inpDir ${inpDir} \ + --filePattern ${filePattern} \ + --outDir ${outDir} diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py index a807b8362..e8bd1f440 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/__main__.py @@ -49,7 +49,7 @@ def _collect_files( if isinstance(inp, (str, pathlib.Path)): dir_path = pathlib.Path(inp) if isinstance(inp, str) else inp if not dir_path.is_dir(): - msg= f"Input path is not a directory: {dir_path}" + msg = f"Input path is not a directory: {dir_path}" raise ValueError(msg) fps = fp.FilePattern(dir_path, pattern) return [files[1][0] for files in fps()] @@ -111,7 +111,7 @@ def main( logger.info(f"filePattern = {file_pattern}") logger.info(f"preview = {preview}") - files = _collect_files(inp_dir, file_pattern) # type: ignore + files = _collect_files(inp_dir, file_pattern) # type: ignore logger.info(f"Found {len(files)} file(s) to process.") if preview: diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py index d364e5a50..155337d18 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py @@ -34,6 +34,7 @@ def get_num_threads() -> int: except (AttributeError, OSError): return 1 + NUM_THREADS: int = get_num_threads() # NUM_WORKERS: default to 1 process @@ -78,7 +79,7 @@ def get_num_series(inp_image: pathlib.Path) -> int: return 1 -def convert_image( # noqa:C901,PLR0912 +def convert_image( # noqa: C901 inp_image: pathlib.Path, file_extension: str, out_dir: pathlib.Path, From eac8cfcf679dab11e03ea8bda3ebd48f2833a43f Mon Sep 17 00:00:00 2001 From: hamshkhawar Date: Tue, 3 Mar 2026 14:27:54 -0500 Subject: [PATCH 16/17] update README --- formats/ome-converter-tool/README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/formats/ome-converter-tool/README.md b/formats/ome-converter-tool/README.md index 40103c67f..d012fb777 100644 --- a/formats/ome-converter-tool/README.md +++ b/formats/ome-converter-tool/README.md @@ -35,4 +35,12 @@ This plugin takes 2 input arguments and 1 output argument: ## Docker Command ```bash -docker run -e POLUS_IMG_EXT=".ome.zarr" --e NUM_THREADS=4 -v /Users/username/:/Users/username/ polusai/ome-converter-tool:0.3.4-dev2 --inpDir=/Users/path/to/Images/ --filePattern=".*.tif" --outDir=/Users/path/to/outputs +docker run \ + -e POLUS_IMG_EXT=".ome.zarr" \ + -e NUM_THREADS=4 \ + -e NUM_WORKERS=2 \ + -v /Users/username/:/Users/username/ \ + polusai/ome-converter-tool:0.3.4-dev2 \ + --inpDir=/Users/path/to/Images/ \ + --filePattern=".*.tif" \ + --outDir=/Users/path/to/outputs From 718199f1285ef0b1a9f06618689ab764c6fc539a Mon Sep 17 00:00:00 2001 From: hamshkhawar Date: Wed, 4 Mar 2026 15:52:33 -0500 Subject: [PATCH 17/17] refactored convert_image to resolve PLR0912 --- formats/ome-converter-tool/mypy.ini | 2 +- .../formats/ome_converter/image_converter.py | 58 ++++++++----------- formats/ome-converter-tool/tests/conftest.py | 6 +- 3 files changed, 28 insertions(+), 38 deletions(-) diff --git a/formats/ome-converter-tool/mypy.ini b/formats/ome-converter-tool/mypy.ini index 16e95711f..2251e5c41 100644 --- a/formats/ome-converter-tool/mypy.ini +++ b/formats/ome-converter-tool/mypy.ini @@ -13,4 +13,4 @@ ignore_missing_imports = True ignore_missing_imports = True # [mypy-polus.images.formats.ome_converter.*] -# ignore_missing_imports = False \ No newline at end of file +# ignore_missing_imports = False diff --git a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py index 155337d18..a2e7e5ab5 100644 --- a/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py +++ b/formats/ome-converter-tool/src/polus/images/formats/ome_converter/image_converter.py @@ -79,7 +79,7 @@ def get_num_series(inp_image: pathlib.Path) -> int: return 1 -def convert_image( # noqa: C901 +def convert_image( inp_image: pathlib.Path, file_extension: str, out_dir: pathlib.Path, @@ -91,6 +91,15 @@ def convert_image( # noqa: C901 logger.info(f"Detected {num_series} series/levels in image {inp_image.name}") files_written = 0 + if inp_image.suffix: + extension_len = 6 + extension = "".join( + s for s in inp_image.suffixes[-2:] if len(s) < extension_len + ) + base_name = inp_image.name.replace(extension, file_extension) + else: + base_name = f"{inp_image.name}{file_extension}" + # Process each series independently for idx in range(num_series): logger.info(f"Processing levels {idx} of {inp_image.name}") @@ -104,42 +113,23 @@ def convert_image( # noqa: C901 # Process each view (t, c, z) for t, c, z in product(range(br.T), range(br.C), range(br.Z)): - # Build the output path - if inp_image.suffix: - extension_len = 6 - extension = "".join( - [ - suffix - for suffix in inp_image.suffixes[-2:] - if len(suffix) < extension_len - ], - ) - out_path = out_dir.joinpath( - inp_image.name.replace(extension, file_extension), - ) - else: - out_path = out_dir.joinpath(f"{inp_image.name}{file_extension}") - # Build suffix components - suffix_parts = [] - if br.C > 1: - suffix_parts.append(f"_c{c}") - if br.T > 1: - suffix_parts.append(f"_t{t}") - if br.Z > 1: - suffix_parts.append(f"_z{z}") - if num_series > 1: - suffix_parts.append(f"_s{idx}") + suffix = "".join( + [ + f"_c{c}" if br.C > 1 else "", + f"_t{t}" if br.T > 1 else "", + f"_z{z}" if br.Z > 1 else "", + f"_s{idx}" if num_series > 1 else "", + ], + ) # Apply combined suffix - if suffix_parts: - suffix = "".join(suffix_parts) - out_path = out_dir.joinpath( - out_path.name.replace( - file_extension, - f"{suffix}{file_extension}", - ), - ) + out_name = ( + base_name.replace(file_extension, f"{suffix}{file_extension}") + if suffix + else base_name + ) + out_path = out_dir.joinpath(out_name) with BioWriter( out_path, diff --git a/formats/ome-converter-tool/tests/conftest.py b/formats/ome-converter-tool/tests/conftest.py index 1da67a3b7..3c84e76fb 100644 --- a/formats/ome-converter-tool/tests/conftest.py +++ b/formats/ome-converter-tool/tests/conftest.py @@ -42,7 +42,7 @@ def get_params(request) -> tuple[int, str]: # noqa: ANN001 return request.param -@pytest.fixture +@pytest.fixture() def synthetic_images( get_params: tuple[int, str], ) -> typing.Generator[tuple[list[numpy.ndarray], pathlib.Path], None, None]: @@ -74,7 +74,7 @@ def synthetic_images( shutil.rmtree(syn_dir) -@pytest.fixture +@pytest.fixture() def output_directory() -> typing.Generator[pathlib.Path, None, None]: """Generate random synthetic images.""" out_dir = pathlib.Path(tempfile.mkdtemp(suffix="_out_dir")) @@ -82,7 +82,7 @@ def output_directory() -> typing.Generator[pathlib.Path, None, None]: shutil.rmtree(out_dir) -@pytest.fixture +@pytest.fixture() def download_images() -> typing.Generator[pathlib.Path, None, None]: """Download test.""" imagelist = {