From f82792469a87dc703c09815ba7046e0b04c08620 Mon Sep 17 00:00:00 2001 From: hamshkhawar Date: Wed, 13 Dec 2023 15:08:34 -0600 Subject: [PATCH 1/3] updated czi plugin --- formats/czi-extract-plugin/.dockerignore | 4 + formats/czi-extract-plugin/.gitignore | 23 ++ formats/czi-extract-plugin/CHANGELOG.md | 10 + formats/czi-extract-plugin/Dockerfile | 19 ++ .../README.md | 13 +- formats/czi-extract-plugin/VERSION | 1 + formats/czi-extract-plugin/build-docker.sh | 4 + formats/czi-extract-plugin/bumpversion.cfg | 27 +++ .../package-release.sh | 0 formats/czi-extract-plugin/plugin.json | 63 ++++++ formats/czi-extract-plugin/pyproject.toml | 34 +++ .../run-plugin.sh | 8 +- .../plugins/formats/czi_extract/__init__.py | 2 + .../plugins/formats/czi_extract/__main__.py | 99 +++++++++ .../polus/plugins/formats/czi_extract/czi.py | 163 ++++++++++++++ formats/czi-extract-plugin/tests/__init__.py | 1 + formats/czi-extract-plugin/tests/fixture.py | 44 ++++ formats/czi-extract-plugin/tests/test_cli.py | 22 ++ formats/czi-extract-plugin/tests/test_czi.py | 56 +++++ formats/polus-czi-extract-plugin/Dockerfile | 18 -- formats/polus-czi-extract-plugin/VERSION | 1 - .../polus-czi-extract-plugin/build-docker.sh | 4 - .../polus-czi-extract-plugin/bumpversion.cfg | 10 - formats/polus-czi-extract-plugin/plugin.json | 30 --- formats/polus-czi-extract-plugin/src/main.py | 204 ------------------ .../src/requirements.txt | 2 - 26 files changed, 585 insertions(+), 277 deletions(-) create mode 100644 formats/czi-extract-plugin/.dockerignore create mode 100644 formats/czi-extract-plugin/.gitignore create mode 100644 formats/czi-extract-plugin/CHANGELOG.md create mode 100644 formats/czi-extract-plugin/Dockerfile rename formats/{polus-czi-extract-plugin => czi-extract-plugin}/README.md (78%) create mode 100644 formats/czi-extract-plugin/VERSION create mode 100755 formats/czi-extract-plugin/build-docker.sh create mode 100644 formats/czi-extract-plugin/bumpversion.cfg rename formats/{polus-czi-extract-plugin => czi-extract-plugin}/package-release.sh (100%) create mode 100644 formats/czi-extract-plugin/plugin.json create mode 100644 formats/czi-extract-plugin/pyproject.toml rename formats/{polus-czi-extract-plugin => czi-extract-plugin}/run-plugin.sh (74%) create mode 100644 formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/__init__.py create mode 100644 formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/__main__.py create mode 100644 formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/czi.py create mode 100644 formats/czi-extract-plugin/tests/__init__.py create mode 100644 formats/czi-extract-plugin/tests/fixture.py create mode 100644 formats/czi-extract-plugin/tests/test_cli.py create mode 100644 formats/czi-extract-plugin/tests/test_czi.py delete mode 100644 formats/polus-czi-extract-plugin/Dockerfile delete mode 100644 formats/polus-czi-extract-plugin/VERSION delete mode 100755 formats/polus-czi-extract-plugin/build-docker.sh delete mode 100644 formats/polus-czi-extract-plugin/bumpversion.cfg delete mode 100644 formats/polus-czi-extract-plugin/plugin.json delete mode 100644 formats/polus-czi-extract-plugin/src/main.py delete mode 100644 formats/polus-czi-extract-plugin/src/requirements.txt diff --git a/formats/czi-extract-plugin/.dockerignore b/formats/czi-extract-plugin/.dockerignore new file mode 100644 index 000000000..7c603f814 --- /dev/null +++ b/formats/czi-extract-plugin/.dockerignore @@ -0,0 +1,4 @@ +.venv +out +tests +__pycache__ diff --git a/formats/czi-extract-plugin/.gitignore b/formats/czi-extract-plugin/.gitignore new file mode 100644 index 000000000..9ed1c3775 --- /dev/null +++ b/formats/czi-extract-plugin/.gitignore @@ -0,0 +1,23 @@ +# Jupyter Notebook +.ipynb_checkpoints +poetry.lock +../../poetry.lock +# Environments +.env +.myenv +.venv +env/ +venv/ +# test data directory +data +# yaml file +.pre-commit-config.yaml +# hidden files +.DS_Store +.ds_store +# flake8 +.flake8 +../../.flake8 +__pycache__ +.mypy_cache +requirements.txt diff --git a/formats/czi-extract-plugin/CHANGELOG.md b/formats/czi-extract-plugin/CHANGELOG.md new file mode 100644 index 000000000..c95c2d3f0 --- /dev/null +++ b/formats/czi-extract-plugin/CHANGELOG.md @@ -0,0 +1,10 @@ +Czi Extraction (v1.1.2-dev) +This plugin is updated only to the new plugin standards +Updated dependencies (bfio, filepattern, preadator) to latest +Replaced docker base image with latest pre-installed bfio package +This plugin is now installable with pip. +Argparse package is replaced with Typer package for command line arguments. +baseCommand added in a plugin manifiest. +--preview flag is added which shows outputs to be generated by this plugin. +Use python -m python -m polus.plugins.formats.czi_extract to run plugin from command line. +Wrote pytests for this plugin diff --git a/formats/czi-extract-plugin/Dockerfile b/formats/czi-extract-plugin/Dockerfile new file mode 100644 index 000000000..b951c001c --- /dev/null +++ b/formats/czi-extract-plugin/Dockerfile @@ -0,0 +1,19 @@ +FROM polusai/bfio:2.3.3 + +# environment variables defined in polusai/bfio +ENV EXEC_DIR="/opt/executables" +ENV POLUS_IMG_EXT=".ome.tif" +ENV POLUS_TAB_EXT=".csv" + +# Work directory defined in the base container +WORKDIR ${EXEC_DIR} + +COPY pyproject.toml ${EXEC_DIR} +COPY VERSION ${EXEC_DIR} +COPY README.md ${EXEC_DIR} +COPY src ${EXEC_DIR}/src + +RUN pip3 install ${EXEC_DIR} --no-cache-dir + +ENTRYPOINT ["python3", "-m", "polus.plugins.formats.czi_extract"] +CMD ["--help"] diff --git a/formats/polus-czi-extract-plugin/README.md b/formats/czi-extract-plugin/README.md similarity index 78% rename from formats/polus-czi-extract-plugin/README.md rename to formats/czi-extract-plugin/README.md index 839c3839e..128b7c13f 100644 --- a/formats/polus-czi-extract-plugin/README.md +++ b/formats/czi-extract-plugin/README.md @@ -1,4 +1,4 @@ -# Polus CZI Extraction Plugin (v1.1.0) +# Polus CZI Extraction Plugin (v1.1.2-dev) This WIPP plugin will import individual fields of view from a CZI file (will only import one scene or collection of images). Images are exported as tiled @@ -39,19 +39,22 @@ the contents of `plugin.json` into the pop-up window and submit. ## Options -This plugin takes one input argument and one output argument: +This plugin takes two input arguments and one output argument: | Name | Description | I/O | Type | | -------- | ----------------------- | ------ | ---- | -| `inpDir` | Input image collection | Input | Path | -| `outDir` | Output image collection | Output | List | +| `--inpDir` | Input image collection | Input | genericData | +| `--filePattern` | Pattern to parse image files | Input | string | +| `--outDir` | Output image collection | Output | collection | +| `--preview` | Generate a JSON file with outputs | Output | JSON | ## Run the plugin ### Run the Docker Container ```bash -docker run -v /path/to/data:/data polus-czi-extract-plugin \ +docker run -v /path/to/data:/data polusai/czi-extract-plugin:1.1.2-dev \ --inpDir /data/input \ + --filePattern ".*.czi" \ --outDir /data/output ``` diff --git a/formats/czi-extract-plugin/VERSION b/formats/czi-extract-plugin/VERSION new file mode 100644 index 000000000..13c00786d --- /dev/null +++ b/formats/czi-extract-plugin/VERSION @@ -0,0 +1 @@ +1.1.2-dev diff --git a/formats/czi-extract-plugin/build-docker.sh b/formats/czi-extract-plugin/build-docker.sh new file mode 100755 index 000000000..f236c611a --- /dev/null +++ b/formats/czi-extract-plugin/build-docker.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +version=$(\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? +serialize = + {major}.{minor}.{patch}-{release}{dev} + {major}.{minor}.{patch} + +[bumpversion:part:release] +optional_value = _ +first_value = dev +values = + dev + _ + +[bumpversion:part:dev] + +[bumpversion:file:pyproject.toml] +search = version = "{current_version}" +replace = version = "{new_version}" + +[bumpversion:file:plugin.json] + +[bumpversion:file:VERSION] + +[bumpversion:file:src/polus/plugins/formats/czi_extract/__init__.py] diff --git a/formats/polus-czi-extract-plugin/package-release.sh b/formats/czi-extract-plugin/package-release.sh similarity index 100% rename from formats/polus-czi-extract-plugin/package-release.sh rename to formats/czi-extract-plugin/package-release.sh diff --git a/formats/czi-extract-plugin/plugin.json b/formats/czi-extract-plugin/plugin.json new file mode 100644 index 000000000..a552517e6 --- /dev/null +++ b/formats/czi-extract-plugin/plugin.json @@ -0,0 +1,63 @@ +{ + "name": "Czi Extraction", + "version": "1.1.2-dev", + "title": "Extract TIFFs From CZI", + "description": "Extracts individual fields of view from a CZI file. Saves as OME TIFF.", + "author": "Nick Schaub (nick.schaub@nih.gov), Hamdah Shafqat Abbasi (hamdahshafqat.abbasi@nih.gov)", + "institution": "National Center for Advancing Translational Sciences, National Institutes of Health", + "repository": "https://github.com/PolusAI/polus-plugins", + "website": "https://ncats.nih.gov/preclinical/core/informatics", + "containerId": "polusai/czi-extract-plugin:1.1.2-dev", + "baseCommand": [ + "python3", + "-m", + "polus.plugins.formats.czi_extract" + ], + "inputs": { + "inpDir": { + "type": "collection", + "title": "Input collection", + "description": "Input image collection to be processed by this plugin.", + "required": "True" + }, + "filePattern": { + "type": "string", + "title": "Filename pattern", + "description": "Filename pattern used to separate data.", + "required": "False" + }, + "preview": { + "type": "boolean", + "title": "Preview", + "description": "Generate an output preview.", + "required": "False" + } + }, + "outputs": { + "outDir": { + "type": "collection", + "description": "Output collection." + } + }, + "ui": { + "inpDir": { + "type": "collection", + "title": "Input collection", + "description": "Input image collection to be processed by this plugin.", + "required": "True" + }, + "filePattern": { + "type": "string", + "title": "Filename pattern", + "description": "Filename pattern used to separate data.", + "required": "False", + "default": ".*.czi" + }, + "preview": { + "type": "boolean", + "title": "Preview example output of this plugin", + "description": "Generate an output preview.", + "required": "False" + } + } +} diff --git a/formats/czi-extract-plugin/pyproject.toml b/formats/czi-extract-plugin/pyproject.toml new file mode 100644 index 000000000..2a9e0c0a4 --- /dev/null +++ b/formats/czi-extract-plugin/pyproject.toml @@ -0,0 +1,34 @@ +[tool.poetry] +name = "polus-plugins-formats-czi-extract" +version = "1.1.2-dev" +description = "Extracts individual fields of view from a CZI file. Saves as OME TIFF." +authors = [ + "Nick Schaub ", + "Hamdah Shafqat abbasi " + ] +readme = "README.md" +packages = [{include = "polus", from = "src"}] + +[tool.poetry.dependencies] +python = ">=3.9,<3.12" +filepattern = "^2.0.4" +typer = "^0.7.0" +tqdm = "^4.64.1" +bfio = {version = "2.3.3", extras = ["all"]} +preadator="0.3.0.dev1" +czifile="^2019.7.2" + +[tool.poetry.group.dev.dependencies] +bump2version = "^1.0.1" +pre-commit = "^3.0.4" +black = "^23.1.0" +flake8 = "^6.0.0" +mypy = "^1.0.0" +pytest = "^7.2.1" +ipykernel = "^6.21.2" +requests = "^2.28.2" +scikit-image = "^0.22.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/formats/polus-czi-extract-plugin/run-plugin.sh b/formats/czi-extract-plugin/run-plugin.sh similarity index 74% rename from formats/polus-czi-extract-plugin/run-plugin.sh rename to formats/czi-extract-plugin/run-plugin.sh index 6846d0b3e..b8e3b6b5c 100755 --- a/formats/polus-czi-extract-plugin/run-plugin.sh +++ b/formats/czi-extract-plugin/run-plugin.sh @@ -6,6 +6,8 @@ datapath=$(readlink --canonicalize ../data) # Inputs inpDir=/data/path_to_files +filePattern=".*.czi" + # Output paths outDir=/data/path_to_output @@ -15,7 +17,7 @@ LOGLEVEL=INFO docker run --mount type=bind,source=${datapath},target=/data/ \ --user $(id -u):$(id -g) \ --env POLUS_LOG=${LOGLEVEL} \ - labshare/polus-polus-czi-extract-plugin:${version} \ + polusai/czi-extract-plugin:${version} \ --inpDir ${inpDir} \ - --outDir ${outDir} - \ No newline at end of file + --filePattern ${filePattern} \ + --outDir ${outDir} diff --git a/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/__init__.py b/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/__init__.py new file mode 100644 index 000000000..da00ef6d3 --- /dev/null +++ b/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/__init__.py @@ -0,0 +1,2 @@ +"""Czi Extract Plugin.""" +__version__ = "1.1.2-dev" diff --git a/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/__main__.py b/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/__main__.py new file mode 100644 index 000000000..bd561e842 --- /dev/null +++ b/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/__main__.py @@ -0,0 +1,99 @@ +"""Czi Extract Plugin.""" +import json +import logging +import os +from multiprocessing import cpu_count +from pathlib import Path +from typing import Any +from typing import Optional + +import filepattern as fp +import polus.plugins.formats.czi_extract.czi as cz +import typer +from preadator import ProcessManager + +# Import environment variables +POLUS_EXT = os.environ.get("POLUS_EXT", ".ome.tif") + +app = typer.Typer() + +# Initialize the logger +logging.basicConfig( + format="%(asctime)s - %(name)-8s - %(levelname)-8s - %(message)s", + datefmt="%d-%b-%y %H:%M:%S", +) +logger = logging.getLogger("polus.plugins.formats.czi_extract") + + +@app.command() +def main( + inp_dir: Path = typer.Option( + ..., + "--inpDir", + "-i", + help="Path to folder with CZI files", + ), + file_pattern: str = typer.Option( + ".*.czi", + "--filePattern", + "-f", + help="Pattern use to parse filenames", + ), + out_dir: Path = typer.Option( + ..., + "--outDir", + "-o", + help="Output directory", + ), + preview: Optional[bool] = typer.Option( + False, + "--preview", + help="Output a JSON preview of files", + ), +) -> None: + """Extracts individual fields of view from a CZI file and saves as OME TIFF.""" + logger.info(f"--inpDir = {inp_dir}") + logger.info(f"--filePattern = {file_pattern}") + logger.info(f"--outDir = {out_dir}") + + inp_dir = inp_dir.resolve() + out_dir = out_dir.resolve() + + assert inp_dir.exists(), f"{inp_dir} does not exist!! Please check input path again" + assert ( + out_dir.exists() + ), f"{out_dir} does not exist!! Please check output path again" + + num_threads = max([cpu_count(), 2]) + + ProcessManager.num_processes(num_threads) + ProcessManager.init_processes(name="Czi Extraction") + + files = fp.FilePattern(inp_dir, file_pattern) + + file_ext = all(Path(f[1][0].name).suffix for f in files()) + assert ( + file_ext is True + ), f"{inp_dir} does not contain all czi files!! Please check input directory again" + + if preview: + with Path.open(Path(out_dir, "preview.json"), "w") as jfile: + out_json: dict[str, Any] = { + "filepattern": file_pattern, + "outDir": [], + } + for file in files(): + out_name = file[1][0].name.replace( + "".join(file[1][0].suffixes), + ".ome.tif", + ) + out_json["outDir"].append(out_name) + json.dump(out_json, jfile, indent=2) + + for file in files(): + ProcessManager.submit_process(cz.extract_fovs, file[1][0], out_dir) + ProcessManager.join_processes() + + +if __name__ == "__main__": + app() diff --git a/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/czi.py b/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/czi.py new file mode 100644 index 000000000..409dadf35 --- /dev/null +++ b/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/czi.py @@ -0,0 +1,163 @@ +"""Czi Extract Plugin.""" +import logging +from pathlib import Path +from typing import Optional + +import czifile +import numpy as np +from bfio import BioReader +from bfio import BioWriter + +logger = logging.getLogger(__name__) + + +def _get_image_dim(s: np.ndarray, dim: str) -> int: + """Get czi image dimension.""" + ind = s.axes.find(dim) + if ind < 0: + return 1 + return s.shape[ind] + + +def _get_image_name( # noqa: PLR0913 + base_name: str, + row: int, + col: int, + z: Optional[int] = None, + c: Optional[int] = None, + t: Optional[int] = None, + padding: int = 3, +) -> str: + """This function generates an image name from image coordinates.""" + name = base_name + name += "_y" + str(row).zfill(padding) + name += "_x" + str(col).zfill(padding) + if z is not None: + name += "_z" + str(z).zfill(padding) + if c is not None: + name += "_c" + str(c).zfill(padding) + if t is not None: + name += "_t" + str(t).zfill(padding) + name += ".ome.tif" + return name + + +def write_thread( + out_file_path: Path, + data: np.ndarray, + metadata: BioReader.metadata, + chan_name: str, +) -> None: + """Thread for saving images. + + This function is intended to be run inside a threadpool to save an image. + + Args: + out_file_path : Path to an output file + data : FOV to save + metadata : Metadata for the image + chan_name: Name of the channel + """ + logger.info(f"Writing: {Path(out_file_path).name}") + with BioWriter(out_file_path, metadata=metadata) as bw: + bw.X = data.shape[1] + bw.Y = data.shape[0] + bw.Z = 1 + bw.C = 1 + bw.cnames = [chan_name] + bw[:] = data + + +def extract_fovs(file_path: Path, out_path: Path) -> None: + """Extract individual FOVs from a czi file. + + When CZI files are loaded by BioFormats, it will generally try to mosaic + images together by stage position if the image was captured with the + intention of mosaicing images together. At the time this function was + written, there was no clear way of extracting individual FOVs so this + algorithm was created. + + Every field of view in each z-slice, channel, and timepoint contained in a + CZI file is saved as an individual image. + + Args: + file_path : Path to CZI file + out_path : Path to output directory + """ + logger.info("Starting extraction from " + str(file_path.name) + "...") + + base_name = Path(file_path.name).stem + + # Load files without mosaicing + czi = czifile.CziFile(file_path, detectmosaic=False) + subblocks = [ + s for s in czi.filtered_subblock_directory if s.mosaic_index is not None + ] + + ind: dict = {"X": [], "Y": [], "Z": [], "C": [], "T": [], "Row": [], "Col": []} + + # Get the indices of each FOV + for s in subblocks: + scene = [dim.start for dim in s.dimension_entries if dim.dimension == "S"] + if scene is not None and scene[0] != 0: + continue + + for dim in s.dimension_entries: + if dim.dimension == "X": + ind["X"].append(dim.start) + elif dim.dimension == "Y": + ind["Y"].append(dim.start) + elif dim.dimension == "Z": + ind["Z"].append(dim.start) + elif dim.dimension == "C": + ind["C"].append(dim.start) + elif dim.dimension == "T": + ind["T"].append(dim.start) + + row_conv = dict( + zip( + np.unique(np.sort(ind["Y"])), + range(0, len(np.unique(ind["Y"]))), + ), + ) + col_conv = dict( + zip( + np.unique(np.sort(ind["X"])), + range(0, len(np.unique(ind["X"]))), + ), + ) + + ind["Row"] = [row_conv[y] for y in ind["Y"]] + ind["Col"] = [col_conv[x] for x in ind["X"]] + + with BioReader(file_path) as br: + metadata = br.metadata + chan_names = br.cnames + + for s, i in zip(subblocks, range(0, len(subblocks))): + z = None if len(ind["Z"]) == 0 else ind["Z"][i] + c = None if len(ind["C"]) == 0 else ind["C"][i] + t = None if len(ind["T"]) == 0 else ind["T"][i] + + out_file_path = out_path.joinpath( + _get_image_name( + base_name, + row=ind["Row"][i], + col=ind["Col"][i], + z=z, + c=c, + t=t, + ), + ) + + dims = [ + _get_image_dim(s, "Y"), + _get_image_dim(s, "X"), + _get_image_dim(s, "Z"), + _get_image_dim(s, "C"), + _get_image_dim(s, "T"), + ] + + data = s.data_segment().data().reshape(dims) + + write_thread(out_file_path, data, metadata, chan_names[c]) diff --git a/formats/czi-extract-plugin/tests/__init__.py b/formats/czi-extract-plugin/tests/__init__.py new file mode 100644 index 000000000..ab483280a --- /dev/null +++ b/formats/czi-extract-plugin/tests/__init__.py @@ -0,0 +1 @@ +"""Czi Extract Plugin.""" diff --git a/formats/czi-extract-plugin/tests/fixture.py b/formats/czi-extract-plugin/tests/fixture.py new file mode 100644 index 000000000..a48a6e861 --- /dev/null +++ b/formats/czi-extract-plugin/tests/fixture.py @@ -0,0 +1,44 @@ +"""Test fixtures. + +Set up all data used in tests. +""" +import shutil +import tempfile +from pathlib import Path +from typing import Union + +import pytest +import requests + + +def clean_directories() -> None: + """Remove all temporary directories.""" + for d in Path(".").cwd().iterdir(): + if d.is_dir() and d.name.startswith("tmp"): + shutil.rmtree(d) + + +@pytest.fixture() +def output_directory() -> Union[str, Path]: + """Create output directory.""" + return Path(tempfile.mkdtemp(dir=Path.cwd())) + + +@pytest.fixture() +def download_czi() -> Union[str, Path]: + """Download czi images.""" + inp_path = Path.cwd().joinpath("tmp_czi") + if not inp_path.exists(): + Path(inp_path).mkdir(parents=True) + + url_list = [ + "https://downloads.openmicroscopy.org/images/Zeiss-CZI/idr0011/Plate1-Blue-A_TS-Stinger/Plate1-Blue-A-08-Scene-3-P2-B2-03.czi", + "https://downloads.openmicroscopy.org/images/Zeiss-CZI/idr0011/Plate1-Blue-A_TS-Stinger/Plate1-Blue-A-02-Scene-1-P2-E1-01.czi", + ] + for url in url_list: + file = Path(url).name + req = requests.get(url, timeout=10) + with Path.open(inp_path.joinpath(file), "wb") as fw: + fw.write(req.content) + + return inp_path diff --git a/formats/czi-extract-plugin/tests/test_cli.py b/formats/czi-extract-plugin/tests/test_cli.py new file mode 100644 index 000000000..c39962131 --- /dev/null +++ b/formats/czi-extract-plugin/tests/test_cli.py @@ -0,0 +1,22 @@ +"""Test Command line Tool.""" +from typer.testing import CliRunner +from polus.plugins.formats.czi_extract.__main__ import app +from tests.fixture import * + + +def test_cli(download_czi, output_directory) -> None: + """Test the command line.""" + runner = CliRunner() + result = runner.invoke( + app, + [ + "--inpDir", + download_czi, + "--filePattern", + ".*.czi", + "--outDir", + output_directory, + ], + ) + + assert result.exit_code == 0 diff --git a/formats/czi-extract-plugin/tests/test_czi.py b/formats/czi-extract-plugin/tests/test_czi.py new file mode 100644 index 000000000..4072267dc --- /dev/null +++ b/formats/czi-extract-plugin/tests/test_czi.py @@ -0,0 +1,56 @@ +"""Cell border segmentation package.""" +from pathlib import Path +import czifile +import skimage +import polus.plugins.formats.czi_extract.czi as cz + +from tests.fixture import * # noqa: F403 +from tests.fixture import clean_directories + + +def test_extract_fovs(download_czi: Path, output_directory: Path) -> None: + """Test extracting fovs from czi image.""" + for im in download_czi.iterdir(): + fname = Path(Path(im).name).stem + cz.extract_fovs(Path(im), output_directory) + czi_image = czifile.CziFile(Path(im), detectmosaic=False) + (_, _, ch, zpos, _, _, _) = czi_image.shape + num_files = ch * zpos + + out_files = [ + f for f in output_directory.iterdir() if f.name.startswith(f"{fname}") + ] + assert len(out_files) == num_files + + out_ext = all([Path(f.name).suffix for f in output_directory.iterdir()]) + + assert out_ext == True + + +def test_get_image_dim(download_czi: Path) -> None: + """Test getting czi image dimensions.""" + im = list(download_czi.iterdir())[0] + czi_image = czifile.CziFile(im, detectmosaic=False) + for s in czi_image.filtered_subblock_directory: + dimensions = ["X", "Y", "Z", "C", "T"] + for d in dimensions: + dim_numb = cz._get_image_dim(s, d) + assert dim_numb != 0 + + +def test_write_thread(output_directory) -> None: + """Test writing ome tif image.""" + for i in range(10): + blobs = skimage.data.binary_blobs( + length=1024, volume_fraction=0.05, blob_size_fraction=0.05 + ) + syn_img = skimage.measure.label(blobs) + outname = f"test_image_{i}.ome.tif" + out_file_path = Path(output_directory, outname) + chan_name = "c" + cz.write_thread(out_file_path, syn_img, None, chan_name) + + out_ext = all([Path(f.name).suffix for f in output_directory.iterdir()]) + assert out_ext == True + + clean_directories() diff --git a/formats/polus-czi-extract-plugin/Dockerfile b/formats/polus-czi-extract-plugin/Dockerfile deleted file mode 100644 index 377ec4c13..000000000 --- a/formats/polus-czi-extract-plugin/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM polusai/bfio:2.1.9 - -# environment variables defined in labshare/polus-bfio-util -# ENV EXEC_DIR="/opt/executables" -# ENV DATA_DIR="/data" -# ENV POLUS_EXT=".ome.tif" -# ENV POLUS_LOG="INFO" # Change to WARNING for fewer logs, and DEBUG for debugging - -# Work directory defined in the base container -# WORKDIR ${EXEC_DIR} - -COPY VERSION ${EXEC_DIR} -COPY src ${EXEC_DIR}/ - -RUN pip3 install -r ${EXEC_DIR}/requirements.txt --no-cache-dir - -# Default command. Additional arguments are provided through the command line -ENTRYPOINT ["python3", "/opt/executables/main.py"] \ No newline at end of file diff --git a/formats/polus-czi-extract-plugin/VERSION b/formats/polus-czi-extract-plugin/VERSION deleted file mode 100644 index 8cfbc905b..000000000 --- a/formats/polus-czi-extract-plugin/VERSION +++ /dev/null @@ -1 +0,0 @@ -1.1.1 \ No newline at end of file diff --git a/formats/polus-czi-extract-plugin/build-docker.sh b/formats/polus-czi-extract-plugin/build-docker.sh deleted file mode 100755 index b8ce5f75f..000000000 --- a/formats/polus-czi-extract-plugin/build-docker.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -version=$( None: - - logger.info('Extracting tiffs and saving as ome.tif...') - files = [f for f in Path(input_dir).iterdir() if f.suffix=='.czi'] - if not files: - logger.error('No CZI files found.') - raise ValueError('No CZI files found.') - - ProcessManager.init_processes() - - for file in files: - ProcessManager.submit_process(extract_fovs,file,output_dir) - - ProcessManager.join_processes() - -if __name__ == "__main__": - # Setup the Argument parsing - logger.info("Parsing arguments...") - parser = argparse.ArgumentParser(prog='main', description='Extract individual fields of view from a czi file.') - - parser.add_argument('--inpDir', dest='input_dir', type=str, - help='Path to folder with CZI files', required=True) - parser.add_argument('--outDir', dest='output_dir', type=str, - help='The output directory for ome.tif files', required=True) - - - args = parser.parse_args() - input_dir = Path(args.input_dir) - output_dir = Path(args.output_dir) - logger.info('input_dir = {}'.format(input_dir)) - logger.info('output_dir = {}'.format(output_dir)) - - main(input_dir, - output_dir) \ No newline at end of file diff --git a/formats/polus-czi-extract-plugin/src/requirements.txt b/formats/polus-czi-extract-plugin/src/requirements.txt deleted file mode 100644 index ed31f8806..000000000 --- a/formats/polus-czi-extract-plugin/src/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -czifile==2019.7.2 -preadator==0.2.0 \ No newline at end of file From 594e140032b1672a9a40cad207c2b456c3addd6e Mon Sep 17 00:00:00 2001 From: hamshkhawar Date: Fri, 15 Dec 2023 10:24:06 -0600 Subject: [PATCH 2/3] bump up preadator package and added short cli test --- formats/czi-extract-plugin/pyproject.toml | 2 +- .../plugins/formats/czi_extract/__main__.py | 33 ++++++++++++++----- formats/czi-extract-plugin/tests/test_cli.py | 18 ++++++++++ 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/formats/czi-extract-plugin/pyproject.toml b/formats/czi-extract-plugin/pyproject.toml index 2a9e0c0a4..5341749d8 100644 --- a/formats/czi-extract-plugin/pyproject.toml +++ b/formats/czi-extract-plugin/pyproject.toml @@ -15,7 +15,7 @@ filepattern = "^2.0.4" typer = "^0.7.0" tqdm = "^4.64.1" bfio = {version = "2.3.3", extras = ["all"]} -preadator="0.3.0.dev1" +preadator="0.4.0.dev2" czifile="^2019.7.2" [tool.poetry.group.dev.dependencies] diff --git a/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/__main__.py b/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/__main__.py index bd561e842..3ad56ab1b 100644 --- a/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/__main__.py +++ b/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/__main__.py @@ -2,6 +2,7 @@ import json import logging import os +from concurrent.futures import as_completed from multiprocessing import cpu_count from pathlib import Path from typing import Any @@ -9,8 +10,9 @@ import filepattern as fp import polus.plugins.formats.czi_extract.czi as cz +import preadator import typer -from preadator import ProcessManager +from tqdm import tqdm # Import environment variables POLUS_EXT = os.environ.get("POLUS_EXT", ".ome.tif") @@ -64,10 +66,7 @@ def main( out_dir.exists() ), f"{out_dir} does not exist!! Please check output path again" - num_threads = max([cpu_count(), 2]) - - ProcessManager.num_processes(num_threads) - ProcessManager.init_processes(name="Czi Extraction") + num_workers = max([cpu_count(), 2]) files = fp.FilePattern(inp_dir, file_pattern) @@ -90,9 +89,27 @@ def main( out_json["outDir"].append(out_name) json.dump(out_json, jfile, indent=2) - for file in files(): - ProcessManager.submit_process(cz.extract_fovs, file[1][0], out_dir) - ProcessManager.join_processes() + with preadator.ProcessManager( + name="Convert czi to individual ome tif", + num_processes=num_workers, + threads_per_process=2, + ) as pm: + threads = [] + for file in files(): + thread = pm.submit_process(cz.extract_fovs, file[1][0], out_dir) + threads.append(thread) + pm.join_processes() + + for f in tqdm( + as_completed(threads), + total=len(threads), + mininterval=5, + desc="Extract czi", + initial=0, + unit_scale=True, + colour="cyan", + ): + f.result() if __name__ == "__main__": diff --git a/formats/czi-extract-plugin/tests/test_cli.py b/formats/czi-extract-plugin/tests/test_cli.py index c39962131..890749b47 100644 --- a/formats/czi-extract-plugin/tests/test_cli.py +++ b/formats/czi-extract-plugin/tests/test_cli.py @@ -20,3 +20,21 @@ def test_cli(download_czi, output_directory) -> None: ) assert result.exit_code == 0 + + +def test_short_cli(download_czi, output_directory) -> None: + """Test the short cli command line.""" + runner = CliRunner() + result = runner.invoke( + app, + [ + "-i", + download_czi, + "-f", + ".*.czi", + "-o", + output_directory, + ], + ) + + assert result.exit_code == 0 From 2a50d9fe1885bd78192c9c4358a2afaaec3e3f53 Mon Sep 17 00:00:00 2001 From: hamshkhawar Date: Wed, 10 Jan 2024 09:11:27 -0600 Subject: [PATCH 3/3] addressed comments on PR --- formats/czi-extract-plugin/CHANGELOG.md | 20 ++--- .../plugins/formats/czi_extract/__main__.py | 29 +------ .../polus/plugins/formats/czi_extract/czi.py | 87 +++++++++++-------- .../tests/{fixture.py => conftest.py} | 5 +- formats/czi-extract-plugin/tests/test_cli.py | 2 +- formats/czi-extract-plugin/tests/test_czi.py | 5 +- 6 files changed, 69 insertions(+), 79 deletions(-) rename formats/czi-extract-plugin/tests/{fixture.py => conftest.py} (94%) diff --git a/formats/czi-extract-plugin/CHANGELOG.md b/formats/czi-extract-plugin/CHANGELOG.md index c95c2d3f0..f948a99eb 100644 --- a/formats/czi-extract-plugin/CHANGELOG.md +++ b/formats/czi-extract-plugin/CHANGELOG.md @@ -1,10 +1,10 @@ -Czi Extraction (v1.1.2-dev) -This plugin is updated only to the new plugin standards -Updated dependencies (bfio, filepattern, preadator) to latest -Replaced docker base image with latest pre-installed bfio package -This plugin is now installable with pip. -Argparse package is replaced with Typer package for command line arguments. -baseCommand added in a plugin manifiest. ---preview flag is added which shows outputs to be generated by this plugin. -Use python -m python -m polus.plugins.formats.czi_extract to run plugin from command line. -Wrote pytests for this plugin +## [1.1.2-dev] - 2024-01-10 +### Added +- Pytests to test this plugin +- This plugin is now installable with pip. + +### Changed +- Updated dependencies (bfio, filepattern, preadator) to latest +- Argparse package is replaced with Typer package for command line arguments +- Replaced docker base image with latest container image with pre-installed bfio + diff --git a/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/__main__.py b/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/__main__.py index 3ad56ab1b..fa128bfb5 100644 --- a/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/__main__.py +++ b/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/__main__.py @@ -2,7 +2,6 @@ import json import logging import os -from concurrent.futures import as_completed from multiprocessing import cpu_count from pathlib import Path from typing import Any @@ -10,9 +9,7 @@ import filepattern as fp import polus.plugins.formats.czi_extract.czi as cz -import preadator import typer -from tqdm import tqdm # Import environment variables POLUS_EXT = os.environ.get("POLUS_EXT", ".ome.tif") @@ -66,8 +63,6 @@ def main( out_dir.exists() ), f"{out_dir} does not exist!! Please check output path again" - num_workers = max([cpu_count(), 2]) - files = fp.FilePattern(inp_dir, file_pattern) file_ext = all(Path(f[1][0].name).suffix for f in files()) @@ -88,29 +83,9 @@ def main( ) out_json["outDir"].append(out_name) json.dump(out_json, jfile, indent=2) - - with preadator.ProcessManager( - name="Convert czi to individual ome tif", - num_processes=num_workers, - threads_per_process=2, - ) as pm: - threads = [] + else: for file in files(): - thread = pm.submit_process(cz.extract_fovs, file[1][0], out_dir) - threads.append(thread) - pm.join_processes() - - for f in tqdm( - as_completed(threads), - total=len(threads), - mininterval=5, - desc="Extract czi", - initial=0, - unit_scale=True, - colour="cyan", - ): - f.result() - + cz.extract_fovs(file[1][0], out_dir) if __name__ == "__main__": app() diff --git a/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/czi.py b/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/czi.py index 409dadf35..9b3ad5363 100644 --- a/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/czi.py +++ b/formats/czi-extract-plugin/src/polus/plugins/formats/czi_extract/czi.py @@ -2,14 +2,17 @@ import logging from pathlib import Path from typing import Optional +from multiprocessing import cpu_count import czifile import numpy as np from bfio import BioReader from bfio import BioWriter +import preadator logger = logging.getLogger(__name__) +num_workers = max([cpu_count(), 2]) def _get_image_dim(s: np.ndarray, dim: str) -> int: """Get czi image dimension.""" @@ -114,50 +117,64 @@ def extract_fovs(file_path: Path, out_path: Path) -> None: elif dim.dimension == "T": ind["T"].append(dim.start) + unique_y = np.unique(ind["Y"]) + unique_x = np.unique(ind["X"]) + row_conv = dict( zip( - np.unique(np.sort(ind["Y"])), - range(0, len(np.unique(ind["Y"]))), + np.sort(unique_y), + range(0, len(unique_y)), ), ) col_conv = dict( zip( - np.unique(np.sort(ind["X"])), - range(0, len(np.unique(ind["X"]))), + np.sort(unique_x), + range(0, len(unique_x)), ), ) ind["Row"] = [row_conv[y] for y in ind["Y"]] ind["Col"] = [col_conv[x] for x in ind["X"]] - with BioReader(file_path) as br: - metadata = br.metadata - chan_names = br.cnames - - for s, i in zip(subblocks, range(0, len(subblocks))): - z = None if len(ind["Z"]) == 0 else ind["Z"][i] - c = None if len(ind["C"]) == 0 else ind["C"][i] - t = None if len(ind["T"]) == 0 else ind["T"][i] - - out_file_path = out_path.joinpath( - _get_image_name( - base_name, - row=ind["Row"][i], - col=ind["Col"][i], - z=z, - c=c, - t=t, - ), - ) - - dims = [ - _get_image_dim(s, "Y"), - _get_image_dim(s, "X"), - _get_image_dim(s, "Z"), - _get_image_dim(s, "C"), - _get_image_dim(s, "T"), - ] - - data = s.data_segment().data().reshape(dims) - - write_thread(out_file_path, data, metadata, chan_names[c]) + if ind.__len__() != 0: + with BioReader(file_path) as br: + metadata = br.metadata + chan_names = br.cnames + + with preadator.ProcessManager( + name="Convert czi to individual ome tif", + num_processes=num_workers, + threads_per_process=2, + ) as pm: + for s, i in zip(subblocks, range(0, len(subblocks))): + z = None if len(ind["Z"]) == 0 else ind["Z"][i] + c = None if len(ind["C"]) == 0 else ind["C"][i] + t = None if len(ind["T"]) == 0 else ind["T"][i] + + out_file_path = out_path.joinpath( + _get_image_name( + base_name, + row=ind["Row"][i], + col=ind["Col"][i], + z=z, + c=c, + t=t, + ), + ) + + dims = [ + _get_image_dim(s, "Y"), + _get_image_dim(s, "X"), + _get_image_dim(s, "Z"), + _get_image_dim(s, "C"), + _get_image_dim(s, "T"), + ] + + data = s.data_segment().data().reshape(dims) + pm.submit_process(write_thread, out_file_path, data, metadata, chan_names[c]) + pm.join_processes() + + else: + msg = logger.info("Unable to extract individual fovs in czi file") + raise ValueError(msg) + diff --git a/formats/czi-extract-plugin/tests/fixture.py b/formats/czi-extract-plugin/tests/conftest.py similarity index 94% rename from formats/czi-extract-plugin/tests/fixture.py rename to formats/czi-extract-plugin/tests/conftest.py index a48a6e861..bc354020e 100644 --- a/formats/czi-extract-plugin/tests/fixture.py +++ b/formats/czi-extract-plugin/tests/conftest.py @@ -10,7 +10,6 @@ import pytest import requests - def clean_directories() -> None: """Remove all temporary directories.""" for d in Path(".").cwd().iterdir(): @@ -21,7 +20,7 @@ def clean_directories() -> None: @pytest.fixture() def output_directory() -> Union[str, Path]: """Create output directory.""" - return Path(tempfile.mkdtemp(dir=Path.cwd())) + return Path(tempfile.mkdtemp()) @pytest.fixture() @@ -41,4 +40,4 @@ def download_czi() -> Union[str, Path]: with Path.open(inp_path.joinpath(file), "wb") as fw: fw.write(req.content) - return inp_path + return inp_path \ No newline at end of file diff --git a/formats/czi-extract-plugin/tests/test_cli.py b/formats/czi-extract-plugin/tests/test_cli.py index 890749b47..62fb97f6c 100644 --- a/formats/czi-extract-plugin/tests/test_cli.py +++ b/formats/czi-extract-plugin/tests/test_cli.py @@ -1,7 +1,7 @@ """Test Command line Tool.""" from typer.testing import CliRunner from polus.plugins.formats.czi_extract.__main__ import app -from tests.fixture import * + def test_cli(download_czi, output_directory) -> None: diff --git a/formats/czi-extract-plugin/tests/test_czi.py b/formats/czi-extract-plugin/tests/test_czi.py index 4072267dc..3f49c87bc 100644 --- a/formats/czi-extract-plugin/tests/test_czi.py +++ b/formats/czi-extract-plugin/tests/test_czi.py @@ -3,9 +3,8 @@ import czifile import skimage import polus.plugins.formats.czi_extract.czi as cz +from tests.conftest import clean_directories -from tests.fixture import * # noqa: F403 -from tests.fixture import clean_directories def test_extract_fovs(download_czi: Path, output_directory: Path) -> None: @@ -38,7 +37,7 @@ def test_get_image_dim(download_czi: Path) -> None: assert dim_numb != 0 -def test_write_thread(output_directory) -> None: +def test_write_thread(output_directory: Path) -> None: """Test writing ome tif image.""" for i in range(10): blobs = skimage.data.binary_blobs(