From 784adc2e3e3de477025fc369a4175594f1400878 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 9 Jun 2026 13:23:34 -0400 Subject: [PATCH 01/79] Transform labels instead of images --- scallops/cli/util.py | 154 ++++++++++- scallops/io.py | 6 +- scallops/tests/test_wdl.py | 86 ++++-- scallops/utils.py | 127 --------- wdl/ops_tasks.wdl | 6 +- wdl/ops_workflow.wdl | 524 ++++++++++++++++++------------------- wdl/utils.wdl | 20 +- 7 files changed, 494 insertions(+), 429 deletions(-) diff --git a/scallops/cli/util.py b/scallops/cli/util.py index 31d11d0..595f0fa 100644 --- a/scallops/cli/util.py +++ b/scallops/cli/util.py @@ -93,7 +93,7 @@ def _dask_workers_threads( } -def _create_default_dask_config(config: dict | None = None) -> set: +def _create_default_dask_config(config: dict | None = None) -> dask.config.set: if config is None: return dask.config.set(DEFAULT_DASK_CONFIG) @@ -450,3 +450,155 @@ def load_json(path_or_str: str) -> dict: with fs.open(path_or_str, "rt") as fp: return json.load(fp) return json.loads(path_or_str) + + +def _write_img_size(file_list: list[str]): + from scallops.io import _images2fov, _localize_path + + local_file_list = [] + cleanup_file_list = [] + for path in file_list: + local_path = _localize_path(path) + if local_path is not None: + cleanup_file_list.append(local_path) + local_file_list.append(local_path) + else: + local_file_list.append(path) + sizes = _images2fov(local_file_list, dask=True).sizes + for path in cleanup_file_list: + os.remove(path) + with open("img_size.txt", "wt") as f: + for dim in ["t", "c", "z", "y", "x"]: + s = sizes[dim] if dim in sizes else 0 + f.write(f"{s}") + f.write("\n") + + +def _write_group_size(metadata: dict): + n_tiles = len(metadata["file_metadata"]) + metadata_fields = [v for v in ("c", "z") if v in metadata["file_metadata"][0]] + if len(metadata_fields) > 0: + from scallops.cli.util import _group_src_attrs + + keys, channel_sources, filepaths = _group_src_attrs( + metadata=metadata, metadata_fields=tuple(metadata_fields) + ) + n_tiles = len(filepaths) + with open("group_size.txt", "wt") as f: + f.write(f"{n_tiles}") + f.write("\n") + + +def _list_images_wdl( + image_pattern: str, + urls: list[str], + groupby: list[str], + reference_time: str | None, + subset: list[str] | None, + batch_size_str: str | None, + save_group_size: bool = False, + expected_cycles_str: int | None = None, +): + """Used by WDL workflow to output info about images""" + from scallops.io import _set_up_experiment + + batch_size = 1 + expected_cycles = None + if expected_cycles_str is not None and expected_cycles_str != "": + expected_cycles = int(expected_cycles_str) + if batch_size_str is not None and batch_size_str != "": + batch_size = int(batch_size_str) + if reference_time == "": + reference_time = None + + if subset is not None and ( + len(subset) == 0 or (len(subset) == 1 and subset[0] == "") + ): + subset = None + if image_pattern != "": + groupby = [g for g in groupby if "{" + g + "}" in image_pattern] + exp_gen = _set_up_experiment( + image_path=urls, files_pattern=image_pattern, group_by=groupby, subset=subset + ) + # "groups.txt": each line passed to --subset in cli + # "groupby.txt": filtered groupby with values not in image_pattern removed + groupby_t = "t" in groupby + times = [] + + if not save_group_size: + with open("group_size.txt", "wt") as f: + f.write("0\n") + + with ( + open("subsets.txt", "wt") as groups_out, + open("subsets_with_t.txt", "wt") as groups_with_t_out, + ): + subset_ids = [] + subset_ids_with_reference_times = [] + first = True + + for g, file_list, metadata in exp_gen: + times = None + if first: + first = False + if save_group_size: + _write_group_size(metadata) + if not groupby_t and "t" in metadata["file_metadata"][0]: + times = [md["t"] for md in metadata["file_metadata"]] + if expected_cycles is not None: + assert len(times) == expected_cycles + t_suffix = "" + if times is not None and len(times) > 0: + t_suffix = ( + f"-{times[0]}" if reference_time is None else f"-{reference_time}" + ) + + subset_ids.append('"' + metadata["id"] + '"') + subset_ids_with_reference_times.append( + '"' + metadata["id"] + t_suffix + '"' + ) + if len(subset_ids) == batch_size: + groups_out.write(" ".join(subset_ids)) + groups_out.write("\n") + + groups_with_t_out.write(" ".join(subset_ids_with_reference_times)) + groups_with_t_out.write("\n") + + subset_ids = [] + subset_ids_with_reference_times = [] + if len(subset_ids) > 0: + groups_out.write(" ".join(subset_ids)) + groups_out.write("\n") + + groups_with_t_out.write(" ".join(subset_ids_with_reference_times)) + groups_with_t_out.write("\n") + + with open("groupby.txt", "wt") as f: + for g in groupby: + f.write(g) + f.write("\n") + groupby_with_t = list(groupby) + + if not groupby_t and times is not None: + groupby_with_t.append("t") + + with open("groupby_with_t.txt", "wt") as f: + for g in groupby_with_t: + f.write(g) + f.write("\n") + + with open("t.txt", "wt") as f: + if times is not None: + for val in times: + f.write(str(val)) + f.write("\n") + + with open("groupby_pattern.txt", "wt") as f: + first = True + for g in groupby: + if not first: + f.write("-") + first = False + f.write("{") + f.write(g) + f.write("}") diff --git a/scallops/io.py b/scallops/io.py index f58c16c..91f4343 100644 --- a/scallops/io.py +++ b/scallops/io.py @@ -1277,8 +1277,10 @@ def _images2fov( if image is None: raise ValueError(f"{file_list[i]} could not be read.") if file_metadata is not None: - if "t" in file_metadata[i] and isinstance( - file_metadata[i]["t"], str + if ( + "t" in file_metadata[i] + and isinstance(file_metadata[i]["t"], str) + and file_metadata[i]["t"].isdigit() ): # convert to int try: # note we replace "_" with "-" so we don't convert "1_2" to 12 e.g. diff --git a/scallops/tests/test_wdl.py b/scallops/tests/test_wdl.py index 0e1b2ee..1fa68bf 100644 --- a/scallops/tests/test_wdl.py +++ b/scallops/tests/test_wdl.py @@ -1,4 +1,3 @@ -import glob import json import os.path from subprocess import check_call @@ -10,6 +9,7 @@ import xarray as xr from scallops import Experiment +from scallops.cli.util import _list_images_wdl from scallops.io import read_image, save_ome_tiff from scallops.tests.test_stitch import _write_image_with_position @@ -27,6 +27,42 @@ def add_physical_size(input_path, output_path): save_ome_tiff(img.values, uri=output_path, ome_xml=img.attrs["processed"].to_xml()) +@pytest.mark.parametrize("reference_time", ["IF", None]) +@pytest.mark.cli_e2e +def test_list_images_wdl(reference_time, tmp_path, monkeypatch): + (tmp_path / "plate1-A1-IF").touch() + (tmp_path / "plate1-A1-FISH").touch() + monkeypatch.chdir(tmp_path) + _list_images_wdl( + image_pattern="{plate}-{well}-{t}", + reference_time=reference_time, + urls=[str(tmp_path)], + groupby=["plate", "well"], + subset=None, + batch_size_str=None, + save_group_size=False, + expected_cycles_str=None, + ) + groups = pd.read_csv(tmp_path / "groups.txt", header=None)[0].values + np.testing.assert_array_equal(groups, ["plate1-A1"]) + groupby = pd.read_csv(tmp_path / "groupby.txt", header=None)[0].values + np.testing.assert_array_equal(groupby, ["plate", "well"]) + groupby_pattern = pd.read_csv(tmp_path / "groupby_pattern.txt", header=None)[ + 0 + ].values + np.testing.assert_array_equal(groupby_pattern, ["{plate}-{well}"]) + + times = pd.read_csv(tmp_path / "t.txt", header=None)[0].values + np.testing.assert_array_equal(times, ["FISH", "IF"]) + groups_with_times = pd.read_csv(tmp_path / "groups_with_t.txt", header=None)[ + 0 + ].values + if reference_time == "IF": + np.testing.assert_array_equal(groups_with_times, ["plate1-A1-IF"]) + else: + np.testing.assert_array_equal(groups_with_times, ["plate1-A1-FISH"]) + + @pytest.mark.cli_e2e def test_stitch_wdl_z_stack(tmp_path): input_path = tmp_path / "input" @@ -119,15 +155,15 @@ def test_stitch_wdl(tmp_path): @pytest.mark.cli_e2e def test_ops_wdl(tmp_path): - sbs_dir = tmp_path / "sbs" - pheno_dir = tmp_path / "pheno" + sbs_dir = tmp_path / "sbs.zarr" + pheno_dir = tmp_path / "pheno.zarr" output = tmp_path / "out" - sbs_dir.mkdir() - pheno_dir.mkdir() output.mkdir() - for p in glob.glob("scallops/tests/data/experimentC/input/*/*Tile-102*"): - add_physical_size(p, str(sbs_dir / os.path.basename(p))) + iss_img = read_image( + "scallops/tests/data/experimentC/input/10X_c1-SBS-1/10X_c1-SBS-1_A1_Tile-102.sbs.tif" + ) + iss_img.attrs["physical_pixel_sizes"] = (1, 1) pheno_img = read_image( "scallops/tests/data/experimentC/10X_c0-DAPI-p65ab/10X_c0-DAPI-p65ab_A1_Tile-102.phenotype.tif" ) @@ -135,45 +171,47 @@ def test_ops_wdl(tmp_path): phenotype_mask = np.ones( (pheno_img.sizes["y"], pheno_img.sizes["x"]), dtype=np.uint8 ) - phenotype_mask[10, 10] = 1 + phenotype_mask[10, 10] = 0 phenotype_tile = np.ones( (pheno_img.sizes["y"], pheno_img.sizes["x"]), dtype=np.uint16 ) phenotype_tile[10, 10] = 2 - exp = Experiment( - images={"A1-102-1": pheno_img, "A1-102-2": pheno_img}, + Experiment( + images={"plateA-A1-IF": pheno_img, "plateA-A1-FISH": pheno_img}, labels={ - "A1-102-1-mask": phenotype_mask, - "A1-102-1-tile": phenotype_tile, - "A1-102-2-mask": phenotype_mask, - "A1-102-2-tile": phenotype_tile, + "plateA-A1-IF-mask": phenotype_mask, + "plateA-A1-IF-tile": phenotype_tile, + "plateA-A1-FISH-mask": phenotype_mask, + "plateA-A1-FISH-tile": phenotype_tile, }, - ) - exp.save(str(pheno_dir)) + ).save(pheno_dir) + + Experiment( + images={"plateA-A1-1": iss_img, "plateA-A1-2": iss_img}, + ).save(sbs_dir) input_json = { "model_dir": "", "iss_url": str(sbs_dir.absolute()), - "iss_image_pattern": "{mag}X_c{t}-{experiment}-{t}_{well}_Tile-{tile}.{datatype}.tif", "output_directory": str(output.absolute()), + "nuclei_segmentation_method": "cellpose", "iss_registration_extra_arguments": "--no-landmarks", "pheno_to_iss_registration_extra_arguments": "--no-landmarks", "pheno_registration_extra_arguments": "--no-landmarks", "phenotype_cyto_channel": [1], - "phenotype_dapi_channel": 0, + "reference_phenotype_time": "IF", "phenotype_url": str(pheno_dir.absolute()), - "phenotype_nuclei_features": ["intensity_0", "intensity_1"], + "phenotype_nuclei_features": { + "IF": ["intensity_0", "intensity_1"], + "FISH": ["intensity_0", "intensity_1"], + }, # 2 batches - "phenotype_cell_features": ["intensity_0"], + "phenotype_cell_features": {"IF": ["intensity_0"]}, # "phenotype_cytosol_features": ["mean_0 area"], # no cytosol features - "phenotype_image_pattern": "{well}-{tile}-{t}", - "groupby": ["well", "tile"], "reads_threshold_peaks": "0", "reads_threshold_peaks_crosstalk": "20", "barcodes": os.path.abspath("scallops/tests/data/experimentC/barcodes.csv"), - "mark_stitch_boundary_cells": False, "reads_labels": "cell", - "merge_extra_arguments": "--format parquet", "docker": "", } diff --git a/scallops/utils.py b/scallops/utils.py index f015589..6663f8d 100644 --- a/scallops/utils.py +++ b/scallops/utils.py @@ -637,130 +637,3 @@ def _dask_from_array_no_copy( meta = x return da.Array(dsk, name, chunks, meta=meta, dtype=getattr(x, "dtype", None)) - - -def _write_img_size(file_list: list[str]): - from scallops.io import _images2fov, _localize_path - - local_file_list = [] - cleanup_file_list = [] - for path in file_list: - local_path = _localize_path(path) - if local_path is not None: - cleanup_file_list.append(local_path) - local_file_list.append(local_path) - else: - local_file_list.append(path) - sizes = _images2fov(local_file_list, dask=True).sizes - for path in cleanup_file_list: - os.remove(path) - with open("img_size.txt", "wt") as f: - for dim in ["t", "c", "z", "y", "x"]: - s = sizes[dim] if dim in sizes else 0 - f.write(f"{s}") - f.write("\n") - - -def _write_group_size(metadata: dict): - n_tiles = len(metadata["file_metadata"]) - metadata_fields = [v for v in ("c", "z") if v in metadata["file_metadata"][0]] - if len(metadata_fields) > 0: - from scallops.cli.util import _group_src_attrs - - keys, channel_sources, filepaths = _group_src_attrs( - metadata=metadata, metadata_fields=tuple(metadata_fields) - ) - n_tiles = len(filepaths) - with open("group_size.txt", "wt") as f: - f.write(f"{n_tiles}") - f.write("\n") - - -def _list_images_wdl( - image_pattern: str, - urls: list[str], - groupby: list[str], - subset: list[str], - batch_size_str: str, - save_group_size: bool = False, - expected_cycles_str: int | None = None, -): - """Used by WDL workflow to output info about images""" - from scallops.io import _set_up_experiment - - batch_size = 0 - expected_cycles = None - if expected_cycles_str != "": - expected_cycles = int(expected_cycles_str) - if batch_size_str != "": - batch_size = int(batch_size_str) - - if len(subset) == 0 or (len(subset) == 1 and subset[0] == ""): - subset = None - if image_pattern != "": - groupby = [g for g in groupby if "{" + g + "}" in image_pattern] - exp_gen = _set_up_experiment( - image_path=urls, files_pattern=image_pattern, group_by=groupby, subset=subset - ) - # "groups.txt" is passed to --subset in cli - # "groupby.txt" filtered groupby - groupby_t = "t" in groupby - t = [] - - if not save_group_size: - with open("group_size.txt", "wt") as f: - f.write("0\n") - if batch_size > 0: - with open("groups.txt", "wt") as f: - ids = [] - first = True - for g, file_list, metadata in exp_gen: - if first: - first = False - if save_group_size: - _write_group_size(metadata) - if not groupby_t and "t" in metadata["file_metadata"][0]: - t = [md["t"] for md in metadata["file_metadata"]] - if expected_cycles is not None: - assert len(t) == expected_cycles - - ids.append('"' + metadata["id"] + '"') - if len(ids) == batch_size: - f.write(" ".join(ids)) - f.write("\n") - ids = [] - if len(ids) > 0: - f.write(" ".join(ids)) - f.write("\n") - else: - with open("groups.txt", "wt") as f: - first = True - for g, file_list, metadata in exp_gen: - f.write(metadata["id"]) - f.write("\n") - if first: - first = False - if save_group_size: - _write_group_size(metadata) - if not groupby_t and "t" in metadata["file_metadata"][0]: - t = [md["t"] for md in metadata["file_metadata"]] - - with open("groupby.txt", "wt") as f: - for g in groupby: - f.write(g) - f.write("\n") - - with open("t.txt", "wt") as f: - for val in t: - f.write(str(val)) - f.write("\n") - - with open("groupby_pattern.txt", "wt") as f: - first = True - for g in groupby: - if not first: - f.write("-") - first = False - f.write("{") - f.write(g) - f.write("}") diff --git a/wdl/ops_tasks.wdl b/wdl/ops_tasks.wdl index f17dc52..396bdb5 100644 --- a/wdl/ops_tasks.wdl +++ b/wdl/ops_tasks.wdl @@ -1,4 +1,4 @@ -version 1.0 +version 1.1 task segment_nuclei { input { @@ -430,7 +430,7 @@ task intersects_boundary { task find_objects { input { - String? labels + Array[String] labels String subset Boolean? force String? label_pattern @@ -454,7 +454,7 @@ task find_objects { fi scallops find-objects \ - --labels "~{labels}" \ + --labels ~{sep=" " labels} \ --subset ~{subset} \ ~{"--label-pattern " + label_pattern} \ --label-suffix ~{suffix} \ diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index 1955d4a..9c14b86 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -1,4 +1,4 @@ -version 1.0 +version 1.1 import "utils.wdl" as utils import "ops_tasks.wdl" as tasks @@ -15,14 +15,14 @@ workflow ops_workflow { String output_directory - # t to align phenotyping rounds to e.g. "IF" + # t to align phenotyping rounds to e.g. "IF". If not specified then first round in natural sorted order is used String? reference_phenotype_time # features String? features_label_filter = "~barcode_count_0.isna()" # valid barcodes - Array[String]? phenotype_cell_features - Array[String]? phenotype_nuclei_features - Array[String]? phenotype_cytosol_features + Map[String, Array[String]]? phenotype_cell_features + Map[String, Array[String]]? phenotype_nuclei_features + Map[String, Array[String]]? phenotype_cytosol_features String? features_extra_arguments # Single string with extra arguments to scallops features cli Int? features_cell_min_area @@ -32,16 +32,15 @@ workflow ops_workflow { Int? features_nuclei_max_area Int? features_cytosol_max_area - Array[Int] phenotype_cyto_channel # indices after registration for cell segmentation - Int phenotype_dapi_channel # index after registration for segmentation and pheno to iss registration - Int? phenotype_dapi_channel_before_registration # for pheno to pheno registration + Array[Int] phenotype_cyto_channel # indices within referent time for cell segmentation + Int? phenotype_dapi_channel # index within t for nuclei segmentation and pheno to iss registration Int? iss_dapi_channel # ISS to ISS and pheno to ISS registration String? iss_registration_extra_arguments # Extra arguments in scallops registration elastix cli for ISS String? pheno_to_iss_registration_extra_arguments String? pheno_registration_extra_arguments - Boolean? register_across_channels + # spot detect Int? iss_expected_cycles @@ -68,7 +67,7 @@ workflow ops_workflow { String model_dir = "" # nuclei segment - String? nuclei_segmentation + String? nuclei_segmentation_method String? nuclei_segmentation_extra_arguments # cell segment @@ -156,7 +155,6 @@ workflow ops_workflow { String cell_intersects_boundary_memory = "32 GiB" String cell_intersects_boundary_disks = "local-disk 200 HDD" - String docker Int preemptible = 0 @@ -187,7 +185,6 @@ workflow ops_workflow { String cell_intersects_boundary_non_reference_t_suffix = "intersects-boundary-t" } - String output_stripped = sub(output_directory, "/+$", "") + "/" String segment_directory = output_stripped + segment_suffix String register_iss_t0_directory = output_stripped + register_iss_suffix @@ -208,7 +205,6 @@ workflow ops_workflow { String merge_features_directory = output_stripped + merge_features_suffix String register_pheno_to_iss_qc_directory = output_stripped + register_pheno_to_iss_qc_suffix String cell_intersects_boundary_directory = output_stripped + cell_intersects_boundary_suffix - String cell_intersects_boundary_directory_non_reference_t = output_stripped + cell_intersects_boundary_non_reference_t_suffix Boolean iss_url_supplied = defined(iss_url) Boolean pheno_url_supplied = defined(phenotype_url) @@ -218,60 +214,35 @@ workflow ops_workflow { urls = [select_first([phenotype_url, iss_url])], image_pattern = if pheno_url_supplied then phenotype_image_pattern else iss_image_pattern, batch_size=batch_size, + reference_time=reference_phenotype_time, groupby=groupby, - subset=subset, + subset = subset, docker=docker, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, max_retries = max_retries } - String image_pattern_after_registration = list_images.groupby_pattern - Array[String] groups = list_images.groups + String groupby_pattern = list_images.groupby_pattern # plate-well + Array[String] subsets = list_images.subsets + Array[String] subset_with_reference_times = list_images.subset_with_reference_times Array[String] times = list_images.t - scatter (group in groups) { + Array[String] groupby_with_time = list_images.filtered_groupby_with_t + scatter (subset_index in range(length(subsets))) { + String subset_ = subsets[subset_index] + String subset_with_reference_time = subset_with_reference_times[subset_index] if(pheno_url_supplied) { - if(length(times)>1) { - call tasks.register_elastix as register_pheno_to_pheno { - input: - moving=select_all([phenotype_url]), - moving_label=phenotype_url, # transform stitch masks - moving_channel=phenotype_dapi_channel_before_registration, # DAPI index in each round - moving_image_pattern=phenotype_image_pattern, - reference_time=reference_phenotype_time, - extra_arguments=pheno_registration_extra_arguments, - unroll_channels=true, - register_across_channels=register_across_channels, - groupby=groupby, - moving_output_directory=register_pheno_to_pheno_directory, - label_output_directory=register_pheno_to_pheno_directory, - transform_output_directory=register_pheno_to_pheno_transform_directory, - subset = group, - force = force_register_pheno_to_pheno, - docker=docker, - zones = zones, - preemptible = preemptible, - aws_queue_arn = aws_queue_arn, - disks = register_pheno_to_pheno_disks, - memory = register_pheno_to_pheno_memory, - cpu = register_pheno_to_pheno_cpu, - max_retries = max_retries - } - } - String register_pheno_to_pheno_output_url = select_first([register_pheno_to_pheno.moving_output_url, phenotype_url]) - String register_pheno_to_pheno_image_pattern = if(length(times)>1) then image_pattern_after_registration else phenotype_image_pattern - if(run_nuclei_segmentation) { call tasks.segment_nuclei { input: - images = register_pheno_to_pheno_output_url, - image_pattern = register_pheno_to_pheno_image_pattern, - method = nuclei_segmentation, - groupby=groupby, + images = select_first([phenotype_url]), + image_pattern = phenotype_image_pattern, + method = nuclei_segmentation_method, + groupby=groupby_with_time, dapi_channel = phenotype_dapi_channel, output_directory=segment_directory, model_dir=model_dir, - subset = group, + subset = subset_with_reference_time, extra_arguments=nuclei_segmentation_extra_arguments, force = force_segment_nuclei, docker=docker, @@ -283,14 +254,16 @@ workflow ops_workflow { cpu = segment_nuclei_cpu, max_retries = max_retries } + } if(run_cell_segmentation) { call tasks.segment_cell { input: - images = register_pheno_to_pheno_output_url, - image_pattern = register_pheno_to_pheno_image_pattern, + images = select_first([phenotype_url]), + image_pattern = phenotype_image_pattern, method = cell_segmentation_method, - groupby=groupby, + groupby = groupby_with_time, + subset = subset_with_reference_time, dapi_channel = phenotype_dapi_channel, cyto_channel=phenotype_cyto_channel, nuclei_label=select_first([segment_nuclei.output_url]), @@ -298,7 +271,7 @@ workflow ops_workflow { threshold_correction_factor = segment_cell_threshold_correction_factor, output_directory=segment_directory, model_dir=model_dir, - subset = group, + extra_arguments=cell_segmentation_extra_arguments, force = force_segment_cell, docker=docker, @@ -310,62 +283,113 @@ workflow ops_workflow { cpu = segment_cell_cpu, max_retries = max_retries } - call tasks.find_objects as find_objects_cell { - input: - labels= segment_cell.output_url, - label_pattern=image_pattern_after_registration, - suffix="cell", - output_directory=cell_objects_directory, - subset = group, - force = force_find_objects, - docker=docker, - zones = zones, - preemptible = preemptible, - aws_queue_arn = aws_queue_arn, - disks = find_objects_disks, - memory = find_objects_memory, - cpu = find_objects_cpu, - max_retries = max_retries + + if(length(times)>1) { + call tasks.register_elastix as register_pheno_to_pheno { + input: + moving=select_all([phenotype_url]), + moving_label=segment_cell.output_url, + moving_channel=phenotype_dapi_channel, + moving_image_pattern=phenotype_image_pattern, + reference_time=reference_phenotype_time, + extra_arguments=pheno_registration_extra_arguments, + output_aligned_channels_only=true, + groupby=groupby, + subset = subset_, + moving_output_directory=register_pheno_to_pheno_directory, + label_output_directory=register_pheno_to_pheno_directory, + transform_output_directory=register_pheno_to_pheno_transform_directory, + + force = force_register_pheno_to_pheno, + docker=docker, + zones = zones, + preemptible = preemptible, + aws_queue_arn = aws_queue_arn, + disks = register_pheno_to_pheno_disks, + memory = register_pheno_to_pheno_memory, + cpu = register_pheno_to_pheno_cpu, + max_retries = max_retries + } } - call tasks.find_objects as find_objects_cytosol { - input: - labels=segment_cell.output_url, - label_pattern=image_pattern_after_registration, - suffix="cytosol", - output_directory=cytosol_objects_directory, - subset = group, - force = force_find_objects, - docker=docker, - zones = zones, - preemptible = preemptible, - aws_queue_arn = aws_queue_arn, - disks = find_objects_disks, - memory = find_objects_memory, - cpu = find_objects_cpu, - max_retries = max_retries + if(run_nuclei_segmentation) { + call tasks.find_objects as find_objects_nuclei { + input: + labels=select_all([segment_nuclei.output_url, register_pheno_to_pheno.label_output_url]), + label_pattern=phenotype_image_pattern, + suffix="nuclei", + output_directory=nuclei_objects_directory, + subset = subset_, + force = force_find_objects, + docker=docker, + zones = zones, + preemptible = preemptible, + aws_queue_arn = aws_queue_arn, + disks = find_objects_disks, + memory = find_objects_memory, + cpu = find_objects_cpu, + max_retries = max_retries + } } + if(run_cell_segmentation) { + call tasks.find_objects as find_objects_cell { + input: + labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), + label_pattern=phenotype_image_pattern, + suffix="cell", + output_directory=cell_objects_directory, + subset = subset_, + force = force_find_objects, + docker=docker, + zones = zones, + preemptible = preemptible, + aws_queue_arn = aws_queue_arn, + disks = find_objects_disks, + memory = find_objects_memory, + cpu = find_objects_cpu, + max_retries = max_retries + } + } + if (run_nuclei_segmentation && run_cell_segmentation) { + call tasks.find_objects as find_objects_cytosol { + input: + labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), + label_pattern=phenotype_image_pattern, + suffix="cytosol", + output_directory=cytosol_objects_directory, + subset = subset_, + force = force_find_objects, + docker=docker, + zones = zones, + preemptible = preemptible, + aws_queue_arn = aws_queue_arn, + disks = find_objects_disks, + memory = find_objects_memory, + cpu = find_objects_cpu, + max_retries = max_retries + } + } + # String register_pheno_to_pheno_output_url = select_first([register_pheno_to_pheno.moving_output_url, phenotype_url]) + # String phenotype_image_pattern = if(length(times)>1) then image_pattern_after_registration else phenotype_image_pattern + # determine whether cells intersect stitch boundary - # using stitch mask as image + # use stitch mask as image and segment output for reference phenotype or transformed phenotype for others + if(mark_stitch_boundary_cells) { - String t0 = if (length(times)>0) then times[0] else "" - String reference_phenotype_time_ = select_first([reference_phenotype_time, t0]) - String output_prefix = if (reference_phenotype_time_!="") then "-" else "" String phenotype_url_stripped = if (pheno_url_supplied) then sub(select_first([phenotype_url]), "/+$", "") else "" call tasks.intersects_boundary as cell_intersects_boundary { - # reference time mask is not transformed - # use mask from stitch output + input: - labels=segment_cell.output_url, + labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), images=phenotype_url_stripped + '/labels/', - image_pattern=image_pattern_after_registration + output_prefix + reference_phenotype_time_ + '-mask', + image_pattern=phenotype_image_pattern, output_directory=cell_intersects_boundary_directory, label_type='cell', objects=find_objects_cell.output_url, groupby=groupby, - subset = group, + subset = subset_, force = force_segment_cell, docker=docker, zones = zones, @@ -376,35 +400,14 @@ workflow ops_workflow { cpu = cell_intersects_boundary_cpu, max_retries = max_retries } - if (length(times)>1) { - call tasks.intersects_boundary as cell_intersects_boundary_t { - # non-reference time masks are transformed - # use masks from registration output - input: - labels= segment_cell.output_url, - images=register_pheno_to_pheno.moving_output_url + '/labels/', - image_pattern=phenotype_image_pattern + '-mask', - output_directory=cell_intersects_boundary_directory_non_reference_t, - label_type='cell', - objects=find_objects_cell.output_url, - subset = group, - groupby=groupby, - force = force_segment_cell, - docker=docker, - zones = zones, - preemptible = preemptible, - aws_queue_arn = aws_queue_arn, - disks = cell_intersects_boundary_disks, - memory = cell_intersects_boundary_memory, - cpu = cell_intersects_boundary_cpu, - max_retries = max_retries - } - } + } } + } if(iss_url_supplied) { + call tasks.register_elastix as register_iss_t0 { input: moving=[select_first([iss_url])], @@ -413,9 +416,8 @@ workflow ops_workflow { groupby=groupby, moving_output_directory=register_iss_t0_directory, transform_output_directory=register_iss_t0_transforms_directory, - register_across_channels=register_across_channels, extra_arguments=iss_registration_extra_arguments, - subset = group, + subset = subset_, force = force_register_iss, docker=docker, zones = zones, @@ -429,21 +431,21 @@ workflow ops_workflow { } if(iss_url_supplied && pheno_url_supplied) { + # transfer phenotype segmentation and DAPI channel to ISS call tasks.register_elastix as register_pheno_to_iss { input: fixed=select_first([iss_url]), fixed_channel=iss_dapi_channel, moving_label=segment_cell.output_url, - moving=select_all([register_pheno_to_pheno_output_url]), - moving_image_pattern=register_pheno_to_pheno_image_pattern, + moving=select_all([phenotype_url]), + moving_image_pattern=phenotype_image_pattern, fixed_image_pattern=iss_image_pattern, moving_channel=phenotype_dapi_channel, output_aligned_channels_only=true, - register_across_channels=register_across_channels, moving_output_directory=register_pheno_to_iss_directory, label_output_directory=register_pheno_to_iss_directory, transform_output_directory=register_pheno_to_iss_transforms_directory, - subset = group, + subset = subset_with_reference_time, groupby=groupby, extra_arguments=pheno_to_iss_registration_extra_arguments, force = force_register_pheno_to_iss, @@ -457,36 +459,19 @@ workflow ops_workflow { max_retries = max_retries } if(run_nuclei_segmentation) { - call tasks.find_objects as find_objects_nuclei { - input: - labels=segment_nuclei.output_url, - label_pattern=image_pattern_after_registration, - suffix="nuclei", - output_directory=nuclei_objects_directory, - subset = group, - force = force_find_objects, - docker=docker, - zones = zones, - preemptible = preemptible, - aws_queue_arn = aws_queue_arn, - disks = find_objects_disks, - memory = find_objects_memory, - cpu = find_objects_cpu, - max_retries = max_retries - } - + # ISS t0 to phenotype reference time call tasks.register_pheno_to_iss_qc as register_pheno_to_iss_qc { input: - images=select_first([register_iss_t0.moving_output_url]), - image_pattern=image_pattern_after_registration, + images=select_first([iss_url]), + image_pattern=iss_image_pattern, stacked_images=register_pheno_to_iss.moving_output_url, - stacked_image_pattern=image_pattern_after_registration, + stacked_image_pattern=phenotype_image_pattern, image_channel=iss_dapi_channel, stacked_image_channel=0, label_type='nuclei', output_directory=register_pheno_to_iss_qc_directory, labels=register_pheno_to_iss.label_output_url, - subset = group, + subset = subset_, groupby=groupby, force = force_register_pheno_to_iss_qc, docker=docker, @@ -498,16 +483,17 @@ workflow ops_workflow { cpu = register_pheno_to_iss_qc_cpu, max_retries = max_retries } - call tasks.register_qc as register_iss_to_iss_qc { + # ISS t0 to other times + call tasks.register_qc as register_iss_to_iss_qc { input: images=select_first([register_iss_t0.moving_output_url]), - image_pattern=image_pattern_after_registration, + image_pattern=groupby_pattern, channel=select_first([iss_dapi_channel, 0]), label_type='nuclei', channel_prefix="ISS", output_directory=register_iss_to_iss_qc_directory, labels=register_pheno_to_iss.label_output_url, - subset = group, + subset = subset_, groupby=groupby, force = force_register_iss_to_iss_qc, docker=docker, @@ -526,14 +512,14 @@ workflow ops_workflow { call tasks.spot_detect { input: images=select_first([register_iss_t0.moving_output_url]), - image_pattern=image_pattern_after_registration, + image_pattern=groupby_pattern, iss_channels=iss_channels, sigma_log=spot_detection_sigma_log, max_filter_width=spot_detection_max_filter_width, peak_neighborhood_size=spot_detection_peak_neighborhood_size, expected_cycles=iss_expected_cycles, output_directory=spot_detect_directory, - subset = group, + subset = subset_, groupby=groupby, extra_arguments=spot_detection_extra_arguments, force = force_spot_detect, @@ -562,7 +548,7 @@ workflow ops_workflow { label_name=reads_labels, mismatches=reads_mismatches, threshold_peaks_crosstalk=reads_threshold_peaks_crosstalk, - subset = group, + subset = subset_, extra_arguments=reads_extra_arguments, force = force_reads, docker=docker, @@ -580,20 +566,16 @@ workflow ops_workflow { call tasks.merge as merge_sbs_metadata { input: iss_reads=select_first([reads.output_url]) + '/labels', -# phenotypes_nuclei=features_nuclei.output_url, -# phenotypes_cell=features_cell.output_url, -# phenotypes_cytosol=features_cytosol.output_url, - objects_nuclei=find_objects_nuclei.output_url, + objects_nuclei=find_objects_nuclei.output_url, # all rounds objects_cell=find_objects_cell.output_url, objects_cytosol=find_objects_cytosol.output_url, cell_intersects_boundary=cell_intersects_boundary.output_url, - cell_intersects_boundary_t=cell_intersects_boundary_t.output_url, register_pheno_to_iss_qc=register_pheno_to_iss_qc.output_url, register_iss_to_iss_qc=register_iss_to_iss_qc.output_url, barcodes=select_first([barcodes]), barcode_column=barcode_column, output_directory=merge_meta_directory, - subset = group, + subset = subset_, extra_arguments=merge_extra_arguments, force = force_merge, docker=docker, @@ -609,132 +591,146 @@ workflow ops_workflow { } if (defined(phenotype_nuclei_features)) { - Array[String] phenotype_nuclei_features_ = select_first([phenotype_nuclei_features]) - # cromwell hack + Map[String,Array[String]] phenotype_nuclei_features_ = select_first([phenotype_nuclei_features]) Int features_nuclei_min_area_ = select_first([features_nuclei_min_area, -1]) Int features_nuclei_max_area_ = select_first([features_nuclei_max_area, -1]) - scatter (index in range(length(phenotype_nuclei_features_))) { + Array[String] phenotype_nuclei_times = keys(phenotype_nuclei_features_) - call tasks.features as features_nuclei { - input: - images = select_first([register_pheno_to_pheno_output_url]), - image_pattern=register_pheno_to_pheno_image_pattern, - objects=merge_sbs_metadata.output_url, - label_filter=features_label_filter, - nuclei_features = phenotype_nuclei_features_[index], - nuclei_min_area = features_nuclei_min_area_, - nuclei_max_area = features_nuclei_max_area_, - features_extra_arguments=features_extra_arguments, - labels= segment_cell.output_url, - model_dir=model_dir, - groupby=groupby, - output_directory=nuclei_features_directory + '-' + index, - subset = group, - force = force_features, - docker=docker, - zones = zones, - preemptible = preemptible, - aws_queue_arn = aws_queue_arn, - disks = features_disks, - memory = features_memory, - cpu = features_cpu, - max_retries = max_retries + scatter (phenotype_time in phenotype_nuclei_times) { + Array[String] nuclei_features = phenotype_nuclei_features_[phenotype_time] + scatter (feature_index in range(length(nuclei_features))) { + call tasks.features as features_nuclei { + input: + images = select_first([phenotype_url]), + image_pattern=phenotype_image_pattern, + objects=merge_sbs_metadata.output_url, + labels=select_all([segment_nuclei.output_url, register_pheno_to_pheno.label_output_url]), + label_filter=features_label_filter, + nuclei_features = nuclei_features[feature_index], + nuclei_min_area = features_nuclei_min_area_, + nuclei_max_area = features_nuclei_max_area_, + features_extra_arguments=features_extra_arguments, + + model_dir=model_dir, + groupby=groupby, + output_directory=nuclei_features_directory + '-' + phenotype_time + '-' + feature_index, + subset = subset_, + force = force_features, + docker=docker, + zones = zones, + preemptible = preemptible, + aws_queue_arn = aws_queue_arn, + disks = features_disks, + memory = features_memory, + cpu = features_cpu, + max_retries = max_retries + } } } } if (defined(phenotype_cell_features)) { - Array[String] phenotype_cell_features_ = select_first([phenotype_cell_features]) - # cromwell hack + Map[String,Array[String]] phenotype_cell_features_ = select_first([phenotype_cell_features]) Int features_cell_min_area_ = select_first([features_cell_min_area, -1]) Int features_cell_max_area_ = select_first([features_cell_max_area, -1]) - scatter (index in range(length(phenotype_cell_features_))) { - call tasks.features as features_cell { - input: - images = select_first([register_pheno_to_pheno_output_url]), - image_pattern=register_pheno_to_pheno_image_pattern, - objects=merge_sbs_metadata.output_url, - label_filter=features_label_filter, - cell_features = phenotype_cell_features_[index], - cell_min_area = features_cell_min_area_, - cell_max_area = features_cell_max_area_, - features_extra_arguments=features_extra_arguments, - labels= segment_cell.output_url, - model_dir=model_dir, - groupby=groupby, - output_directory=cell_features_directory + '-' + index, - subset = group, - force = force_features, - docker=docker, - zones = zones, - preemptible = preemptible, - aws_queue_arn = aws_queue_arn, - disks = features_disks, - memory = features_memory, - cpu = features_cpu, - max_retries = max_retries + Array[String] phenotype_cell_times = keys(phenotype_cell_features_) + + scatter (phenotype_time in phenotype_cell_times) { + Array[String] cell_features = phenotype_cell_features_[phenotype_time] + scatter (feature_index in range(length(cell_features))) { + call tasks.features as features_cell { + input: + images = select_first([phenotype_url]), + image_pattern=phenotype_image_pattern, + objects=merge_sbs_metadata.output_url, + labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), + label_filter=features_label_filter, + cell_features = cell_features[feature_index], + cell_min_area = features_cell_min_area_, + cell_max_area = features_cell_max_area_, + features_extra_arguments=features_extra_arguments, + + model_dir=model_dir, + groupby=groupby, + output_directory=cell_features_directory + '-' + phenotype_time + '-' + feature_index, + subset = subset_, + force = force_features, + docker=docker, + zones = zones, + preemptible = preemptible, + aws_queue_arn = aws_queue_arn, + disks = features_disks, + memory = features_memory, + cpu = features_cpu, + max_retries = max_retries + } } } } if (defined(phenotype_cytosol_features)) { - Array[String] phenotype_cytosol_features_ = select_first([phenotype_cytosol_features]) - # cromwell hack + Map[String,Array[String]] phenotype_cytosol_features_ = select_first([phenotype_cytosol_features]) Int features_cytosol_min_area_ = select_first([features_cytosol_min_area, -1]) Int features_cytosol_max_area_ = select_first([features_cytosol_max_area, -1]) - scatter (index in range(length(phenotype_cytosol_features_))) { - call tasks.features as features_cytosol { - input: - images = select_first([register_pheno_to_pheno_output_url]), - image_pattern=register_pheno_to_pheno_image_pattern, - objects=merge_sbs_metadata.output_url, - label_filter=features_label_filter, - cytosol_features = phenotype_cytosol_features_[index], - cytosol_min_area = features_cytosol_min_area_, - cytosol_max_area = features_cytosol_max_area_, - labels = segment_cell.output_url, - features_extra_arguments=features_extra_arguments, - model_dir=model_dir, - groupby=groupby, - output_directory=cytosol_features_directory + '-' + index, - subset = group, - force = force_features, - docker=docker, - zones = zones, - preemptible = preemptible, - aws_queue_arn = aws_queue_arn, - disks = features_disks, - memory = features_memory, - cpu = features_cpu, - max_retries = max_retries + Array[String] phenotype_cytosol_times = keys(phenotype_cytosol_features_) + + scatter (phenotype_time in phenotype_cytosol_times) { + Array[String] cytosol_features = phenotype_cytosol_features_[phenotype_time] + scatter (feature_index in range(length(cytosol_features))) { + call tasks.features as features_cytosol { + input: + images = select_first([phenotype_url]), + image_pattern=phenotype_image_pattern, + objects=merge_sbs_metadata.output_url, + labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), + label_filter=features_label_filter, + output_directory=cytosol_features_directory + '-' + phenotype_time + '-' + feature_index, + cytosol_features = cytosol_features[feature_index], + cytosol_min_area = features_cytosol_min_area_, + cytosol_max_area = features_cytosol_max_area_, + features_extra_arguments=features_extra_arguments, + + model_dir=model_dir, + groupby=groupby, + + subset = subset_, + force = force_features, + docker=docker, + zones = zones, + preemptible = preemptible, + aws_queue_arn = aws_queue_arn, + disks = features_disks, + memory = features_memory, + cpu = features_cpu, + max_retries = max_retries + } } } } - if (defined(barcodes)) { + if (defined(barcodes)) { - call tasks.merge as merge_features { - input: - phenotypes_nuclei=features_nuclei.output_url, - phenotypes_cell=features_cell.output_url, - phenotypes_cytosol=features_cytosol.output_url, - iss_reads=merge_sbs_metadata.output_url, - output_directory=merge_features_directory, - subset = group, - extra_arguments=merge_extra_arguments, - force = force_merge, - docker=docker, - zones = zones, - preemptible = preemptible, - aws_queue_arn = aws_queue_arn, - disks = merge_disks, - memory = merge_memory, - cpu = merge_cpu, - max_retries = max_retries - } + call tasks.merge as merge_features { + input: + phenotypes_nuclei=features_nuclei.output_url, + phenotypes_cell=features_cell.output_url, + phenotypes_cytosol=features_cytosol.output_url, + iss_reads=merge_sbs_metadata.output_url, + output_directory=merge_features_directory, + subset = subset_, + extra_arguments=merge_extra_arguments, + force = force_merge, + docker=docker, + zones = zones, + preemptible = preemptible, + aws_queue_arn = aws_queue_arn, + disks = merge_disks, + memory = merge_memory, + cpu = merge_cpu, + max_retries = max_retries } - + } } output { @@ -754,7 +750,5 @@ workflow ops_workflow { Array[Array[String]?] features_cytosol_output_url = features_cytosol.output_url Array[String?] merge_sbs_metadata_output_url = merge_sbs_metadata.output_url Array[String?] merge_features_output_url = merge_features.output_url - Array[String] list_images_groups = list_images.groups - } } diff --git a/wdl/utils.wdl b/wdl/utils.wdl index 35e8f01..77060f4 100644 --- a/wdl/utils.wdl +++ b/wdl/utils.wdl @@ -1,10 +1,11 @@ -version 1.0 +version 1.1 task list_images { input { Boolean? save_group_size Array[String] urls String? image_pattern + String? reference_time Array[String] groupby Array[String]? subset Int? expected_cycles @@ -23,26 +24,31 @@ task list_images { command <<< set -e python <>> output { - Array[String] groups = read_lines('groups.txt') + Array[String] subsets = read_lines('subsets.txt') + Array[String] subset_with_reference_times = read_lines('subsets_with_t.txt') Array[String] t = read_lines('t.txt') - Array[String] filtered_groupby = read_lines('groupby.txt') - String groupby_pattern = read_lines('groupby_pattern.txt')[0] - Int group_size = read_int('group_size.txt') + String groupby_pattern = read_lines('groupby_pattern.txt')[0] # e.g. {plate}-{well} + + Array[String] filtered_groupby_with_t = read_lines('groupby_with_t.txt') # e.g. [plate, well, t] + Array[String] filtered_groupby = read_lines('groupby.txt') # e.g. [plate, well] + + Int group_size = read_int('group_size.txt') } meta { From 0fb20ab7fd266e8cd09071a3ee2543b46d258b26 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 9 Jun 2026 13:24:26 -0400 Subject: [PATCH 02/79] Transform labels instead of images --- scallops/tests/test_wdl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scallops/tests/test_wdl.py b/scallops/tests/test_wdl.py index 1fa68bf..943e549 100644 --- a/scallops/tests/test_wdl.py +++ b/scallops/tests/test_wdl.py @@ -194,7 +194,6 @@ def test_ops_wdl(tmp_path): "model_dir": "", "iss_url": str(sbs_dir.absolute()), "output_directory": str(output.absolute()), - "nuclei_segmentation_method": "cellpose", "iss_registration_extra_arguments": "--no-landmarks", "pheno_to_iss_registration_extra_arguments": "--no-landmarks", "pheno_registration_extra_arguments": "--no-landmarks", From 92a4b3592eb810d3d146f428b92bbaffee87843c Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 9 Jun 2026 15:00:23 -0400 Subject: [PATCH 03/79] Transform labels instead of images --- scallops/cli/util.py | 17 ++++++++++++++++- wdl/ops_workflow.wdl | 19 ++++++++----------- wdl/utils.wdl | 3 +-- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/scallops/cli/util.py b/scallops/cli/util.py index 595f0fa..28553b9 100644 --- a/scallops/cli/util.py +++ b/scallops/cli/util.py @@ -523,7 +523,7 @@ def _list_images_wdl( # "groups.txt": each line passed to --subset in cli # "groupby.txt": filtered groupby with values not in image_pattern removed groupby_t = "t" in groupby - times = [] + times = None if not save_group_size: with open("group_size.txt", "wt") as f: @@ -602,3 +602,18 @@ def _list_images_wdl( f.write("{") f.write(g) f.write("}") + groupby_with_reference_time = list(groupby) + if reference_time is not None: + groupby_with_reference_time.append(reference_time) + elif times is not None and len(times) > 0: + groupby_with_reference_time.append(times[0]) + + with open("groupby_pattern_with_reference_t.txt", "wt") as f: + first = True + for g in groupby_with_reference_time: + if not first: + f.write("-") + first = False + f.write("{") + f.write(g) + f.write("}") diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index 9c14b86..eed2343 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -223,11 +223,12 @@ workflow ops_workflow { aws_queue_arn = aws_queue_arn, max_retries = max_retries } - String groupby_pattern = list_images.groupby_pattern # plate-well - Array[String] subsets = list_images.subsets - Array[String] subset_with_reference_times = list_images.subset_with_reference_times - Array[String] times = list_images.t - Array[String] groupby_with_time = list_images.filtered_groupby_with_t + String groupby_pattern = list_images.groupby_pattern # "{plate}-{well}" + Array[String] subsets = list_images.subsets # e.g. ["plate1-A1", "plate1-A2", ...] + Array[String] subset_with_reference_times = list_images.subset_with_reference_times # e.g. ["plate1-A1-IF", "plate1-A2-IF", ...] + Array[String] times = list_images.t # e.g. ["FISH", "IF"] + Array[String] groupby_with_time = list_images.filtered_groupby_with_t # e.g. ['plate', 'well', 't'] + String groupby_pattern_with_reference_t = list_images.groupby_pattern_with_reference_t # e.g. "{plate}-{well}-IF" scatter (subset_index in range(length(subsets))) { String subset_ = subsets[subset_index] String subset_with_reference_time = subset_with_reference_times[subset_index] @@ -369,10 +370,6 @@ workflow ops_workflow { max_retries = max_retries } } - # String register_pheno_to_pheno_output_url = select_first([register_pheno_to_pheno.moving_output_url, phenotype_url]) - # String phenotype_image_pattern = if(length(times)>1) then image_pattern_after_registration else phenotype_image_pattern - - # determine whether cells intersect stitch boundary # use stitch mask as image and segment output for reference phenotype or transformed phenotype for others @@ -438,14 +435,14 @@ workflow ops_workflow { fixed_channel=iss_dapi_channel, moving_label=segment_cell.output_url, moving=select_all([phenotype_url]), - moving_image_pattern=phenotype_image_pattern, + moving_image_pattern=groupby_pattern_with_reference_t, fixed_image_pattern=iss_image_pattern, moving_channel=phenotype_dapi_channel, output_aligned_channels_only=true, moving_output_directory=register_pheno_to_iss_directory, label_output_directory=register_pheno_to_iss_directory, transform_output_directory=register_pheno_to_iss_transforms_directory, - subset = subset_with_reference_time, + subset = subset_, groupby=groupby, extra_arguments=pheno_to_iss_registration_extra_arguments, force = force_register_pheno_to_iss, diff --git a/wdl/utils.wdl b/wdl/utils.wdl index 77060f4..393b806 100644 --- a/wdl/utils.wdl +++ b/wdl/utils.wdl @@ -43,8 +43,7 @@ task list_images { Array[String] subset_with_reference_times = read_lines('subsets_with_t.txt') Array[String] t = read_lines('t.txt') String groupby_pattern = read_lines('groupby_pattern.txt')[0] # e.g. {plate}-{well} - - + String groupby_pattern_with_reference_t = read_lines('groupby_pattern_with_reference_t.txt')[0] # e.g. {plate}-{well}-IF Array[String] filtered_groupby_with_t = read_lines('groupby_with_t.txt') # e.g. [plate, well, t] Array[String] filtered_groupby = read_lines('groupby.txt') # e.g. [plate, well] From fd1fa6442ebd14a0babad19b842232a5584546a5 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 9 Jun 2026 15:34:36 -0400 Subject: [PATCH 04/79] Transform labels instead of images --- scallops/cli/util.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scallops/cli/util.py b/scallops/cli/util.py index 28553b9..74f514e 100644 --- a/scallops/cli/util.py +++ b/scallops/cli/util.py @@ -602,18 +602,19 @@ def _list_images_wdl( f.write("{") f.write(g) f.write("}") - groupby_with_reference_time = list(groupby) + reference_time_suffix = "" if reference_time is not None: - groupby_with_reference_time.append(reference_time) + reference_time_suffix = f"-{reference_time}" elif times is not None and len(times) > 0: - groupby_with_reference_time.append(times[0]) + reference_time_suffix = f"-{times[0]}" with open("groupby_pattern_with_reference_t.txt", "wt") as f: first = True - for g in groupby_with_reference_time: + for g in groupby: if not first: f.write("-") first = False f.write("{") f.write(g) f.write("}") + f.write(reference_time_suffix) From e6f66ab5b0c38ba879d0e160f421989a60fbc45c Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 9 Jun 2026 15:43:34 -0400 Subject: [PATCH 05/79] Transform labels instead of images --- scallops/cli/util.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scallops/cli/util.py b/scallops/cli/util.py index 74f514e..b7360e1 100644 --- a/scallops/cli/util.py +++ b/scallops/cli/util.py @@ -520,8 +520,7 @@ def _list_images_wdl( exp_gen = _set_up_experiment( image_path=urls, files_pattern=image_pattern, group_by=groupby, subset=subset ) - # "groups.txt": each line passed to --subset in cli - # "groupby.txt": filtered groupby with values not in image_pattern removed + groupby_t = "t" in groupby times = None @@ -617,4 +616,4 @@ def _list_images_wdl( f.write("{") f.write(g) f.write("}") - f.write(reference_time_suffix) + f.write(reference_time_suffix) From ce6634f14cb7f9bf66c4ec11696389dfcee58a78 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Thu, 11 Jun 2026 12:37:53 -0400 Subject: [PATCH 06/79] registration transform across times --- scallops/cli/register.py | 151 +++++++++++++++++++++++---------------- 1 file changed, 88 insertions(+), 63 deletions(-) diff --git a/scallops/cli/register.py b/scallops/cli/register.py index 70be43d..21245a3 100644 --- a/scallops/cli/register.py +++ b/scallops/cli/register.py @@ -5,7 +5,7 @@ import os from collections.abc import Sequence from itertools import zip_longest -from typing import Literal +from typing import Any, Literal import fsspec import itk @@ -13,6 +13,8 @@ import xarray as xr import zarr from dask.bag import from_sequence +from natsort import natsorted +from xarray import DataArray from zarr import Group from scallops.cli.util import ( @@ -54,6 +56,48 @@ logger = _get_cli_logger() +def _output_exists( + register_self, label_output_root, moving_label_keys, image_output_root, image_key +): + labels_exist = True + if label_output_root is not None: + if not register_self: + for key in moving_label_keys: + key = os.path.basename(key) + if not is_ome_zarr_array(label_output_root.get(f"labels/{key}")): + labels_exist = False + break + # TODO check for transformed labels when register_self + + image_exists = True + if image_output_root is not None: + if not is_ome_zarr_array(image_output_root.get(f"images/{image_key}")): + image_exists = False + elif label_output_root is None: + image_exists = False + return labels_exist and image_exists + + +def _get_reference_timepoint( + reference_timepoint: str, moving_image: Sequence[DataArray] +) -> tuple[int, Any]: + reference_timepoint_value = reference_timepoint + if isinstance(reference_timepoint, str): + reference_timepoint_found = False + for i in range(len(moving_image)): + if moving_image[i].coords["t"].values[0] == reference_timepoint: + reference_timepoint = i + reference_timepoint_found = True + break + if not reference_timepoint_found: + raise ValueError(f"Reference timepoint not found: {reference_timepoint}.") + else: + reference_timepoint_value = ( + moving_image[reference_timepoint].coords["t"].values[0] + ) + return reference_timepoint, reference_timepoint_value + + def single_registration( fixed_tuple: tuple[tuple[str, ...], list[str | Group], dict] | None, moving_tuple: tuple[tuple[str, ...], list[str | Group], dict], @@ -130,51 +174,44 @@ def single_registration( transform_dest = f"{transform_output_dir}{transform_fs.sep}{image_key}" moving_label_keys = [] + # when registering self, transforms written to output_dir/time={t}/ + moving_image = _images2fov( + moving_file_list, + moving_metadata, + dask=True, + concat_dims=("c",), + ) + if register_self: + reference_timepoint, reference_timepoint_value = _get_reference_timepoint( + reference_timepoint=reference_timepoint, moving_image=moving_image + ) if moving_labels is not None: - matching_label_prefix = image_key - if register_self: - matching_label_prefix = f"{matching_label_prefix}-{reference_timepoint}" + matching_label_prefix = ( + f"{image_key}-{reference_timepoint_value}" if register_self else image_key + ) + for moving_label in moving_labels: moving_label_keys.extend( get_matching_names( image_key=matching_label_prefix, image_dir=moving_label, labels=True ) ) - moving_label_keys = sorted(moving_label_keys) + moving_label_keys = natsorted(moving_label_keys) if len(moving_label_keys) == 0: logger.warning(f"No labels found for {image_key}") - if not force: - labels_exist = True - if label_output_root is not None: - if not register_self: - for key in moving_label_keys: - key = os.path.basename(key) - if not is_ome_zarr_array(label_output_root.get(f"labels/{key}")): - labels_exist = False - break - # TODO check for transformed labels when register_self - - image_exists = True - if image_output_root is not None: - if not is_ome_zarr_array(image_output_root.get(f"images/{image_key}")): - image_exists = False - elif label_output_root is None: - image_exists = False - if labels_exist and image_exists: - logger.info(f"Skipping registration for {image_key}") - return image_key - - if register_self: - logger.info(f"Running registration for {image_key} t={reference_timepoint}") - logger.info( - f"{len(moving_file_list):,} {pluralize('input', len(moving_file_list))}:" - f" {', '.join([s.name.replace('/images/', '') if isinstance(s, zarr.Group) else str(s) for s in moving_file_list])}" - ) - else: - logger.info(f"Running registration for {image_key}") + if not force and _output_exists( + register_self, + label_output_root, + moving_label_keys, + image_output_root, + image_key, + ): + logger.info(f"Skipping registration for {image_key}") + return image_key if not register_self: + logger.info(f"Running registration for {image_key}") _, fixed_file_list, fixed_metadata = fixed_tuple assert fixed_metadata["id"] == moving_metadata["id"], ( @@ -192,12 +229,6 @@ def single_registration( fixed_image = _z_projection(fixed_image, z_index).isel( t=0, c=fixed_channel, missing_dims="ignore" ) - moving_image = _images2fov( - moving_file_list, - moving_metadata, - dask=True, - concat_dims=("c",), - ) parameter_object = _load_itk_parameters(itk_parameters) parameter_object_across_channels = ( @@ -353,17 +384,11 @@ def single_registration( ) else: # align to t=reference_timepoint - if isinstance(reference_timepoint, str): - reference_timepoint_found = False - for i in range(len(moving_image)): - if moving_image[i].coords["t"].values[0] == reference_timepoint: - reference_timepoint = i - reference_timepoint_found = True - break - if not reference_timepoint_found: - raise ValueError( - f"Reference timepoint not found: {reference_timepoint}." - ) + logger.info(f"Running registration for {image_key} t={reference_timepoint}") + logger.info( + f"{len(moving_file_list):,} {pluralize('input', len(moving_file_list))}:" + f" {', '.join([s.name.replace('/images/', '') if isinstance(s, zarr.Group) else str(s) for s in moving_file_list])}" + ) set_automatic_transform_initialization(parameter_object, False) if output_aligned_channels_only and not isinstance(moving_image, xr.DataArray): @@ -423,7 +448,7 @@ def _transform_labels_t( ): # transform_dest structure is image_key/t=1 # assume labels are named image_key-t-suffix - print("moving_label_keys", moving_label_keys) + for transform_file in transform_fs.ls(transform_dest, detail=True, refresh=True): if transform_file["type"] == "directory": transform_name = transform_file["name"] @@ -457,7 +482,10 @@ def _transform_labels_t( def get_matching_names( - image_key: str, image_dir: str | Group, labels: bool = True + image_key: str, + image_dir: str | Group, + labels: bool = True, + label_suffixes: Sequence[str] = {"cell", "nuclei", "cytosol"}, ) -> list[str]: """Get matching keys for the given image key and directory. @@ -472,24 +500,21 @@ def get_matching_names( # look for f'labels/image_key-{suffix} or f'images/image_key zarr_dir = "labels" if labels else "images" if isinstance(image_dir, Group): - protocol = _get_fs_protocol(_get_fs(image_dir)) + fs = _get_fs(image_dir) image_dir = f"{_get_store_path(image_dir)}{image_dir.name}" - if protocol != "file": - image_dir = f"{protocol}://{image_dir}" + image_dir = fs.unstrip_protocol(image_dir) image_fs, _ = fsspec.core.url_to_fs(image_dir) image_dir = image_dir.rstrip(image_fs.sep) glob_pattern = f"{image_dir}{image_fs.sep}{zarr_dir}{image_fs.sep}{image_key}" - if labels: - glob_pattern += "-*" - paths = image_fs.glob(glob_pattern) - protocol = _get_fs_protocol(image_fs) - if protocol != "file": - paths = [f"{protocol}://{x}" for x in paths] + results = [] - for path in paths: + for path in image_fs.glob(glob_pattern): + path = image_fs.unstrip_protocol(path) name = os.path.basename(path) + if labels and name.split("-")[-1] not in label_suffixes: + continue if not name.startswith(".") and is_ome_zarr_array(zarr.open(path, mode="r")): results.append(path) return results From cc2367555324d48d0e1f41776417ead83020524b Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Fri, 12 Jun 2026 16:24:23 -0400 Subject: [PATCH 07/79] Registration transform across times --- scallops/cli/register.py | 240 ++++++++++------ scallops/cli/register_main.py | 10 +- scallops/cli/segment.py | 78 ++++- scallops/cli/segment_main.py | 11 +- scallops/cli/util.py | 284 ++++++++++++------- scallops/tests/miniwdl_local/local_runner.py | 7 +- scallops/tests/test_register_cli.py | 61 ++-- scallops/zarr_io.py | 121 ++++---- wdl/ops_tasks.wdl | 10 +- wdl/ops_workflow.wdl | 65 +++-- wdl/utils.wdl | 65 +++-- 11 files changed, 609 insertions(+), 343 deletions(-) diff --git a/scallops/cli/register.py b/scallops/cli/register.py index 21245a3..e40bedd 100644 --- a/scallops/cli/register.py +++ b/scallops/cli/register.py @@ -13,7 +13,7 @@ import xarray as xr import zarr from dask.bag import from_sequence -from natsort import natsorted +from natsort import index_natsorted, natsorted from xarray import DataArray from zarr import Group @@ -48,6 +48,7 @@ _get_fs, _get_store_path, _write_zarr_image, + _write_zarr_labels, is_ome_zarr_array, open_ome_zarr, read_ome_zarr_array, @@ -78,24 +79,40 @@ def _output_exists( return labels_exist and image_exists -def _get_reference_timepoint( - reference_timepoint: str, moving_image: Sequence[DataArray] +def _get_timepoint_index_and_value( + timepoint: str | int, image: Sequence[DataArray] | xr.DataArray ) -> tuple[int, Any]: - reference_timepoint_value = reference_timepoint - if isinstance(reference_timepoint, str): - reference_timepoint_found = False - for i in range(len(moving_image)): - if moving_image[i].coords["t"].values[0] == reference_timepoint: - reference_timepoint = i - reference_timepoint_found = True - break - if not reference_timepoint_found: - raise ValueError(f"Reference timepoint not found: {reference_timepoint}.") + timepoint_value = None + timepoint_index = None + if isinstance(timepoint, str): + if isinstance(image, Sequence): + for i in range(len(image)): + if ( + "t" in image[i].coords + and image[i].coords["t"].values[0] == timepoint + ): + timepoint_index = i + timepoint_value = image[i].coords["t"].values[0] + break + elif "t" in image.coords: + times = list(image.coords["t"].values) + if timepoint in times: + timepoint_index = times.index(timepoint) + timepoint_value = times[timepoint_index] + + if timepoint_index is None: + raise ValueError(f"Reference timepoint not found: {timepoint}.") + elif isinstance(timepoint, int): + timepoint_index = timepoint + if isinstance(image, Sequence): + if "t" in image[timepoint_index].coords: + timepoint_value = image[timepoint_index].coords["t"].values[0] + elif "t" in image.coords: + times = list(image.coords["t"].values) + timepoint_value = times[timepoint_index] else: - reference_timepoint_value = ( - moving_image[reference_timepoint].coords["t"].values[0] - ) - return reference_timepoint, reference_timepoint_value + raise ValueError() + return timepoint_index, timepoint_value def single_registration( @@ -111,7 +128,8 @@ def single_registration( moving_labels: list[str], moving_image_spacing: tuple[float, float] | None, fixed_image_spacing: tuple[float, float] | None, - reference_timepoint: int | str, + moving_timepoint: int | str, + fixed_timepoint: int | str, unroll_channels: bool = False, force: bool = False, z_index: int | str = "max", @@ -149,8 +167,8 @@ def single_registration( :param moving_image_spacing: Spacing of the moving image. :param fixed_image_spacing: Spacing of the fixed image. :param force: Whether to overwrite existing output - :param reference_timepoint: Index or value of timepoint to register to when registering - across timepoints + :param moving_timepoint: Index or value of moving timepoint to use + :param fixed_timepoint: Index or value of fixed timepoint to use :param unroll_channels: Whether to unroll channels across timepoints. :param landmarks_initialize: Use landmarks to initialize registration. :param landmark_slice_size: The slice size in physical coordinates @@ -174,31 +192,28 @@ def single_registration( transform_dest = f"{transform_output_dir}{transform_fs.sep}{image_key}" moving_label_keys = [] - # when registering self, transforms written to output_dir/time={t}/ + # when registering self, transforms written to output_dir/time={t_value}/ moving_image = _images2fov( moving_file_list, moving_metadata, dask=True, concat_dims=("c",), ) - if register_self: - reference_timepoint, reference_timepoint_value = _get_reference_timepoint( - reference_timepoint=reference_timepoint, moving_image=moving_image - ) - if moving_labels is not None: - matching_label_prefix = ( - f"{image_key}-{reference_timepoint_value}" if register_self else image_key - ) + moving_timepoint, moving_timepoint_value = _get_timepoint_index_and_value( + moving_timepoint, moving_image + ) + if moving_labels is not None: for moving_label in moving_labels: moving_label_keys.extend( get_matching_names( - image_key=matching_label_prefix, image_dir=moving_label, labels=True + image_key=image_key, image_dir=moving_label, labels=True ) ) moving_label_keys = natsorted(moving_label_keys) if len(moving_label_keys) == 0: - logger.warning(f"No labels found for {image_key}") + logger.warning(f"No labels found for {image_key}.") + return image_key if not force and _output_exists( register_self, @@ -224,12 +239,16 @@ def single_registration( concat_dims=("c",), dask=True, ) - if isinstance(fixed_image, Sequence): - fixed_image = fixed_image[0] - fixed_image = _z_projection(fixed_image, z_index).isel( - t=0, c=fixed_channel, missing_dims="ignore" + fixed_timepoint, fixed_timepoint_value = _get_timepoint_index_and_value( + fixed_timepoint, fixed_image ) - + if isinstance(fixed_image, Sequence): + fixed_image = fixed_image[fixed_timepoint] + else: + fixed_image = fixed_image.isel( + t=fixed_timepoint, c=fixed_channel, missing_dims="ignore" + ) + fixed_image = _z_projection(fixed_image, z_index) parameter_object = _load_itk_parameters(itk_parameters) parameter_object_across_channels = ( _load_itk_parameters(itk_channel_parameters) @@ -242,8 +261,15 @@ def single_registration( transform_fs.makedirs(transform_dest, exist_ok=True) if not register_self: - if isinstance(moving_image, Sequence): - moving_image = moving_image[0] + moving_image = ( + moving_image[moving_timepoint].isel(c=moving_channel, missing_dims="ignore") + if isinstance(moving_image, Sequence) + else moving_image.isel( + t=moving_timepoint, c=moving_channel, missing_dims="ignore" + ) + ) + moving_image_align = _z_projection(moving_image, z_index) + if ( moving_image_spacing is None and get_image_spacing(moving_image.attrs) is None @@ -257,9 +283,6 @@ def single_registration( f"Physical size not found for fixed image for {image_key}." ) - moving_image_align = _z_projection(moving_image, z_index).isel( - t=0, c=moving_channel, missing_dims="ignore" - ) if "c" in moving_image_align.dims and moving_image_align.sizes["c"] > 1: moving_image_align = moving_image_align.median(dim="c", keep_attrs=True) @@ -383,8 +406,8 @@ def single_registration( output_root=label_output_root, ) - else: # align to t=reference_timepoint - logger.info(f"Running registration for {image_key} t={reference_timepoint}") + else: # align to t=moving_timepoint + logger.info(f"Running registration for {image_key} t={moving_timepoint}") logger.info( f"{len(moving_file_list):,} {pluralize('input', len(moving_file_list))}:" f" {', '.join([s.name.replace('/images/', '') if isinstance(s, zarr.Group) else str(s) for s in moving_file_list])}" @@ -398,10 +421,10 @@ def single_registration( moving_channel = 0 moving_image = new_moving_image if not no_version: - moving_image[reference_timepoint].attrs.update(cli_metadata()) + moving_image[moving_timepoint].attrs.update(cli_metadata()) _itk_align_reference_time_zarr( unroll_channels=unroll_channels, - reference_timepoint=reference_timepoint, + reference_timepoint=moving_timepoint, moving_image=moving_image, moving_channel=moving_channel, parameter_object=parameter_object, @@ -421,64 +444,90 @@ def single_registration( parameter_object_across_channels=parameter_object_across_channels, ) moving_image_attrs = moving_image[0].attrs.copy() + chunksize = moving_image[0].data.chunksize[-2:] del moving_image - + if moving_image_spacing is None: + moving_image_spacing = get_image_spacing(moving_image_attrs) if len(moving_label_keys) > 0: _transform_labels_t( - image_key=image_key, transform_fs=transform_fs, transform_dest=transform_dest, - moving_label_keys=moving_label_keys, + label_output_root=label_output_root, moving_image_attrs=moving_image_attrs, + moving_label_keys=moving_label_keys, moving_image_spacing=moving_image_spacing, - label_output_root=label_output_root, + moving_timepoint_value=moving_timepoint_value, + chunksize=chunksize, ) return image_key def _transform_labels_t( - image_key: str, transform_fs, transform_dest: str, - moving_label_keys: Sequence[str], - moving_image_attrs, - moving_image_spacing, label_output_root, + moving_image_attrs: dict, + moving_label_keys: Sequence[str], + moving_image_spacing: tuple[int, int], + moving_timepoint_value: str, + chunksize: tuple[int, int], ): # transform_dest structure is image_key/t=1 - # assume labels are named image_key-t-suffix - - for transform_file in transform_fs.ls(transform_dest, detail=True, refresh=True): - if transform_file["type"] == "directory": - transform_name = transform_file["name"] - basename = os.path.basename(transform_name) - if basename.startswith("t="): - time = basename[2:] - moving_label_keys_t = [] - output_label_prefix = f"{image_key}-{time}" - output_names = [] - # e.g. transform plateA-A1-IF-cell to plateA-A1-FISH-cell - for moving_label_key in moving_label_keys: - moving_label_key_basename = os.path.basename(moving_label_key) - output_label_suffix = "-" + moving_label_key_basename.split("-")[-1] - output_name = f"{output_label_prefix}{output_label_suffix}" - moving_label_keys_t.append(moving_label_key) - output_names.append(output_name) - - if len(moving_label_keys_t) > 0: - transform_parameter_object = _load_itk_parameters_from_dir( - transform_fs.unstrip_protocol(transform_name) + if len(moving_label_keys) > 0: + times = [] + transform_file_paths = [] + for transform_file in transform_fs.ls( + transform_dest, detail=True, refresh=True + ): + if transform_file["type"] == "directory": + transform_file_path = transform_file["name"] + basename = os.path.basename(transform_file_path) + if basename.startswith("t="): + times.append(basename[2:]) + transform_file_paths.append( + transform_fs.unstrip_protocol(transform_file_path) ) - if transform_parameter_object.GetNumberOfParameterMaps() > 0: - _transform_labels( + index = index_natsorted(times) + times = [times[val] for val in index] + transform_file_paths = [transform_file_paths[val] for val in index] + storage_options = {"chunks": chunksize} + for moving_label_key in moving_label_keys: + moving_label_key_basename = os.path.basename(moving_label_key) + transformed_labels = [] + times_ = [] + for i in range(len(times)): + transform_parameter_object = _load_itk_parameters_from_dir( + transform_file_paths[i] + ) + if transform_parameter_object.GetNumberOfParameterMaps() > 0: + src = read_ome_zarr_array(moving_label_key).squeeze() + times_.append(times[i]) + if src.sizes.get("t", 0) > 0 and "t" in src.coords: + src = src.sel(t=moving_timepoint_value) + transformed_labels.append( + itk_transform_labels( + image=src, transform_parameter_object=transform_parameter_object, - attrs=moving_image_attrs, - matching_keys=moving_label_keys_t, - output_names=output_names, - moving_image_spacing=moving_image_spacing, - output_root=label_output_root, + image_spacing=moving_image_spacing, ) + ) + + transformed_labels = xr.DataArray( + np.stack(transformed_labels), + dims=["t", "y", "x"], + coords={"t": times_}, + attrs=moving_image_attrs, + ) + + _write_zarr_labels( + name=moving_label_key_basename, + root=label_output_root, + metadata=None, + group_metadata=None, + labels=transformed_labels, + storage_options=storage_options, + ) def get_matching_names( @@ -506,15 +555,19 @@ def get_matching_names( image_fs, _ = fsspec.core.url_to_fs(image_dir) image_dir = image_dir.rstrip(image_fs.sep) - glob_pattern = f"{image_dir}{image_fs.sep}{zarr_dir}{image_fs.sep}{image_key}" - + if labels: + glob_pattern = f"{glob_pattern}-*" # for suffix results = [] + for path in image_fs.glob(glob_pattern): path = image_fs.unstrip_protocol(path) name = os.path.basename(path) - if labels and name.split("-")[-1] not in label_suffixes: - continue + + if labels: + tokens = name.split("-") + if tokens[-1] not in label_suffixes and tokens[:-1] == image_key: + continue if not name.startswith(".") and is_ome_zarr_array(zarr.open(path, mode="r")): results.append(path) return results @@ -782,10 +835,16 @@ def run_itk_registration(arguments: argparse.Namespace) -> None: moving_image_pattern = arguments.moving_image_pattern fixed_image_pattern = arguments.fixed_image_pattern group_by = arguments.groupby - reference_timepoint = arguments.time - if reference_timepoint is not None: + fixed_timepoint = arguments.fixed_time if arguments.fixed_time is not None else 0 + moving_timepoint = arguments.moving_time if arguments.moving_time is not None else 0 + if isinstance(fixed_timepoint, str) and fixed_timepoint.isdigit(): + try: + fixed_timepoint = int(fixed_timepoint) + except ValueError: + pass + if isinstance(moving_timepoint, str) and moving_timepoint.isdigit(): try: - reference_timepoint = int(reference_timepoint) + moving_timepoint = int(moving_timepoint) except ValueError: pass unroll_channels = arguments.unroll_channels @@ -895,7 +954,8 @@ def run_itk_registration(arguments: argparse.Namespace) -> None: image_output_root=image_output_root, moving_image_spacing=moving_image_spacing, fixed_image_spacing=fixed_image_spacing, - reference_timepoint=reference_timepoint, + fixed_timepoint=fixed_timepoint, + moving_timepoint=moving_timepoint, landmarks_initialize=landmarks_initialize, landmark_slice_size=landmark_slice_size, landmark_min_count=landmark_min_count, diff --git a/scallops/cli/register_main.py b/scallops/cli/register_main.py index 598d481..8bde07e 100644 --- a/scallops/cli/register_main.py +++ b/scallops/cli/register_main.py @@ -152,10 +152,12 @@ def _create_elastix_parser(subparsers: ArgumentParser, default_help: bool) -> No ) parser.add_argument( - "--time", - "-t", - default="0", - help="Time index (0-based) or value for alignment across timepoints", + "--moving-time", + help="Time index (0-based) or value for moving image", + ) + parser.add_argument( + "--fixed-time", + help="Time index (0-based) or value for fixed image", ) parser.add_argument( "--unroll-channels", diff --git a/scallops/cli/segment.py b/scallops/cli/segment.py index b4afe86..1447867 100644 --- a/scallops/cli/segment.py +++ b/scallops/cli/segment.py @@ -12,15 +12,19 @@ import argparse import importlib +from collections.abc import Sequence from typing import Callable, Literal, Optional import dask.array as da import fsspec import numpy as np +import xarray as xr import zarr +from array_api_compat import get_namespace from dask.bag import from_sequence from zarr import Group +from scallops.cli.register import _get_timepoint_index_and_value from scallops.cli.util import ( _create_dask_client, _create_default_dask_config, @@ -51,6 +55,7 @@ def segment_nuclei( dapi_channel: int, method: Callable, root: Group, + timepoint: int | str, z_index: int | str, min_area: float | None = None, max_area: float | None = None, @@ -71,6 +76,7 @@ def segment_nuclei( :param method: Segmentation method. :param root: Zarr hierarchy root. :param z_index: Either 'max' or z-index + :param timepoint: Time to use. :param min_area: Minimum area threshold for filtering labels. :param max_area: Maximum area threshold for filtering labels. :param chunks: Tuple specifying chunking size for Dask arrays. @@ -88,7 +94,19 @@ def segment_nuclei( logger.info(f"Skipping nuclei segmentation for {image_key}") return root logger.info(f"Running nuclei segmentation for {image_key}") - image = _images2fov(file_list, metadata, dask=True).squeeze() + image = _images2fov( + file_list, + metadata, + concat_dims=("c",), + dask=True, + ) + timepoint, timepoint_value = _get_timepoint_index_and_value(timepoint, image) + + image = ( + image[timepoint] + if isinstance(image, Sequence) + else image.isel(t=timepoint, missing_dims="ignore") + ) image = _z_projection(image, z_index) nuclei_seg_args = {} @@ -125,10 +143,17 @@ def segment_nuclei( group_metadata = { "image-label": {"source": {"image": f"../../images/{image_key}"}} } - additional_metadata = label_metadata.get(key) if label_metadata else None + additional_metadata = label_metadata.get(key) if label_metadata else dict() storage_options = None if isinstance(label_data, np.ndarray): storage_options = {"chunks": image.data.chunksize[-2:]} + if timepoint_value is not None: + label_data = xr.DataArray( + get_namespace(label_data).expand_dims(label_data, 0), + dims=["t", "y", "x"], + coords={"t": [timepoint_value]}, + ) + additional_metadata.update(label_data.attrs) _write_zarr_labels( name=f"{image_key}-{key}", root=root, @@ -150,6 +175,7 @@ def segment_cells( method: Callable, root: Group, z_index: int | str, + timepoint: int | str, min_area: float | None = None, max_area: float | None = None, chunks: None | tuple[int, int] = None, @@ -160,7 +186,6 @@ def segment_cells( cell_segmentation_rolling_ball: bool = False, cell_segmentation_sigma: Optional[float] = None, closing_radius: Optional[int] = None, - cell_segmentation_t: Optional[list[int]] = None, force: bool = False, shrink_primary: bool = False, no_version: bool = False, @@ -178,6 +203,7 @@ def segment_cells( :param method: Segmentation method. :param root: Zarr hierarchy root. :param z_index: Either 'max' or z-index + :param timepoint: Time to use. :param min_area: Minimum area threshold for filtering labels. :param max_area: Maximum area threshold for filtering labels. :param chunks: Tuple specifying chunking size for Dask arrays. @@ -188,7 +214,6 @@ def segment_cells( :param cell_segmentation_rolling_ball: Use rolling ball mask for cell segmentation. :param cell_segmentation_sigma: Standard deviation for smoothing in cell segmentation. :param closing_radius: Radius for closing operation in cell segmentation. - :param cell_segmentation_t: List of timepoints to consider for cell segmentation. :param force: Whether to overwrite existing output :param shrink_primary: Whether to shrink primary labels. :param no_version: Whether to skip version/CLI information in output. @@ -197,7 +222,20 @@ def segment_cells( if not force and is_ome_zarr_array(root.get(f"labels/{image_key}-cell")): logger.info(f"Skipping cell segmentation for {image_key}") return root - image = _images2fov(file_list, metadata, dask=True).squeeze() + image = _images2fov( + file_list, + metadata, + concat_dims=("c",), + dask=True, + ) + + timepoint, timepoint_value = _get_timepoint_index_and_value(timepoint, image) + + image = ( + image[timepoint] + if isinstance(image, Sequence) + else image.isel(t=timepoint, missing_dims="ignore") + ) image = _z_projection(image, z_index) if cyto_channel is None: cyto_channel = np.delete(np.arange(image.sizes["c"]), dapi_channel) @@ -210,7 +248,14 @@ def segment_cells( if method.__name__ in ["segment_cells_watershed", "segment_cells_propagation"]: nuclei = read_ome_zarr_array( nuclei_image_root["labels"][image_key + "-nuclei"] - ).values + ).squeeze() + if ( + timepoint_value is not None + and nuclei.sizes.get("t", 0) > 0 + and "t" in nuclei.coords + ): + nuclei = nuclei.sel(t=timepoint_value) + nuclei = nuclei.values assert nuclei.shape == ( image.sizes["y"], image.sizes["x"], @@ -223,7 +268,6 @@ def segment_cells( cell_seg_args["rolling_ball"] = cell_segmentation_rolling_ball cell_seg_args["sigma"] = cell_segmentation_sigma cell_seg_args["closing_radius"] = closing_radius - cell_seg_args["t"] = cell_segmentation_t if method.__name__ == "segment_cells_watershed": cell_seg_args["watershed_method"] = watershed_method if chunks is not None: @@ -267,10 +311,17 @@ def segment_cells( group_metadata = { "image-label": {"source": {"image": f"../../images/{image_key}"}} } - additional_metadata = label_metadata.get(key) if label_metadata else None + additional_metadata = label_metadata.get(key) if label_metadata else dict() storage_options = None if isinstance(label_data, np.ndarray): storage_options = {"chunks": image.data.chunksize[-2:]} + if timepoint_value is not None: + label_data = xr.DataArray( + get_namespace(label_data).expand_dims(label_data, 0), + dims=["t", "y", "x"], + coords={"t": [timepoint_value]}, + ) + additional_metadata.update(label_data.attrs) _write_zarr_labels( name=f"{image_key}-{key}", root=root, @@ -332,7 +383,12 @@ def run_pipeline(arguments: argparse.Namespace, nuclei: bool): output_root = open_ome_zarr(output, mode="a") kwargs = dict() - + timepoint = arguments.time if arguments.time is not None else 0 + if isinstance(timepoint, str) and timepoint.isdigit(): + try: + timepoint = int(timepoint) + except ValueError: + pass if not nuclei: kwargs["nuclei_min_area"] = arguments.nuclei_min_area kwargs["nuclei_max_area"] = arguments.nuclei_max_area @@ -353,7 +409,6 @@ def run_pipeline(arguments: argparse.Namespace, nuclei: bool): "Please provide sigma for `local` threshold" ) - kwargs["cell_segmentation_t"] = arguments.cell_segmentation_t if method in ["watershed", "watershed-intensity", "propagation"]: nuclei_label = arguments.nuclei_label if nuclei_label is None: @@ -381,10 +436,12 @@ def run_pipeline(arguments: argparse.Namespace, nuclei: bool): kwargs["clip"] = arguments.stardist_clip kwargs["pmin"] = arguments.stardist_pmin kwargs["pmax"] = arguments.stardist_pmax + method = getattr( importlib.import_module("scallops.segmentation." + method), f"{'segment_nuclei_' if nuclei else 'segment_cells_'}{method}", ) + image_seq = from_sequence( _set_up_experiment(data_path, image_pattern, group_by, subset=subset) ) @@ -399,6 +456,7 @@ def run_pipeline(arguments: argparse.Namespace, nuclei: bool): root=output_root, min_area=min_area, max_area=max_area, + timepoint=timepoint, chunks=chunks, chunk_overlap=chunk_overlap, z_index=z_index, diff --git a/scallops/cli/segment_main.py b/scallops/cli/segment_main.py index fe06e6e..d6b513f 100644 --- a/scallops/cli/segment_main.py +++ b/scallops/cli/segment_main.py @@ -71,6 +71,10 @@ def _add_common_args(parser: ArgumentParser) -> None: default=0, help="Channel index (0-based) where DAPI is found", ) + parser.add_argument( + "--time", + help="Time index (0-based) or value.", + ) parser.add_argument( "--min-area", @@ -252,13 +256,6 @@ def _add_cell_parser(subparsers: ArgumentParser, default_help: bool = True) -> N type=int, ) - parser.add_argument( - "--time", - dest="cell_segmentation_t", - help="Time indices (0-based) to include when computing cell segmentation mask. Defaults to all time points.", - type=int, - action="append", - ) parser.add_argument( "--shrink-nuclei", help="Shrink nuclei prior to subtraction of nuclei from cells to identify the " diff --git a/scallops/cli/util.py b/scallops/cli/util.py index b7360e1..928df57 100644 --- a/scallops/cli/util.py +++ b/scallops/cli/util.py @@ -24,6 +24,7 @@ import dask.array as da import fsspec import numpy as np +import pandas as pd import xarray as xr import zarr from distributed import Client @@ -474,7 +475,7 @@ def _write_img_size(file_list: list[str]): f.write("\n") -def _write_group_size(metadata: dict): +def _n_files_in_group(metadata: dict) -> int: n_tiles = len(metadata["file_metadata"]) metadata_fields = [v for v in ("c", "z") if v in metadata["file_metadata"][0]] if len(metadata_fields) > 0: @@ -484,115 +485,94 @@ def _write_group_size(metadata: dict): metadata=metadata, metadata_fields=tuple(metadata_fields) ) n_tiles = len(filepaths) - with open("group_size.txt", "wt") as f: - f.write(f"{n_tiles}") - f.write("\n") + return n_tiles def _list_images_wdl( - image_pattern: str, - urls: list[str], + image_pattern1: str, + urls1: list[str], + reference_time1: str | None, + n_cycles1: str | None, + image_pattern2: str, + urls2: list[str], + reference_time2: str | None, + n_cycles2: str | None, groupby: list[str], - reference_time: str | None, subset: list[str] | None, - batch_size_str: str | None, + batch_size: str | None, save_group_size: bool = False, - expected_cycles_str: int | None = None, ): """Used by WDL workflow to output info about images""" - from scallops.io import _set_up_experiment - batch_size = 1 - expected_cycles = None - if expected_cycles_str is not None and expected_cycles_str != "": - expected_cycles = int(expected_cycles_str) - if batch_size_str is not None and batch_size_str != "": - batch_size = int(batch_size_str) - if reference_time == "": - reference_time = None + urls1 = [url.strip() for url in urls1 if url.strip() != ""] + reference_time1 = None if reference_time1 == "" else reference_time1 + n_cycles1 = int(n_cycles1) if n_cycles1 is not None and n_cycles1 != "" else None + + urls2 = [url.strip() for url in urls2 if url.strip() != ""] + reference_time2 = None if reference_time2 == "" else reference_time2 + n_cycles2 = int(n_cycles2) if n_cycles2 is not None and n_cycles2 != "" else None + batch_size = int(batch_size) if batch_size is not None and batch_size != "" else 1 if subset is not None and ( len(subset) == 0 or (len(subset) == 1 and subset[0] == "") ): subset = None - if image_pattern != "": - groupby = [g for g in groupby if "{" + g + "}" in image_pattern] - exp_gen = _set_up_experiment( - image_path=urls, files_pattern=image_pattern, group_by=groupby, subset=subset + if image_pattern1 != "": + groupby1 = [g for g in groupby if "{" + g + "}" in image_pattern1] + if image_pattern2 != "": + groupby2 = [g for g in groupby if "{" + g + "}" in image_pattern2] + if len(urls1) > 0 and len(urls2) > 0: + groupby = groupby1 + assert groupby1 == groupby2 + elif len(urls1) > 0: + groupby = groupby1 + elif len(urls2) > 0: + groupby = groupby2 + else: + raise ValueError() + + result1 = _list_images( + urls=urls1, + image_pattern=image_pattern1, + groupby=groupby, + subset=subset, + save_group_size=save_group_size, + reference_time=reference_time1, + n_cycles=n_cycles1, ) - - groupby_t = "t" in groupby - times = None - - if not save_group_size: - with open("group_size.txt", "wt") as f: - f.write("0\n") - - with ( - open("subsets.txt", "wt") as groups_out, - open("subsets_with_t.txt", "wt") as groups_with_t_out, - ): - subset_ids = [] - subset_ids_with_reference_times = [] - first = True - - for g, file_list, metadata in exp_gen: - times = None - if first: - first = False - if save_group_size: - _write_group_size(metadata) - if not groupby_t and "t" in metadata["file_metadata"][0]: - times = [md["t"] for md in metadata["file_metadata"]] - if expected_cycles is not None: - assert len(times) == expected_cycles - t_suffix = "" - if times is not None and len(times) > 0: - t_suffix = ( - f"-{times[0]}" if reference_time is None else f"-{reference_time}" - ) - - subset_ids.append('"' + metadata["id"] + '"') - subset_ids_with_reference_times.append( - '"' + metadata["id"] + t_suffix + '"' - ) - if len(subset_ids) == batch_size: - groups_out.write(" ".join(subset_ids)) - groups_out.write("\n") - - groups_with_t_out.write(" ".join(subset_ids_with_reference_times)) - groups_with_t_out.write("\n") - - subset_ids = [] - subset_ids_with_reference_times = [] - if len(subset_ids) > 0: - groups_out.write(" ".join(subset_ids)) - groups_out.write("\n") - - groups_with_t_out.write(" ".join(subset_ids_with_reference_times)) - groups_with_t_out.write("\n") - - with open("groupby.txt", "wt") as f: - for g in groupby: - f.write(g) + result2 = _list_images( + urls=urls2, + image_pattern=image_pattern2, + groupby=groupby, + subset=subset, + save_group_size=save_group_size, + reference_time=reference_time2, + n_cycles=n_cycles2, + ) + results = [result1, result2] + + if len(urls1) > 0 and len(urls2) > 0: + df1 = result1["subset_df"] + df2 = result2["subset_df"] + subset_ids = df1.index.intersection(df2.index) + result1["subset_df"] = df1.loc[subset_ids] + result2["subset_df"] = df2.loc[subset_ids] + elif len(urls1) > 0: + subset_ids = result1["subset_df"].index.values + elif len(urls2) > 0: + subset_ids = result2["subset_df"].index.values + + with open("subsets.txt", "wt") as f: # ["plate1-A1", "plate1-A2", ...] + for i in range(0, len(subset_ids), batch_size): + selected = subset_ids[i : i + batch_size] + f.write(" ".join(selected)) f.write("\n") - groupby_with_t = list(groupby) - - if not groupby_t and times is not None: - groupby_with_t.append("t") - with open("groupby_with_t.txt", "wt") as f: - for g in groupby_with_t: + with open("groupby_array.txt", "wt") as f: # ['plate', 'well'] + for g in groupby: f.write(g) f.write("\n") - - with open("t.txt", "wt") as f: - if times is not None: - for val in times: - f.write(str(val)) - f.write("\n") - - with open("groupby_pattern.txt", "wt") as f: + with open("groupby_pattern.txt", "wt") as f: # "{plate}-{well}" first = True for g in groupby: if not first: @@ -601,19 +581,115 @@ def _list_images_wdl( f.write("{") f.write(g) f.write("}") + for index in range(len(results)): + result = results[index] + url_val = index + 1 + subset_df = result["subset_df"] + + with open(f"group_size_{url_val}.txt", "wt") as f: + f.write(f"{result['group_size']}") + f.write("\n") + + with open(f"reference_time_{url_val}.txt", "wt") as f: # IF + f.write(result["reference_time"]) + f.write("\n") + with open(f"times_{url_val}.txt", "wt") as f: # ["FISH", "IF"] + if result["times"] is not None: + for val in result["times"]: + f.write(str(val)) + f.write("\n") + + with open( + f"subsets_with_reference_time_{url_val}.txt", "wt" + ) as f: # ["plate1-A1-IF", "plate1-A2-IF", ...] + subset_ids_with_reference_times = ( + subset_df["subset_ids_with_reference_times"].values + if subset_df is not None + else [] + ) + for i in range(0, len(subset_ids_with_reference_times), batch_size): + selected = subset_ids_with_reference_times[i : i + batch_size] + f.write(" ".join(selected)) + f.write("\n") + + with open( + f"image_pattern_with_reference_time_{url_val}.txt", "wt" + ) as f: # "{plate}-{well}-IF" + first = True + for g in groupby: + if not first: + f.write("-") + first = False + f.write("{") + f.write(g) + f.write("}") + f.write(f"-{result['reference_time']}") + + +def _list_images( + urls: Sequence[str], + image_pattern: str, + groupby: list[str], + subset: list[str] | None, + save_group_size: bool, + reference_time: str | None, + n_cycles: int | None, +): reference_time_suffix = "" - if reference_time is not None: - reference_time_suffix = f"-{reference_time}" - elif times is not None and len(times) > 0: - reference_time_suffix = f"-{times[0]}" + group_size = 0 + if len(urls) == 0: + return dict( + group_size=group_size, + subset_df=None, + times=None, + reference_time_suffix=reference_time_suffix, + ) + from scallops.io import _set_up_experiment - with open("groupby_pattern_with_reference_t.txt", "wt") as f: - first = True - for g in groupby: - if not first: - f.write("-") + exp_gen = _set_up_experiment( + image_path=urls, files_pattern=image_pattern, group_by=groupby, subset=subset + ) + + groupby_t = "t" in groupby + times = None + subset_ids = [] + subset_ids_with_reference_times = [] + first = True + + for g, file_list, metadata in exp_gen: + times = None + if first: first = False - f.write("{") - f.write(g) - f.write("}") - f.write(reference_time_suffix) + if save_group_size: + group_size = _n_files_in_group(metadata) + if not groupby_t and "t" in metadata["file_metadata"][0]: + times = [md["t"] for md in metadata["file_metadata"]] + if n_cycles is not None: + assert len(times) == n_cycles + t_suffix = "" + if times is not None and len(times) > 0: + t_suffix = ( + f"-{times[0]}" if reference_time is None else f"-{reference_time}" + ) + + subset_ids.append('"' + metadata["id"] + '"') + subset_ids_with_reference_times.append('"' + metadata["id"] + t_suffix + '"') + subset_df = pd.DataFrame( + index=subset_ids, + data=dict(subset_ids_with_reference_times=subset_ids_with_reference_times), + ) + + groupby_with_t = list(groupby) + if not groupby_t and times is not None: + groupby_with_t.append("t") + + if reference_time is None: + reference_time = times[0] if times is not None and len(times) > 0 else "0" + + return dict( + group_size=group_size, + subset_df=subset_df, + groupby=groupby, + times=times, + reference_time=reference_time, + ) diff --git a/scallops/tests/miniwdl_local/local_runner.py b/scallops/tests/miniwdl_local/local_runner.py index c9b1392..6207931 100644 --- a/scallops/tests/miniwdl_local/local_runner.py +++ b/scallops/tests/miniwdl_local/local_runner.py @@ -15,8 +15,13 @@ def global_init(cls, cfg, logger): """ Perform any necessary process-wide initialization of the container backend """ + cpu_count = os.environ.get("SCALLOPS_MINIWDL_CPU") + if cpu_count is None: + cpu_count = psutil.cpu_count() + else: + cpu_count = int(cpu_count) cls._resource_limits = { - "cpu": psutil.cpu_count(), + "cpu": cpu_count, "mem_bytes": psutil.virtual_memory().total, } diff --git a/scallops/tests/test_register_cli.py b/scallops/tests/test_register_cli.py index d2cf5b8..9be1113 100644 --- a/scallops/tests/test_register_cli.py +++ b/scallops/tests/test_register_cli.py @@ -118,7 +118,7 @@ def test_register_itk_cli_known_shift(tmp_path): "scallops", "registration", "elastix", - "--time", + "--moving-time", "1", "--moving", str(data_path), @@ -147,10 +147,11 @@ def test_register_itk_cli_t_reference(tmp_path, array_A1_102_nuclei): tmp_path, "registration-input.zarr" ) exp = Experiment() - reference_t = 2 + + reference_t_index = 2 test_t = 10 array_A1_102_nuclei = array_A1_102_nuclei.squeeze() - exp.labels[f"A1-102-{reference_t}-mask"] = array_A1_102_nuclei + exp.labels["A1-102-nuclei"] = array_A1_102_nuclei exp.save(registration_input_moving_labels_path) cmd = [ @@ -179,8 +180,8 @@ def test_register_itk_cli_t_reference(tmp_path, array_A1_102_nuclei): registration_input_moving_labels_path, "--label-output", elastix_output_dir, - "--time", - str(reference_t), + "--moving-time", + str(reference_t_index), ] subprocess.check_call(cmd) result_exp = read_experiment(elastix_output_dir) @@ -195,16 +196,24 @@ def test_register_itk_cli_t_reference(tmp_path, array_A1_102_nuclei): .images["A1-102"] .squeeze() ) - assert len(result_exp.labels.keys()) == 8 + times = list(original_image.t.values) + del times[reference_t_index] + times = [str(t) for t in times] + transformed_times = list(result_exp.labels["A1-102-nuclei"].coords["t"].values) + transformed_times = [str(t) for t in transformed_times] + assert times == transformed_times, ( + f"{', '.join(times)} != {', '.join(transformed_times)}" + ) + assert len(result_exp.labels.keys()) == 1 np.testing.assert_array_equal(transformed_image.t.values, original_image.t.values) np.testing.assert_array_equal(transformed_image.c.values, original_image.c.values) np.testing.assert_array_equal( - transformed_image.isel(t=reference_t), - original_image.isel(t=reference_t), + transformed_image.isel(t=reference_t_index), + original_image.isel(t=reference_t_index), err_msg="Reference t not equal via CLI", ) for t in range(original_image.sizes["t"]): - if t != reference_t: + if t != reference_t_index: with np.testing.assert_raises(AssertionError): np.testing.assert_array_equal( transformed_image.isel(t=t), original_image.isel(t=t) @@ -232,8 +241,9 @@ def test_register_itk_cli_t_reference(tmp_path, array_A1_102_nuclei): image_spacing=(1, 1), ) assert warped_labels.min() == 0 + np.testing.assert_array_equal( - result_exp.labels[f"A1-102-{test_t}-mask"].values, + result_exp.labels["A1-102-nuclei"].sel(t=str(test_t)).values, warped_labels, err_msg=f"t {test_t} labels not equal using itk_transform_labels and CLI", ) @@ -245,7 +255,7 @@ def test_register_itk_cli_t_reference(tmp_path, array_A1_102_nuclei): moving_channel=[0], parameter_object=parameter_object, moving_image_spacing=(1, 1), - reference_timepoint=reference_t, + reference_timepoint=reference_t_index, ) xr.testing.assert_equal(result_np, transformed_image) @@ -308,19 +318,25 @@ def test_register_transform_labels_moving_only(tmp_path): output_zarr = tmp_path / "out.zarr" output_transforms = tmp_path / "transforms" - img = read_image( - "scallops/tests/data/experimentC/10X_c0-DAPI-p65ab/10X_c0-DAPI-p65ab_A1_Tile-102.phenotype.tif" - ) - img.attrs["physical_pixel_sizes"] = (1, 1) - rng = np.random.default_rng(0) + img = rng.integers(low=0, high=10, size=(2, 2, 100, 100)) + img = xr.DataArray( + img, + dims=["t", "c", "y", "x"], + coords={"t": ["IF", "FISH"]}, + attrs={"physical_pixel_sizes": (1, 1)}, + ) - segmentation = rng.integers(low=0, high=10, size=(img.sizes["y"], img.sizes["x"])) + segmentation = rng.integers(low=0, high=10, size=(100, 100)) Experiment( - images={"plateA-A1-IF": img, "plateA-A1-FISH": img}, + images={"plateA-A1": img}, labels={ - "plateA-A1-IF-cell": segmentation, + "plateA-A1-cell": xr.DataArray( + np.expand_dims(segmentation, 0), + dims=["t", "y", "x"], + coords={"t": ["IF"]}, + ), }, ).save(image_zarr) cmd = [ @@ -330,7 +346,7 @@ def test_register_transform_labels_moving_only(tmp_path): "--moving", str(image_zarr), "--moving-image-pattern", - "{plate}-{well}-{t}", + "{plate}-{well}", "--moving-label", str(image_zarr), "--subset", @@ -344,7 +360,7 @@ def test_register_transform_labels_moving_only(tmp_path): "--label-output", str(output_zarr), "--output-aligned-channels-only", - "--time", + "--moving-time", "IF", "--transform-output", str(output_transforms), @@ -352,7 +368,8 @@ def test_register_transform_labels_moving_only(tmp_path): create_itk_param_file(tmp_path), ] subprocess.check_call(cmd) - transformed_labels = read_image(output_zarr / "labels" / "plateA-A1-FISH-cell") + transformed_labels = read_image(output_zarr / "labels" / "plateA-A1-cell") + assert list(transformed_labels.coords["t"].values) == ["FISH"] assert transformed_labels.max() > 0 transformed_image = read_image(output_zarr / "images" / "plateA-A1") assert transformed_image.shape[0] == 2 diff --git a/scallops/zarr_io.py b/scallops/zarr_io.py index 9f04010..77facc6 100644 --- a/scallops/zarr_io.py +++ b/scallops/zarr_io.py @@ -285,6 +285,65 @@ def _attrs_axes_coordinates( return image_attrs, axes, coordinate_transformations +def _write_zarr_labels( + name: str, + root: zarr.Group | str | Path, + labels: np.ndarray | xr.DataArray | da.Array, + metadata: dict[str, Any] | None = None, + group_metadata: dict[str, Any] | None = None, + compute: bool = True, + storage_options: JSONDict | None = None, +) -> list[Delayed]: + """Write label in zarr format. + + :param name: Zarr group name to store label + :param root: Root zarr group. + :param labels: Labels to write. + :param metadata: Optional label metadata. + :param group_metadata: Optional group level metadata. + :param compute: If true compute immediately otherwise a list + of :class:`dask.delayed.Delayed` is returned. + :param storage_options: Optional storage options. + :return: Empty list if the compute flag is True, otherwise it returns a list + of :class:`dask.delayed.Delayed` representing the value to be computed by dask. + """ + + # stored in labels/key + if isinstance(root, (str, Path)): + root = open_ome_zarr(root, mode="a") + + labels_grp = root.require_group("labels", overwrite=False) + dest_grp = labels_grp.create_group(name.replace("/", "-"), overwrite=True) + + label_attrs = None + coords = None + dims = None + if isinstance(labels, xr.DataArray): + data = labels.data + label_attrs = labels.attrs.copy() + coords = labels.coords + dims = labels.dims + else: + data = labels + + # need 'image-label' attr to be recognized as label + group_metadata = group_metadata.copy() if group_metadata is not None else dict() + if "image-label" not in group_metadata: + group_metadata["image-label"] = {} + metadata = metadata.copy() if metadata is not None else {} + + return write_zarr( + grp=dest_grp, + data=data, + image_attrs=label_attrs, + coords=coords, + dims=dims, + metadata=metadata, + zarr_format="ome_zarr", + compute=compute, + ) + + def _write_zarr_image( name: str | None, root: zarr.Group | str | Path, @@ -509,68 +568,6 @@ def rechunk(image: xr.DataArray | da.Array) -> xr.DataArray | da.Array: return image -def _write_zarr_labels( - name: str, - root: zarr.Group | str | Path, - labels: np.ndarray | xr.DataArray | da.Array, - metadata: dict[str, Any] = None, - group_metadata: dict[str, Any] = None, - compute: bool = True, - storage_options: JSONDict | None = None, -) -> list[Delayed]: - """Write label in zarr format. - - :param name: Zarr group name to store label - :param root: Root zarr group. - :param labels: Labels to write. - :param metadata: Optional label metadata. - :param group_metadata: Optional group level metadata. - :param compute: If true compute immediately otherwise a list - of :class:`dask.delayed.Delayed` is returned. - :param storage_options: Optional storage options. - :return: Empty list if the compute flag is True, otherwise it returns a list - of :class:`dask.delayed.Delayed` representing the value to be computed by dask. - """ - - # stored in labels/key - name = name.replace("/", "-") - if isinstance(root, (str, Path)): - root = open_ome_zarr(root, mode="a") - labels_grp = root.require_group("labels") - grp = labels_grp.create_group(name, overwrite=True) - if not isinstance(labels, xr.DataArray): - if labels.ndim == 2: - label_axes = ["y", "x"] - elif labels.ndim == 5: - label_axes = ["t", "c", "z", "y", "x"] - else: - raise ValueError("Axes can't be inferred for 3D or 4D labels") - else: - label_axes = labels.dims - labels = labels.data - - # need 'image-label' attr to be recognized as label - group_metadata = group_metadata.copy() if group_metadata is not None else dict() - if "image-label" not in group_metadata: - group_metadata["image-label"] = {} - grp.attrs.update(group_metadata) - metadata = metadata.copy() if metadata is not None else {} - if isinstance(labels, da.Array) or ( - isinstance(labels, xr.DataArray) and isinstance(labels.data, da.Array) - ): - labels = rechunk(labels) - return write_image( - labels, - grp, - scaler=None, - # scale_factors=[], - axes=label_axes, - metadata=metadata, - compute=compute, - storage_options=storage_options, - ) - - def _read_zarr_attrs(attrs) -> tuple[dict, dict, list[str]]: """Read attributes from Zarr. diff --git a/wdl/ops_tasks.wdl b/wdl/ops_tasks.wdl index 396bdb5..d1adc5a 100644 --- a/wdl/ops_tasks.wdl +++ b/wdl/ops_tasks.wdl @@ -7,6 +7,7 @@ task segment_nuclei { String? image_pattern Array[String] groupby Int? dapi_channel + String? time String output_directory String subset Boolean? force @@ -33,6 +34,7 @@ task segment_nuclei { --groupby ~{sep=" " groupby} \ ~{if defined(image_pattern) then '--image-pattern "' + image_pattern + '"' else ''} \ ~{'--dapi-channel ' + dapi_channel} \ + ~{'--time ' + time} \ --output "~{output_directory}" \ --subset ~{subset} \ ~{if defined(extra_arguments) then extra_arguments else ''} \ @@ -64,6 +66,7 @@ task segment_cell { Array[String] groupby Int? dapi_channel Array[Int] cyto_channel + String? time Int? chunks String? nuclei_label String? threshold @@ -94,6 +97,7 @@ task segment_cell { --groupby ~{sep=" " groupby} \ ~{if defined(image_pattern) then '--image-pattern "' + image_pattern + '"' else ''} \ ~{'--dapi-channel ' + dapi_channel} \ + ~{'--time ' + time} \ --cyto-channel ~{sep=" " cyto_channel} \ ~{"--nuclei-label " + nuclei_label} \ ~{"--method " + method} \ @@ -134,7 +138,8 @@ task register_elastix { Boolean? output_aligned_channels_only Boolean? unroll_channels String? fixed - String? reference_time + String? moving_time + String? fixed_time Int? fixed_channel Boolean? register_across_channels String transform_output_directory @@ -174,7 +179,8 @@ task register_elastix { --subset ~{subset} \ ~{if defined(label_output_directory) then '--label-output "' + label_output_directory + '"' else ''} \ ~{true="--unroll-channels" false="" unroll_channels} \ - ~{if defined(reference_time) then '--time "' + reference_time + '"' else ''} \ + ~{if defined(moving_time) then '--moving-time "' + moving_time + '"' else ''} \ + ~{if defined(fixed_time) then '--fixed-time "' + fixed_time + '"' else ''} \ ~{true="--force" false="" force} \ ~{true="--align-across-channels" false="" register_across_channels} \ ~{true="--output-aligned-channels-only" false="" output_aligned_channels_only} \ diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index eed2343..ca801af 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -211,10 +211,14 @@ workflow ops_workflow { call utils.list_images { input: - urls = [select_first([phenotype_url, iss_url])], - image_pattern = if pheno_url_supplied then phenotype_image_pattern else iss_image_pattern, + urls1 = select_all([phenotype_url]), + image_pattern1 = phenotype_image_pattern, + reference_time1=reference_phenotype_time, + + urls2 = select_all([iss_url]), + image_pattern2 = iss_image_pattern, batch_size=batch_size, - reference_time=reference_phenotype_time, + groupby=groupby, subset = subset, docker=docker, @@ -223,27 +227,40 @@ workflow ops_workflow { aws_queue_arn = aws_queue_arn, max_retries = max_retries } - String groupby_pattern = list_images.groupby_pattern # "{plate}-{well}" - Array[String] subsets = list_images.subsets # e.g. ["plate1-A1", "plate1-A2", ...] - Array[String] subset_with_reference_times = list_images.subset_with_reference_times # e.g. ["plate1-A1-IF", "plate1-A2-IF", ...] - Array[String] times = list_images.t # e.g. ["FISH", "IF"] - Array[String] groupby_with_time = list_images.filtered_groupby_with_t # e.g. ['plate', 'well', 't'] - String groupby_pattern_with_reference_t = list_images.groupby_pattern_with_reference_t # e.g. "{plate}-{well}-IF" + Array[String] subsets = list_images.subsets + String groupby_pattern = list_images.groupby_pattern # e.g. {plate}-{well} + Array[String] groupby_array = list_images.groupby_array # e.g. ["plate", "well"] + + + # Array[String] subsets_with_reference_times_pheno = list_images.subsets_with_reference_times_1 + # Array[String] subsets_with_reference_times_iss = list_images.subsets_with_reference_times_2 + + Array[String] times_pheno = list_images.times_1 + Array[String] times_iss = list_images.times_2 + + String reference_time_pheno = list_images.reference_time_1 + String reference_time_iss = list_images.reference_time_2 + + #String image_pattern_with_reference_time_pheno = list_images.image_pattern_with_reference_time_1 # e.g. {plate}-{well}-IF + # String image_pattern_with_reference_time_iss = list_images.image_pattern_with_reference_time_2 # e.g. {plate}-{well}-1 scatter (subset_index in range(length(subsets))) { String subset_ = subsets[subset_index] - String subset_with_reference_time = subset_with_reference_times[subset_index] + # String subset_with_reference_times_pheno = subsets_with_reference_times_pheno[subset_index] + # String subset_with_reference_times_iss = subsets_with_reference_times_iss[subset_index] if(pheno_url_supplied) { if(run_nuclei_segmentation) { call tasks.segment_nuclei { input: images = select_first([phenotype_url]), image_pattern = phenotype_image_pattern, + time=reference_time_pheno, + subset = subset_, method = nuclei_segmentation_method, - groupby=groupby_with_time, + groupby=groupby, dapi_channel = phenotype_dapi_channel, output_directory=segment_directory, model_dir=model_dir, - subset = subset_with_reference_time, + extra_arguments=nuclei_segmentation_extra_arguments, force = force_segment_nuclei, docker=docker, @@ -262,9 +279,10 @@ workflow ops_workflow { input: images = select_first([phenotype_url]), image_pattern = phenotype_image_pattern, + time=reference_time_pheno, method = cell_segmentation_method, - groupby = groupby_with_time, - subset = subset_with_reference_time, + groupby = groupby, + subset = subset_, dapi_channel = phenotype_dapi_channel, cyto_channel=phenotype_cyto_channel, nuclei_label=select_first([segment_nuclei.output_url]), @@ -285,14 +303,15 @@ workflow ops_workflow { max_retries = max_retries } - if(length(times)>1) { + if(length(times_pheno)>1) { call tasks.register_elastix as register_pheno_to_pheno { input: moving=select_all([phenotype_url]), moving_label=segment_cell.output_url, moving_channel=phenotype_dapi_channel, moving_image_pattern=phenotype_image_pattern, - reference_time=reference_phenotype_time, + moving_time=reference_time_pheno, + extra_arguments=pheno_registration_extra_arguments, output_aligned_channels_only=true, groupby=groupby, @@ -375,7 +394,7 @@ workflow ops_workflow { # use stitch mask as image and segment output for reference phenotype or transformed phenotype for others if(mark_stitch_boundary_cells) { - String phenotype_url_stripped = if (pheno_url_supplied) then sub(select_first([phenotype_url]), "/+$", "") else "" + String phenotype_url_stripped = sub(select_first([phenotype_url]), "/+$", "") call tasks.intersects_boundary as cell_intersects_boundary { input: @@ -435,9 +454,11 @@ workflow ops_workflow { fixed_channel=iss_dapi_channel, moving_label=segment_cell.output_url, moving=select_all([phenotype_url]), - moving_image_pattern=groupby_pattern_with_reference_t, + moving_image_pattern=phenotype_image_pattern, fixed_image_pattern=iss_image_pattern, moving_channel=phenotype_dapi_channel, + moving_time=reference_time_pheno, + fixed_time=reference_time_iss, output_aligned_channels_only=true, moving_output_directory=register_pheno_to_iss_directory, label_output_directory=register_pheno_to_iss_directory, @@ -459,10 +480,10 @@ workflow ops_workflow { # ISS t0 to phenotype reference time call tasks.register_pheno_to_iss_qc as register_pheno_to_iss_qc { input: - images=select_first([iss_url]), - image_pattern=iss_image_pattern, - stacked_images=register_pheno_to_iss.moving_output_url, - stacked_image_pattern=phenotype_image_pattern, + images=register_pheno_to_iss.moving_output_url, + image_pattern=groupby_pattern, + stacked_images=select_first([iss_url]), + stacked_image_pattern=groupby_pattern, image_channel=iss_dapi_channel, stacked_image_channel=0, label_type='nuclei', diff --git a/wdl/utils.wdl b/wdl/utils.wdl index 393b806..0daf6aa 100644 --- a/wdl/utils.wdl +++ b/wdl/utils.wdl @@ -1,14 +1,22 @@ -version 1.1 +version 1.0 task list_images { input { - Boolean? save_group_size - Array[String] urls - String? image_pattern - String? reference_time + + Array[String]? urls1 + String? image_pattern1 + String? reference_time1 + Int? n_cycles1 + + Array[String]? urls2 + String? image_pattern2 + String? reference_time2 + Int? n_cycles2 + Array[String] groupby Array[String]? subset - Int? expected_cycles + + Boolean? save_group_size Int? batch_size String docker String zones @@ -26,28 +34,47 @@ task list_images { python <>> output { Array[String] subsets = read_lines('subsets.txt') - Array[String] subset_with_reference_times = read_lines('subsets_with_t.txt') - Array[String] t = read_lines('t.txt') String groupby_pattern = read_lines('groupby_pattern.txt')[0] # e.g. {plate}-{well} - String groupby_pattern_with_reference_t = read_lines('groupby_pattern_with_reference_t.txt')[0] # e.g. {plate}-{well}-IF - Array[String] filtered_groupby_with_t = read_lines('groupby_with_t.txt') # e.g. [plate, well, t] - Array[String] filtered_groupby = read_lines('groupby.txt') # e.g. [plate, well] + Array[String] groupby_array = read_lines('groupby_array.txt') # e.g. ["plate", "well"] + + Int group_size_1 = read_int('group_size_1.txt') + Int group_size_2 = read_int('group_size_2.txt') + Array[String] subsets_with_reference_times_1 = read_lines('subsets_with_reference_time_1.txt') + Array[String] subsets_with_reference_times_2 = read_lines('subsets_with_reference_time_2.txt') + + Array[String] times_1 = read_lines('times_1.txt') + Array[String] times_2 = read_lines('times_2.txt') + + String reference_time_1 = read_lines('reference_time_1.txt')[0] + String reference_time_2 = read_lines('reference_time_2.txt')[0] + + String image_pattern_with_reference_time_1 = read_lines('image_pattern_with_reference_time_1.txt')[0] # e.g. {plate}-{well}-IF + String image_pattern_with_reference_time_2 = read_lines('image_pattern_with_reference_time_2.txt')[0] # e.g. {plate}-{well}-IF - Int group_size = read_int('group_size.txt') } meta { From e52424364b22177cc21620ea40e51866dcdb7191 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Fri, 12 Jun 2026 16:38:02 -0400 Subject: [PATCH 08/79] Registration transform across times --- scallops/cli/register.py | 11 +++++------ scallops/tests/test_register_cli.py | 6 +++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/scallops/cli/register.py b/scallops/cli/register.py index e40bedd..8f695db 100644 --- a/scallops/cli/register.py +++ b/scallops/cli/register.py @@ -68,7 +68,6 @@ def _output_exists( if not is_ome_zarr_array(label_output_root.get(f"labels/{key}")): labels_exist = False break - # TODO check for transformed labels when register_self image_exists = True if image_output_root is not None: @@ -262,13 +261,13 @@ def single_registration( if not register_self: moving_image = ( - moving_image[moving_timepoint].isel(c=moving_channel, missing_dims="ignore") + moving_image[moving_timepoint] if isinstance(moving_image, Sequence) - else moving_image.isel( - t=moving_timepoint, c=moving_channel, missing_dims="ignore" - ) + else moving_image.isel(t=moving_timepoint, missing_dims="ignore") + ) + moving_image_align = _z_projection( + moving_image.isel(c=moving_channel, missing_dims="ignore"), z_index ) - moving_image_align = _z_projection(moving_image, z_index) if ( moving_image_spacing is None diff --git a/scallops/tests/test_register_cli.py b/scallops/tests/test_register_cli.py index 9be1113..14a6118 100644 --- a/scallops/tests/test_register_cli.py +++ b/scallops/tests/test_register_cli.py @@ -401,6 +401,9 @@ def _warp(img): dims=["c", "y", "x"], ) moving_image = moving_image.isel(c=[0, 1, 2]) + moving_image.coords["c"] = ["a", "b", "c"] + moving_image.attrs["physical_pixel_sizes"] = (2, 2) + moving_image.attrs["physical_pixel_units"] = ("micrometer", "micrometer") moving_labels = array_A1_102_nuclei.squeeze().values moving_labels = warp(moving_labels, st, order=0, preserve_range=True) @@ -413,9 +416,6 @@ def _warp(img): ), dims=["y", "x"], ) - moving_image.coords["c"] = ["a", "b", "c"] - moving_image.attrs["physical_pixel_sizes"] = (2, 2) - moving_image.attrs["physical_pixel_units"] = ("micrometer", "micrometer") fixed_image = xr.DataArray( resize( From 6fc0e5546c42308807297ac84d17158baaad88e4 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Fri, 12 Jun 2026 16:52:24 -0400 Subject: [PATCH 09/79] storage options --- scallops/zarr_io.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/scallops/zarr_io.py b/scallops/zarr_io.py index 77facc6..56acd35 100644 --- a/scallops/zarr_io.py +++ b/scallops/zarr_io.py @@ -317,7 +317,7 @@ def _write_zarr_labels( label_attrs = None coords = None - dims = None + dims = ["y", "x"] if isinstance(labels, xr.DataArray): data = labels.data label_attrs = labels.attrs.copy() @@ -341,6 +341,7 @@ def _write_zarr_labels( metadata=metadata, zarr_format="ome_zarr", compute=compute, + storage_options=storage_options, ) @@ -405,6 +406,7 @@ def write_zarr( metadata: dict[str, Any] | None = None, zarr_format: Literal["ome_zarr", "zarr"] = "ome_zarr", compute: bool = True, + storage_options: JSONDict | None = None, ) -> list[Delayed]: """Write data to a Zarr group with optional metadata and scaling. @@ -425,6 +427,7 @@ def write_zarr( :param compute: If True, compute immediately. Otherwise, return a list of dask.delayed. Delayed objects representing the value to be computed by dask. Default is True. + :param storage_options: Optional storage options. :return: Empty list if the compute flag is True, otherwise a list of dask.delayed.Delayed objects. @@ -467,19 +470,26 @@ def write_zarr( dask_delayed = [] fmt = _current_format() if zarr_format == "zarr": # No axis validation + chunks_opt = None + if storage_options is not None: + chunks_opt = storage_options.pop("chunks", None) if isinstance(data, da.Array): d = da.to_zarr( arr=data, url=grp.store, component=str(Path(grp.path, "0")), compute=compute, + storage_options=storage_options, **_da_to_zarr_kwargs(fmt), ) if not compute: dask_delayed.append(d) elif not isinstance(data, zarr.Array): + kwds = _da_to_zarr_kwargs(fmt) + if storage_options is not None: + kwds.update(storage_options) grp.create_dataset( - "0", data=data, overwrite=True, **_da_to_zarr_kwargs(fmt) + "0", data=data, overwrite=True, chunks=chunks_opt, **kwds ) datasets = [{"path": "0"}] @@ -529,6 +539,7 @@ def _write_metadata_delayed(grp, d): if coordinate_transformations is not None else None ), + storage_options=storage_options, ) From baa48d33dade54f512dde06e5d575138c5d6b46b Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Fri, 12 Jun 2026 16:56:08 -0400 Subject: [PATCH 10/79] pattern --- wdl/ops_workflow.wdl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index ca801af..fe2121a 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -336,7 +336,7 @@ workflow ops_workflow { call tasks.find_objects as find_objects_nuclei { input: labels=select_all([segment_nuclei.output_url, register_pheno_to_pheno.label_output_url]), - label_pattern=phenotype_image_pattern, + label_pattern=groupby_pattern, suffix="nuclei", output_directory=nuclei_objects_directory, subset = subset_, @@ -355,7 +355,7 @@ workflow ops_workflow { call tasks.find_objects as find_objects_cell { input: labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), - label_pattern=phenotype_image_pattern, + label_pattern=groupby_pattern, suffix="cell", output_directory=cell_objects_directory, subset = subset_, @@ -374,7 +374,7 @@ workflow ops_workflow { call tasks.find_objects as find_objects_cytosol { input: labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), - label_pattern=phenotype_image_pattern, + label_pattern=groupby_pattern, suffix="cytosol", output_directory=cytosol_objects_directory, subset = subset_, @@ -400,7 +400,7 @@ workflow ops_workflow { input: labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), images=phenotype_url_stripped + '/labels/', - image_pattern=phenotype_image_pattern, + image_pattern=groupby_pattern, output_directory=cell_intersects_boundary_directory, label_type='cell', objects=find_objects_cell.output_url, From a6cdfb7fa5615d7ccd221371b3080dc092be8b40 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Mon, 15 Jun 2026 12:36:50 -0400 Subject: [PATCH 11/79] handle timepoint in segmentation --- scallops/cli/find_objects.py | 80 ++++++++++++++++++++++--------- scallops/features/find_objects.py | 2 +- 2 files changed, 59 insertions(+), 23 deletions(-) diff --git a/scallops/cli/find_objects.py b/scallops/cli/find_objects.py index 88bf71e..c7fffec 100644 --- a/scallops/cli/find_objects.py +++ b/scallops/cli/find_objects.py @@ -8,6 +8,7 @@ import argparse import json +import dask.array as da import fsspec import zarr from zarr import Group @@ -29,10 +30,11 @@ def _execute( label_tuple: tuple[tuple[str, ...], list[str | Group], dict], + timepoint: str | None, output_dir: str, output_sep: str, - force: bool = False, - no_version: bool = False, + force: bool, + no_version: bool, ): group, file_list, metadata = label_tuple assert len(file_list) == 1 @@ -41,16 +43,31 @@ def _execute( path = ( f"{output_dir}{output_sep}{label_name}{output_sep}{image_key}-objects.parquet" ) + if timepoint is not None: + path = ( + f"{path}/t={timepoint}/{label_name}{output_sep}{image_key}-objects.parquet" + ) fs = fsspec.url_to_fs(path)[0] if fs.exists(path): if force: fs.rm(path, recursive=True) else: - logger.info(f"Skipping finding objects for {metadata['id']}.") + logger.info( + f"Skipping find objects for {metadata['id']}{' at t=' + timepoint if timepoint is not None else ''}." + ) return - logger.info(f"Finding objects for {metadata['id']}.") - array = file_list[0][list(file_list[0].keys())[0]] + logger.info( + f"Find objects for {metadata['id']}{' at t=' + timepoint if timepoint is not None else ''}." + ) + g = file_list[0] + array = da.from_zarr(g[list(g.keys())[0]]) + + if timepoint is not None: + timepoint_index = g.attrs["multiscales"][0]["metadata"]["t"].index(timepoint) + array = array[timepoint_index] + df = find_objects(array) + df.index.name = "label" prefix = _label_name_to_prefix.get(label_name) if prefix is not None: @@ -64,7 +81,9 @@ def _execute( if not no_version else None, ) - logger.info(f"Saved objects for {metadata['id']} to {path}.") + logger.info( + f"Saved objects for {metadata['id']}{' at t=' + timepoint if timepoint is not None else ''} to {path}." + ) def run_pipeline_find_objects(arguments: argparse.Namespace) -> None: @@ -91,33 +110,50 @@ def run_pipeline_find_objects(arguments: argparse.Namespace) -> None: output_fs, _ = fsspec.core.url_to_fs(output_dir) output_dir = output_dir.rstrip(output_fs.sep) - paths = [] + _, _, keys = _create_file_regex(label_pattern) + keys = list(keys) + label_tuples = [] + timepoints = [] + for path in labels_paths: label_root = zarr.open(path, mode="r") labels_group = label_root.get("labels") - if labels_group is None: - raise ValueError(f"Labels group not found for {path}") - paths.append(labels_group) - _, _, keys = _create_file_regex(label_pattern) - gen = _set_up_experiment( - image_path=paths, - files_pattern=label_pattern, - group_by=list(keys), - subset=subset, - ) + if labels_group is not None: + gen = _set_up_experiment( + image_path=labels_group, + files_pattern=label_pattern, + group_by=keys, + subset=subset, + ) + for label_tuple in gen: + label_key, file_list, metadata = label_tuple + + if ( + label_suffix is None + or label_key[len(label_key) - 1] in label_suffix + ): + assert len(file_list) == 1 + g = file_list[0] + zarr_metadata = g.attrs["multiscales"][0]["metadata"] + + if "t" not in zarr_metadata: + label_tuples.append(label_tuple) + timepoints.append(None) + else: + for timepoint_ in zarr_metadata["t"]: + label_tuples.append(label_tuple) + timepoints.append(timepoint_) with ( _create_default_dask_config(), _create_dask_client(dask_server_url, **dask_cluster_parameters), ): - [ + for i in range(len(label_tuples)): _execute( - label_tuple=g, + label_tuple=label_tuples[i], + timepoint=timepoints[i], output_dir=output_dir, output_sep=output_fs.sep, force=force, no_version=no_version, ) - for g in gen - if label_suffix is None or g[0][len(g[0]) - 1] in label_suffix - ] diff --git a/scallops/features/find_objects.py b/scallops/features/find_objects.py index 9c4f175..b315be1 100644 --- a/scallops/features/find_objects.py +++ b/scallops/features/find_objects.py @@ -254,7 +254,7 @@ def _agg_objects(grouped): return objects_df -def find_objects(label_image: da.Array) -> dd.DataFrame: +def find_objects(label_image: da.Array | zarr.Array) -> dd.DataFrame: """Find objects in a labeled array. :param label_image: Image labels noted by integers. From c9db0a2ca52d97fd98ecc913faee374c7fd2f34f Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Mon, 15 Jun 2026 12:44:46 -0400 Subject: [PATCH 12/79] handle timepoint in segmentation --- scallops/cli/pooled_if_sbs.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scallops/cli/pooled_if_sbs.py b/scallops/cli/pooled_if_sbs.py index 6f45daf..ceb30bc 100644 --- a/scallops/cli/pooled_if_sbs.py +++ b/scallops/cli/pooled_if_sbs.py @@ -973,9 +973,11 @@ def reads_pipeline( points_path = f"{points_path}/{image_key}-peaks.parquet" peaks = dd.read_parquet(points_path) maxed = read_ome_zarr_array(spots_root["images"][image_key + "-max"], dask=True) - labels = read_ome_zarr_array( - labels_root[image_key + "-" + label_name], dask=True - ).data.compute() + labels = ( + read_ome_zarr_array(labels_root[image_key + "-" + label_name], dask=True) + .data.squeeze() + .compute() + ) if expand_labels_distance is not None and expand_labels_distance > 0: labels = expand_labels(labels, distance=expand_labels_distance) iss_cycles = maxed.coords["t"].values From 3815857f498e13215a5e8c799d704324a3723279 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Mon, 15 Jun 2026 13:08:30 -0400 Subject: [PATCH 13/79] handle timepoint in segmentation --- scallops/cli/find_objects.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scallops/cli/find_objects.py b/scallops/cli/find_objects.py index c7fffec..d292da9 100644 --- a/scallops/cli/find_objects.py +++ b/scallops/cli/find_objects.py @@ -41,12 +41,10 @@ def _execute( label_name = group[len(group) - 1] image_key = "-".join(group[:-1]) # exclude suffix from key path = ( - f"{output_dir}{output_sep}{label_name}{output_sep}{image_key}-objects.parquet" + (f"{output_dir}{output_sep}{label_name}{output_sep}{image_key}-objects.parquet") + if timepoint is None + else f"{output_dir}{output_sep}{label_name}{output_sep}t={timepoint}{output_sep}{image_key}-objects.parquet" ) - if timepoint is not None: - path = ( - f"{path}/t={timepoint}/{label_name}{output_sep}{image_key}-objects.parquet" - ) fs = fsspec.url_to_fs(path)[0] if fs.exists(path): if force: From 3cea7ecea96bceaa98080b76d809aab53bee1190 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 16 Jun 2026 07:54:18 -0400 Subject: [PATCH 14/79] handle timepoint in segmentation --- scallops/cli/extract_crops.py | 13 +- scallops/cli/features.py | 526 ++++++++++++++++++---------------- scallops/cli/features_main.py | 3 +- scallops/cli/find_objects.py | 21 +- scallops/cli/util.py | 12 +- scallops/features/generate.py | 9 +- wdl/ops_tasks.wdl | 4 +- wdl/ops_workflow.wdl | 11 +- wdl/utils.wdl | 3 + 9 files changed, 322 insertions(+), 280 deletions(-) diff --git a/scallops/cli/extract_crops.py b/scallops/cli/extract_crops.py index 5c364e7..c5f7d03 100644 --- a/scallops/cli/extract_crops.py +++ b/scallops/cli/extract_crops.py @@ -10,7 +10,7 @@ from skimage.util import img_as_ubyte from zarr import Group -from scallops.cli.features import _read_merged_or_objects, get_labels +from scallops.cli.features import _read_merged_or_objects from scallops.cli.util import ( _get_cli_logger, cli_metadata, @@ -69,14 +69,11 @@ def single_crop( output_fs.makedirs(output_dir, exist_ok=True) image = _images2fov(file_list, metadata, dask=True).squeeze().data logger.info(f"{image_key} image shape {image.shape}") - zarr_labels = get_labels( - labels_group=labels_group, - name=image_key, - suffix=label_name, # e.g. nuclei - ) - - if zarr_labels is None: + g = labels_group.get(f"{image_key}-{label_name}") + if g is None: raise ValueError(f"Unable to read {label_name} labels for {image_key}.") + zarr_labels = g[list(g.keys())[0]] + merged_df = _read_merged_or_objects( merge_dir=merge_dir, merge_dir_sep=merge_dir_sep, diff --git a/scallops/cli/features.py b/scallops/cli/features.py index 77917cc..1e4ecce 100644 --- a/scallops/cli/features.py +++ b/scallops/cli/features.py @@ -8,15 +8,13 @@ import argparse import json -import warnings from collections.abc import Sequence from itertools import zip_longest -from typing import get_type_hints +from typing import Any, get_type_hints import dask.array import dask.array as da import fsspec -import numpy as np import pandas as pd import pyarrow as pa import pyarrow.parquet as pq @@ -26,6 +24,7 @@ from natsort import natsorted from zarr import Group +from scallops.cli.find_objects import get_path from scallops.cli.util import ( _create_dask_client, _create_default_dask_config, @@ -52,27 +51,28 @@ pluralize, read_anndata_zarr, ) -from scallops.zarr_io import _read_ome_zarr_array logger = _get_cli_logger() def _read_merged_or_objects( - merge_dir: str, - merge_dir_sep: str, + path: str, + path_sep: str, + timepoint: str | None, label_name: str, image_key: str, label_filter: str | None, ): - merge_dir = merge_dir.rstrip(merge_dir_sep) - merge_paths = [ - f"{merge_dir}{merge_dir_sep}{label_name}{merge_dir_sep}{image_key}.parquet", - f"{merge_dir}{merge_dir_sep}{image_key}.zarr", - f"{merge_dir}{merge_dir_sep}{image_key}.parquet", - f"{merge_dir}{merge_dir_sep}{label_name}{merge_dir_sep}{image_key}-objects.parquet", + path = path.rstrip(path_sep) + paths = [ + f"{path}{path_sep}{label_name}{path_sep}{image_key}.parquet", + f"{path}{path_sep}{image_key}.zarr", + f"{path}{path_sep}{image_key}.parquet", + get_path(path, path_sep, label_name, image_key, timepoint), ] + merge_path = None - for path in merge_paths: + for path in paths: if fsspec.core.url_to_fs(path)[0].exists(path): merge_path = path break @@ -103,51 +103,6 @@ def _read_merged_or_objects( return merged_df -def get_labels(labels_group: Group, name: str, suffix: str) -> zarr.Array | None: - """Retrieve labels from a zarr group. - - :param labels_group: The zarr group containing labels. - :param name: The identifier associated with the labels. - :param suffix: The suffix used to identify the specific set of labels (e.g., 'nuclei'). - :return: The retrieved labels as a DataArray or None if the labels are not found. - """ - try: - g = labels_group[f"{name}-{suffix}"] - return g[list(g.keys())[0]] - except KeyError as e: - logger.warning(f'"{name}-{suffix}" not found in {labels_group}.') - raise e - - -def _read_image(file_list: list[str], metadata: dict) -> xr.DataArray: - """Read image files and preprocess them into a standardized format. - - This function reads image files specified in the file_list and processes them into an - xarray.DataArray with dimensions adjusted as needed. It handles missing dimensions and stacks - time and channel dimensions for further processing. - - :param file_list: List of file paths to the image files. - :param metadata: Dictionary containing metadata associated with the images. - :return: DataArray containing the processed image data. - """ - image = _images2fov(file_list, metadata, dask=True) - dims = tuple([d for d in ["t", "c", "z"] if d in image.dims]) - - if len(dims) > 0: - image = image.stack(t_c_z=dims, create_index=False).transpose( - *("y", "x", "t_c_z") - ) - with warnings.catch_warnings(): - # ignore UserWarning: rename 't_c_z' to 'c' does not create an index anymore. - # Try using swap_dims instead or use set_index after rename to create an indexed coordinate. - warnings.filterwarnings("ignore", "rename .*", UserWarning) - image = image.rename({"t_c_z": "c"}) - else: - # add trailing c dimension - image = image.expand_dims("c", -1) - return image - - def _get_feature_channel_indices(tokens): """Get indices of channel parameters in a feature name. @@ -172,10 +127,47 @@ def _get_feature_channel_indices(tokens): ] +def _find_labels( + label_paths: list[str], + image_key: str, + label_name: str, + image_key_no_t: str | None, + selected_timepoint: Any, +): + timepoints = None + g = None + for label_path in label_paths: + label_root = zarr.open(label_path, mode="r") + labels_group = label_root.get("labels") + if labels_group is not None: + g = labels_group.get(f"{image_key}-{label_name}") + if g is not None: + timepoints = ( + g.attrs["multiscales"][0]["metadata"]["t"] + if "t" in g.attrs["multiscales"][0]["metadata"] + else [None] + ) + return g, timepoints + if g is None and image_key_no_t is not None: + g = labels_group.get(f"{image_key_no_t}-{label_name}") + zarr_metadata = g.attrs["multiscales"][0]["metadata"] + + if "t" in zarr_metadata: + timepoints = zarr_metadata["t"] + if selected_timepoint in timepoints: + index = timepoints.index(selected_timepoint) + timepoints = [timepoints[index]] + return g, timepoints + else: + timepoints = [None] + return g, timepoints + return g, timepoints + + def single_feature( stacked_image_tuple: tuple[tuple[str, ...], list[str | Group], dict] | None, image_tuple: tuple[tuple[str, ...], list[str | Group], dict], - labels_group: Group, + label_paths: list[str], output_dir: str, output_sep: str, objects_dir: str, @@ -240,221 +232,246 @@ def single_feature( output_fs, _ = fsspec.core.url_to_fs(output_dir) - zarr_inputs = True - - for f in file_list: - if not isinstance(f, (zarr.Group, zarr.Array)): - zarr_inputs = False - break - - if zarr_inputs and stacked_image_tuple is not None: - for f in stacked_file_list: - if not isinstance(f, (zarr.Group, zarr.Array)): - zarr_inputs = False - break - if not zarr_inputs: - image = _read_image(file_list, metadata) - else: - image = [] - for f in file_list: - array, _, _, _ = _read_ome_zarr_array(f) - image.append(array) + image = _images2fov(file_list, metadata, dask=True) + image_dims = tuple([d for d in ["t", "c", "z"] if d in image.dims]) n_channels1 = None + stacked_image = None if stacked_image_tuple is not None: - if not zarr_inputs: - stacked_image = _read_image(stacked_file_list, stacked_metadata) - n_channels1 = image.sizes["c"] - # clear coords to avoid issues with xr.concat - for c in list(image.coords.keys()): - del image.coords[c] - for c in list(stacked_image.coords.keys()): - del stacked_image.coords[c] - image = xr.concat((image, stacked_image), dim="c") - else: - n_channels1 = 0 - for img in image: - n_channels1 += np.prod(img.shape[:-2]) - n_channels1 = int(n_channels1) - for f in stacked_file_list: - array, _, _, _ = _read_ome_zarr_array(f) - image.append(array) + stacked_image = _images2fov(stacked_file_list, stacked_metadata, dask=True) + n_channels1 = image.sizes["c"] + stacked_image_dims = tuple( + [d for d in ["t", "c", "z"] if d in stacked_image.dims] + ) + image_key_no_t = None + selected_timepoint = None + if "t" in metadata["group_metadata"]["group"]: + image_key_no_t = [] + for key in metadata["group_metadata"]["group"]: + if key != "t": + image_key_no_t.append(str(metadata["group_metadata"]["group"][key])) + else: + selected_timepoint = metadata["group_metadata"]["group"][key] + image_key_no_t = "-".join(image_key_no_t).replace("/", "-") for label_name in label_name_to_features: + label_prefix = _label_name_to_prefix[label_name] features = label_name_to_features[label_name] - output_parquet_path = ( - f"{output_dir}{output_sep}{label_name}{output_sep}{image_key}.parquet" - ) - - if not force and is_parquet_file(output_parquet_path): - logger.info(f"Skipping features for {image_key} {label_name}") - continue - zarr_labels = get_labels( - labels_group=labels_group, - name=image_key, - suffix=label_name, # e.g. nuclei + g, timepoints = _find_labels( + label_paths=label_paths, + image_key=image_key, + label_name=label_name, + image_key_no_t=image_key_no_t, + selected_timepoint=selected_timepoint, ) - - if zarr_labels is None: - logger.info(f"Unable to read {label_name} labels for {image_key}.") + if g is None: + logger.info(f"No labels found for {image_key}") continue - label_prefix = _label_name_to_prefix[label_name] - merged_df = None - if objects_dir is not None: - merged_df = _read_merged_or_objects( - merge_dir=objects_dir, - merge_dir_sep=objects_dir_sep, - label_name=label_name, - image_key=image_key, - label_filter=label_filter, + labels_array = da.from_array(g[list(g.keys())[0]]) + for timepoint in timepoints: + image_ = ( + image.sel(t=timepoint) + if timepoint is not None and image.sizes.get("t", 0) > 1 + else image + ) + image_ = ( + image_.stack(t_c_z=image_dims, create_index=False) + .transpose(*("y", "x", "t_c_z")) + .rename({"t_c_z": "c"}) + if len(image_dims) > 0 + else image_.expand_dims("c", -1) ) + if stacked_image is not None: + stacked_image_ = ( + stacked_image.stack(t_c_z=stacked_image_dims, create_index=False) + .transpose(*("y", "x", "t_c_z")) + .rename({"t_c_z": "c"}) + if len(stacked_image_dims) > 0 + else stacked_image.expand_dims("c", -1) + ) - if merged_df is None: - logger.info(f"Find {label_name} objects for {image_key}.") - merged_df = find_objects(zarr_labels) - objects_path = f"{output_dir}{output_sep}{label_name}{output_sep}{image_key}-objects.parquet" - merged_df.index.name = "label" - merged_df.columns = f"{label_prefix}_" + merged_df.columns - _to_parquet( - merged_df, - objects_path, - write_index=True, - compute=True, - custom_metadata=dict(scallops=json.dumps(cli_metadata())) - if not no_version - else None, + intensity_image = ( + xr.concat((image_, stacked_image_), dim="c", join="outer") + if stacked_image is not None + else image_ ) - merged_df = pd.read_parquet(objects_path) - - features = normalize_features(features) - # strip nuclei_, etc. from features_plot - features_plot_label = [] - for feature in features_plot: - tokens = feature.lower().split("_") - if tokens[0] == label_prefix: - features_plot_label.append("_".join(tokens[1:])) - - features_plot_label = normalize_features(features_plot_label) - if stacked_image_tuple is not None: - stacked_features = set() - stacked_features_plot = set() - for feature in features: # add offset for image1 - tokens = feature.lower().split("_") - if ( - tokens[0] in _features_multichannel.keys() - or tokens[0] in _features_single_channel.keys() - ): - for token_index in range( - 1, - 3 if tokens[0] in _features_multichannel.keys() else 2, - ): - c = tokens[token_index] - if c[0] == "s": - if c in channel_names: - channel_names[str(n_channels1 + int(c[1:]))] = ( - channel_names.pop(c) - ) - tokens[token_index] = str(n_channels1 + int(c[1:])) - - new_feature = "_".join(tokens) - if feature in features_plot_label: - stacked_features_plot.add(new_feature) - stacked_features.add(new_feature) - - features = stacked_features - features_plot_label = stacked_features_plot - - features = list(set(natsorted(features))) - logger.info( - f"{image_key} {label_name} {len(features):,} {pluralize('feature', len(features))}: " - f"{', '.join(features)}" - ) - if label_filter is not None: - merged_df = merged_df.query(label_filter) - min_max_area = label_name_to_min_max_area.get(label_name) - area_column = f"{label_prefix}_AreaShape_Area" - n_labels = len(merged_df) - prefix = "" - if min_max_area[0] is not None or min_max_area[1] is not None: - area_query = [] - if min_max_area[0] is not None: - area_query.append(f"{area_column}>={min_max_area[0]}") - if min_max_area[1] is not None: - area_query.append(f"{area_column}<={min_max_area[1]}") - merged_df = merged_df.query("&".join(area_query)) - n_labels_filtered = n_labels - len(merged_df) - prefix = f"Removed {n_labels_filtered:,} out of " - logger.info( - f"{prefix}{n_labels:,} {pluralize('label', n_labels)}. " - f"Area: {merged_df[area_column].min():,.0f} to {merged_df[area_column].max():,.0f}." - ) + features_path = get_path( + output_dir, output_sep, label_name, image_key, timepoint, ".parquet" + ) + if not force and is_parquet_file(features_path): + logger.info( + f"Skipping features for {image_key} {label_name}{' at t=' + timepoint if timepoint is not None else ''}." + ) + continue + + merged_df = None + if objects_dir is not None: + merged_df = _read_merged_or_objects( + path=objects_dir, + timepoint=timepoint, + path_sep=objects_dir_sep, + label_name=label_name, + image_key=image_key, + label_filter=label_filter, + ) + label_image = labels_array[ + timepoints.index(timepoint) + if timepoint is not None and labels_array.ndim == 3 + else labels_array + ] - df = label_features( - objects_df=merged_df, - label_image=zarr_labels if zarr_inputs else da.from_zarr(zarr_labels), - intensity_image=image if zarr_inputs else image.data, - features=features, - normalize=normalize, - bounding_box_columns=[ - f"{label_prefix}_AreaShape_BoundingBoxMinimum_Y", - f"{label_prefix}_AreaShape_BoundingBoxMinimum_X", - f"{label_prefix}_AreaShape_BoundingBoxMaximum_Y", - f"{label_prefix}_AreaShape_BoundingBoxMaximum_X", - ], - channel_names=channel_names, - ) - # df will be None if only area and coordinates requested - - if df is not None: - fs = fsspec.url_to_fs(output_parquet_path)[0] - if fs.exists(output_parquet_path): - fs.rm(output_parquet_path, recursive=True) - df.index.name = "label" - df.columns = f"{label_prefix}_" + df.columns - - if isinstance(df, pd.DataFrame): - table = pa.Table.from_pandas(df, preserve_index=True) - if not no_version: - table = table.replace_schema_metadata( - { - "scallops".encode(): json.dumps(cli_metadata()).encode(), - **table.schema.metadata, - } - ) - fs, output_file = fsspec.url_to_fs(output_parquet_path) - pq.write_table( - table, - output_parquet_path, - filesystem=fs, + if merged_df is None: + logger.info( + f"Find {label_name} objects for {image_key}{' at t=' + timepoint if timepoint is not None else ''}." + ) + merged_df = find_objects(label_image) + objects_path = get_path( + output_dir, output_sep, label_name, image_key, timepoint ) - else: + merged_df.index.name = "label" + merged_df.columns = f"{label_prefix}_" + merged_df.columns _to_parquet( - df, - output_parquet_path, + merged_df, + objects_path, write_index=True, compute=True, custom_metadata=dict(scallops=json.dumps(cli_metadata())) if not no_version else None, ) + merged_df = pd.read_parquet(objects_path) - if len(features_plot_label) > 0: - features_plot_label = [ - label_prefix + "_" + feature for feature in features_plot_label - ] - df_features = pd.read_parquet( - output_parquet_path, columns=features_plot_label + features = normalize_features(features) + # strip nuclei_, etc. from features_plot + features_plot_label = [] + for feature in features_plot: + tokens = feature.lower().split("_") + if tokens[0] == label_prefix: + features_plot_label.append("_".join(tokens[1:])) + + features_plot_label = normalize_features(features_plot_label) + if stacked_image_tuple is not None: + stacked_features = set() + stacked_features_plot = set() + for feature in features: # add offset for image1 + tokens = feature.lower().split("_") + + if ( + tokens[0] in _features_multichannel.keys() + or tokens[0] in _features_single_channel.keys() + ): + for token_index in range( + 1, + 3 if tokens[0] in _features_multichannel.keys() else 2, + ): + c = tokens[token_index] + if c[0] == "s": + if c in channel_names: + channel_names[str(n_channels1 + int(c[1:]))] = ( + channel_names.pop(c) + ) + tokens[token_index] = str(n_channels1 + int(c[1:])) + + new_feature = "_".join(tokens) + if feature in features_plot_label: + stacked_features_plot.add(new_feature) + stacked_features.add(new_feature) + + features = stacked_features + features_plot_label = stacked_features_plot + + features = list(set(natsorted(features))) + + if label_filter is not None: + merged_df = merged_df.query(label_filter) + min_max_area = label_name_to_min_max_area.get(label_name) + area_column = f"{label_prefix}_AreaShape_Area" + n_labels = len(merged_df) + prefix = "" + if min_max_area[0] is not None or min_max_area[1] is not None: + area_query = [] + if min_max_area[0] is not None: + area_query.append(f"{area_column}>={min_max_area[0]}") + if min_max_area[1] is not None: + area_query.append(f"{area_column}<={min_max_area[1]}") + merged_df = merged_df.query("&".join(area_query)) + n_labels_filtered = n_labels - len(merged_df) + prefix = f"Removed {n_labels_filtered:,} out of " + logger.info( + f"{prefix}{n_labels:,} {pluralize('label', n_labels)}. " + f"Area: {merged_df[area_column].min():,.0f} to {merged_df[area_column].max():,.0f}." ) - centroid_columns = [ - label_prefix + "_centroid-1", - label_name + "_centroid-0", - ] - df = merged_df[centroid_columns].join(df_features) - pdf_path = ( - f"{output_dir}{output_sep}{label_name}{output_sep}{image_key}.pdf" + + df = label_features( + objects_df=merged_df, + label_image=label_image, + intensity_image=intensity_image, + features=features, + normalize=normalize, + bounding_box_columns=[ + f"{label_prefix}_AreaShape_BoundingBoxMinimum_Y", + f"{label_prefix}_AreaShape_BoundingBoxMinimum_X", + f"{label_prefix}_AreaShape_BoundingBoxMaximum_Y", + f"{label_prefix}_AreaShape_BoundingBoxMaximum_X", + ], + channel_names=channel_names, ) - _plot_features(df, features_plot_label, pdf_path, centroid_columns) + # df will be None if only area and coordinates requested + + if df is not None: + fs = fsspec.url_to_fs(features_path)[0] + if fs.exists(features_path): + fs.rm(features_path, recursive=True) + df.index.name = "label" + df.columns = f"{label_prefix}_" + df.columns + + if isinstance(df, pd.DataFrame): + table = pa.Table.from_pandas(df, preserve_index=True) + if not no_version: + table = table.replace_schema_metadata( + { + "scallops".encode(): json.dumps( + cli_metadata() + ).encode(), + **table.schema.metadata, + } + ) + fs, output_file = fsspec.url_to_fs(features_path) + pq.write_table( + table, + features_path, + filesystem=fs, + ) + + else: + _to_parquet( + df, + features_path, + write_index=True, + compute=True, + custom_metadata=dict(scallops=json.dumps(cli_metadata())) + if not no_version + else None, + ) + + if len(features_plot_label) > 0: + features_plot_label = [ + label_prefix + "_" + feature for feature in features_plot_label + ] + df_features = pd.read_parquet( + features_path, columns=features_plot_label + ) + centroid_columns = [ + label_prefix + "_centroid-1", + label_name + "_centroid-0", + ] + df = merged_df[centroid_columns].join(df_features) + pdf_path = get_path( + output_dir, output_sep, label_name, image_key, timepoint, ".pdf" + ) + + _plot_features(df, features_plot_label, pdf_path, centroid_columns) return [] @@ -481,6 +498,7 @@ def run_pipeline_compute_features(arguments: argparse.Namespace) -> None: channel_names = arguments.channel_rename stack_images = arguments.stack_images label_filter = arguments.label_filter + label_paths = arguments.labels normalize = not arguments.no_normalize stack_image_pattern = arguments.stack_image_pattern cell_features = arguments.features_cell @@ -522,11 +540,9 @@ def run_pipeline_compute_features(arguments: argparse.Namespace) -> None: output_dir = output_dir.rstrip(output_fs.sep) for label in label_name_to_features: output_fs.makedirs(output_dir + output_fs.sep + label, exist_ok=True) - labels_path = arguments.labels + no_version = arguments.no_version - assert labels_path is not None, "No labels provided" - label_root = zarr.open(labels_path, mode="r") - labels_group = label_root["labels"] + if channel_names is not None: # keys are strings in json try: @@ -566,7 +582,7 @@ def run_pipeline_compute_features(arguments: argparse.Namespace) -> None: output_sep=output_fs.sep, objects_dir=objects_dir, objects_dir_sep=objects_dir_sep, - labels_group=labels_group, + label_paths=label_paths, label_filter=label_filter, label_name_to_min_max_area=label_name_to_min_max_area, label_name_to_features=label_name_to_features, diff --git a/scallops/cli/features_main.py b/scallops/cli/features_main.py index e7c5d19..0117d13 100644 --- a/scallops/cli/features_main.py +++ b/scallops/cli/features_main.py @@ -73,7 +73,8 @@ def new_format_help(x): "--labels", dest="labels", required=True, - help="Path to zarr directory containing labels", + nargs="+", + help="Path(s) to zarr directory containing labels", ) generic_features_help = ( diff --git a/scallops/cli/find_objects.py b/scallops/cli/find_objects.py index d292da9..c746517 100644 --- a/scallops/cli/find_objects.py +++ b/scallops/cli/find_objects.py @@ -28,6 +28,21 @@ logger = _get_cli_logger() +def get_path( + output_dir: str, + output_sep: str, + label_name: str, + image_key: str, + timepoint: str | None, + suffix="-objects.parquet", +): + return ( + (f"{output_dir}{output_sep}{label_name}{output_sep}{image_key}{suffix}") + if timepoint is None + else f"{output_dir}{output_sep}{label_name}{output_sep}t={timepoint}{output_sep}{image_key}{suffix}" + ) + + def _execute( label_tuple: tuple[tuple[str, ...], list[str | Group], dict], timepoint: str | None, @@ -40,11 +55,7 @@ def _execute( assert len(file_list) == 1 label_name = group[len(group) - 1] image_key = "-".join(group[:-1]) # exclude suffix from key - path = ( - (f"{output_dir}{output_sep}{label_name}{output_sep}{image_key}-objects.parquet") - if timepoint is None - else f"{output_dir}{output_sep}{label_name}{output_sep}t={timepoint}{output_sep}{image_key}-objects.parquet" - ) + path = get_path(output_dir, output_sep, label_name, image_key, timepoint) fs = fsspec.url_to_fs(path)[0] if fs.exists(path): if force: diff --git a/scallops/cli/util.py b/scallops/cli/util.py index 928df57..2266e89 100644 --- a/scallops/cli/util.py +++ b/scallops/cli/util.py @@ -586,6 +586,13 @@ def _list_images_wdl( url_val = index + 1 subset_df = result["subset_df"] + with open( + f"groupby_array_with_time_{url_val}.txt", "wt" + ) as f: # ['plate', 'well', 't'] + for g in result["groupby_with_time"]: + f.write(g) + f.write("\n") + with open(f"group_size_{url_val}.txt", "wt") as f: f.write(f"{result['group_size']}") f.write("\n") @@ -679,9 +686,9 @@ def _list_images( data=dict(subset_ids_with_reference_times=subset_ids_with_reference_times), ) - groupby_with_t = list(groupby) + groupby_with_time = list(groupby) if not groupby_t and times is not None: - groupby_with_t.append("t") + groupby_with_time.append("t") if reference_time is None: reference_time = times[0] if times is not None and len(times) > 0 else "0" @@ -690,6 +697,7 @@ def _list_images( group_size=group_size, subset_df=subset_df, groupby=groupby, + groupby_with_time=groupby_with_time, times=times, reference_time=reference_time, ) diff --git a/scallops/features/generate.py b/scallops/features/generate.py index 59b9234..036bd62 100644 --- a/scallops/features/generate.py +++ b/scallops/features/generate.py @@ -16,6 +16,7 @@ import numpy as np import pandas as pd import skimage.util +import xarray as xr import zarr from dask import delayed @@ -123,8 +124,8 @@ def _create_dd_metadata( def label_features( objects_df: pd.DataFrame, - label_image: da.Array | zarr.Array, - intensity_image: da.Array | zarr.Array | Sequence[zarr.Array] | None, + label_image: da.Array | xr.DataArray | zarr.Array, + intensity_image: da.Array | zarr.Array | xr.DataArray | Sequence[zarr.Array] | None, features: Iterable[str], channel_names: dict[int | str, str] | None = None, normalize: bool = True, @@ -146,6 +147,10 @@ def label_features( :return: DataFrame with extracted features. """ is_numpy = False + if isinstance(label_image, xr.DataArray): + label_image = label_image.data + if isinstance(intensity_image, xr.DataArray): + intensity_image = intensity_image.data if isinstance(label_image, np.ndarray): is_numpy = True label_image = da.from_array(label_image) diff --git a/wdl/ops_tasks.wdl b/wdl/ops_tasks.wdl index d1adc5a..c01f669 100644 --- a/wdl/ops_tasks.wdl +++ b/wdl/ops_tasks.wdl @@ -379,7 +379,7 @@ task intersects_boundary { String images String? image_pattern String label_type - String labels + Array[String] labels String subset String? objects String output_directory @@ -405,7 +405,7 @@ task intersects_boundary { scallops features \ --features-~{label_type} "intersects-boundary_0" \ - --labels "~{labels}" \ + --labels ~{sep=" " labels} \ --groupby ~{sep=" " groupby} \ --subset ~{subset} \ --output "~{output_directory}" \ diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index fe2121a..5320ce6 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -240,11 +240,11 @@ workflow ops_workflow { String reference_time_pheno = list_images.reference_time_1 String reference_time_iss = list_images.reference_time_2 - + Array[String] phenotype_group_by_with_time = list_images.groupby_array_with_time_2 #String image_pattern_with_reference_time_pheno = list_images.image_pattern_with_reference_time_1 # e.g. {plate}-{well}-IF # String image_pattern_with_reference_time_iss = list_images.image_pattern_with_reference_time_2 # e.g. {plate}-{well}-1 scatter (subset_index in range(length(subsets))) { - String subset_ = subsets[subset_index] + String subset_ = subsets[subset_index] # e.g. plate1-A1 # String subset_with_reference_times_pheno = subsets_with_reference_times_pheno[subset_index] # String subset_with_reference_times_iss = subsets_with_reference_times_iss[subset_index] if(pheno_url_supplied) { @@ -304,6 +304,7 @@ workflow ops_workflow { } if(length(times_pheno)>1) { + call tasks.register_elastix as register_pheno_to_pheno { input: moving=select_all([phenotype_url]), @@ -400,12 +401,12 @@ workflow ops_workflow { input: labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), images=phenotype_url_stripped + '/labels/', - image_pattern=groupby_pattern, + image_pattern=phenotype_image_pattern + '-mask', output_directory=cell_intersects_boundary_directory, label_type='cell', objects=find_objects_cell.output_url, - groupby=groupby, - subset = subset_, + groupby=phenotype_group_by_with_time, + subset = subset_ + '-*', force = force_segment_cell, docker=docker, zones = zones, diff --git a/wdl/utils.wdl b/wdl/utils.wdl index 0daf6aa..3a8a0ee 100644 --- a/wdl/utils.wdl +++ b/wdl/utils.wdl @@ -61,6 +61,9 @@ task list_images { String groupby_pattern = read_lines('groupby_pattern.txt')[0] # e.g. {plate}-{well} Array[String] groupby_array = read_lines('groupby_array.txt') # e.g. ["plate", "well"] + Array[String] groupby_array_with_time_1 = read_lines('groupby_array_with_time_1.txt') # e.g. ["plate", "well", "t"] + Array[String] groupby_array_with_time_2 = read_lines('groupby_array_with_time_2.txt') # e.g. ["plate", "well", "t"] + Int group_size_1 = read_int('group_size_1.txt') Int group_size_2 = read_int('group_size_2.txt') Array[String] subsets_with_reference_times_1 = read_lines('subsets_with_reference_time_1.txt') From 13a32ca7c9406342a4ada8c4912c7d61ecb23a5c Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 16 Jun 2026 09:04:44 -0400 Subject: [PATCH 15/79] Segmentation time --- wdl/ops_tasks.wdl | 2 +- wdl/ops_workflow.wdl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/wdl/ops_tasks.wdl b/wdl/ops_tasks.wdl index c01f669..0446745 100644 --- a/wdl/ops_tasks.wdl +++ b/wdl/ops_tasks.wdl @@ -407,7 +407,7 @@ task intersects_boundary { --features-~{label_type} "intersects-boundary_0" \ --labels ~{sep=" " labels} \ --groupby ~{sep=" " groupby} \ - --subset ~{subset} \ + --subset "~{subset}" \ --output "~{output_directory}" \ --images "~{images}" \ --objects "~{objects}" \ diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index 5320ce6..34ef69f 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -483,7 +483,7 @@ workflow ops_workflow { input: images=register_pheno_to_iss.moving_output_url, image_pattern=groupby_pattern, - stacked_images=select_first([iss_url]), + stacked_images=select_first([register_pheno_to_iss.moving_output_url]), stacked_image_pattern=groupby_pattern, image_channel=iss_dapi_channel, stacked_image_channel=0, From 33ce260e486a6105a6ee57413802f7ed29956ec9 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 16 Jun 2026 10:39:23 -0400 Subject: [PATCH 16/79] Segmentation time --- scallops/cli/features.py | 48 ++++++++++++++------------ scallops/tests/test_wdl.py | 20 +++++------ wdl/ops_tasks.wdl | 71 +++++++++----------------------------- wdl/ops_workflow.wdl | 7 ++-- 4 files changed, 55 insertions(+), 91 deletions(-) diff --git a/scallops/cli/features.py b/scallops/cli/features.py index 1e4ecce..0e2c990 100644 --- a/scallops/cli/features.py +++ b/scallops/cli/features.py @@ -8,6 +8,7 @@ import argparse import json +import warnings from collections.abc import Sequence from itertools import zip_longest from typing import Any, get_type_hints @@ -164,6 +165,21 @@ def _find_labels( return g, timepoints +def _stack_and_rename(image: xr.DataArray) -> xr.DataArray: + image_dims = tuple([d for d in ["t", "c", "z"] if d in image.dims]) + with warnings.catch_warnings(): + # ignore UserWarning: rename 't_c_z' to 'c' does not create an index anymore. + # Try using swap_dims instead or use set_index after rename to create an indexed coordinate. + warnings.filterwarnings("ignore", "rename .*", UserWarning) + return ( + image.stack(t_c_z=image_dims, create_index=False) + .transpose(*("y", "x", "t_c_z")) + .rename({"t_c_z": "c"}) + if len(image_dims) > 0 + else image.expand_dims("c", -1) + ) + + def single_feature( stacked_image_tuple: tuple[tuple[str, ...], list[str | Group], dict] | None, image_tuple: tuple[tuple[str, ...], list[str | Group], dict], @@ -233,15 +249,13 @@ def single_feature( output_fs, _ = fsspec.core.url_to_fs(output_dir) image = _images2fov(file_list, metadata, dask=True) - image_dims = tuple([d for d in ["t", "c", "z"] if d in image.dims]) + n_channels1 = None stacked_image = None if stacked_image_tuple is not None: stacked_image = _images2fov(stacked_file_list, stacked_metadata, dask=True) n_channels1 = image.sizes["c"] - stacked_image_dims = tuple( - [d for d in ["t", "c", "z"] if d in stacked_image.dims] - ) + image_key_no_t = None selected_timepoint = None if "t" in metadata["group_metadata"]["group"]: @@ -273,21 +287,9 @@ def single_feature( if timepoint is not None and image.sizes.get("t", 0) > 1 else image ) - image_ = ( - image_.stack(t_c_z=image_dims, create_index=False) - .transpose(*("y", "x", "t_c_z")) - .rename({"t_c_z": "c"}) - if len(image_dims) > 0 - else image_.expand_dims("c", -1) - ) + image_ = _stack_and_rename(image_) if stacked_image is not None: - stacked_image_ = ( - stacked_image.stack(t_c_z=stacked_image_dims, create_index=False) - .transpose(*("y", "x", "t_c_z")) - .rename({"t_c_z": "c"}) - if len(stacked_image_dims) > 0 - else stacked_image.expand_dims("c", -1) - ) + stacked_image_ = _stack_and_rename(stacked_image) intensity_image = ( xr.concat((image_, stacked_image_), dim="c", join="outer") @@ -314,11 +316,11 @@ def single_feature( image_key=image_key, label_filter=label_filter, ) - label_image = labels_array[ - timepoints.index(timepoint) - if timepoint is not None and labels_array.ndim == 3 - else labels_array - ] + if timepoint is not None and labels_array.ndim == 3: + timepoint_index = timepoints.index(timepoint) + label_image = labels_array[timepoint_index] + else: + label_image = labels_array if merged_df is None: logger.info( diff --git a/scallops/tests/test_wdl.py b/scallops/tests/test_wdl.py index 943e549..96786dd 100644 --- a/scallops/tests/test_wdl.py +++ b/scallops/tests/test_wdl.py @@ -1,3 +1,4 @@ +import glob import json import os.path from subprocess import check_call @@ -155,15 +156,17 @@ def test_stitch_wdl(tmp_path): @pytest.mark.cli_e2e def test_ops_wdl(tmp_path): - sbs_dir = tmp_path / "sbs.zarr" - pheno_dir = tmp_path / "pheno.zarr" + sbs_dir = tmp_path / "sbs" output = tmp_path / "out" + pheno_dir = tmp_path / "pheno.zarr" + sbs_dir.mkdir() output.mkdir() + for p in glob.glob("scallops/tests/data/experimentC/input/*/*Tile-102*"): + cycles = os.path.basename(p).split("_")[1] + cycles = cycles.split("-")[0] + dest = f"plateA-A1-{cycles[1:]}.tif" + add_physical_size(p, str(sbs_dir / dest)) - iss_img = read_image( - "scallops/tests/data/experimentC/input/10X_c1-SBS-1/10X_c1-SBS-1_A1_Tile-102.sbs.tif" - ) - iss_img.attrs["physical_pixel_sizes"] = (1, 1) pheno_img = read_image( "scallops/tests/data/experimentC/10X_c0-DAPI-p65ab/10X_c0-DAPI-p65ab_A1_Tile-102.phenotype.tif" ) @@ -186,13 +189,10 @@ def test_ops_wdl(tmp_path): }, ).save(pheno_dir) - Experiment( - images={"plateA-A1-1": iss_img, "plateA-A1-2": iss_img}, - ).save(sbs_dir) - input_json = { "model_dir": "", "iss_url": str(sbs_dir.absolute()), + "iss_image_pattern": "{plate}-{well}-{t}.tif", "output_directory": str(output.absolute()), "iss_registration_extra_arguments": "--no-landmarks", "pheno_to_iss_registration_extra_arguments": "--no-landmarks", diff --git a/wdl/ops_tasks.wdl b/wdl/ops_tasks.wdl index 0446745..3ec391b 100644 --- a/wdl/ops_tasks.wdl +++ b/wdl/ops_tasks.wdl @@ -272,16 +272,17 @@ task register_pheno_to_iss_qc { } -task register_qc { +task register_iss_iss_qc { input { String images String? image_pattern String label_type String labels - Int channel + Int dapi_channel + Int n_timepoints String subset String output_directory - String channel_prefix + Array[String] groupby Boolean? force @@ -294,6 +295,7 @@ task register_qc { String memory Int max_retries } + Int n_channels = n_timepoints*5 command <<< set -ex @@ -302,58 +304,17 @@ task register_qc { ulimit -n 100000 fi - python < 0: - cmd.append("--subset") - cmd += subset - cmd += ["--output", output_directory] - cmd += ["--images", images] - cmd += ["--channel-rename", f"{json.dumps(channel_rename)}"] - - if force == "true": - cmd.append("--force") - print(" ".join(cmd)) - check_call(cmd) - - CODE + scallops features \ + --features-~{label_type} "correlationpearsonbox_~{dapi_channel}_5:~{n_channels}:5" \ + --labels "~{labels}" \ + --groupby ~{sep=" " groupby} \ + --subset ~{subset} \ + --output "~{output_directory}" \ + --images "~{images}" \ + ~{'--image-pattern ' + image_pattern} \ + ~{true="--force" false="" force} + + >>> diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index 34ef69f..100f474 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -503,13 +503,14 @@ workflow ops_workflow { max_retries = max_retries } # ISS t0 to other times - call tasks.register_qc as register_iss_to_iss_qc { + call tasks.register_iss_iss_qc as register_iss_to_iss_qc { input: images=select_first([register_iss_t0.moving_output_url]), image_pattern=groupby_pattern, - channel=select_first([iss_dapi_channel, 0]), + dapi_channel=select_first([iss_dapi_channel, 0]), + n_timepoints=length(times_iss), label_type='nuclei', - channel_prefix="ISS", + output_directory=register_iss_to_iss_qc_directory, labels=register_pheno_to_iss.label_output_url, subset = subset_, From 5f88d479781f3f205ae3b7c90fc8787ffea02214 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 16 Jun 2026 10:52:24 -0400 Subject: [PATCH 17/79] Segmentation time --- wdl/ops_tasks.wdl | 4 ++-- wdl/ops_workflow.wdl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/wdl/ops_tasks.wdl b/wdl/ops_tasks.wdl index 3ec391b..4f50e53 100644 --- a/wdl/ops_tasks.wdl +++ b/wdl/ops_tasks.wdl @@ -460,7 +460,7 @@ task features { Int? cytosol_max_area String? features_extra_arguments String? model_dir - String? labels + Array[String] labels String? objects String images String subset @@ -498,7 +498,7 @@ task features { ~{if defined(cell_max_area) && select_first([cell_max_area])>0 then '--cell-max-area ' + cell_max_area else ''} \ ~{if defined(cytosol_max_area) && select_first([cytosol_max_area])>0 then '--cytosol-max-area ' + cytosol_max_area else ''} \ ~{if defined(features_extra_arguments) then features_extra_arguments else ''} \ - --labels "~{labels}" \ + --labels ~{sep=" " labels} \ ~{"--objects " + objects} \ ~{"--label-filter " + '"' + label_filter + '"'} \ --subset ~{subset} \ diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index 100f474..2974fda 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -697,7 +697,7 @@ workflow ops_workflow { Array[String] phenotype_cytosol_times = keys(phenotype_cytosol_features_) scatter (phenotype_time in phenotype_cytosol_times) { - Array[String] cytosol_features = phenotype_cytosol_features_[phenotype_time] + Array[String] cytosol_features = phenotype_cytosol_features_[phenotype_time] scatter (feature_index in range(length(cytosol_features))) { call tasks.features as features_cytosol { input: From 970a5bd8c3983ed21f4fca229a811509374f42db Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 16 Jun 2026 11:29:01 -0400 Subject: [PATCH 18/79] Segmentation time --- wdl/ops_workflow.wdl | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index 2974fda..6133f9f 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -626,15 +626,16 @@ workflow ops_workflow { objects=merge_sbs_metadata.output_url, labels=select_all([segment_nuclei.output_url, register_pheno_to_pheno.label_output_url]), label_filter=features_label_filter, + groupby=phenotype_group_by_with_time, nuclei_features = nuclei_features[feature_index], nuclei_min_area = features_nuclei_min_area_, nuclei_max_area = features_nuclei_max_area_, features_extra_arguments=features_extra_arguments, model_dir=model_dir, - groupby=groupby, + output_directory=nuclei_features_directory + '-' + phenotype_time + '-' + feature_index, - subset = subset_, + subset = subset_ + '-' + phenotype_time, force = force_features, docker=docker, zones = zones, @@ -666,15 +667,16 @@ workflow ops_workflow { objects=merge_sbs_metadata.output_url, labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), label_filter=features_label_filter, + groupby=phenotype_group_by_with_time, cell_features = cell_features[feature_index], cell_min_area = features_cell_min_area_, cell_max_area = features_cell_max_area_, features_extra_arguments=features_extra_arguments, model_dir=model_dir, - groupby=groupby, + output_directory=cell_features_directory + '-' + phenotype_time + '-' + feature_index, - subset = subset_, + subset = subset_ + '-' + phenotype_time, force = force_features, docker=docker, zones = zones, @@ -706,6 +708,7 @@ workflow ops_workflow { objects=merge_sbs_metadata.output_url, labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), label_filter=features_label_filter, + groupby=phenotype_group_by_with_time, output_directory=cytosol_features_directory + '-' + phenotype_time + '-' + feature_index, cytosol_features = cytosol_features[feature_index], cytosol_min_area = features_cytosol_min_area_, @@ -713,9 +716,8 @@ workflow ops_workflow { features_extra_arguments=features_extra_arguments, model_dir=model_dir, - groupby=groupby, - subset = subset_, + subset = subset_ + '-' + phenotype_time, force = force_features, docker=docker, zones = zones, From eda1e3874b3f579c460cd7535f56e1905ebcb904 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Wed, 17 Jun 2026 12:42:24 -0400 Subject: [PATCH 19/79] Segmentation time --- scallops/cli/extract_crops.py | 245 ++++++++++++++++------------- scallops/cli/extract_crops_main.py | 24 +-- scallops/cli/features.py | 141 +++++++++-------- scallops/cli/features_main.py | 4 +- scallops/features/generate.py | 2 +- 5 files changed, 219 insertions(+), 197 deletions(-) diff --git a/scallops/cli/extract_crops.py b/scallops/cli/extract_crops.py index f8f089a..79a6796 100644 --- a/scallops/cli/extract_crops.py +++ b/scallops/cli/extract_crops.py @@ -8,9 +8,13 @@ import pyarrow.parquet as pq from array_api_compat import get_namespace from skimage.util import img_as_ubyte -from zarr import Group -from scallops.cli.features import _read_merged_or_objects +from scallops.cli.features import ( + _find_labels, + _image_key_without_time_and_selected_time, + _read_merged_or_objects, +) +from scallops.cli.find_objects import get_path from scallops.cli.util import ( _get_cli_logger, cli_metadata, @@ -27,23 +31,25 @@ logger = _get_cli_logger() -def _norm_block(image: np.ndarray, percentiles) -> np.ndarray: - xp = get_namespace(image) - percentiles = xp.percentile(image, percentiles, axis=(1, 2), keepdims=True) - image = (image - percentiles[0]) / (percentiles[1] - percentiles[0]) - image = xp.clip(image, 0, 1) - return image +def _norm_block(intensity_image: np.ndarray, percentiles) -> np.ndarray: + xp = get_namespace(intensity_image) + percentiles = xp.percentile( + intensity_image, percentiles, axis=(1, 2), keepdims=True + ) + intensity_image = (intensity_image - percentiles[0]) / ( + percentiles[1] - percentiles[0] + ) + intensity_image = xp.clip(intensity_image, 0, 1) + return intensity_image def single_crop( group: str, # NOT USED file_list: list[str], metadata: dict, - labels_group: Group, + label_paths: list[str], output_dir: str, - output_sep: str, - merge_dir: str, - merge_dir_sep: str, + merge_dirs: list[str], crop_size: tuple[int, int], output_format: Literal["tiff", "npy"], label_name: str, @@ -57,108 +63,135 @@ def single_crop( force: bool, ): image_key = metadata["id"] + image = _images2fov(file_list, metadata, dask=True).squeeze().data - output_dir = f"{output_dir}{output_sep}{label_name}{output_sep}{image_key}" + image_key_no_t, selected_timepoint = _image_key_without_time_and_selected_time( + metadata + ) - output_parquet_path = f"{output_dir}.parquet" - if not force and is_parquet_file(output_parquet_path): - logger.info(f"Skipping features for {image_key} {label_name}") + g, timepoints = _find_labels( + label_paths=label_paths, + image_key=image_key, + label_name=label_name, + image_key_no_t=image_key_no_t, + selected_timepoint=selected_timepoint, + ) + if g is None: + logger.info(f"No labels found for {image_key}") return - + labels_array = da.from_array(g[list(g.keys())[0]]) output_fs, _ = fsspec.core.url_to_fs(output_dir) - output_fs.makedirs(output_dir, exist_ok=True) - image = _images2fov(file_list, metadata, dask=True).squeeze().data - logger.info(f"{image_key} image shape {image.shape}") - g = labels_group.get(f"{image_key}-{label_name}") - if g is None: - raise ValueError(f"Unable to read {label_name} labels for {image_key}.") - zarr_labels = g[list(g.keys())[0]] + output_sep = output_fs.sep + for timepoint in timepoints: + features_path = get_path( + output_dir, output_sep, label_name, image_key, timepoint, ".parquet" + ) + output_dir = get_path( + output_dir, output_sep, label_name, image_key, timepoint, "" + ) - merged_df = _read_merged_or_objects( - merge_dir=merge_dir, - merge_dir_sep=merge_dir_sep, - label_name=label_name, - image_key=image_key, - label_filter=label_filter, - ) - if merged_df is None: - raise ValueError(f"Unable to read merged data for {image_key}.") - n_labels_before_filtering = len(merged_df) - if label_filter is not None: - merged_df = merged_df.query(label_filter) - label_prefix = _label_name_to_prefix[label_name] - area_column = f"{label_prefix}_AreaShape_Area" - merged_df = merged_df.query(f"{area_column}>=2") - n_labels_filtered = n_labels_before_filtering - len(merged_df) - logger.info( - f"Removed {n_labels_filtered:,} out of {n_labels_before_filtering:,} labels for {image_key}." - ) - if len(merged_df) == 0: - raise ValueError(f"No labels found for {image_key}.") - # e.g. CHAMMI-75 + if not force and is_parquet_file(features_path): + logger.info( + f"Skipping crops for {image_key} {label_name}{' at t=' + timepoint if timepoint is not None else ''}." + ) + continue + if timepoint is not None and labels_array.ndim == 3: + timepoint_index = timepoints.index(timepoint) + label_image = labels_array[timepoint_index] + else: + label_image = labels_array + intensity_image = ( + image.sel(t=timepoint) + if timepoint is not None and image.sizes.get("t", 0) > 1 + else image + ) + merged_df = _read_merged_or_objects( + paths=merge_dirs, + label_name=label_name, + timepoint=timepoint, + image_key=image_key, + label_filter=label_filter, + ) + if merged_df is None: + raise ValueError(f"Unable to read merged data for {image_key}.") + n_labels_before_filtering = len(merged_df) + if label_filter is not None: + merged_df = merged_df.query(label_filter) + label_prefix = _label_name_to_prefix[label_name] + area_column = f"{label_prefix}_AreaShape_Area" + merged_df = merged_df.query(f"{area_column}>=2") + n_labels_filtered = n_labels_before_filtering - len(merged_df) + logger.info( + f"Removed {n_labels_filtered:,} out of {n_labels_before_filtering:,} labels for {image_key}." + ) + if len(merged_df) == 0: + raise ValueError(f"No labels found for {image_key}.") + # e.g. CHAMMI-75 - if percentile_normalize is not None: - if local_percentile_normalize: - chunksize = list(image.chunksize) - for i in range(len(chunksize) - 2): - chunksize[i] = -1 - if chunks is not None: - chunksize[-2] = chunks - chunksize[-1] = chunks + if percentile_normalize is not None: + if local_percentile_normalize: + chunksize = list(intensity_image.chunksize) + for i in range(len(chunksize) - 2): + chunksize[i] = -1 + if chunks is not None: + chunksize[-2] = chunks + chunksize[-1] = chunks + else: + logger.info( + f"{image_key} chunk size: {chunksize[-2]:,} by {chunksize[-1]:,}" + ) + intensity_image = intensity_image.rechunk(tuple(chunksize)) + depth = None + if local_normalize_overlap is not None and local_normalize_overlap > 0: + depth = { + intensity_image.ndim - 2: local_normalize_overlap, + intensity_image.ndim - 1: local_normalize_overlap, + } + intensity_image = da.map_overlap( + _norm_block, + intensity_image, + percentiles=percentile_normalize, + depth=depth, + dtype=float, + ) else: - logger.info( - f"{image_key} chunk size: {chunksize[-2]:,} by {chunksize[-1]:,}" + percentiles = da.percentile( + intensity_image, percentile_normalize, axis=(1, 2), keepdims=True ) - image = image.rechunk(tuple(chunksize)) - depth = None - if local_normalize_overlap is not None and local_normalize_overlap > 0: - depth = { - image.ndim - 2: local_normalize_overlap, - image.ndim - 1: local_normalize_overlap, - } - image = da.map_overlap( - _norm_block, - image, - percentiles=percentile_normalize, - depth=depth, - dtype=float, - ) - else: - percentiles = da.percentile( - image, percentile_normalize, axis=(1, 2), keepdims=True - ) - image = (image - percentiles[0]) / (percentiles[1] - percentiles[0]) - image = da.clip(image, 0, 1) - image = da.map_blocks(img_as_ubyte, image) - label_col = "label" if "label" in merged_df.columns else None - merged_df = to_label_crops( - label_image=da.from_zarr(zarr_labels), - intensity_image=image, - df=merged_df, - label_col=label_col, - output_dir=output_dir, - crop_size=crop_size, - output_format=output_format, - centroid_cols=[ - f"{label_prefix}_AreaShape_Center_Y", - f"{label_prefix}_AreaShape_Center_X", - ], - gaussian_sigma=gaussian_sigma, - ) + intensity_image = (intensity_image - percentiles[0]) / ( + percentiles[1] - percentiles[0] + ) + intensity_image = da.clip(intensity_image, 0, 1) + intensity_image = da.map_blocks(img_as_ubyte, intensity_image) + label_col = "label" if "label" in merged_df.columns else None + merged_df = to_label_crops( + label_image=label_image, + intensity_image=intensity_image, + df=merged_df, + label_col=label_col, + output_dir=output_dir, + crop_size=crop_size, + output_format=output_format, + centroid_cols=[ + f"{label_prefix}_AreaShape_Center_Y", + f"{label_prefix}_AreaShape_Center_X", + ], + gaussian_sigma=gaussian_sigma, + ) - output_metadata = cli_metadata() if not no_version else dict() + output_metadata = cli_metadata() if not no_version else dict() - table = pa.Table.from_pandas(merged_df, preserve_index=True) - table = table.replace_schema_metadata( - { - "scallops".encode(): json.dumps(output_metadata).encode(), - **table.schema.metadata, - } - ) + table = pa.Table.from_pandas(merged_df, preserve_index=True) + table = table.replace_schema_metadata( + { + "scallops".encode(): json.dumps(output_metadata).encode(), + **table.schema.metadata, + } + ) - fs, output_parquet_path = fsspec.url_to_fs(output_parquet_path) - pq.write_table( - table, - output_parquet_path, - filesystem=fs, - ) + fs, features_path = fsspec.url_to_fs(features_path) + pq.write_table( + table, + features_path, + filesystem=fs, + ) diff --git a/scallops/cli/extract_crops_main.py b/scallops/cli/extract_crops_main.py index db8afa0..9481be8 100644 --- a/scallops/cli/extract_crops_main.py +++ b/scallops/cli/extract_crops_main.py @@ -1,7 +1,5 @@ import argparse -import fsspec -import zarr from dask.bag import from_sequence from scallops.cli.arg_parser import _sort_groups @@ -39,12 +37,13 @@ def run_pipeline_extract_crops(arguments: argparse.Namespace): image_patterns = arguments.image_pattern output_dir = arguments.output - merge_dir = arguments.merge + merge_dirs = arguments.merge subset = arguments.subset force = arguments.force groupby = arguments.groupby crop_size = arguments.crop_size crop_size = (crop_size, crop_size) + label_paths = arguments.labels label_filter = arguments.label_filter percentile_min = arguments.percentile_min @@ -69,20 +68,7 @@ def run_pipeline_extract_crops(arguments: argparse.Namespace): if dask_server_url is None and arguments.dask_cluster is None: dask_cluster_parameters = _dask_workers_threads() - merge_dir_sep = None - if merge_dir is not None: - merge_dir_sep = fsspec.core.url_to_fs(merge_dir)[0].sep - merge_dir = merge_dir.rstrip(merge_dir_sep) - - output_fs, _ = fsspec.core.url_to_fs(output_dir) - output_dir = output_dir.rstrip(output_fs.sep) - - labels_path = arguments.labels no_version = arguments.no_version - assert labels_path is not None, "No labels provided" - label_root = zarr.open(labels_path, mode="r") - labels_group = label_root["labels"] - image_seq = from_sequence( _set_up_experiment( images_paths, @@ -101,10 +87,8 @@ def run_pipeline_extract_crops(arguments: argparse.Namespace): image_seq.starmap( single_crop, output_dir=output_dir, - output_sep=output_fs.sep, - merge_dir=merge_dir, - merge_dir_sep=merge_dir_sep, - labels_group=labels_group, + merge_dirs=merge_dirs, + label_paths=label_paths, label_filter=label_filter, label_name=label_name, percentile_normalize=percentile_normalize, diff --git a/scallops/cli/features.py b/scallops/cli/features.py index 0e2c990..a093a32 100644 --- a/scallops/cli/features.py +++ b/scallops/cli/features.py @@ -57,51 +57,61 @@ def _read_merged_or_objects( - path: str, - path_sep: str, + paths: list[str], timepoint: str | None, label_name: str, image_key: str, label_filter: str | None, ): - path = path.rstrip(path_sep) - paths = [ - f"{path}{path_sep}{label_name}{path_sep}{image_key}.parquet", - f"{path}{path_sep}{image_key}.zarr", - f"{path}{path_sep}{image_key}.parquet", - get_path(path, path_sep, label_name, image_key, timepoint), - ] - - merge_path = None + found_paths = [] for path in paths: - if fsspec.core.url_to_fs(path)[0].exists(path): - merge_path = path - break - if merge_path is None: - return None + path_sep = fsspec.core.url_to_fs(path)[0].sep + path = path.rstrip(path_sep) + + test_paths = [ + f"{path}{path_sep}{label_name}{path_sep}{image_key}.parquet", + f"{path}{path_sep}{image_key}.zarr", + f"{path}{path_sep}{image_key}.parquet", + get_path(path, path_sep, label_name, image_key, timepoint), + ] - area_column = f"{_label_name_to_prefix[label_name]}_AreaShape_Area" - if merge_path.lower().endswith(".zarr"): - data = read_anndata_zarr(merge_path, dask=True) - merged_df = data.obs - columns = {area_column} - assert area_column in data.var.index - if label_filter is not None: - query_columns = _get_names_from_pd_query(label_filter) - columns.update( - c - for c in query_columns - if c not in merged_df.columns and c in data.var.index - ) - columns = list(columns) - values = data[:, columns].X.compute() - for i in range(len(columns)): - merged_df[columns[i]] = values[:, i] + for test_path in test_paths: + if fsspec.core.url_to_fs(path)[0].exists(test_path): + found_paths.append(test_path) - else: - merged_df = pd.read_parquet(merge_path) + if len(found_paths) == 0: + return None - return merged_df + area_column = f"{_label_name_to_prefix[label_name]}_AreaShape_Area" + merged_dfs = [] + for path in found_paths: + if path.lower().endswith(".zarr"): + data = read_anndata_zarr(path, dask=True) + merged_df = data.obs + columns = {area_column} + assert area_column in data.var.index + if label_filter is not None: + query_columns = _get_names_from_pd_query(label_filter) + columns.update( + c + for c in query_columns + if c not in merged_df.columns and c in data.var.index + ) + columns = list(columns) + values = data[:, columns].X.compute() + for i in range(len(columns)): + merged_df[columns[i]] = values[:, i] + + else: + merged_df = pd.read_parquet(path) + if "label" in merged_df.columns: + merged_df = merged_df.set_index("label") + merged_dfs.append(merged_df) + return ( + merged_dfs[0] + if len(merged_dfs) == 1 + else pd.concat(merged_dfs, axis=1, join="inner") + ) def _get_feature_channel_indices(tokens): @@ -180,14 +190,27 @@ def _stack_and_rename(image: xr.DataArray) -> xr.DataArray: ) +def _image_key_without_time_and_selected_time(metadata): + image_key_no_t = None + selected_timepoint = None + if "t" in metadata["group_metadata"]["group"]: + image_key_no_t = [] + for key in metadata["group_metadata"]["group"]: + if key != "t": + image_key_no_t.append(str(metadata["group_metadata"]["group"][key])) + else: + selected_timepoint = metadata["group_metadata"]["group"][key] + image_key_no_t = "-".join(image_key_no_t).replace("/", "-") + + return image_key_no_t, selected_timepoint + + def single_feature( stacked_image_tuple: tuple[tuple[str, ...], list[str | Group], dict] | None, image_tuple: tuple[tuple[str, ...], list[str | Group], dict], label_paths: list[str], output_dir: str, - output_sep: str, - objects_dir: str, - objects_dir_sep: str, + merge_paths: list[str], label_name_to_features: dict[str, set[str]], label_name_to_min_max_area: dict[str, tuple[float | None, float | None]], features_plot: set[str], @@ -215,13 +238,10 @@ def single_feature( - A tuple of metadata strings. - A list of file paths or Zarr groups containing the primary image data. - A dictionary with additional metadata. - :param labels_group: Zarr group containing labels used to identify regions of interest in the image + :param label_paths: Zarr paths containing labels used to identify regions of interest in the image for feature computation. :param output_dir: Directory path where the computed feature files will be saved. - :param output_sep: Separator string used to construct the output file names. This helps in organizing - the output files systematically. - :param objects_dir: Directory path containing find objects output. - :param objects_dir_sep: File separator for `objects_dir` + :param merge_paths: Directory path containing output to merge :param label_name_to_features: Dictionary mapping label names (keys) to sets of feature names (values). Label names correspond to components in the labeled image (e.g. nuclei), and feature names specify the features to compute. @@ -247,7 +267,8 @@ def single_feature( ) output_fs, _ = fsspec.core.url_to_fs(output_dir) - + output_sep = output_fs.sep + output_dir = output_dir.rstrip(output_fs.sep) image = _images2fov(file_list, metadata, dask=True) n_channels1 = None @@ -255,18 +276,9 @@ def single_feature( if stacked_image_tuple is not None: stacked_image = _images2fov(stacked_file_list, stacked_metadata, dask=True) n_channels1 = image.sizes["c"] - - image_key_no_t = None - selected_timepoint = None - if "t" in metadata["group_metadata"]["group"]: - image_key_no_t = [] - for key in metadata["group_metadata"]["group"]: - if key != "t": - image_key_no_t.append(str(metadata["group_metadata"]["group"][key])) - else: - selected_timepoint = metadata["group_metadata"]["group"][key] - image_key_no_t = "-".join(image_key_no_t).replace("/", "-") - + image_key_no_t, selected_timepoint = _image_key_without_time_and_selected_time( + metadata + ) for label_name in label_name_to_features: label_prefix = _label_name_to_prefix[label_name] features = label_name_to_features[label_name] @@ -307,11 +319,10 @@ def single_feature( continue merged_df = None - if objects_dir is not None: + if len(merge_paths) > 0: merged_df = _read_merged_or_objects( - path=objects_dir, + paths=merge_paths, timepoint=timepoint, - path_sep=objects_dir_sep, label_name=label_name, image_key=image_key, label_filter=label_filter, @@ -493,7 +504,7 @@ def run_pipeline_compute_features(arguments: argparse.Namespace) -> None: image_patterns = arguments.image_pattern output_dir = arguments.output - objects_dir = arguments.objects + merge_paths = arguments.merge subset = arguments.subset force = arguments.force groupby = arguments.groupby @@ -527,10 +538,6 @@ def run_pipeline_compute_features(arguments: argparse.Namespace) -> None: threads_per_worker=4 if "sizeshape" in unique_features else 1 ) - objects_dir_sep = None - if objects_dir is not None: - objects_dir_sep = fsspec.core.url_to_fs(objects_dir)[0].sep - objects_dir = objects_dir.rstrip(objects_dir_sep) label_name_to_min_max_area = dict( nuclei=[arguments.nuclei_min_area, arguments.nuclei_max_area], cytosol=[arguments.cytosol_min_area, arguments.cytosol_max_area], @@ -581,9 +588,7 @@ def run_pipeline_compute_features(arguments: argparse.Namespace) -> None: img_tuple[0], img_tuple[1], output_dir=output_dir, - output_sep=output_fs.sep, - objects_dir=objects_dir, - objects_dir_sep=objects_dir_sep, + merge_paths=merge_paths, label_paths=label_paths, label_filter=label_filter, label_name_to_min_max_area=label_name_to_min_max_area, diff --git a/scallops/cli/features_main.py b/scallops/cli/features_main.py index 0117d13..0062e86 100644 --- a/scallops/cli/features_main.py +++ b/scallops/cli/features_main.py @@ -121,8 +121,8 @@ def new_format_help(x): ) required.add_argument( "--merge", - required=False, - help="Path to directory containing output from `merge`", + nargs="*", + help="Path(s) to directory containing output from `merge`", ) parser.add_argument( "--label-filter", diff --git a/scallops/features/generate.py b/scallops/features/generate.py index 036bd62..df20ba7 100644 --- a/scallops/features/generate.py +++ b/scallops/features/generate.py @@ -134,7 +134,7 @@ def label_features( ) -> dd.DataFrame | pd.DataFrame: """Extract features from labeled regions in the image. - :param objects_df: Data frame containing labeled regions from `find_objects`. + :param objects_df: Data frame containing labeled regions from `find_objects` with frame index set to label id. :param label_image: Labeled regions. :param intensity_image: Intensity image with dimensions (y, x, c) or zarr array(s) with dimensions with leading dimensions unrolled to channel dimension From 6a7cd74e3c0cfd3ef4ad563d44f59e2f14031958 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Wed, 17 Jun 2026 13:02:15 -0400 Subject: [PATCH 20/79] Segmentation time --- scallops/cli/features.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scallops/cli/features.py b/scallops/cli/features.py index a093a32..d4fd211 100644 --- a/scallops/cli/features.py +++ b/scallops/cli/features.py @@ -210,7 +210,7 @@ def single_feature( image_tuple: tuple[tuple[str, ...], list[str | Group], dict], label_paths: list[str], output_dir: str, - merge_paths: list[str], + merge_paths: list[str] | None, label_name_to_features: dict[str, set[str]], label_name_to_min_max_area: dict[str, tuple[float | None, float | None]], features_plot: set[str], @@ -319,7 +319,7 @@ def single_feature( continue merged_df = None - if len(merge_paths) > 0: + if merge_paths is not None and len(merge_paths) > 0: merged_df = _read_merged_or_objects( paths=merge_paths, timepoint=timepoint, From c376bccf0f8dc1d0b168e23a0bf434f73cc7cc1d Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Thu, 18 Jun 2026 14:11:14 -0400 Subject: [PATCH 21/79] Segmentation time --- scallops/cli/extract_crops.py | 7 ++-- scallops/cli/extract_crops_main.py | 2 +- scallops/cli/features.py | 66 +++++++++++++++++++++--------- scallops/cli/features_main.py | 7 +--- scallops/cli/find_objects.py | 13 ++++-- scallops/cli/pooled_if_sbs.py | 66 ++++++++++++++++-------------- wdl/ops_tasks.wdl | 6 +-- wdl/ops_workflow.wdl | 56 +++++++++++-------------- 8 files changed, 127 insertions(+), 96 deletions(-) diff --git a/scallops/cli/extract_crops.py b/scallops/cli/extract_crops.py index 79a6796..b8a07f8 100644 --- a/scallops/cli/extract_crops.py +++ b/scallops/cli/extract_crops.py @@ -65,7 +65,7 @@ def single_crop( image_key = metadata["id"] image = _images2fov(file_list, metadata, dask=True).squeeze().data - image_key_no_t, selected_timepoint = _image_key_without_time_and_selected_time( + image_key_without_t, selected_timepoint = _image_key_without_time_and_selected_time( metadata ) @@ -73,7 +73,7 @@ def single_crop( label_paths=label_paths, image_key=image_key, label_name=label_name, - image_key_no_t=image_key_no_t, + image_key_without_t=image_key_without_t, selected_timepoint=selected_timepoint, ) if g is None: @@ -110,10 +110,11 @@ def single_crop( label_name=label_name, timepoint=timepoint, image_key=image_key, + image_key_without_t=image_key_without_t, label_filter=label_filter, ) if merged_df is None: - raise ValueError(f"Unable to read merged data for {image_key}.") + raise ValueError(f"Unable to read metadata for {image_key}.") n_labels_before_filtering = len(merged_df) if label_filter is not None: merged_df = merged_df.query(label_filter) diff --git a/scallops/cli/extract_crops_main.py b/scallops/cli/extract_crops_main.py index 9481be8..dc2c823 100644 --- a/scallops/cli/extract_crops_main.py +++ b/scallops/cli/extract_crops_main.py @@ -66,7 +66,7 @@ def run_pipeline_extract_crops(arguments: argparse.Namespace): label_name = arguments.label_name # cell, cytosol, nuclei chunks = arguments.chunks if dask_server_url is None and arguments.dask_cluster is None: - dask_cluster_parameters = _dask_workers_threads() + dask_cluster_parameters = _dask_workers_threads(threads_per_worker=4) no_version = arguments.no_version image_seq = from_sequence( diff --git a/scallops/cli/features.py b/scallops/cli/features.py index d4fd211..af12a60 100644 --- a/scallops/cli/features.py +++ b/scallops/cli/features.py @@ -61,30 +61,46 @@ def _read_merged_or_objects( timepoint: str | None, label_name: str, image_key: str, + image_key_without_t: str | None, label_filter: str | None, ): - found_paths = [] + found_paths = [] # tuple of (path, time) + for path in paths: path_sep = fsspec.core.url_to_fs(path)[0].sep path = path.rstrip(path_sep) test_paths = [ - f"{path}{path_sep}{label_name}{path_sep}{image_key}.parquet", - f"{path}{path_sep}{image_key}.zarr", - f"{path}{path_sep}{image_key}.parquet", - get_path(path, path_sep, label_name, image_key, timepoint), + (f"{path}{path_sep}{label_name}{path_sep}{image_key}.parquet", None), + (f"{path}{path_sep}{image_key}.zarr", None), + (f"{path}{path_sep}{image_key}.parquet", None), + ( + get_path( + path, + path_sep, + label_name, + image_key_without_t + if image_key_without_t is not None + else image_key, + timepoint, + "-objects.parquet", + ), + timepoint if image_key_without_t is not None else None, + ), ] - for test_path in test_paths: + for test_path, t in test_paths: if fsspec.core.url_to_fs(path)[0].exists(test_path): - found_paths.append(test_path) + logger.info(f"Reading {test_path}") + found_paths.append((test_path, t)) if len(found_paths) == 0: return None area_column = f"{_label_name_to_prefix[label_name]}_AreaShape_Area" merged_dfs = [] - for path in found_paths: + # add suffix for time specific paths + for path, t in found_paths: if path.lower().endswith(".zarr"): data = read_anndata_zarr(path, dask=True) merged_df = data.obs @@ -106,6 +122,8 @@ def _read_merged_or_objects( merged_df = pd.read_parquet(path) if "label" in merged_df.columns: merged_df = merged_df.set_index("label") + if t is not None: + merged_df.columns = merged_df.columns + f"_{t}" merged_dfs.append(merged_df) return ( merged_dfs[0] @@ -142,7 +160,7 @@ def _find_labels( label_paths: list[str], image_key: str, label_name: str, - image_key_no_t: str | None, + image_key_without_t: str | None, selected_timepoint: Any, ): timepoints = None @@ -159,8 +177,8 @@ def _find_labels( else [None] ) return g, timepoints - if g is None and image_key_no_t is not None: - g = labels_group.get(f"{image_key_no_t}-{label_name}") + if g is None and image_key_without_t is not None: + g = labels_group.get(f"{image_key_without_t}-{label_name}") zarr_metadata = g.attrs["multiscales"][0]["metadata"] if "t" in zarr_metadata: @@ -191,18 +209,20 @@ def _stack_and_rename(image: xr.DataArray) -> xr.DataArray: def _image_key_without_time_and_selected_time(metadata): - image_key_no_t = None + image_key_without_t = None selected_timepoint = None if "t" in metadata["group_metadata"]["group"]: - image_key_no_t = [] + image_key_without_t = [] for key in metadata["group_metadata"]["group"]: if key != "t": - image_key_no_t.append(str(metadata["group_metadata"]["group"][key])) + image_key_without_t.append( + str(metadata["group_metadata"]["group"][key]) + ) else: selected_timepoint = metadata["group_metadata"]["group"][key] - image_key_no_t = "-".join(image_key_no_t).replace("/", "-") + image_key_without_t = "-".join(image_key_without_t).replace("/", "-") - return image_key_no_t, selected_timepoint + return image_key_without_t, selected_timepoint def single_feature( @@ -276,7 +296,7 @@ def single_feature( if stacked_image_tuple is not None: stacked_image = _images2fov(stacked_file_list, stacked_metadata, dask=True) n_channels1 = image.sizes["c"] - image_key_no_t, selected_timepoint = _image_key_without_time_and_selected_time( + image_key_without_t, selected_timepoint = _image_key_without_time_and_selected_time( metadata ) for label_name in label_name_to_features: @@ -286,7 +306,7 @@ def single_feature( label_paths=label_paths, image_key=image_key, label_name=label_name, - image_key_no_t=image_key_no_t, + image_key_without_t=image_key_without_t, selected_timepoint=selected_timepoint, ) if g is None: @@ -325,8 +345,11 @@ def single_feature( timepoint=timepoint, label_name=label_name, image_key=image_key, + image_key_without_t=image_key_without_t, label_filter=label_filter, ) + if merged_df is None: + raise ValueError(f"Metadata not found for {image_key}") if timepoint is not None and labels_array.ndim == 3: timepoint_index = timepoints.index(timepoint) label_image = labels_array[timepoint_index] @@ -339,7 +362,12 @@ def single_feature( ) merged_df = find_objects(label_image) objects_path = get_path( - output_dir, output_sep, label_name, image_key, timepoint + output_dir, + output_sep, + label_name, + image_key, + timepoint, + "-objects.parquet", ) merged_df.index.name = "label" diff --git a/scallops/cli/features_main.py b/scallops/cli/features_main.py index 0062e86..9786ed0 100644 --- a/scallops/cli/features_main.py +++ b/scallops/cli/features_main.py @@ -108,11 +108,6 @@ def new_format_help(x): help=generic_features_help, ) - parser.add_argument( - "--objects", - required=False, - help="Path to directory containing output from `find-objects` or `merge`", - ) parser.add_argument( "--stack-images", help="Path to additional images to stack with `images`. Add `s` prefix to refer" @@ -122,7 +117,7 @@ def new_format_help(x): required.add_argument( "--merge", nargs="*", - help="Path(s) to directory containing output from `merge`", + help="Path(s) to directory containing output from `find-objects`, `merge`, or `features`", ) parser.add_argument( "--label-filter", diff --git a/scallops/cli/find_objects.py b/scallops/cli/find_objects.py index c746517..2501201 100644 --- a/scallops/cli/find_objects.py +++ b/scallops/cli/find_objects.py @@ -33,8 +33,8 @@ def get_path( output_sep: str, label_name: str, image_key: str, - timepoint: str | None, - suffix="-objects.parquet", + timepoint: str | None = None, + suffix="", ): return ( (f"{output_dir}{output_sep}{label_name}{output_sep}{image_key}{suffix}") @@ -55,7 +55,14 @@ def _execute( assert len(file_list) == 1 label_name = group[len(group) - 1] image_key = "-".join(group[:-1]) # exclude suffix from key - path = get_path(output_dir, output_sep, label_name, image_key, timepoint) + path = get_path( + output_dir, + output_sep, + label_name, + image_key, + timepoint, + suffix="-objects.parquet", + ) fs = fsspec.url_to_fs(path)[0] if fs.exists(path): if force: diff --git a/scallops/cli/pooled_if_sbs.py b/scallops/cli/pooled_if_sbs.py index ceb30bc..b207b29 100644 --- a/scallops/cli/pooled_if_sbs.py +++ b/scallops/cli/pooled_if_sbs.py @@ -572,7 +572,7 @@ def merge_sbs_phenotype_pipeline( prefixes = [] for i in range(len(phenotype_paths)): - df = dd.read_parquet(phenotype_paths[i]) + df = dd.read_parquet(path) _metadata_cols = df.columns[ df.columns.str.contains(_metadata_columns_whitelist_str) ].tolist() @@ -584,7 +584,7 @@ def merge_sbs_phenotype_pipeline( if phenotype_suffix is not None: df.columns = df.columns + phenotype_suffix[i] - prefixes.append(phenotype_paths[i].split("/")[-3]) + prefixes.append(path.split("/")[-3]) if output_format == "zarr": # read index and metadata if len(_metadata_cols) > 0: @@ -595,7 +595,7 @@ def merge_sbs_phenotype_pipeline( ) for key in rename_features: feature_names_i[feature_names_i.index(key)] = rename_features[key] - df = dd.read_parquet(phenotype_paths[i], columns=_metadata_cols) + df = dd.read_parquet(path, columns=_metadata_cols) feature_names += feature_names_i unique_columns.update(feature_names_i) @@ -671,34 +671,38 @@ def merge_sbs_phenotype_pipeline( ) -def _find_phenotype_paths( - phenotype_paths, phenotype_filesystems, phenotype_suffix, image_key -): - _phenotype_paths = [] - _phenotype_suffix = [] +def _find_phenotype_paths(paths: list[str], image_key: str, image_metadata: dict): + found_paths = [] + for path in paths: + path_ = path + path = path.format(**image_metadata["file_metadata"][0]) + fs, path = fsspec.url_to_fs(path) + if path_ == path and "*" not in path: # directory + # match */A1-*.parquet and */A1.parquet + sep = fs.sep + matches = fs.glob(f"{path}{sep}*{sep}{image_key}-*.parquet") + fs.glob( + f"{path}{sep}*{sep}{image_key}.parquet" + ) - for i in range(len(phenotype_paths)): - # match */A1-*.parquet and */A1.parquet - sep = phenotype_filesystems[i].sep - matches = phenotype_filesystems[i].glob( - f"{phenotype_paths[i]}{sep}*{sep}{image_key}-*.parquet" - ) + phenotype_filesystems[i].glob( - f"{phenotype_paths[i]}{sep}*{sep}{image_key}.parquet" - ) + if len(matches) == 0: + # match A1-*.parquet and A1.parquet + matches = fs.glob(f"{path}{sep}{image_key}-*.parquet") + fs.glob( + f"{path}{sep}{image_key}.parquet" + ) - if len(matches) == 0: - # match A1-*.parquet and A1.parquet - matches = phenotype_filesystems[i].glob( - f"{phenotype_paths[i]}{sep}{image_key}-*.parquet" - ) + phenotype_filesystems[i].glob( - f"{phenotype_paths[i]}{sep}{image_key}.parquet" - ) + for x in matches: + path.append(fs.unstrip_protocol(x)) + # if phenotype_suffix is not None: + # _phenotype_suffix.append(phenotype_suffix[i]) - for x in matches: - _phenotype_paths.append(phenotype_filesystems[i].unstrip_protocol(x)) - if phenotype_suffix is not None: - _phenotype_suffix.append(phenotype_suffix[i]) - return _phenotype_paths, _phenotype_suffix + else: + if "*" in path: + matches = fs.glob(path) + for match in matches: + found_paths.append(fs.unstrip_protocol(match)) + elif fs.exists(path): + found_paths.append(fs.unstrip_protocol(path)) + return found_paths def merge_main(arguments: argparse.Namespace): @@ -745,8 +749,9 @@ def merge_main(arguments: argparse.Namespace): if len(set(phenotype_paths)) != len(phenotype_paths): raise ValueError("Duplicate phenotype paths") for i in range(len(phenotype_paths)): - phenotype_fs, _ = fsspec.core.url_to_fs(phenotype_paths[i]) - phenotype_paths[i] = phenotype_paths[i].rstrip(phenotype_fs.sep) + path = phenotype_paths[i] + phenotype_fs, _ = fsspec.core.url_to_fs(path) + path = path.rstrip(phenotype_fs.sep) phenotype_filesystems.append(phenotype_fs) paths = [] @@ -757,6 +762,7 @@ def merge_main(arguments: argparse.Namespace): sbs = sbs.rstrip(sbs_fs.sep) sbs_matches = sbs_fs.glob(sbs + sbs_fs.sep + "*.parquet") sbs_matches = [sbs_fs.unstrip_protocol(m) for m in sbs_matches] + print("sbs_matches", sbs_matches) for sbs_path in sbs_matches: name = os.path.splitext(os.path.basename(sbs_path))[0] if not name.startswith("."): # ignore hidden files diff --git a/wdl/ops_tasks.wdl b/wdl/ops_tasks.wdl index 4f50e53..2a35a24 100644 --- a/wdl/ops_tasks.wdl +++ b/wdl/ops_tasks.wdl @@ -371,7 +371,7 @@ task intersects_boundary { --subset "~{subset}" \ --output "~{output_directory}" \ --images "~{images}" \ - --objects "~{objects}" \ + --merge "~{objects}" \ --no-normalize \ ~{'--image-pattern ' + image_pattern} \ ~{true="--force" false="" force} @@ -461,7 +461,7 @@ task features { String? features_extra_arguments String? model_dir Array[String] labels - String? objects + String? merge String images String subset Boolean? force @@ -498,8 +498,8 @@ task features { ~{if defined(cell_max_area) && select_first([cell_max_area])>0 then '--cell-max-area ' + cell_max_area else ''} \ ~{if defined(cytosol_max_area) && select_first([cytosol_max_area])>0 then '--cytosol-max-area ' + cytosol_max_area else ''} \ ~{if defined(features_extra_arguments) then features_extra_arguments else ''} \ + --merge ~{merge} \ --labels ~{sep=" " labels} \ - ~{"--objects " + objects} \ ~{"--label-filter " + '"' + label_filter + '"'} \ --subset ~{subset} \ ~{"--image-pattern " + image_pattern} \ diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index 6133f9f..b893206 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -77,6 +77,7 @@ workflow ops_workflow { String? cell_segmentation_extra_arguments Boolean mark_stitch_boundary_cells = true + String intersects_stitch_boundary_label = "cell" # nuclei # merge String? merge_extra_arguments @@ -151,9 +152,9 @@ workflow ops_workflow { String merge_memory = "256 GiB" String merge_disks = "local-disk 20 HDD" - Int cell_intersects_boundary_cpu = 16 - String cell_intersects_boundary_memory = "32 GiB" - String cell_intersects_boundary_disks = "local-disk 200 HDD" + Int intersects_boundary_cpu = 16 + String intersects_boundary_memory = "32 GiB" + String intersects_boundary_disks = "local-disk 200 HDD" String docker @@ -167,9 +168,7 @@ workflow ops_workflow { String register_iss_transforms_suffix = "iss-transforms-t0" String register_pheno_to_iss_suffix = "pheno-to-iss-registered.zarr" String register_pheno_to_iss_transforms_suffix = "pheno-to-iss-transforms" - String nuclei_objects_suffix = "objects-nuclei" - String cell_objects_suffix = "objects-cell" - String cytosol_objects_suffix = "objects-cytosol" + String objects_suffix = "objects" String nuclei_features_suffix = "features-nuclei" String cell_features_suffix = "features-cell" String cytosol_features_suffix = "features-cytosol" @@ -181,8 +180,7 @@ workflow ops_workflow { String reads_suffix = "reads" String merge_meta_suffix = "merge-sbs-metadata" String merge_features_suffix = "merge-features" - String cell_intersects_boundary_suffix = "intersects-boundary" - String cell_intersects_boundary_non_reference_t_suffix = "intersects-boundary-t" + String intersects_boundary_suffix = "intersects-boundary" } String output_stripped = sub(output_directory, "/+$", "") + "/" @@ -194,9 +192,7 @@ workflow ops_workflow { String nuclei_features_directory = output_stripped + nuclei_features_suffix String cell_features_directory = output_stripped + cell_features_suffix String cytosol_features_directory = output_stripped + cytosol_features_suffix - String nuclei_objects_directory = output_stripped + nuclei_objects_suffix - String cell_objects_directory = output_stripped + cell_objects_suffix - String cytosol_objects_directory = output_stripped + cytosol_objects_suffix + String objects_directory = output_stripped + objects_suffix String register_pheno_to_pheno_directory = output_stripped + register_pheno_to_pheno_suffix String register_pheno_to_pheno_transform_directory = output_stripped + register_pheno_to_pheno_transform_suffix String spot_detect_directory = output_stripped + spot_detect_suffix @@ -204,7 +200,7 @@ workflow ops_workflow { String merge_meta_directory = output_stripped + merge_meta_suffix String merge_features_directory = output_stripped + merge_features_suffix String register_pheno_to_iss_qc_directory = output_stripped + register_pheno_to_iss_qc_suffix - String cell_intersects_boundary_directory = output_stripped + cell_intersects_boundary_suffix + String intersects_boundary_directory = output_stripped + intersects_boundary_suffix Boolean iss_url_supplied = defined(iss_url) Boolean pheno_url_supplied = defined(phenotype_url) @@ -339,7 +335,7 @@ workflow ops_workflow { labels=select_all([segment_nuclei.output_url, register_pheno_to_pheno.label_output_url]), label_pattern=groupby_pattern, suffix="nuclei", - output_directory=nuclei_objects_directory, + output_directory=objects_directory, subset = subset_, force = force_find_objects, docker=docker, @@ -358,7 +354,7 @@ workflow ops_workflow { labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), label_pattern=groupby_pattern, suffix="cell", - output_directory=cell_objects_directory, + output_directory=objects_directory, subset = subset_, force = force_find_objects, docker=docker, @@ -377,7 +373,7 @@ workflow ops_workflow { labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), label_pattern=groupby_pattern, suffix="cytosol", - output_directory=cytosol_objects_directory, + output_directory=objects_directory, subset = subset_, force = force_find_objects, docker=docker, @@ -396,25 +392,25 @@ workflow ops_workflow { if(mark_stitch_boundary_cells) { String phenotype_url_stripped = sub(select_first([phenotype_url]), "/+$", "") - call tasks.intersects_boundary as cell_intersects_boundary { + call tasks.intersects_boundary as intersects_boundary { input: labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), images=phenotype_url_stripped + '/labels/', image_pattern=phenotype_image_pattern + '-mask', - output_directory=cell_intersects_boundary_directory, - label_type='cell', - objects=find_objects_cell.output_url, + output_directory=intersects_boundary_directory, + label_type=intersects_stitch_boundary_label, + objects=if(intersects_stitch_boundary_label=='cell') then find_objects_cell.output_url else find_objects_nuclei.output_url, groupby=phenotype_group_by_with_time, subset = subset_ + '-*', - force = force_segment_cell, + force = if(intersects_stitch_boundary_label=='cell') then force_segment_cell else force_segment_nuclei, docker=docker, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, - disks = cell_intersects_boundary_disks, - memory = cell_intersects_boundary_memory, - cpu = cell_intersects_boundary_cpu, + disks = intersects_boundary_disks, + memory = intersects_boundary_memory, + cpu = intersects_boundary_cpu, max_retries = max_retries } @@ -586,10 +582,8 @@ workflow ops_workflow { call tasks.merge as merge_sbs_metadata { input: iss_reads=select_first([reads.output_url]) + '/labels', - objects_nuclei=find_objects_nuclei.output_url, # all rounds - objects_cell=find_objects_cell.output_url, - objects_cytosol=find_objects_cytosol.output_url, - cell_intersects_boundary=cell_intersects_boundary.output_url, + objects_nuclei=if(run_cell_segmentation) then find_objects_nuclei.output_url else find_objects_cell.output_url, + cell_intersects_boundary=intersects_boundary.output_url, register_pheno_to_iss_qc=register_pheno_to_iss_qc.output_url, register_iss_to_iss_qc=register_iss_to_iss_qc.output_url, barcodes=select_first([barcodes]), @@ -623,7 +617,7 @@ workflow ops_workflow { input: images = select_first([phenotype_url]), image_pattern=phenotype_image_pattern, - objects=merge_sbs_metadata.output_url, + merge=merge_sbs_metadata.output_url, labels=select_all([segment_nuclei.output_url, register_pheno_to_pheno.label_output_url]), label_filter=features_label_filter, groupby=phenotype_group_by_with_time, @@ -634,7 +628,7 @@ workflow ops_workflow { model_dir=model_dir, - output_directory=nuclei_features_directory + '-' + phenotype_time + '-' + feature_index, + output_directory=nuclei_features_directory + '-' + phenotype_time + '-batch' + feature_index, subset = subset_ + '-' + phenotype_time, force = force_features, docker=docker, @@ -664,7 +658,7 @@ workflow ops_workflow { input: images = select_first([phenotype_url]), image_pattern=phenotype_image_pattern, - objects=merge_sbs_metadata.output_url, + merge=merge_sbs_metadata.output_url, labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), label_filter=features_label_filter, groupby=phenotype_group_by_with_time, @@ -705,7 +699,7 @@ workflow ops_workflow { input: images = select_first([phenotype_url]), image_pattern=phenotype_image_pattern, - objects=merge_sbs_metadata.output_url, + merge=merge_sbs_metadata.output_url, labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), label_filter=features_label_filter, groupby=phenotype_group_by_with_time, From 166a8b5d94a1c71e1b9eb857b18b1323692eec0a Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Thu, 25 Jun 2026 10:40:23 -0400 Subject: [PATCH 22/79] read merged --- scallops/cli/features.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/scallops/cli/features.py b/scallops/cli/features.py index af12a60..725274b 100644 --- a/scallops/cli/features.py +++ b/scallops/cli/features.py @@ -63,6 +63,7 @@ def _read_merged_or_objects( image_key: str, image_key_without_t: str | None, label_filter: str | None, + add_timepoint_suffix: bool, ): found_paths = [] # tuple of (path, time) @@ -122,7 +123,7 @@ def _read_merged_or_objects( merged_df = pd.read_parquet(path) if "label" in merged_df.columns: merged_df = merged_df.set_index("label") - if t is not None: + if add_timepoint_suffix and t is not None: merged_df.columns = merged_df.columns + f"_{t}" merged_dfs.append(merged_df) return ( @@ -347,6 +348,7 @@ def single_feature( image_key=image_key, image_key_without_t=image_key_without_t, label_filter=label_filter, + add_timepoint_suffix=False, ) if merged_df is None: raise ValueError(f"Metadata not found for {image_key}") @@ -428,6 +430,13 @@ def single_feature( merged_df = merged_df.query(label_filter) min_max_area = label_name_to_min_max_area.get(label_name) area_column = f"{label_prefix}_AreaShape_Area" + bounding_box_columns = [ + f"{label_prefix}_AreaShape_BoundingBoxMinimum_Y", + f"{label_prefix}_AreaShape_BoundingBoxMinimum_X", + f"{label_prefix}_AreaShape_BoundingBoxMaximum_Y", + f"{label_prefix}_AreaShape_BoundingBoxMaximum_X", + ] + n_labels = len(merged_df) prefix = "" if min_max_area[0] is not None or min_max_area[1] is not None: @@ -450,12 +459,7 @@ def single_feature( intensity_image=intensity_image, features=features, normalize=normalize, - bounding_box_columns=[ - f"{label_prefix}_AreaShape_BoundingBoxMinimum_Y", - f"{label_prefix}_AreaShape_BoundingBoxMinimum_X", - f"{label_prefix}_AreaShape_BoundingBoxMaximum_Y", - f"{label_prefix}_AreaShape_BoundingBoxMaximum_X", - ], + bounding_box_columns=bounding_box_columns, channel_names=channel_names, ) # df will be None if only area and coordinates requested @@ -466,7 +470,6 @@ def single_feature( fs.rm(features_path, recursive=True) df.index.name = "label" df.columns = f"{label_prefix}_" + df.columns - if isinstance(df, pd.DataFrame): table = pa.Table.from_pandas(df, preserve_index=True) if not no_version: @@ -500,6 +503,10 @@ def single_feature( features_plot_label = [ label_prefix + "_" + feature for feature in features_plot_label ] + if timepoint is not None: + features_plot_label = [ + f"{c}_{timepoint}" for c in features_plot_label + ] df_features = pd.read_parquet( features_path, columns=features_plot_label ) From 24d84f17f0b26b7e2dab0606d24056eac11c9277 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Thu, 25 Jun 2026 13:19:57 -0400 Subject: [PATCH 23/79] read merged --- wdl/ops_tasks.wdl | 29 ++++++++++++++--------------- wdl/ops_workflow.wdl | 5 +++-- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/wdl/ops_tasks.wdl b/wdl/ops_tasks.wdl index 2a35a24..34a026a 100644 --- a/wdl/ops_tasks.wdl +++ b/wdl/ops_tasks.wdl @@ -657,16 +657,17 @@ task reads { task merge { input { String? iss_reads + + String? objects + String? cell_intersects_boundary + String? register_pheno_to_iss_qc + String? register_iss_to_iss_qc + String? register_pheno_to_pheno_qc + + Array[String]? phenotypes_nuclei Array[String]? phenotypes_cell Array[String]? phenotypes_cytosol - String? objects_nuclei - String? objects_cell - String? register_pheno_to_iss_qc - String? register_iss_to_iss_qc - String? objects_cytosol - String? cell_intersects_boundary - String? cell_intersects_boundary_t String output_directory String? barcodes @@ -686,7 +687,7 @@ task merge { } command <<< - set -e + set -ex scallops pooled-sbs merge \ @@ -694,16 +695,14 @@ task merge { ~{"--barcodes " + barcodes} \ --output "~{output_directory}" \ --phenotype \ - ~{sep=" " phenotypes_nuclei} \ - ~{sep=" " phenotypes_cell} \ - ~{sep=" " phenotypes_cytosol} \ - ~{objects_nuclei} \ - ~{objects_cell} \ - ~{objects_cytosol} \ + ~{objects} \ ~{cell_intersects_boundary} \ - ~{cell_intersects_boundary_t} \ ~{register_pheno_to_iss_qc} \ ~{register_iss_to_iss_qc} \ + ~{register_pheno_to_pheno_qc} \ + ~{sep=" " phenotypes_nuclei} \ + ~{sep=" " phenotypes_cell} \ + ~{sep=" " phenotypes_cytosol} \ --subset ~{subset} \ ~{"--barcode-col " + barcode_column} \ ~{if defined(extra_arguments) then extra_arguments else ''} \ diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index b893206..2e88b28 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -175,7 +175,7 @@ workflow ops_workflow { String register_pheno_to_pheno_suffix = "pheno-registered.zarr" String register_pheno_to_pheno_transform_suffix = "pheno-to-pheno-transforms" String register_pheno_to_iss_qc_suffix = "pheno-to-iss-qc" - String register_iss_to_iss_qc_directory = "iss-to-iss-qc" + String register_iss_to_iss_qc_suffix = "iss-to-iss-qc" String spot_detect_suffix = "spot-detect.zarr" String reads_suffix = "reads" String merge_meta_suffix = "merge-sbs-metadata" @@ -201,6 +201,7 @@ workflow ops_workflow { String merge_features_directory = output_stripped + merge_features_suffix String register_pheno_to_iss_qc_directory = output_stripped + register_pheno_to_iss_qc_suffix String intersects_boundary_directory = output_stripped + intersects_boundary_suffix + String register_iss_to_iss_qc_directory = output_stripped + register_iss_to_iss_qc_suffix Boolean iss_url_supplied = defined(iss_url) Boolean pheno_url_supplied = defined(phenotype_url) @@ -582,7 +583,7 @@ workflow ops_workflow { call tasks.merge as merge_sbs_metadata { input: iss_reads=select_first([reads.output_url]) + '/labels', - objects_nuclei=if(run_cell_segmentation) then find_objects_nuclei.output_url else find_objects_cell.output_url, + objects=if(run_cell_segmentation) then find_objects_nuclei.output_url else find_objects_cell.output_url, cell_intersects_boundary=intersects_boundary.output_url, register_pheno_to_iss_qc=register_pheno_to_iss_qc.output_url, register_iss_to_iss_qc=register_iss_to_iss_qc.output_url, From e54d030f9bc06b0f42abed89aa80b5f9ecfb324f Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Fri, 26 Jun 2026 13:38:32 -0400 Subject: [PATCH 24/79] features by t --- scallops/cli/extract_crops.py | 16 +++- scallops/cli/features.py | 21 ++++- scallops/cli/pooled_if_sbs.py | 144 +++++++++++++++------------------- 3 files changed, 93 insertions(+), 88 deletions(-) diff --git a/scallops/cli/extract_crops.py b/scallops/cli/extract_crops.py index b8a07f8..d95d21c 100644 --- a/scallops/cli/extract_crops.py +++ b/scallops/cli/extract_crops.py @@ -112,6 +112,7 @@ def single_crop( image_key=image_key, image_key_without_t=image_key_without_t, label_filter=label_filter, + add_timepoint_suffix=False, ) if merged_df is None: raise ValueError(f"Unable to read metadata for {image_key}.") @@ -120,6 +121,16 @@ def single_crop( merged_df = merged_df.query(label_filter) label_prefix = _label_name_to_prefix[label_name] area_column = f"{label_prefix}_AreaShape_Area" + centroid_cols = [ + f"{label_prefix}_AreaShape_Center_Y", + f"{label_prefix}_AreaShape_Center_X", + ] + if timepoint is not None: + if area_column not in merged_df.columns: + area_column = f"{area_column}_{timepoint}" + if centroid_cols[0] not in merged_df.columns: + centroid_cols = [f"{c}_{timepoint}" for c in centroid_cols] + merged_df = merged_df.query(f"{area_column}>=2") n_labels_filtered = n_labels_before_filtering - len(merged_df) logger.info( @@ -173,10 +184,7 @@ def single_crop( output_dir=output_dir, crop_size=crop_size, output_format=output_format, - centroid_cols=[ - f"{label_prefix}_AreaShape_Center_Y", - f"{label_prefix}_AreaShape_Center_X", - ], + centroid_cols=centroid_cols, gaussian_sigma=gaussian_sigma, ) diff --git a/scallops/cli/features.py b/scallops/cli/features.py index 725274b..00892d0 100644 --- a/scallops/cli/features.py +++ b/scallops/cli/features.py @@ -89,10 +89,11 @@ def _read_merged_or_objects( timepoint if image_key_without_t is not None else None, ), ] + if image_key_without_t is not None: + test_paths.append((f"{path}{path_sep}{image_key_without_t}.parquet", None)) for test_path, t in test_paths: if fsspec.core.url_to_fs(path)[0].exists(test_path): - logger.info(f"Reading {test_path}") found_paths.append((test_path, t)) if len(found_paths) == 0: @@ -100,7 +101,7 @@ def _read_merged_or_objects( area_column = f"{_label_name_to_prefix[label_name]}_AreaShape_Area" merged_dfs = [] - # add suffix for time specific paths + for path, t in found_paths: if path.lower().endswith(".zarr"): data = read_anndata_zarr(path, dask=True) @@ -331,8 +332,14 @@ def single_feature( ) features_path = get_path( - output_dir, output_sep, label_name, image_key, timepoint, ".parquet" + output_dir, + output_sep, + label_name, + image_key_without_t if image_key_without_t is not None else image_key, + timepoint, + ".parquet", ) + if not force and is_parquet_file(features_path): logger.info( f"Skipping features for {image_key} {label_name}{' at t=' + timepoint if timepoint is not None else ''}." @@ -340,6 +347,7 @@ def single_feature( continue merged_df = None + print(image_key, image_key_without_t) if merge_paths is not None and len(merge_paths) > 0: merged_df = _read_merged_or_objects( paths=merge_paths, @@ -436,7 +444,14 @@ def single_feature( f"{label_prefix}_AreaShape_BoundingBoxMaximum_Y", f"{label_prefix}_AreaShape_BoundingBoxMaximum_X", ] + if timepoint is not None: + if area_column not in merged_df.columns: + area_column = f"{area_column}_{timepoint}" + if bounding_box_columns[0] not in merged_df.columns: + bounding_box_columns = [ + f"{c}_{timepoint}" for c in bounding_box_columns + ] n_labels = len(merged_df) prefix = "" if min_max_area[0] is not None or min_max_area[1] is not None: diff --git a/scallops/cli/pooled_if_sbs.py b/scallops/cli/pooled_if_sbs.py index 4d1be20..a174af1 100644 --- a/scallops/cli/pooled_if_sbs.py +++ b/scallops/cli/pooled_if_sbs.py @@ -12,6 +12,7 @@ import logging import os import re +from collections import defaultdict from typing import Literal import anndata @@ -401,7 +402,7 @@ def _fix_cycles(sbs_cycles): for i in range(len(sbs_cycles)): sbs_cycles[i] = sbs_cycles[i] + 1 logger.info( - f"Timepoint indices (0-based): {', '.join([str(t - 1) for t in sbs_cycles])}" + f"ISS timepoint indices (0-based): {', '.join([str(t - 1) for t in sbs_cycles])}" ) return sbs_cycles @@ -501,7 +502,7 @@ def merge_sbs_phenotype_pipeline( image_key: str, sbs_path: str | None, phenotype_paths: list[str], - phenotype_suffix: list[str], + phenotype_suffixes: list[str], df_barcode: pd.DataFrame | None, output_dir: str, join_sbs: Literal["left", "right", "inner", "outer", "cross"] = "inner", @@ -515,7 +516,7 @@ def merge_sbs_phenotype_pipeline( :param image_key: Image identifier. :param sbs_path: SBS label assignments path in Parquet format. :param phenotype_paths: List of parquet paths containing phenotype data. - :param phenotype_suffix: List of suffixes for phenotype columns. + :param phenotype_suffixes: List of suffixes. :param df_barcode: DataFrame containing barcode information. :param output_dir: Directory to save the merged results. :param join_sbs: Type of join to perform for SBS data. @@ -532,18 +533,14 @@ def merge_sbs_phenotype_pipeline( ): logger.info(f"Skipping merge for {image_key}") return [] - paths_and_suffixes = [] - for i in range(len(phenotype_paths)): - path = phenotype_paths[i] - if phenotype_suffix is not None: - path += f" ({phenotype_suffix[i]})" - paths_and_suffixes.append(path) - logger.info(f"Running merge for {image_key} with {', '.join(paths_and_suffixes)}.") + + logger.info(f"Running merge for {image_key} with {', '.join(phenotype_paths)}.") image_metadata = None sbs_cycles = None unique_columns = set() if df_barcode is not None: unique_columns.update(df_barcode.columns.tolist()) + df_labels = None if sbs_path is not None: fs, sbs_path_ = fsspec.core.url_to_fs(sbs_path) iss_dataset = pq.ParquetDataset(sbs_path_, filesystem=fs) @@ -573,6 +570,7 @@ def merge_sbs_phenotype_pipeline( prefixes = [] for i in range(len(phenotype_paths)): + path = phenotype_paths[i] df = dd.read_parquet(path) _metadata_cols = df.columns[ df.columns.str.contains(_metadata_columns_whitelist_str) @@ -582,8 +580,8 @@ def merge_sbs_phenotype_pipeline( ].tolist() metadata_columns.append(_metadata_cols) feature_columns.append(_feature_cols) - if phenotype_suffix is not None: - df.columns = df.columns + phenotype_suffix[i] + if phenotype_suffixes[i] is not None: + df.columns = df.columns + "_" + phenotype_suffixes[i] prefixes.append(path.split("/")[-3]) @@ -676,38 +674,32 @@ def merge_sbs_phenotype_pipeline( ) -def _find_phenotype_paths(paths: list[str], image_key: str, image_metadata: dict): +def _find_phenotype_paths_and_suffixes( + paths: list[str], image_key: str +) -> tuple[list[str], list[str]]: found_paths = [] + found_suffixes = [] for path in paths: - path_ = path - path = path.format(**image_metadata["file_metadata"][0]) fs, path = fsspec.url_to_fs(path) - if path_ == path and "*" not in path: # directory - # match */A1-*.parquet and */A1.parquet - sep = fs.sep - matches = fs.glob(f"{path}{sep}*{sep}{image_key}-*.parquet") + fs.glob( - f"{path}{sep}*{sep}{image_key}.parquet" - ) + prefix = f"{path}{fs.sep}**{fs.sep}{image_key}" + maxdepth = 3 + matches = fs.glob(f"{prefix}.parquet", maxdepth=maxdepth) + fs.glob( + f"{prefix}-objects.parquet", maxdepth=maxdepth + ) - if len(matches) == 0: - # match A1-*.parquet and A1.parquet - matches = fs.glob(f"{path}{sep}{image_key}-*.parquet") + fs.glob( - f"{path}{sep}{image_key}.parquet" - ) + if len(matches) == 0: + raise ValueError(f"No matches found in {path}") - for x in matches: - path.append(fs.unstrip_protocol(x)) - # if phenotype_suffix is not None: - # _phenotype_suffix.append(phenotype_suffix[i]) + for x in matches: + x = fs.unstrip_protocol(x) + # if parent starts with t=xxx, add xxx suffix + tokens = x.split("/") + suffix = tokens[-2] + suffix = suffix[2:] if suffix.startswith("t=") else tokens[-3] + found_suffixes.append(suffix) + found_paths.append(x) - else: - if "*" in path: - matches = fs.glob(path) - for match in matches: - found_paths.append(fs.unstrip_protocol(match)) - elif fs.exists(path): - found_paths.append(fs.unstrip_protocol(path)) - return found_paths + return found_paths, found_suffixes def merge_main(arguments: argparse.Namespace): @@ -728,11 +720,7 @@ def merge_main(arguments: argparse.Namespace): output_dir = arguments.output output_format = arguments.format join_phenotype = arguments.join_phenotype - phenotype_suffix = arguments.phenotype_suffix - if phenotype_suffix is not None: - assert len(phenotype_paths) == len(phenotype_suffix), ( - "Length of phenotype and suffix must match" - ) + join_sbs = arguments.join_sbs subset = arguments.subset force = arguments.force @@ -750,64 +738,58 @@ def merge_main(arguments: argparse.Namespace): assert "barcode" in df_barcode.columns, ( f"`barcode` column not found in {arguments.barcodes}" ) - phenotype_filesystems = [] + if len(set(phenotype_paths)) != len(phenotype_paths): raise ValueError("Duplicate phenotype paths") - for i in range(len(phenotype_paths)): - path = phenotype_paths[i] - phenotype_fs, _ = fsspec.core.url_to_fs(path) - path = path.rstrip(phenotype_fs.sep) - phenotype_filesystems.append(phenotype_fs) - paths = [] if subset is not None: subset = _create_subset_function(subset) + values = [] + search_paths = [] if sbs is not None: - sbs_fs, _ = fsspec.core.url_to_fs(sbs) - sbs = sbs.rstrip(sbs_fs.sep) - sbs_matches = sbs_fs.glob(sbs + sbs_fs.sep + "*.parquet") - sbs_matches = [sbs_fs.unstrip_protocol(m) for m in sbs_matches] - print("sbs_matches", sbs_matches) - for sbs_path in sbs_matches: - name = os.path.splitext(os.path.basename(sbs_path))[0] - if not name.startswith("."): # ignore hidden files - image_key, _ = os.path.splitext(name) - if subset is None or subset(image_key): - _phenotype_paths, _phenotype_suffix = _find_phenotype_paths( - phenotype_paths, - phenotype_filesystems, - phenotype_suffix, - image_key, - ) - if len(_phenotype_paths) > 0: - paths.append( - ( - image_key, - sbs_path, - _phenotype_paths, - _phenotype_suffix - if phenotype_suffix is not None - else None, - ) - ) + search_paths.append(sbs) + else: + search_paths = phenotype_paths + matches = [] + for search_path in search_paths: + fs, search_path = fsspec.core.url_to_fs(search_path) + search_path = search_path.rstrip(fs.sep) + matches += fs.glob(search_path + fs.sep + "*.parquet") + image_key_to_matches = defaultdict(list) + for path in matches: + fs, path = fsspec.core.url_to_fs(path) + path = fs.unstrip_protocol(path) + name = os.path.splitext(os.path.basename(path))[0] + if not name.startswith("."): # ignore hidden files + image_key, _ = os.path.splitext(name) + if subset is None or subset(image_key): + image_key_to_matches[image_key].append(path) + + for image_key in image_key_to_matches: + phenotype_paths_, phenotype_suffixes_ = _find_phenotype_paths_and_suffixes( + phenotype_paths, image_key + ) + + if len(phenotype_paths_) > 0: + sbs_path = image_key_to_matches[image_key][0] if sbs is not None else None + values.append((image_key, sbs_path, phenotype_paths_, phenotype_suffixes_)) output_fs, _ = fsspec.core.url_to_fs(output_dir) output_dir = output_dir.rstrip(output_fs.sep) output_fs.makedirs(output_dir, exist_ok=True) - if len(paths) == 0: + if len(values) == 0: logger.warning("No files found to merge") else: with ( _create_default_dask_config(), _create_dask_client(dask_scheduler_url, **dask_cluster_parameters), ): - for path in paths: - image_key, sbs_path, phenotype_paths, phenotype_suffix = path + for image_key, sbs_path, phenotype_paths, phenotype_suffixes in values: merge_sbs_phenotype_pipeline( image_key=image_key, sbs_path=sbs_path, phenotype_paths=phenotype_paths, - phenotype_suffix=phenotype_suffix, + phenotype_suffixes=phenotype_suffixes, df_barcode=df_barcode, output_dir=output_dir + output_fs.sep, join_sbs=join_sbs, From 38c5ea77fc5b1fbbc00ec78df9657d8ea13a61f0 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Mon, 29 Jun 2026 10:31:18 -0400 Subject: [PATCH 25/79] features by t --- scallops/cli/pooled_if_sbs.py | 6 +- scallops/cli/pooled_if_sbs_main.py | 6 +- scallops/tests/test_wdl.py | 130 ++++++++++++++++++++--------- wdl/ops_tasks.wdl | 20 +++-- wdl/ops_workflow.wdl | 42 +++++----- wdl/stitch_tasks.wdl | 2 +- 6 files changed, 125 insertions(+), 81 deletions(-) diff --git a/scallops/cli/pooled_if_sbs.py b/scallops/cli/pooled_if_sbs.py index a174af1..7f0d64d 100644 --- a/scallops/cli/pooled_if_sbs.py +++ b/scallops/cli/pooled_if_sbs.py @@ -745,11 +745,7 @@ def merge_main(arguments: argparse.Namespace): if subset is not None: subset = _create_subset_function(subset) values = [] - search_paths = [] - if sbs is not None: - search_paths.append(sbs) - else: - search_paths = phenotype_paths + search_paths = [sbs] if sbs is not None else phenotype_paths matches = [] for search_path in search_paths: fs, search_path = fsspec.core.url_to_fs(search_path) diff --git a/scallops/cli/pooled_if_sbs_main.py b/scallops/cli/pooled_if_sbs_main.py index 1244d8d..31758f3 100644 --- a/scallops/cli/pooled_if_sbs_main.py +++ b/scallops/cli/pooled_if_sbs_main.py @@ -98,9 +98,7 @@ def _create_merge_parser(subparsers, default_help): ), ) required = parser.add_argument_group("required arguments") - required.add_argument( - "--sbs", required=True, help="Directory containing SBS parquet files." - ) + required.add_argument( "--phenotype", nargs="+", @@ -108,7 +106,7 @@ def _create_merge_parser(subparsers, default_help): help="Directories with phenotype parquet files.", ) output_dir_arg(required) - + parser.add_argument("--sbs", help="Directory containing SBS parquet files.") barcodes_arg(parser, False) parser.add_argument( "--join-sbs", choices=["inner", "outer"], default="outer", help="SBS join type." diff --git a/scallops/tests/test_wdl.py b/scallops/tests/test_wdl.py index 96786dd..599c173 100644 --- a/scallops/tests/test_wdl.py +++ b/scallops/tests/test_wdl.py @@ -154,8 +154,9 @@ def test_stitch_wdl(tmp_path): np.testing.assert_array_equal(image.coords["c"].values, ["a", "b"]) +@pytest.mark.parametrize("phenotype_rounds", [1, 2, None]) @pytest.mark.cli_e2e -def test_ops_wdl(tmp_path): +def test_ops_wdl(phenotype_rounds, tmp_path): sbs_dir = tmp_path / "sbs" output = tmp_path / "out" pheno_dir = tmp_path / "pheno.zarr" @@ -179,34 +180,65 @@ def test_ops_wdl(tmp_path): (pheno_img.sizes["y"], pheno_img.sizes["x"]), dtype=np.uint16 ) phenotype_tile[10, 10] = 2 - Experiment( - images={"plateA-A1-IF": pheno_img, "plateA-A1-FISH": pheno_img}, - labels={ - "plateA-A1-IF-mask": phenotype_mask, - "plateA-A1-IF-tile": phenotype_tile, - "plateA-A1-FISH-mask": phenotype_mask, - "plateA-A1-FISH-tile": phenotype_tile, - }, - ).save(pheno_dir) + reference_phenotype_time = "IF" + + phenotype_cell_features = {"IF": ["intensity_0"]} + phenotype_image_pattern = "{plate}-{well}-{t}" + if phenotype_rounds == 1: + phenotype_nuclei_features = { + "IF": ["intensity_0", "intensity_1"], + } + Experiment( + images={"plateA-A1-IF": pheno_img}, + labels={ + "plateA-A1-IF-mask": phenotype_mask, + "plateA-A1-IF-tile": phenotype_tile, + }, + ).save(pheno_dir) + elif phenotype_rounds == 2: + phenotype_nuclei_features = { + "IF": ["intensity_0", "intensity_1"], + "FISH": ["intensity_0", "intensity_1"], + } + Experiment( + images={"plateA-A1-IF": pheno_img, "plateA-A1-FISH": pheno_img}, + labels={ + "plateA-A1-IF-mask": phenotype_mask, + "plateA-A1-IF-tile": phenotype_tile, + "plateA-A1-FISH-mask": phenotype_mask, + "plateA-A1-FISH-tile": phenotype_tile, + }, + ).save(pheno_dir) + else: + phenotype_nuclei_features = { + "": ["intensity_0", "intensity_1"], + } + reference_phenotype_time = None + phenotype_image_pattern = "{plate}-{well}" + phenotype_cell_features = {"": ["intensity_0"]} + Experiment( + images={"plateA-A1": pheno_img}, + labels={ + "plateA-A1-mask": phenotype_mask, + "plateA-A1-tile": phenotype_tile, + }, + ).save(pheno_dir) input_json = { "model_dir": "", "iss_url": str(sbs_dir.absolute()), "iss_image_pattern": "{plate}-{well}-{t}.tif", + "phenotype_image_pattern": phenotype_image_pattern, "output_directory": str(output.absolute()), "iss_registration_extra_arguments": "--no-landmarks", "pheno_to_iss_registration_extra_arguments": "--no-landmarks", "pheno_registration_extra_arguments": "--no-landmarks", "phenotype_cyto_channel": [1], - "reference_phenotype_time": "IF", + "reference_phenotype_time": reference_phenotype_time, "phenotype_url": str(pheno_dir.absolute()), - "phenotype_nuclei_features": { - "IF": ["intensity_0", "intensity_1"], - "FISH": ["intensity_0", "intensity_1"], - }, + "phenotype_nuclei_features": phenotype_nuclei_features, # 2 batches - "phenotype_cell_features": {"IF": ["intensity_0"]}, - # "phenotype_cytosol_features": ["mean_0 area"], # no cytosol features + "phenotype_cell_features": phenotype_cell_features, "reads_threshold_peaks": "0", "reads_threshold_peaks_crosstalk": "20", "barcodes": os.path.abspath("scallops/tests/data/experimentC/barcodes.csv"), @@ -230,32 +262,48 @@ def test_ops_wdl(tmp_path): check_call(cmd, env=env) merge_sbs_metadata_df = pd.read_parquet( - output / "merge-sbs-metadata" / "A1-102.parquet" + output / "merge-sbs-metadata" / "plateA-A1.parquet" ) assert len(merge_sbs_metadata_df) > len( merge_sbs_metadata_df.query("~barcode_count_0.isna()") ) + assert ( + len( + merge_sbs_metadata_df.columns[ + merge_sbs_metadata_df.columns.str.contains("iss_to_iss_qc") + ] + ) + > 0 + ) + assert ( + len( + merge_sbs_metadata_df.columns[ + merge_sbs_metadata_df.columns.str.contains("Intensity") + ] + ) + == 0 + ) - for col in [ - "Nuclei_AreaShape_Area", - "Cells_AreaShape_Area", - ]: - assert col in merge_sbs_metadata_df.columns - for col in [ - "Nuclei_Intensity_MeanIntensity_Channel0", - "Nuclei_Intensity_MeanIntensity_Channel1", - "Cells_Intensity_MeanIntensity_Channel0", - ]: - assert col not in merge_sbs_metadata_df.columns - merge_features_df = pd.read_parquet(output / "merge-features" / "A1-102.parquet") - for col in [ - "Nuclei_AreaShape_Area", - "Cells_AreaShape_Area", - "Nuclei_Intensity_MeanIntensity_Channel0", - "Nuclei_Intensity_MeanIntensity_Channel1", - "Cells_Intensity_MeanIntensity_Channel0", - ]: - assert col in merge_features_df.columns - assert len( - merge_features_df.query("~Nuclei_Intensity_MeanIntensity_Channel0.isna()") - ) == len(merge_sbs_metadata_df.query("~barcode_count_0.isna()")) + merge_features_df = pd.read_parquet(output / "merge-features" / "plateA-A1.parquet") + assert ( + len( + merge_features_df.columns[ + merge_sbs_metadata_df.columns.str.contains("iss_to_iss_qc") + ] + ) + > 0 + ) + assert ( + len( + merge_features_df.columns[ + merge_sbs_metadata_df.columns.str.contains("Intensity") + ] + ) + > 0 + ) + intensity_column = merge_features_df.columns[ + merge_sbs_metadata_df.columns.str.contains("Intensity") + ][0] + assert len(merge_features_df.query(f"~{intensity_column}.isna()")) == len( + merge_sbs_metadata_df.query("~barcode_count_0.isna()") + ) diff --git a/wdl/ops_tasks.wdl b/wdl/ops_tasks.wdl index 34a026a..9ca4203 100644 --- a/wdl/ops_tasks.wdl +++ b/wdl/ops_tasks.wdl @@ -664,10 +664,10 @@ task merge { String? register_iss_to_iss_qc String? register_pheno_to_pheno_qc - - Array[String]? phenotypes_nuclei - Array[String]? phenotypes_cell - Array[String]? phenotypes_cytosol + Array[Array[String]]? phenotypes_nuclei + Array[Array[String]]? phenotypes_cell + Array[Array[String]]? phenotypes_cytosol + String? merge_metadata String output_directory String? barcodes @@ -685,11 +685,13 @@ task merge { String memory Int max_retries } + Array[String] phenotypes_nuclei_ = if defined(phenotypes_nuclei) then flatten(select_first([phenotypes_nuclei])) else [] + Array[String] phenotypes_cell_ = if defined(phenotypes_cell) then flatten(select_first([phenotypes_cell])) else [] + Array[String] phenotypes_cytosol_ = if defined(phenotypes_cytosol) then flatten(select_first([phenotypes_cytosol])) else [] command <<< set -ex - scallops pooled-sbs merge \ ~{"--sbs " + iss_reads} \ ~{"--barcodes " + barcodes} \ @@ -700,9 +702,10 @@ task merge { ~{register_pheno_to_iss_qc} \ ~{register_iss_to_iss_qc} \ ~{register_pheno_to_pheno_qc} \ - ~{sep=" " phenotypes_nuclei} \ - ~{sep=" " phenotypes_cell} \ - ~{sep=" " phenotypes_cytosol} \ + ~{merge_metadata} \ + ~{sep=" " phenotypes_nuclei_} \ + ~{sep=" " phenotypes_cell_} \ + ~{sep=" " phenotypes_cytosol_} \ --subset ~{subset} \ ~{"--barcode-col " + barcode_column} \ ~{if defined(extra_arguments) then extra_arguments else ''} \ @@ -711,7 +714,6 @@ task merge { output { String output_url = "~{output_directory}" - } runtime { diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index 2e88b28..796e533 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -670,7 +670,7 @@ workflow ops_workflow { model_dir=model_dir, - output_directory=cell_features_directory + '-' + phenotype_time + '-' + feature_index, + output_directory=cell_features_directory + '-' + phenotype_time + '-batch' + feature_index, subset = subset_ + '-' + phenotype_time, force = force_features, docker=docker, @@ -726,29 +726,29 @@ workflow ops_workflow { } } } - if (defined(barcodes)) { - call tasks.merge as merge_features { - input: - phenotypes_nuclei=features_nuclei.output_url, - phenotypes_cell=features_cell.output_url, - phenotypes_cytosol=features_cytosol.output_url, - iss_reads=merge_sbs_metadata.output_url, - output_directory=merge_features_directory, - subset = subset_, - extra_arguments=merge_extra_arguments, - force = force_merge, - docker=docker, - zones = zones, - preemptible = preemptible, - aws_queue_arn = aws_queue_arn, - disks = merge_disks, - memory = merge_memory, - cpu = merge_cpu, - max_retries = max_retries - } + + call tasks.merge as merge_features { + input: + phenotypes_nuclei=features_nuclei.output_url, + phenotypes_cell=features_cell.output_url, + phenotypes_cytosol=features_cytosol.output_url, + merge_metadata=merge_sbs_metadata.output_url, + output_directory=merge_features_directory, + subset = subset_, + extra_arguments=merge_extra_arguments, + force = force_merge, + docker=docker, + zones = zones, + preemptible = preemptible, + aws_queue_arn = aws_queue_arn, + disks = merge_disks, + memory = merge_memory, + cpu = merge_cpu, + max_retries = max_retries } + } output { Array[String?] segment_nuclei_output_url = segment_nuclei.output_url diff --git a/wdl/stitch_tasks.wdl b/wdl/stitch_tasks.wdl index 1381a20..e2f09ac 100644 --- a/wdl/stitch_tasks.wdl +++ b/wdl/stitch_tasks.wdl @@ -93,7 +93,7 @@ task stitch { } command <<< - set -e + set -ex python < Date: Mon, 29 Jun 2026 10:35:11 -0400 Subject: [PATCH 26/79] features by t --- wdl/ops_workflow.wdl | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index 796e533..c3829fb 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -228,22 +228,15 @@ workflow ops_workflow { String groupby_pattern = list_images.groupby_pattern # e.g. {plate}-{well} Array[String] groupby_array = list_images.groupby_array # e.g. ["plate", "well"] - - # Array[String] subsets_with_reference_times_pheno = list_images.subsets_with_reference_times_1 - # Array[String] subsets_with_reference_times_iss = list_images.subsets_with_reference_times_2 - Array[String] times_pheno = list_images.times_1 Array[String] times_iss = list_images.times_2 String reference_time_pheno = list_images.reference_time_1 String reference_time_iss = list_images.reference_time_2 - Array[String] phenotype_group_by_with_time = list_images.groupby_array_with_time_2 - #String image_pattern_with_reference_time_pheno = list_images.image_pattern_with_reference_time_1 # e.g. {plate}-{well}-IF - # String image_pattern_with_reference_time_iss = list_images.image_pattern_with_reference_time_2 # e.g. {plate}-{well}-1 + Array[String] phenotype_group_by_with_time = list_images.groupby_array_with_time_2 # e.g. ["plate", "well", "t"] scatter (subset_index in range(length(subsets))) { String subset_ = subsets[subset_index] # e.g. plate1-A1 - # String subset_with_reference_times_pheno = subsets_with_reference_times_pheno[subset_index] - # String subset_with_reference_times_iss = subsets_with_reference_times_iss[subset_index] + if(pheno_url_supplied) { if(run_nuclei_segmentation) { call tasks.segment_nuclei { From 622de5d2e926208ace7173bb5484ad303cb3218b Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Mon, 29 Jun 2026 10:44:53 -0400 Subject: [PATCH 27/79] features by t --- scallops/cli/pooled_if_sbs.py | 11 +++++++++-- scallops/cli/pooled_if_sbs_main.py | 4 +--- scallops/tests/test_wdl.py | 6 +++--- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/scallops/cli/pooled_if_sbs.py b/scallops/cli/pooled_if_sbs.py index 7f0d64d..4780a0e 100644 --- a/scallops/cli/pooled_if_sbs.py +++ b/scallops/cli/pooled_if_sbs.py @@ -693,9 +693,16 @@ def _find_phenotype_paths_and_suffixes( for x in matches: x = fs.unstrip_protocol(x) # if parent starts with t=xxx, add xxx suffix + # otherwise suffix will only be added for duplicate columns tokens = x.split("/") - suffix = tokens[-2] - suffix = suffix[2:] if suffix.startswith("t=") else tokens[-3] + suffix_ = tokens[-2] + suffix = None + if suffix_.startswith("t="): + suffix = suffix_[2:] + else: + suffix_ = tokens[-3] + if "qc" in suffix_: + suffix = suffix_ found_suffixes.append(suffix) found_paths.append(x) diff --git a/scallops/cli/pooled_if_sbs_main.py b/scallops/cli/pooled_if_sbs_main.py index 31758f3..2530ff2 100644 --- a/scallops/cli/pooled_if_sbs_main.py +++ b/scallops/cli/pooled_if_sbs_main.py @@ -117,9 +117,7 @@ def _create_merge_parser(subparsers, default_help): default="outer", help="Phenotype join type.", ) - parser.add_argument( - "--phenotype-suffix", nargs="*", help="Suffix for phenotype columns." - ) + parser.add_argument( "--format", help="Output file format.", diff --git a/scallops/tests/test_wdl.py b/scallops/tests/test_wdl.py index 599c173..6ce7fe6 100644 --- a/scallops/tests/test_wdl.py +++ b/scallops/tests/test_wdl.py @@ -288,7 +288,7 @@ def test_ops_wdl(phenotype_rounds, tmp_path): assert ( len( merge_features_df.columns[ - merge_sbs_metadata_df.columns.str.contains("iss_to_iss_qc") + merge_features_df.columns.str.contains("iss_to_iss_qc") ] ) > 0 @@ -296,13 +296,13 @@ def test_ops_wdl(phenotype_rounds, tmp_path): assert ( len( merge_features_df.columns[ - merge_sbs_metadata_df.columns.str.contains("Intensity") + merge_features_df.columns.str.contains("Intensity") ] ) > 0 ) intensity_column = merge_features_df.columns[ - merge_sbs_metadata_df.columns.str.contains("Intensity") + merge_features_df.columns.str.contains("Intensity") ][0] assert len(merge_features_df.query(f"~{intensity_column}.isna()")) == len( merge_sbs_metadata_df.query("~barcode_count_0.isna()") From d2eabbe2bae5450be173567fd838004a93e2954a Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Mon, 29 Jun 2026 11:27:37 -0400 Subject: [PATCH 28/79] features by t --- scallops/tests/test_wdl.py | 2 +- wdl/ops_workflow.wdl | 30 ++++++++++++++---------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/scallops/tests/test_wdl.py b/scallops/tests/test_wdl.py index 6ce7fe6..95a84e6 100644 --- a/scallops/tests/test_wdl.py +++ b/scallops/tests/test_wdl.py @@ -154,7 +154,7 @@ def test_stitch_wdl(tmp_path): np.testing.assert_array_equal(image.coords["c"].values, ["a", "b"]) -@pytest.mark.parametrize("phenotype_rounds", [1, 2, None]) +@pytest.mark.parametrize("phenotype_rounds", [1]) @pytest.mark.cli_e2e def test_ops_wdl(phenotype_rounds, tmp_path): sbs_dir = tmp_path / "sbs" diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index c3829fb..108e476 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -390,14 +390,14 @@ workflow ops_workflow { input: labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), - images=phenotype_url_stripped + '/labels/', - image_pattern=phenotype_image_pattern + '-mask', + images=phenotype_url_stripped + "/labels/", + image_pattern=phenotype_image_pattern + "-mask", output_directory=intersects_boundary_directory, label_type=intersects_stitch_boundary_label, - objects=if(intersects_stitch_boundary_label=='cell') then find_objects_cell.output_url else find_objects_nuclei.output_url, + objects=if(intersects_stitch_boundary_label=="cell") then find_objects_cell.output_url else find_objects_nuclei.output_url, groupby=phenotype_group_by_with_time, - subset = subset_ + '-*', - force = if(intersects_stitch_boundary_label=='cell') then force_segment_cell else force_segment_nuclei, + subset = if(sub(phenotype_group_by_with_time, "{t}", "xxx")!=phenotype_group_by_with_time) then subset_ + "-*" else subset_, + force = if(intersects_stitch_boundary_label=="cell") then force_segment_cell else force_segment_nuclei, docker=docker, zones = zones, preemptible = preemptible, @@ -477,7 +477,7 @@ workflow ops_workflow { stacked_image_pattern=groupby_pattern, image_channel=iss_dapi_channel, stacked_image_channel=0, - label_type='nuclei', + label_type="nuclei", output_directory=register_pheno_to_iss_qc_directory, labels=register_pheno_to_iss.label_output_url, subset = subset_, @@ -499,7 +499,7 @@ workflow ops_workflow { image_pattern=groupby_pattern, dapi_channel=select_first([iss_dapi_channel, 0]), n_timepoints=length(times_iss), - label_type='nuclei', + label_type="nuclei", output_directory=register_iss_to_iss_qc_directory, labels=register_pheno_to_iss.label_output_url, @@ -575,7 +575,7 @@ workflow ops_workflow { call tasks.merge as merge_sbs_metadata { input: - iss_reads=select_first([reads.output_url]) + '/labels', + iss_reads=select_first([reads.output_url]) + "/labels", objects=if(run_cell_segmentation) then find_objects_nuclei.output_url else find_objects_cell.output_url, cell_intersects_boundary=intersects_boundary.output_url, register_pheno_to_iss_qc=register_pheno_to_iss_qc.output_url, @@ -619,11 +619,9 @@ workflow ops_workflow { nuclei_min_area = features_nuclei_min_area_, nuclei_max_area = features_nuclei_max_area_, features_extra_arguments=features_extra_arguments, - model_dir=model_dir, - - output_directory=nuclei_features_directory + '-' + phenotype_time + '-batch' + feature_index, - subset = subset_ + '-' + phenotype_time, + output_directory=nuclei_features_directory + "-" + phenotype_time + "-batch" + feature_index, + subset = if(phenotype_time!="") then subset_ + "-" + phenotype_time else subset_, force = force_features, docker=docker, zones = zones, @@ -663,8 +661,8 @@ workflow ops_workflow { model_dir=model_dir, - output_directory=cell_features_directory + '-' + phenotype_time + '-batch' + feature_index, - subset = subset_ + '-' + phenotype_time, + output_directory=cell_features_directory + "-" + phenotype_time + "-batch" + feature_index, + subset = if(phenotype_time!="") then subset_ + "-" + phenotype_time else subset_, force = force_features, docker=docker, zones = zones, @@ -697,7 +695,7 @@ workflow ops_workflow { labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), label_filter=features_label_filter, groupby=phenotype_group_by_with_time, - output_directory=cytosol_features_directory + '-' + phenotype_time + '-' + feature_index, + output_directory=cytosol_features_directory + "-" + phenotype_time + "-" + feature_index, cytosol_features = cytosol_features[feature_index], cytosol_min_area = features_cytosol_min_area_, cytosol_max_area = features_cytosol_max_area_, @@ -705,7 +703,7 @@ workflow ops_workflow { model_dir=model_dir, - subset = subset_ + '-' + phenotype_time, + subset = if(phenotype_time!="") then subset_ + "-" + phenotype_time else subset_, force = force_features, docker=docker, zones = zones, From 8cc4561dcd766ead0c7d166c04079b4632b3323d Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Mon, 29 Jun 2026 11:27:56 -0400 Subject: [PATCH 29/79] features by t --- scallops/tests/test_wdl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scallops/tests/test_wdl.py b/scallops/tests/test_wdl.py index 95a84e6..8f87754 100644 --- a/scallops/tests/test_wdl.py +++ b/scallops/tests/test_wdl.py @@ -154,7 +154,7 @@ def test_stitch_wdl(tmp_path): np.testing.assert_array_equal(image.coords["c"].values, ["a", "b"]) -@pytest.mark.parametrize("phenotype_rounds", [1]) +@pytest.mark.parametrize("phenotype_rounds", [2, None]) @pytest.mark.cli_e2e def test_ops_wdl(phenotype_rounds, tmp_path): sbs_dir = tmp_path / "sbs" From f00406b0b8ef84ce35b0f747156ca91d2895f2e9 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Mon, 29 Jun 2026 13:44:56 -0400 Subject: [PATCH 30/79] features by t --- scallops/cli/pooled_if_sbs.py | 28 +++++++++++++------------- scallops/cli/util.py | 38 ++--------------------------------- scallops/tests/test_wdl.py | 12 +++-------- wdl/ops_workflow.wdl | 2 +- wdl/utils.wdl | 5 +---- 5 files changed, 21 insertions(+), 64 deletions(-) diff --git a/scallops/cli/pooled_if_sbs.py b/scallops/cli/pooled_if_sbs.py index 4780a0e..41b998f 100644 --- a/scallops/cli/pooled_if_sbs.py +++ b/scallops/cli/pooled_if_sbs.py @@ -477,20 +477,19 @@ def read_values(url, index, columns): return anndata.AnnData(obs=obs, var=pd.DataFrame(index=feature_names), X=data) -def _rename_unique(columns, unique_values, prefix): +def _rename_unique(columns, unique_values, suffix): rename = dict() - replace_chars = " |-" + for value in columns: new_value = value - new_value = re.sub(replace_chars, "_", new_value) if value in unique_values: - new_value = f"{value}_{prefix}" + new_value = f"{value}_{suffix}" if new_value in unique_values: counter = 0 - new_value = f"{value}_{prefix}_{counter}" + new_value = f"{value}_{suffix}_{counter}" while new_value in unique_values: counter += 1 - new_value = f"{value}_{prefix}_{counter}" + new_value = f"{value}_{suffix}_{counter}" if value != new_value: rename[value] = new_value @@ -560,18 +559,16 @@ def merge_sbs_phenotype_pipeline( sbs_cycles = np.arange(1, sbs_cycles + 1) unique_columns.update(df_labels.columns.tolist()) df_phenotypes = [] - # can have duplicate columns if features is called in multiple batches + # can have duplicate columns if same feature called in multiple batches feature_names = [] metadata_columns = [] feature_columns = [] # used to read in subset of columns when merging to zarr # df_labels has 'mismatch', 'barcode_Q_mean', 'barcode_Q_min', 'barcode_peak', 'barcode_count', 'barcode_0', ... - prefixes = [] - for i in range(len(phenotype_paths)): path = phenotype_paths[i] - df = dd.read_parquet(path) + df = dd.read_parquet(path, dataset={"partitioning": None}) _metadata_cols = df.columns[ df.columns.str.contains(_metadata_columns_whitelist_str) ].tolist() @@ -583,22 +580,25 @@ def merge_sbs_phenotype_pipeline( if phenotype_suffixes[i] is not None: df.columns = df.columns + "_" + phenotype_suffixes[i] - prefixes.append(path.split("/")[-3]) + duplicate_suffix = path.split("/")[-3] if output_format == "zarr": # read index and metadata if len(_metadata_cols) > 0: df = df.drop(_metadata_cols, axis=1) feature_names_i = df.columns.tolist() rename_features = _rename_unique( - feature_names_i, unique_columns, prefixes[i] + feature_names_i, unique_columns, duplicate_suffix ) for key in rename_features: feature_names_i[feature_names_i.index(key)] = rename_features[key] - df = dd.read_parquet(path, columns=_metadata_cols) + df = dd.read_parquet( + path, columns=_metadata_cols, dataset={"partitioning": None} + ) feature_names += feature_names_i unique_columns.update(feature_names_i) - rename_cols = _rename_unique(df.columns, unique_columns, prefixes[i]) + rename_cols = _rename_unique(df.columns, unique_columns, duplicate_suffix) + if len(rename_cols) > 0: df = df.rename(columns=rename_cols) df_phenotypes.append(df) diff --git a/scallops/cli/util.py b/scallops/cli/util.py index 2266e89..0552d87 100644 --- a/scallops/cli/util.py +++ b/scallops/cli/util.py @@ -584,11 +584,10 @@ def _list_images_wdl( for index in range(len(results)): result = results[index] url_val = index + 1 - subset_df = result["subset_df"] with open( f"groupby_array_with_time_{url_val}.txt", "wt" - ) as f: # ['plate', 'well', 't'] + ) as f: # ['plate', 'well', 't'] if t found in image-pattern for g in result["groupby_with_time"]: f.write(g) f.write("\n") @@ -606,32 +605,6 @@ def _list_images_wdl( f.write(str(val)) f.write("\n") - with open( - f"subsets_with_reference_time_{url_val}.txt", "wt" - ) as f: # ["plate1-A1-IF", "plate1-A2-IF", ...] - subset_ids_with_reference_times = ( - subset_df["subset_ids_with_reference_times"].values - if subset_df is not None - else [] - ) - for i in range(0, len(subset_ids_with_reference_times), batch_size): - selected = subset_ids_with_reference_times[i : i + batch_size] - f.write(" ".join(selected)) - f.write("\n") - - with open( - f"image_pattern_with_reference_time_{url_val}.txt", "wt" - ) as f: # "{plate}-{well}-IF" - first = True - for g in groupby: - if not first: - f.write("-") - first = False - f.write("{") - f.write(g) - f.write("}") - f.write(f"-{result['reference_time']}") - def _list_images( urls: Sequence[str], @@ -660,7 +633,6 @@ def _list_images( groupby_t = "t" in groupby times = None subset_ids = [] - subset_ids_with_reference_times = [] first = True for g, file_list, metadata in exp_gen: @@ -673,17 +645,11 @@ def _list_images( times = [md["t"] for md in metadata["file_metadata"]] if n_cycles is not None: assert len(times) == n_cycles - t_suffix = "" - if times is not None and len(times) > 0: - t_suffix = ( - f"-{times[0]}" if reference_time is None else f"-{reference_time}" - ) subset_ids.append('"' + metadata["id"] + '"') - subset_ids_with_reference_times.append('"' + metadata["id"] + t_suffix + '"') + subset_df = pd.DataFrame( index=subset_ids, - data=dict(subset_ids_with_reference_times=subset_ids_with_reference_times), ) groupby_with_time = list(groupby) diff --git a/scallops/tests/test_wdl.py b/scallops/tests/test_wdl.py index 8f87754..db4293c 100644 --- a/scallops/tests/test_wdl.py +++ b/scallops/tests/test_wdl.py @@ -181,7 +181,6 @@ def test_ops_wdl(phenotype_rounds, tmp_path): ) phenotype_tile[10, 10] = 2 reference_phenotype_time = "IF" - phenotype_cell_features = {"IF": ["intensity_0"]} phenotype_image_pattern = "{plate}-{well}-{t}" if phenotype_rounds == 1: @@ -209,7 +208,7 @@ def test_ops_wdl(phenotype_rounds, tmp_path): "plateA-A1-FISH-tile": phenotype_tile, }, ).save(pheno_dir) - else: + else: # no t in pattern phenotype_nuclei_features = { "": ["intensity_0", "intensity_1"], } @@ -270,7 +269,7 @@ def test_ops_wdl(phenotype_rounds, tmp_path): assert ( len( merge_sbs_metadata_df.columns[ - merge_sbs_metadata_df.columns.str.contains("iss_to_iss_qc") + merge_sbs_metadata_df.columns.str.contains("-qc") ] ) > 0 @@ -286,12 +285,7 @@ def test_ops_wdl(phenotype_rounds, tmp_path): merge_features_df = pd.read_parquet(output / "merge-features" / "plateA-A1.parquet") assert ( - len( - merge_features_df.columns[ - merge_features_df.columns.str.contains("iss_to_iss_qc") - ] - ) - > 0 + len(merge_features_df.columns[merge_features_df.columns.str.contains("qc")]) > 0 ) assert ( len( diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index 108e476..fb55175 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -396,7 +396,7 @@ workflow ops_workflow { label_type=intersects_stitch_boundary_label, objects=if(intersects_stitch_boundary_label=="cell") then find_objects_cell.output_url else find_objects_nuclei.output_url, groupby=phenotype_group_by_with_time, - subset = if(sub(phenotype_group_by_with_time, "{t}", "xxx")!=phenotype_group_by_with_time) then subset_ + "-*" else subset_, + subset = if(sub(phenotype_image_pattern, "{t}", "")!=phenotype_image_pattern) then subset_ + "-*" else subset_, force = if(intersects_stitch_boundary_label=="cell") then force_segment_cell else force_segment_nuclei, docker=docker, zones = zones, diff --git a/wdl/utils.wdl b/wdl/utils.wdl index 3a8a0ee..fc5c378 100644 --- a/wdl/utils.wdl +++ b/wdl/utils.wdl @@ -66,8 +66,6 @@ task list_images { Int group_size_1 = read_int('group_size_1.txt') Int group_size_2 = read_int('group_size_2.txt') - Array[String] subsets_with_reference_times_1 = read_lines('subsets_with_reference_time_1.txt') - Array[String] subsets_with_reference_times_2 = read_lines('subsets_with_reference_time_2.txt') Array[String] times_1 = read_lines('times_1.txt') Array[String] times_2 = read_lines('times_2.txt') @@ -75,8 +73,7 @@ task list_images { String reference_time_1 = read_lines('reference_time_1.txt')[0] String reference_time_2 = read_lines('reference_time_2.txt')[0] - String image_pattern_with_reference_time_1 = read_lines('image_pattern_with_reference_time_1.txt')[0] # e.g. {plate}-{well}-IF - String image_pattern_with_reference_time_2 = read_lines('image_pattern_with_reference_time_2.txt')[0] # e.g. {plate}-{well}-IF + } From 942d4fd0748cf435353e4c9ee263bc91d75e7000 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Mon, 29 Jun 2026 16:07:12 -0400 Subject: [PATCH 31/79] features by t --- scallops/cli/extract_crops_main.py | 2 ++ scallops/tests/test_wdl.py | 21 ++++++++++----------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/scallops/cli/extract_crops_main.py b/scallops/cli/extract_crops_main.py index dc2c823..0720720 100644 --- a/scallops/cli/extract_crops_main.py +++ b/scallops/cli/extract_crops_main.py @@ -121,6 +121,7 @@ def _create_parser(subparsers: argparse.ArgumentParser, default_help: bool) -> N "--labels", dest="labels", required=True, + nargs="+", help="Path to zarr directory containing labels", ) @@ -129,6 +130,7 @@ def _create_parser(subparsers: argparse.ArgumentParser, default_help: bool) -> N required.add_argument( "--merge", required=False, + nargs="*", help="Path to directory containing output from `merge`", ) parser.add_argument( diff --git a/scallops/tests/test_wdl.py b/scallops/tests/test_wdl.py index db4293c..e33adfc 100644 --- a/scallops/tests/test_wdl.py +++ b/scallops/tests/test_wdl.py @@ -35,15 +35,21 @@ def test_list_images_wdl(reference_time, tmp_path, monkeypatch): (tmp_path / "plate1-A1-FISH").touch() monkeypatch.chdir(tmp_path) _list_images_wdl( - image_pattern="{plate}-{well}-{t}", - reference_time=reference_time, - urls=[str(tmp_path)], + image_pattern1="{plate}-{well}-{t}", + reference_time1=reference_time, + urls1=[str(tmp_path)], + n_cycles1=None, groupby=["plate", "well"], subset=None, batch_size_str=None, save_group_size=False, expected_cycles_str=None, + image_pattern2="", + urls2=[], + reference_time2=None, + n_cycles2=None, ) + groups = pd.read_csv(tmp_path / "groups.txt", header=None)[0].values np.testing.assert_array_equal(groups, ["plate1-A1"]) groupby = pd.read_csv(tmp_path / "groupby.txt", header=None)[0].values @@ -55,13 +61,6 @@ def test_list_images_wdl(reference_time, tmp_path, monkeypatch): times = pd.read_csv(tmp_path / "t.txt", header=None)[0].values np.testing.assert_array_equal(times, ["FISH", "IF"]) - groups_with_times = pd.read_csv(tmp_path / "groups_with_t.txt", header=None)[ - 0 - ].values - if reference_time == "IF": - np.testing.assert_array_equal(groups_with_times, ["plate1-A1-IF"]) - else: - np.testing.assert_array_equal(groups_with_times, ["plate1-A1-FISH"]) @pytest.mark.cli_e2e @@ -154,7 +153,7 @@ def test_stitch_wdl(tmp_path): np.testing.assert_array_equal(image.coords["c"].values, ["a", "b"]) -@pytest.mark.parametrize("phenotype_rounds", [2, None]) +@pytest.mark.parametrize("phenotype_rounds", [2]) @pytest.mark.cli_e2e def test_ops_wdl(phenotype_rounds, tmp_path): sbs_dir = tmp_path / "sbs" From 968219fd32c89e625820b8dd35e16f87aed08559 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Mon, 29 Jun 2026 16:41:24 -0400 Subject: [PATCH 32/79] features by t --- wdl/ops_workflow.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index fb55175..250c8ad 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -233,7 +233,7 @@ workflow ops_workflow { String reference_time_pheno = list_images.reference_time_1 String reference_time_iss = list_images.reference_time_2 - Array[String] phenotype_group_by_with_time = list_images.groupby_array_with_time_2 # e.g. ["plate", "well", "t"] + Array[String] phenotype_group_by_with_time = list_images.groupby_array_with_time_1 # e.g. ["plate", "well", "t"] scatter (subset_index in range(length(subsets))) { String subset_ = subsets[subset_index] # e.g. plate1-A1 From d1f58029601b7a7e90073ea910b20758f742b1b3 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 30 Jun 2026 11:08:28 -0400 Subject: [PATCH 33/79] features by t --- scallops/cli/segment.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scallops/cli/segment.py b/scallops/cli/segment.py index 1447867..842d824 100644 --- a/scallops/cli/segment.py +++ b/scallops/cli/segment.py @@ -143,7 +143,9 @@ def segment_nuclei( group_metadata = { "image-label": {"source": {"image": f"../../images/{image_key}"}} } - additional_metadata = label_metadata.get(key) if label_metadata else dict() + additional_metadata = label_metadata.get(key) + if additional_metadata is None: + additional_metadata = dict() storage_options = None if isinstance(label_data, np.ndarray): storage_options = {"chunks": image.data.chunksize[-2:]} From b99c1104e3b21faf0945e87e88f12032a0c10568 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 30 Jun 2026 11:11:01 -0400 Subject: [PATCH 34/79] features by t --- scallops/tests/test_features.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scallops/tests/test_features.py b/scallops/tests/test_features.py index 50edd55..735d525 100644 --- a/scallops/tests/test_features.py +++ b/scallops/tests/test_features.py @@ -468,7 +468,7 @@ def test_features_cli_multi_images(tmp_path, array_A1_102_cells, array_A1_102_al output_path, "--features-cell", "colocalization_0_2", - "--objects", + "--merged", objects_path, "--channel-rename", '{"0":"A", "2":"B"}', @@ -530,7 +530,7 @@ def test_features_cli(tmp_path, array_A1_102_cells, array_A1_102_alnpheno): "intensity_*", "sizeshape", "colocalization_*_*", - "--objects", + "--merged", objects_output_path, ] From e38302df71577b554607c0873b090b36a19b8aec Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 30 Jun 2026 11:25:56 -0400 Subject: [PATCH 35/79] features by t --- scallops/tests/test_features.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scallops/tests/test_features.py b/scallops/tests/test_features.py index 735d525..13338d8 100644 --- a/scallops/tests/test_features.py +++ b/scallops/tests/test_features.py @@ -468,7 +468,7 @@ def test_features_cli_multi_images(tmp_path, array_A1_102_cells, array_A1_102_al output_path, "--features-cell", "colocalization_0_2", - "--merged", + "merge", objects_path, "--channel-rename", '{"0":"A", "2":"B"}', @@ -530,7 +530,7 @@ def test_features_cli(tmp_path, array_A1_102_cells, array_A1_102_alnpheno): "intensity_*", "sizeshape", "colocalization_*_*", - "--merged", + "merge", objects_output_path, ] From 216f43dc4dae1bb4a90c7e6d185413fb3279bfdc Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 30 Jun 2026 11:28:17 -0400 Subject: [PATCH 36/79] features by t --- scallops/tests/test_features.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scallops/tests/test_features.py b/scallops/tests/test_features.py index 13338d8..48305d7 100644 --- a/scallops/tests/test_features.py +++ b/scallops/tests/test_features.py @@ -468,7 +468,7 @@ def test_features_cli_multi_images(tmp_path, array_A1_102_cells, array_A1_102_al output_path, "--features-cell", "colocalization_0_2", - "merge", + "--merge", objects_path, "--channel-rename", '{"0":"A", "2":"B"}', @@ -530,7 +530,7 @@ def test_features_cli(tmp_path, array_A1_102_cells, array_A1_102_alnpheno): "intensity_*", "sizeshape", "colocalization_*_*", - "merge", + "--merge", objects_output_path, ] From bca1ac4919b28a3674f8abc9bfda7cfa76f68d46 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 30 Jun 2026 13:03:31 -0400 Subject: [PATCH 37/79] features by t --- scallops/cli/util.py | 16 +++++++++------- wdl/stitch_workflow.wdl | 23 +++++++++++++---------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/scallops/cli/util.py b/scallops/cli/util.py index 0552d87..dd89a09 100644 --- a/scallops/cli/util.py +++ b/scallops/cli/util.py @@ -588,16 +588,17 @@ def _list_images_wdl( with open( f"groupby_array_with_time_{url_val}.txt", "wt" ) as f: # ['plate', 'well', 't'] if t found in image-pattern - for g in result["groupby_with_time"]: - f.write(g) - f.write("\n") + if result["groupby_with_time"] is not None: + for g in result["groupby_with_time"]: + f.write(g) + f.write("\n") with open(f"group_size_{url_val}.txt", "wt") as f: f.write(f"{result['group_size']}") f.write("\n") with open(f"reference_time_{url_val}.txt", "wt") as f: # IF - f.write(result["reference_time"]) + f.write(str(result["reference_time"])) f.write("\n") with open(f"times_{url_val}.txt", "wt") as f: # ["FISH", "IF"] if result["times"] is not None: @@ -615,14 +616,15 @@ def _list_images( reference_time: str | None, n_cycles: int | None, ): - reference_time_suffix = "" group_size = 0 if len(urls) == 0: return dict( group_size=group_size, subset_df=None, + groupby=groupby, + groupby_with_time=None, times=None, - reference_time_suffix=reference_time_suffix, + reference_time=reference_time, ) from scallops.io import _set_up_experiment @@ -646,7 +648,7 @@ def _list_images( if n_cycles is not None: assert len(times) == n_cycles - subset_ids.append('"' + metadata["id"] + '"') + subset_ids.append(metadata["id"]) subset_df = pd.DataFrame( index=subset_ids, diff --git a/wdl/stitch_workflow.wdl b/wdl/stitch_workflow.wdl index 2da7bbf..f6491b0 100644 --- a/wdl/stitch_workflow.wdl +++ b/wdl/stitch_workflow.wdl @@ -61,11 +61,13 @@ workflow stitch_workflow { call utils.list_images { input: - urls = urls, - image_pattern = image_pattern, + urls1 = urls, + image_pattern1 = image_pattern, + n_cycles1=expected_cycles, + subset=subset, save_group_size=true, - expected_cycles=expected_cycles, + groupby=groupby, docker=docker, zones = zones, @@ -73,9 +75,10 @@ workflow stitch_workflow { aws_queue_arn = aws_queue_arn, max_retries = max_retries } - Array[String] groups = list_images.groups + Array[String] subsets = list_images.subsets - Int group_size = list_images.group_size # assume <= 500 is 10x + Array[String] groupby_array = list_images.groupby_array # e.g. ["plate", "well"] + Int group_size = list_images.group_size_1 # assume <= 500 is 10x String illumination_correction_memory_default = if(group_size<=500) then "16 GiB" else "32 GiB" Int illumination_correction_cpu_default = if(group_size<=500) then 8 else 16 @@ -83,7 +86,7 @@ workflow stitch_workflow { String stitch_memory_default = if(group_size<=500) then "16 GiB" else "96 GiB" Int stitch_cpu_default = if(group_size<=500) then 8 else 48 - scatter (group in groups) { + scatter (subset_ in subsets) { if(run_illumination_correction) { call tasks.illumination_correction { input: @@ -93,8 +96,8 @@ workflow stitch_workflow { z_index=z_index, expected_images=expected_images, extra_arguments=illumination_correction_extra_arguments, - subset = group, - groupby=list_images.filtered_groupby, + subset = subset_, + groupby=groupby_array, force=force_illumination_correction, output_directory=illumination_correction_output_directory, docker=docker, @@ -113,8 +116,8 @@ workflow stitch_workflow { images=urls, image_pattern=image_pattern, z_index=z_index, - subset = group, - groupby=list_images.filtered_groupby, + subset = subset_, + groupby=groupby_array, expected_images=expected_images, output_directory=stitch_output_directory, force=force_stitch, From 9f60bc7dba2e5a079199fa6b67ba8016d11f0df6 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Wed, 1 Jul 2026 09:41:27 -0400 Subject: [PATCH 38/79] segment t --- scallops/cli/segment.py | 78 +++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 38 deletions(-) diff --git a/scallops/cli/segment.py b/scallops/cli/segment.py index 842d824..a21981e 100644 --- a/scallops/cli/segment.py +++ b/scallops/cli/segment.py @@ -132,38 +132,18 @@ def segment_nuclei( labels_dict = dict(nuclei=nuclei) if all_nuclei is not nuclei: labels_dict["nuclei.all"] = all_nuclei - spacing = get_image_spacing(image.attrs) + label_metadata = dict() - if spacing is not None: - for key in labels_dict.keys(): - label_metadata[key] = dict(physical_pixel_sizes=spacing) - if not no_version: - label_metadata.update(cli_metadata()) - for key, label_data in labels_dict.items(): - group_metadata = { - "image-label": {"source": {"image": f"../../images/{image_key}"}} - } - additional_metadata = label_metadata.get(key) - if additional_metadata is None: - additional_metadata = dict() - storage_options = None - if isinstance(label_data, np.ndarray): - storage_options = {"chunks": image.data.chunksize[-2:]} - if timepoint_value is not None: - label_data = xr.DataArray( - get_namespace(label_data).expand_dims(label_data, 0), - dims=["t", "y", "x"], - coords={"t": [timepoint_value]}, - ) - additional_metadata.update(label_data.attrs) - _write_zarr_labels( - name=f"{image_key}-{key}", - root=root, - metadata=additional_metadata, - group_metadata=group_metadata, - labels=label_data, - storage_options=storage_options, - ) + + _write_labels( + root=root, + timepoint_value=timepoint_value, + labels_dict=labels_dict, + label_metadata=label_metadata, + image=image, + image_key=image_key, + no_version=no_version, + ) return root @@ -300,12 +280,32 @@ def segment_cells( if cell_threshold is not None: label_metadata = dict(cell=dict(threshold=cell_threshold)) + _write_labels( + root=root, + timepoint_value=timepoint_value, + labels_dict=labels_dict, + label_metadata=label_metadata, + image=image, + image_key=image_key, + no_version=no_version, + ) + + return root + + +def _write_labels( + root: zarr.Group, + timepoint_value: str | None, + labels_dict: dict, + label_metadata: dict, + image: xr.DataArray, + image_key: str, + no_version: bool = False, +): if not no_version: label_metadata.update(cli_metadata()) spacing = get_image_spacing(image.attrs) if spacing is not None: - if label_metadata is None: - label_metadata = dict() for key in labels_dict.keys(): label_metadata[key] = dict(physical_pixel_sizes=spacing) @@ -313,10 +313,15 @@ def segment_cells( group_metadata = { "image-label": {"source": {"image": f"../../images/{image_key}"}} } - additional_metadata = label_metadata.get(key) if label_metadata else dict() + additional_metadata = label_metadata.get(key) + if additional_metadata is None: + additional_metadata = dict() storage_options = None if isinstance(label_data, np.ndarray): - storage_options = {"chunks": image.data.chunksize[-2:]} + chunks = image.data.chunksize[-2:] + if timepoint_value is not None: + chunks = (1,) + chunks + storage_options = {"chunks": chunks} if timepoint_value is not None: label_data = xr.DataArray( get_namespace(label_data).expand_dims(label_data, 0), @@ -333,8 +338,6 @@ def segment_cells( storage_options=storage_options, ) - return root - def run_pipeline(arguments: argparse.Namespace, nuclei: bool): """Run nuclei or cell segmentation pipeline. @@ -438,7 +441,6 @@ def run_pipeline(arguments: argparse.Namespace, nuclei: bool): kwargs["clip"] = arguments.stardist_clip kwargs["pmin"] = arguments.stardist_pmin kwargs["pmax"] = arguments.stardist_pmax - method = getattr( importlib.import_module("scallops.segmentation." + method), f"{'segment_nuclei_' if nuclei else 'segment_cells_'}{method}", From 19cb9638f29d557c7461583dac9c661161f41b34 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Wed, 1 Jul 2026 09:45:46 -0400 Subject: [PATCH 39/79] added suffix --- scallops/cli/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scallops/cli/register.py b/scallops/cli/register.py index 8f695db..fda3b8f 100644 --- a/scallops/cli/register.py +++ b/scallops/cli/register.py @@ -533,7 +533,7 @@ def get_matching_names( image_key: str, image_dir: str | Group, labels: bool = True, - label_suffixes: Sequence[str] = {"cell", "nuclei", "cytosol"}, + label_suffixes: Sequence[str] = {"cell", "nuclei", "cytosol", "mask", "tile"}, ) -> list[str]: """Get matching keys for the given image key and directory. From 6713cd530df16409ad59b8abe3757d48141c7d9b Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Wed, 1 Jul 2026 10:06:03 -0400 Subject: [PATCH 40/79] features by t --- scallops/tests/test_segmentation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scallops/tests/test_segmentation.py b/scallops/tests/test_segmentation.py index 7aa8e4e..ef7bb93 100644 --- a/scallops/tests/test_segmentation.py +++ b/scallops/tests/test_segmentation.py @@ -289,7 +289,9 @@ def test_segment_cells_cmd(experiment_c_dask, tmp_path): subprocess.check_call(seg_args) experiment = read_experiment(tmp_path) assert len(experiment.labels.keys()) == 3 - np.testing.assert_equal(cell_labels, experiment.labels["A1-102-cell"].values) + np.testing.assert_equal( + cell_labels.squeeze(), experiment.labels["A1-102-cell"].values.squeeze() + ) @pytest.mark.segmentation_watershed From b9e10c7b796a56c5eaec49348ce7b9a276b694c5 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Wed, 1 Jul 2026 10:07:04 -0400 Subject: [PATCH 41/79] features by t --- scallops/tests/test_wdl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scallops/tests/test_wdl.py b/scallops/tests/test_wdl.py index e33adfc..7511426 100644 --- a/scallops/tests/test_wdl.py +++ b/scallops/tests/test_wdl.py @@ -153,7 +153,7 @@ def test_stitch_wdl(tmp_path): np.testing.assert_array_equal(image.coords["c"].values, ["a", "b"]) -@pytest.mark.parametrize("phenotype_rounds", [2]) +@pytest.mark.parametrize("phenotype_rounds", [2, None]) @pytest.mark.cli_e2e def test_ops_wdl(phenotype_rounds, tmp_path): sbs_dir = tmp_path / "sbs" From 6b9baa9feff7e6b0fb3b4085902774dc358936f1 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Wed, 1 Jul 2026 10:34:56 -0400 Subject: [PATCH 42/79] features by t --- scallops/tests/test_wdl.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scallops/tests/test_wdl.py b/scallops/tests/test_wdl.py index 7511426..9f0ebb7 100644 --- a/scallops/tests/test_wdl.py +++ b/scallops/tests/test_wdl.py @@ -41,9 +41,8 @@ def test_list_images_wdl(reference_time, tmp_path, monkeypatch): n_cycles1=None, groupby=["plate", "well"], subset=None, - batch_size_str=None, + batch_size=None, save_group_size=False, - expected_cycles_str=None, image_pattern2="", urls2=[], reference_time2=None, From 829fd5e06fd58a4f5b5e5a61cb72af1a6961a67d Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Wed, 1 Jul 2026 14:39:34 -0400 Subject: [PATCH 43/79] features by t --- scallops/tests/test_wdl.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scallops/tests/test_wdl.py b/scallops/tests/test_wdl.py index 9f0ebb7..7428d71 100644 --- a/scallops/tests/test_wdl.py +++ b/scallops/tests/test_wdl.py @@ -28,7 +28,7 @@ def add_physical_size(input_path, output_path): save_ome_tiff(img.values, uri=output_path, ome_xml=img.attrs["processed"].to_xml()) -@pytest.mark.parametrize("reference_time", ["IF", None]) +@pytest.mark.parametrize("reference_time", ["IF"]) @pytest.mark.cli_e2e def test_list_images_wdl(reference_time, tmp_path, monkeypatch): (tmp_path / "plate1-A1-IF").touch() @@ -49,16 +49,16 @@ def test_list_images_wdl(reference_time, tmp_path, monkeypatch): n_cycles2=None, ) - groups = pd.read_csv(tmp_path / "groups.txt", header=None)[0].values - np.testing.assert_array_equal(groups, ["plate1-A1"]) - groupby = pd.read_csv(tmp_path / "groupby.txt", header=None)[0].values + subsets = pd.read_csv(tmp_path / "subsets.txt", header=None)[0].values + np.testing.assert_array_equal(subsets, ["plate1-A1"]) + groupby = pd.read_csv(tmp_path / "groupby_array.txt", header=None)[0].values np.testing.assert_array_equal(groupby, ["plate", "well"]) groupby_pattern = pd.read_csv(tmp_path / "groupby_pattern.txt", header=None)[ 0 ].values np.testing.assert_array_equal(groupby_pattern, ["{plate}-{well}"]) - times = pd.read_csv(tmp_path / "t.txt", header=None)[0].values + times = pd.read_csv(tmp_path / "times_1.txt", header=None)[0].values np.testing.assert_array_equal(times, ["FISH", "IF"]) From 7d02c5b4a79a5b2b10371293c3450faa1472bf9d Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Wed, 1 Jul 2026 16:41:17 -0400 Subject: [PATCH 44/79] features by t --- scallops/tests/test_wdl.py | 42 +++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/scallops/tests/test_wdl.py b/scallops/tests/test_wdl.py index 7428d71..f24f6a4 100644 --- a/scallops/tests/test_wdl.py +++ b/scallops/tests/test_wdl.py @@ -8,6 +8,7 @@ import pandas as pd import pytest import xarray as xr +from scipy.ndimage import shift from scallops import Experiment from scallops.cli.util import _list_images_wdl @@ -28,15 +29,15 @@ def add_physical_size(input_path, output_path): save_ome_tiff(img.values, uri=output_path, ome_xml=img.attrs["processed"].to_xml()) -@pytest.mark.parametrize("reference_time", ["IF"]) @pytest.mark.cli_e2e -def test_list_images_wdl(reference_time, tmp_path, monkeypatch): - (tmp_path / "plate1-A1-IF").touch() - (tmp_path / "plate1-A1-FISH").touch() +def test_list_images_wdl(tmp_path, monkeypatch): + (tmp_path / "input").mkdir() + (tmp_path / "input" / "plate1-A1-IF").touch() + (tmp_path / "input" / "plate1-A1-FISH").touch() monkeypatch.chdir(tmp_path) _list_images_wdl( - image_pattern1="{plate}-{well}-{t}", - reference_time1=reference_time, + image_pattern1="input/{plate}-{well}-{t}", + reference_time1="IF", urls1=[str(tmp_path)], n_cycles1=None, groupby=["plate", "well"], @@ -49,17 +50,14 @@ def test_list_images_wdl(reference_time, tmp_path, monkeypatch): n_cycles2=None, ) - subsets = pd.read_csv(tmp_path / "subsets.txt", header=None)[0].values - np.testing.assert_array_equal(subsets, ["plate1-A1"]) - groupby = pd.read_csv(tmp_path / "groupby_array.txt", header=None)[0].values - np.testing.assert_array_equal(groupby, ["plate", "well"]) - groupby_pattern = pd.read_csv(tmp_path / "groupby_pattern.txt", header=None)[ - 0 - ].values - np.testing.assert_array_equal(groupby_pattern, ["{plate}-{well}"]) - - times = pd.read_csv(tmp_path / "times_1.txt", header=None)[0].values - np.testing.assert_array_equal(times, ["FISH", "IF"]) + subsets = pd.read_csv("subsets.txt", header=None)[0].values.tolist() + assert subsets == ["plate1-A1"], subsets + groupby = pd.read_csv("groupby_array.txt", header=None)[0].values.tolist() + assert groupby == ["plate", "well"], groupby + groupby_pattern = pd.read_csv("groupby_pattern.txt", header=None)[0].values.tolist() + assert groupby_pattern == ["{plate}-{well}"], groupby_pattern + times = pd.read_csv("times_1.txt", header=None)[0].values.tolist() + assert times == ["FISH", "IF"], times @pytest.mark.cli_e2e @@ -168,7 +166,7 @@ def test_ops_wdl(phenotype_rounds, tmp_path): pheno_img = read_image( "scallops/tests/data/experimentC/10X_c0-DAPI-p65ab/10X_c0-DAPI-p65ab_A1_Tile-102.phenotype.tif" - ) + ).squeeze() pheno_img.attrs["physical_pixel_sizes"] = (1, 1) phenotype_mask = np.ones( (pheno_img.sizes["y"], pheno_img.sizes["x"]), dtype=np.uint8 @@ -197,8 +195,14 @@ def test_ops_wdl(phenotype_rounds, tmp_path): "IF": ["intensity_0", "intensity_1"], "FISH": ["intensity_0", "intensity_1"], } + fish_image = xr.DataArray( + shift(pheno_img.data, (0, 20, 30)), + dims=("c", "y", "x"), + attrs={"physical_pixel_sizes": (1, 1)}, + ) + Experiment( - images={"plateA-A1-IF": pheno_img, "plateA-A1-FISH": pheno_img}, + images={"plateA-A1-IF": pheno_img, "plateA-A1-FISH": fish_image}, labels={ "plateA-A1-IF-mask": phenotype_mask, "plateA-A1-IF-tile": phenotype_tile, From 630b7d88bdabd59f7f3051358746a8254e1c95f9 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Thu, 2 Jul 2026 17:11:59 -0400 Subject: [PATCH 45/79] features by t --- scallops/cli/register.py | 93 +++++++++++++++----------- wdl/ops_tasks.wdl | 108 ++++++++++++++++++++++++++---- wdl/ops_workflow.wdl | 138 +++++++++++++++++++++++++++------------ 3 files changed, 245 insertions(+), 94 deletions(-) diff --git a/scallops/cli/register.py b/scallops/cli/register.py index fda3b8f..d91746a 100644 --- a/scallops/cli/register.py +++ b/scallops/cli/register.py @@ -88,26 +88,28 @@ def _get_timepoint_index_and_value( for i in range(len(image)): if ( "t" in image[i].coords - and image[i].coords["t"].values[0] == timepoint + and str(image[i].coords["t"].values[0]) == timepoint ): timepoint_index = i timepoint_value = image[i].coords["t"].values[0] break elif "t" in image.coords: - times = list(image.coords["t"].values) - if timepoint in times: - timepoint_index = times.index(timepoint) - timepoint_value = times[timepoint_index] + times = image.coords["t"].values + for i in range(len(times)): + if str(times[i]) == timepoint: + timepoint_index = i + timepoint_value = times[i] + break if timepoint_index is None: raise ValueError(f"Reference timepoint not found: {timepoint}.") - elif isinstance(timepoint, int): + elif isinstance(timepoint, int): # index timepoint_index = timepoint if isinstance(image, Sequence): if "t" in image[timepoint_index].coords: timepoint_value = image[timepoint_index].coords["t"].values[0] elif "t" in image.coords: - times = list(image.coords["t"].values) + times = image.coords["t"].values timepoint_value = times[timepoint_index] else: raise ValueError() @@ -403,6 +405,8 @@ def single_registration( output_names=None, moving_image_spacing=moving_image_spacing, output_root=label_output_root, + moving_timepoint=moving_timepoint_value, + output_timepoint=fixed_timepoint_value, ) else: # align to t=moving_timepoint @@ -421,6 +425,7 @@ def single_registration( moving_image = new_moving_image if not no_version: moving_image[moving_timepoint].attrs.update(cli_metadata()) + # image will be in moving timepoint space _itk_align_reference_time_zarr( unroll_channels=unroll_channels, reference_timepoint=moving_timepoint, @@ -442,22 +447,23 @@ def single_registration( landmark_min_count=landmark_min_count, parameter_object_across_channels=parameter_object_across_channels, ) - moving_image_attrs = moving_image[0].attrs.copy() - chunksize = moving_image[0].data.chunksize[-2:] - del moving_image - if moving_image_spacing is None: - moving_image_spacing = get_image_spacing(moving_image_attrs) - if len(moving_label_keys) > 0: - _transform_labels_t( - transform_fs=transform_fs, - transform_dest=transform_dest, - label_output_root=label_output_root, - moving_image_attrs=moving_image_attrs, - moving_label_keys=moving_label_keys, - moving_image_spacing=moving_image_spacing, - moving_timepoint_value=moving_timepoint_value, - chunksize=chunksize, - ) + # moving_image_attrs = moving_image[0].attrs.copy() + # chunksize = moving_image[0].data.chunksize[-2:] + # del moving_image + # if moving_image_spacing is None: + # moving_image_spacing = get_image_spacing(moving_image_attrs) + + # if len(moving_label_keys) > 0: + # _transform_labels_t( + # transform_fs=transform_fs, + # transform_dest=transform_dest, + # label_output_root=label_output_root, + # moving_image_attrs=moving_image_attrs, + # moving_label_keys=moving_label_keys, + # moving_image_spacing=moving_image_spacing, + # moving_timepoint_value=moving_timepoint_value, + # chunksize=chunksize, + # ) return image_key @@ -622,6 +628,8 @@ def _transform_labels( output_root: zarr.Group, moving_image_spacing: None | tuple[float, float], attrs: None | dict, + moving_timepoint: str | None, + output_timepoint: str | None, ): """Transform and save labels. @@ -640,20 +648,36 @@ def _transform_labels( for i in range(len(matching_keys)): key = matching_keys[i] name = os.path.basename(key) - array = read_ome_zarr_array(key) - + moving_labels = read_ome_zarr_array(key) + if moving_labels.sizes.get("t", 0) > 0 and "t" in moving_labels.coords: + moving_times = moving_labels.coords["t"].values + time_index = -1 + for j in range(len(moving_times)): + if str(moving_times[i]) == moving_timepoint: + time_index = j + break + if time_index == -1: + continue + moving_labels = moving_labels.isel(t=time_index) if attrs is not None: - array.attrs = attrs # e.g. copy physical size + moving_labels.attrs = attrs # e.g. copy physical size to = "" if name != output_names[i]: to = f" to {output_names[i]}" logger.info(f"Running transformation for {name}{to}.") transformed_array = itk_transform_labels( - image=array, + image=moving_labels, transform_parameter_object=transform_parameter_object, image_spacing=moving_image_spacing, ) - del array + if output_timepoint is not None: + transformed_array = xr.DataArray( + np.expand_dims(transformed_array, 0), + dims=["t", "y", "x"], + coords={"t": [output_timepoint]}, + ) + + del moving_labels _write_zarr_image( name=output_names[i], @@ -725,6 +749,8 @@ def single_transformix( output_root=output_root, moving_image_spacing=image_spacing, attrs=None, + moving_timepoint=None, # TODO + output_timepoint=None, ) else: # see if transform dir has subdirectory describing channel transformation @@ -836,16 +862,7 @@ def run_itk_registration(arguments: argparse.Namespace) -> None: group_by = arguments.groupby fixed_timepoint = arguments.fixed_time if arguments.fixed_time is not None else 0 moving_timepoint = arguments.moving_time if arguments.moving_time is not None else 0 - if isinstance(fixed_timepoint, str) and fixed_timepoint.isdigit(): - try: - fixed_timepoint = int(fixed_timepoint) - except ValueError: - pass - if isinstance(moving_timepoint, str) and moving_timepoint.isdigit(): - try: - moving_timepoint = int(moving_timepoint) - except ValueError: - pass + unroll_channels = arguments.unroll_channels transform_output_dir = arguments.transform_output_dir subset = arguments.subset diff --git a/wdl/ops_tasks.wdl b/wdl/ops_tasks.wdl index 9ca4203..b430142 100644 --- a/wdl/ops_tasks.wdl +++ b/wdl/ops_tasks.wdl @@ -205,13 +205,89 @@ task register_elastix { } } +task register_pheno_to_pheno_qc { + input { + String phenotype_time + String images # non-reference time + String image_pattern + String label_type + Array[String?]? stacked_images # reference time transformed + String stacked_image_pattern + + Array[String?]? labels # reference labels transformed + Int image_channel + Int stacked_image_channel + String subset + String output_directory + Array[String] groupby + Boolean? force + + String docker + String zones + Int preemptible + String aws_queue_arn + Int cpu + String disks + String memory + Int max_retries + } + + command <<< + set -ex + + if [[ "$SCALLOPS_TEST" != "1" ]]; then + ulimit -n 100000 + fi + + phenotype_time="~{phenotype_time}" + pattern="-~{phenotype_time}.zarr" + stacked_images=('~{sep="' '" stacked_images}') + filtered_images=() + for item in "${stacked_images[@]}"; do + if [[ "$item" == *$pattern ]]; then + filtered_images+=("$item") + fi + done + filtered_images="${filtered_images[*]}" + + scallops features \ + --features-~{label_type} "correlationpearsonbox_~{image_channel}_s~{stacked_image_channel}" \ + --labels $filtered_images \ + --groupby ~{sep=" " groupby} \ + --subset ~{subset} \ + --output "~{output_directory}" \ + --images ~{images} \ + --stack-images $filtered_images \ + --image-pattern ~{image_pattern} \ + --stack-image-pattern ~{stacked_image_pattern} \ + ~{true="--force" false="" force} + >>> + + output { + String output_url = "~{output_directory}" + + } + + runtime { + docker:docker + disks: disks + zones: zones + memory: memory + cpu : cpu + preemptible: preemptible + queueArn: aws_queue_arn + maxRetries : max_retries + } +} + + task register_pheno_to_iss_qc { input { String images String? image_pattern String label_type String labels - String? stacked_images + String stacked_images String? stacked_image_pattern Int? image_channel Int? stacked_image_channel @@ -235,7 +311,7 @@ task register_pheno_to_iss_qc { set -ex if [[ "$SCALLOPS_TEST" != "1" ]]; then - ulimit -n 100000 + ulimit -n 100000 fi @@ -272,14 +348,14 @@ task register_pheno_to_iss_qc { } -task register_iss_iss_qc { +task register_qc { input { String images String? image_pattern String label_type String labels - Int dapi_channel - Int n_timepoints + Int? dapi_channel + Int? n_timepoints String subset String output_directory @@ -295,7 +371,8 @@ task register_iss_iss_qc { String memory Int max_retries } - Int n_channels = n_timepoints*5 + Int n_channels = select_first([n_timepoints, 0])*5 + String feature = if(n_channels>0) then "correlationpearsonbox_~{dapi_channel}_5:~{n_channels}:5" else "correlationpearsonbox_0_*" command <<< set -ex @@ -305,7 +382,7 @@ task register_iss_iss_qc { fi scallops features \ - --features-~{label_type} "correlationpearsonbox_~{dapi_channel}_5:~{n_channels}:5" \ + --features-~{label_type} "~{feature}" \ --labels "~{labels}" \ --groupby ~{sep=" " groupby} \ --subset ~{subset} \ @@ -313,9 +390,6 @@ task register_iss_iss_qc { --images "~{images}" \ ~{'--image-pattern ' + image_pattern} \ ~{true="--force" false="" force} - - - >>> output { @@ -341,6 +415,7 @@ task intersects_boundary { String? image_pattern String label_type Array[String] labels + Array[String?]? additional_labels String subset String? objects String output_directory @@ -361,12 +436,13 @@ task intersects_boundary { set -ex if [[ "$SCALLOPS_TEST" != "1" ]]; then - ulimit -n 100000 + ulimit -n 100000 fi scallops features \ --features-~{label_type} "intersects-boundary_0" \ --labels ~{sep=" " labels} \ + {sep=" " additional_labels} \ --groupby ~{sep=" " groupby} \ --subset "~{subset}" \ --output "~{output_directory}" \ @@ -397,6 +473,7 @@ task intersects_boundary { task find_objects { input { + Array[String?]? additional_labels Array[String] labels String subset Boolean? force @@ -417,11 +494,12 @@ task find_objects { set -ex if [[ "$SCALLOPS_TEST" != "1" ]]; then - ulimit -n 100000 + ulimit -n 100000 fi scallops find-objects \ --labels ~{sep=" " labels} \ + ~{sep=" " additional_labels} \ --subset ~{subset} \ ~{"--label-pattern " + label_pattern} \ --label-suffix ~{suffix} \ @@ -461,6 +539,7 @@ task features { String? features_extra_arguments String? model_dir Array[String] labels + Array[String?]? additional_labels String? merge String images String subset @@ -500,6 +579,7 @@ task features { ~{if defined(features_extra_arguments) then features_extra_arguments else ''} \ --merge ~{merge} \ --labels ~{sep=" " labels} \ + ~{sep=" " additional_labels} \ ~{"--label-filter " + '"' + label_filter + '"'} \ --subset ~{subset} \ ~{"--image-pattern " + image_pattern} \ @@ -662,7 +742,7 @@ task merge { String? cell_intersects_boundary String? register_pheno_to_iss_qc String? register_iss_to_iss_qc - String? register_pheno_to_pheno_qc + Array[String?]? register_pheno_to_pheno_qc Array[Array[String]]? phenotypes_nuclei Array[Array[String]]? phenotypes_cell @@ -701,7 +781,7 @@ task merge { ~{cell_intersects_boundary} \ ~{register_pheno_to_iss_qc} \ ~{register_iss_to_iss_qc} \ - ~{register_pheno_to_pheno_qc} \ + ~{sep=" " register_pheno_to_pheno_qc} \ ~{merge_metadata} \ ~{sep=" " phenotypes_nuclei_} \ ~{sep=" " phenotypes_cell_} \ diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index 250c8ad..cfbac1e 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -98,7 +98,7 @@ workflow ops_workflow { Boolean force_find_objects = false Boolean force_register_pheno_to_iss_qc = false Boolean force_register_iss_to_iss_qc = false - + Boolean force_register_pheno_to_pheno_qc = false # general options Array[String]? subset Int? batch_size # for processing multiple images in one batch @@ -144,6 +144,10 @@ workflow ops_workflow { Int register_pheno_to_iss_qc_cpu = 48 String register_pheno_to_iss_qc_disks = "local-disk 200 HDD" + String register_pheno_to_pheno_qc_memory = "96 GiB" + Int register_pheno_to_pheno_qc_cpu = 48 + String register_pheno_to_pheno_qc_disks = "local-disk 200 HDD" + String register_iss_to_iss_qc_memory = "48 GiB" Int register_iss_to_iss_qc_cpu = 24 String register_iss_to_iss_qc_disks = "local-disk 200 HDD" @@ -172,10 +176,12 @@ workflow ops_workflow { String nuclei_features_suffix = "features-nuclei" String cell_features_suffix = "features-cell" String cytosol_features_suffix = "features-cytosol" - String register_pheno_to_pheno_suffix = "pheno-registered.zarr" - String register_pheno_to_pheno_transform_suffix = "pheno-to-pheno-transforms" + String register_pheno_to_iss_qc_suffix = "pheno-to-iss-qc" String register_iss_to_iss_qc_suffix = "iss-to-iss-qc" + String register_pheno_to_pheno_qc_suffix = "pheno-to-pheno-qc" + String register_pheno_to_pheno_suffix = "pheno-registered" + String register_pheno_to_pheno_transform_suffix = "pheno-to-pheno-transforms" String spot_detect_suffix = "spot-detect.zarr" String reads_suffix = "reads" String merge_meta_suffix = "merge-sbs-metadata" @@ -202,6 +208,7 @@ workflow ops_workflow { String register_pheno_to_iss_qc_directory = output_stripped + register_pheno_to_iss_qc_suffix String intersects_boundary_directory = output_stripped + intersects_boundary_suffix String register_iss_to_iss_qc_directory = output_stripped + register_iss_to_iss_qc_suffix + String register_pheno_to_pheno_qc_directory = output_stripped + register_pheno_to_pheno_qc_suffix Boolean iss_url_supplied = defined(iss_url) Boolean pheno_url_supplied = defined(phenotype_url) @@ -294,39 +301,48 @@ workflow ops_workflow { } if(length(times_pheno)>1) { - - call tasks.register_elastix as register_pheno_to_pheno { - input: - moving=select_all([phenotype_url]), - moving_label=segment_cell.output_url, - moving_channel=phenotype_dapi_channel, - moving_image_pattern=phenotype_image_pattern, - moving_time=reference_time_pheno, - - extra_arguments=pheno_registration_extra_arguments, - output_aligned_channels_only=true, - groupby=groupby, - subset = subset_, - moving_output_directory=register_pheno_to_pheno_directory, - label_output_directory=register_pheno_to_pheno_directory, - transform_output_directory=register_pheno_to_pheno_transform_directory, - - force = force_register_pheno_to_pheno, - docker=docker, - zones = zones, - preemptible = preemptible, - aws_queue_arn = aws_queue_arn, - disks = register_pheno_to_pheno_disks, - memory = register_pheno_to_pheno_memory, - cpu = register_pheno_to_pheno_cpu, - max_retries = max_retries + # transfer segmentation labels from reference_time_pheno to other times + scatter(phenotype_time in times_pheno) { + if(phenotype_time != reference_time_pheno) { + call tasks.register_elastix as register_pheno_to_pheno { + input: + moving=select_all([phenotype_url]), + moving_label=segment_cell.output_url, + moving_channel=phenotype_dapi_channel, + moving_image_pattern=phenotype_image_pattern, + moving_time=reference_time_pheno, + + fixed_channel=phenotype_dapi_channel, + fixed_image_pattern=phenotype_image_pattern, + fixed_time=phenotype_time, + + extra_arguments=pheno_registration_extra_arguments, + output_aligned_channels_only=true, + groupby=groupby, + subset = subset_, + moving_output_directory=register_pheno_to_pheno_directory + "-" + phenotype_time + ".zarr", + label_output_directory=register_pheno_to_pheno_directory + "-" + phenotype_time + ".zarr", + transform_output_directory=register_pheno_to_pheno_transform_directory + "-" + phenotype_time, + + force = force_register_pheno_to_pheno, + docker=docker, + zones = zones, + preemptible = preemptible, + aws_queue_arn = aws_queue_arn, + disks = register_pheno_to_pheno_disks, + memory = register_pheno_to_pheno_memory, + cpu = register_pheno_to_pheno_cpu, + max_retries = max_retries + } + } } } if(run_nuclei_segmentation) { call tasks.find_objects as find_objects_nuclei { input: - labels=select_all([segment_nuclei.output_url, register_pheno_to_pheno.label_output_url]), + labels=select_all([segment_nuclei.output_url]), + additional_labels=register_pheno_to_pheno.label_output_url, label_pattern=groupby_pattern, suffix="nuclei", output_directory=objects_directory, @@ -345,7 +361,8 @@ workflow ops_workflow { if(run_cell_segmentation) { call tasks.find_objects as find_objects_cell { input: - labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), + labels=select_all([segment_cell.output_url]), + additional_labels=register_pheno_to_pheno.label_output_url, label_pattern=groupby_pattern, suffix="cell", output_directory=objects_directory, @@ -364,7 +381,8 @@ workflow ops_workflow { if (run_nuclei_segmentation && run_cell_segmentation) { call tasks.find_objects as find_objects_cytosol { input: - labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), + labels=select_all([segment_cell.output_url]), + additional_labels=register_pheno_to_pheno.label_output_url, label_pattern=groupby_pattern, suffix="cytosol", output_directory=objects_directory, @@ -389,7 +407,8 @@ workflow ops_workflow { call tasks.intersects_boundary as intersects_boundary { input: - labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), + labels=select_all([segment_cell.output_url]), + additional_labels=register_pheno_to_pheno.label_output_url, images=phenotype_url_stripped + "/labels/", image_pattern=phenotype_image_pattern + "-mask", output_directory=intersects_boundary_directory, @@ -443,13 +462,14 @@ workflow ops_workflow { input: fixed=select_first([iss_url]), fixed_channel=iss_dapi_channel, + moving_label=segment_cell.output_url, moving=select_all([phenotype_url]), moving_image_pattern=phenotype_image_pattern, fixed_image_pattern=iss_image_pattern, moving_channel=phenotype_dapi_channel, moving_time=reference_time_pheno, - fixed_time=reference_time_iss, + output_aligned_channels_only=true, moving_output_directory=register_pheno_to_iss_directory, label_output_directory=register_pheno_to_iss_directory, @@ -492,8 +512,39 @@ workflow ops_workflow { cpu = register_pheno_to_iss_qc_cpu, max_retries = max_retries } + if(length(times_pheno)>1) { + scatter(phenotype_time in times_pheno) { + if(phenotype_time != reference_time_pheno) { + call tasks.register_pheno_to_pheno_qc as register_pheno_to_pheno_qc { + input: + phenotype_time=phenotype_time, + images = select_first([phenotype_url]), + image_pattern=if(sub(phenotype_image_pattern, "{t}", phenotype_time)!=phenotype_image_pattern) then sub(phenotype_image_pattern, "{t}", phenotype_time) else phenotype_image_pattern, + + stacked_images=register_pheno_to_pheno.moving_output_url, # task filter this + stacked_image_pattern=groupby_pattern, + groupby=groupby, + labels=register_pheno_to_pheno.label_output_url, # task filters this + subset =subset_, + image_channel=select_first([phenotype_dapi_channel, 0]), + stacked_image_channel=0, + label_type="nuclei", + output_directory=register_pheno_to_pheno_qc_directory + "-" + phenotype_time, + force = force_register_pheno_to_pheno_qc, + docker=docker, + zones = zones, + preemptible = preemptible, + aws_queue_arn = aws_queue_arn, + disks = register_pheno_to_pheno_qc_disks, + memory = register_pheno_to_pheno_qc_memory, + cpu = register_pheno_to_pheno_qc_cpu, + max_retries = max_retries + } + } + } + } # ISS t0 to other times - call tasks.register_iss_iss_qc as register_iss_to_iss_qc { + call tasks.register_qc as register_iss_to_iss_qc { input: images=select_first([register_iss_t0.moving_output_url]), image_pattern=groupby_pattern, @@ -579,6 +630,7 @@ workflow ops_workflow { objects=if(run_cell_segmentation) then find_objects_nuclei.output_url else find_objects_cell.output_url, cell_intersects_boundary=intersects_boundary.output_url, register_pheno_to_iss_qc=register_pheno_to_iss_qc.output_url, + register_pheno_to_pheno_qc=register_pheno_to_pheno_qc.output_url, register_iss_to_iss_qc=register_iss_to_iss_qc.output_url, barcodes=select_first([barcodes]), barcode_column=barcode_column, @@ -612,16 +664,18 @@ workflow ops_workflow { images = select_first([phenotype_url]), image_pattern=phenotype_image_pattern, merge=merge_sbs_metadata.output_url, - labels=select_all([segment_nuclei.output_url, register_pheno_to_pheno.label_output_url]), + additional_labels=register_pheno_to_pheno.label_output_url, + labels=select_all([segment_nuclei.output_url]), label_filter=features_label_filter, groupby=phenotype_group_by_with_time, + subset = if(phenotype_time!="") then subset_ + "-" + phenotype_time else subset_, nuclei_features = nuclei_features[feature_index], nuclei_min_area = features_nuclei_min_area_, nuclei_max_area = features_nuclei_max_area_, features_extra_arguments=features_extra_arguments, model_dir=model_dir, output_directory=nuclei_features_directory + "-" + phenotype_time + "-batch" + feature_index, - subset = if(phenotype_time!="") then subset_ + "-" + phenotype_time else subset_, + force = force_features, docker=docker, zones = zones, @@ -651,7 +705,8 @@ workflow ops_workflow { images = select_first([phenotype_url]), image_pattern=phenotype_image_pattern, merge=merge_sbs_metadata.output_url, - labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), + additional_labels=register_pheno_to_pheno.label_output_url, + labels=select_all([segment_cell.output_url]), label_filter=features_label_filter, groupby=phenotype_group_by_with_time, cell_features = cell_features[feature_index], @@ -692,7 +747,8 @@ workflow ops_workflow { images = select_first([phenotype_url]), image_pattern=phenotype_image_pattern, merge=merge_sbs_metadata.output_url, - labels=select_all([segment_cell.output_url, register_pheno_to_pheno.label_output_url]), + additional_labels=register_pheno_to_pheno.label_output_url, + labels=select_all([segment_cell.output_url]), label_filter=features_label_filter, groupby=phenotype_group_by_with_time, output_directory=cytosol_features_directory + "-" + phenotype_time + "-" + feature_index, @@ -718,7 +774,6 @@ workflow ops_workflow { } } - call tasks.merge as merge_features { input: phenotypes_nuclei=features_nuclei.output_url, @@ -739,7 +794,6 @@ workflow ops_workflow { max_retries = max_retries } - } output { Array[String?] segment_nuclei_output_url = segment_nuclei.output_url @@ -747,7 +801,7 @@ workflow ops_workflow { Array[String?] register_iss_t0_output_url = register_iss_t0.moving_output_url Array[String?] register_pheno_to_iss_output_url = register_pheno_to_iss.moving_output_url Array[String?] register_pheno_to_iss_qc_output_url = register_pheno_to_iss_qc.output_url - Array[String?] register_pheno_to_pheno_moving_output_url = register_pheno_to_pheno.moving_output_url + Array[Array[String?]?] register_pheno_to_pheno_moving_output_url = register_pheno_to_pheno.moving_output_url Array[String?] spot_detect_output_url = spot_detect.output_url Array[String?] reads_output_url = reads.output_url Array[String?] find_objects_nuclei_output_url = find_objects_nuclei.output_url From 04838ba3d0646aa57d642bdb69611aa4b2e6dcbc Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Mon, 6 Jul 2026 08:14:05 -0400 Subject: [PATCH 46/79] features by t --- scallops/cli/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scallops/cli/register.py b/scallops/cli/register.py index d91746a..6a80fd0 100644 --- a/scallops/cli/register.py +++ b/scallops/cli/register.py @@ -653,7 +653,7 @@ def _transform_labels( moving_times = moving_labels.coords["t"].values time_index = -1 for j in range(len(moving_times)): - if str(moving_times[i]) == moving_timepoint: + if str(moving_times[j]) == moving_timepoint: time_index = j break if time_index == -1: From 9f7b21e00ccf287a25fae4f1a6700d4cb329ffd3 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Mon, 6 Jul 2026 17:36:35 -0400 Subject: [PATCH 47/79] features by t --- scallops/cli/features.py | 2 +- scallops/io.py | 3 +- scallops/tests/miniwdl_local/local_runner.py | 5 +- wdl/ops_tasks.wdl | 69 ++++++------- wdl/ops_workflow.wdl | 101 +++++++++---------- wdl/stitch_tasks.wdl | 6 +- wdl/stitch_workflow.wdl | 10 +- wdl/utils.wdl | 6 +- 8 files changed, 92 insertions(+), 110 deletions(-) diff --git a/scallops/cli/features.py b/scallops/cli/features.py index 00892d0..fdfd34b 100644 --- a/scallops/cli/features.py +++ b/scallops/cli/features.py @@ -368,7 +368,7 @@ def single_feature( if merged_df is None: logger.info( - f"Find {label_name} objects for {image_key}{' at t=' + timepoint if timepoint is not None else ''}." + f"Find {label_name} objects for {image_key}{' at t=' if timepoint is not None else ''}{timepoint if timepoint is not None else ''}." ) merged_df = find_objects(label_image) objects_path = get_path( diff --git a/scallops/io.py b/scallops/io.py index 91f4343..2a73f3b 100644 --- a/scallops/io.py +++ b/scallops/io.py @@ -1325,8 +1325,7 @@ def _images2fov( for dim in concat_dims if len(set([image.coords[dim].values[0] for image in images])) > 1 ] - if len(concat_dims) > 0: - _match_size(images) + if len(concat_dims) == 1: image = xr.concat( images, diff --git a/scallops/tests/miniwdl_local/local_runner.py b/scallops/tests/miniwdl_local/local_runner.py index 6207931..44de024 100644 --- a/scallops/tests/miniwdl_local/local_runner.py +++ b/scallops/tests/miniwdl_local/local_runner.py @@ -16,10 +16,7 @@ def global_init(cls, cfg, logger): Perform any necessary process-wide initialization of the container backend """ cpu_count = os.environ.get("SCALLOPS_MINIWDL_CPU") - if cpu_count is None: - cpu_count = psutil.cpu_count() - else: - cpu_count = int(cpu_count) + cpu_count = psutil.cpu_count() if cpu_count is None else int(cpu_count) cls._resource_limits = { "cpu": cpu_count, "mem_bytes": psutil.virtual_memory().total, diff --git a/wdl/ops_tasks.wdl b/wdl/ops_tasks.wdl index b430142..2a0d01b 100644 --- a/wdl/ops_tasks.wdl +++ b/wdl/ops_tasks.wdl @@ -13,7 +13,7 @@ task segment_nuclei { Boolean? force String model_dir String? extra_arguments - String docker + String container String zones Int preemptible String aws_queue_arn @@ -47,7 +47,7 @@ task segment_nuclei { } runtime { - docker:docker + #container:container disks: disks zones: zones memory: memory @@ -77,7 +77,7 @@ task segment_cell { String? extra_arguments Boolean? force - String docker + String container String zones Int preemptible String aws_queue_arn @@ -116,7 +116,7 @@ task segment_cell { } runtime { - docker:docker + #container:container disks: disks zones: zones memory: memory @@ -150,7 +150,7 @@ task register_elastix { String? moving_label String? extra_arguments - String docker + String container String zones Int preemptible String aws_queue_arn @@ -194,7 +194,7 @@ task register_elastix { } runtime { - docker:docker + #container:container disks: disks zones: zones memory: memory @@ -207,14 +207,14 @@ task register_elastix { task register_pheno_to_pheno_qc { input { - String phenotype_time + String images # non-reference time String image_pattern String label_type - Array[String?]? stacked_images # reference time transformed + String stacked_images # reference time transformed String stacked_image_pattern - Array[String?]? labels # reference labels transformed + String labels # reference labels transformed Int image_channel Int stacked_image_channel String subset @@ -222,7 +222,7 @@ task register_pheno_to_pheno_qc { Array[String] groupby Boolean? force - String docker + String container String zones Int preemptible String aws_queue_arn @@ -239,25 +239,16 @@ task register_pheno_to_pheno_qc { ulimit -n 100000 fi - phenotype_time="~{phenotype_time}" - pattern="-~{phenotype_time}.zarr" - stacked_images=('~{sep="' '" stacked_images}') - filtered_images=() - for item in "${stacked_images[@]}"; do - if [[ "$item" == *$pattern ]]; then - filtered_images+=("$item") - fi - done - filtered_images="${filtered_images[*]}" + scallops features \ --features-~{label_type} "correlationpearsonbox_~{image_channel}_s~{stacked_image_channel}" \ - --labels $filtered_images \ + --labels ~{labels} \ --groupby ~{sep=" " groupby} \ --subset ~{subset} \ --output "~{output_directory}" \ --images ~{images} \ - --stack-images $filtered_images \ + --stack-images ~{stacked_images} \ --image-pattern ~{image_pattern} \ --stack-image-pattern ~{stacked_image_pattern} \ ~{true="--force" false="" force} @@ -269,7 +260,7 @@ task register_pheno_to_pheno_qc { } runtime { - docker:docker + #container:container disks: disks zones: zones memory: memory @@ -296,7 +287,7 @@ task register_pheno_to_iss_qc { Array[String] groupby Boolean? force - String docker + String container String zones Int preemptible String aws_queue_arn @@ -336,7 +327,7 @@ task register_pheno_to_iss_qc { } runtime { - docker:docker + #container:container disks: disks zones: zones memory: memory @@ -362,7 +353,7 @@ task register_qc { Array[String] groupby Boolean? force - String docker + String container String zones Int preemptible String aws_queue_arn @@ -398,7 +389,7 @@ task register_qc { } runtime { - docker:docker + #container:container disks: disks zones: zones memory: memory @@ -422,7 +413,7 @@ task intersects_boundary { Array[String] groupby Boolean? force - String docker + String container String zones Int preemptible String aws_queue_arn @@ -460,7 +451,7 @@ task intersects_boundary { } runtime { - docker:docker + #container:container disks: disks zones: zones memory: memory @@ -480,7 +471,7 @@ task find_objects { String? label_pattern String suffix String output_directory - String docker + String container String zones Int preemptible String aws_queue_arn @@ -513,7 +504,7 @@ task find_objects { } runtime { - docker:docker + #container:container disks: disks zones: zones memory: memory @@ -547,7 +538,7 @@ task features { String? image_pattern Array[String] groupby String output_directory - String docker + String container String zones Int preemptible String aws_queue_arn @@ -595,7 +586,7 @@ task features { } runtime { - docker:docker + #container:container disks: disks zones: zones memory: memory @@ -621,7 +612,7 @@ task spot_detect { Int? chunks String output_directory String? extra_arguments - String docker + String container String zones Int preemptible String aws_queue_arn @@ -656,7 +647,7 @@ task spot_detect { } runtime { - docker:docker + #container:container disks: disks zones: zones memory: memory @@ -685,7 +676,7 @@ task reads { String? barcode_column String label_name String? extra_arguments - String docker + String container String zones Int preemptible String aws_queue_arn @@ -723,7 +714,7 @@ task reads { } runtime { - docker:docker + #container:container disks: disks zones: zones memory: memory @@ -756,7 +747,7 @@ task merge { String? extra_arguments Boolean? force - String docker + String container String zones Int preemptible String aws_queue_arn @@ -797,7 +788,7 @@ task merge { } runtime { - docker:docker + #container:container disks: disks zones: zones memory: memory diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index cfbac1e..ab5e91a 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -160,7 +160,7 @@ workflow ops_workflow { String intersects_boundary_memory = "32 GiB" String intersects_boundary_disks = "local-disk 200 HDD" - String docker + String container Int preemptible = 0 String zones = "us-west1-a us-west1-b us-west1-c" @@ -225,7 +225,7 @@ workflow ops_workflow { groupby=groupby, subset = subset, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -260,7 +260,7 @@ workflow ops_workflow { extra_arguments=nuclei_segmentation_extra_arguments, force = force_segment_nuclei, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -290,7 +290,7 @@ workflow ops_workflow { extra_arguments=cell_segmentation_extra_arguments, force = force_segment_cell, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -325,7 +325,7 @@ workflow ops_workflow { transform_output_directory=register_pheno_to_pheno_transform_directory + "-" + phenotype_time, force = force_register_pheno_to_pheno, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -334,6 +334,31 @@ workflow ops_workflow { cpu = register_pheno_to_pheno_cpu, max_retries = max_retries } + call tasks.register_pheno_to_pheno_qc as register_pheno_to_pheno_qc { + input: + + images = select_first([phenotype_url]), + image_pattern=if(sub(phenotype_image_pattern, "{t}", phenotype_time)!=phenotype_image_pattern) then sub(phenotype_image_pattern, "{t}", phenotype_time) else phenotype_image_pattern, + + stacked_images=register_pheno_to_pheno.moving_output_url, + stacked_image_pattern=groupby_pattern, + groupby=groupby, + labels=register_pheno_to_pheno.label_output_url, + subset =subset_, + image_channel=select_first([phenotype_dapi_channel, 0]), + stacked_image_channel=0, + label_type="nuclei", + output_directory=register_pheno_to_pheno_qc_directory + "-" + phenotype_time, + force = force_register_pheno_to_pheno_qc, + container=container, + zones = zones, + preemptible = preemptible, + aws_queue_arn = aws_queue_arn, + disks = register_pheno_to_pheno_qc_disks, + memory = register_pheno_to_pheno_qc_memory, + cpu = register_pheno_to_pheno_qc_cpu, + max_retries = max_retries + } } } } @@ -348,7 +373,7 @@ workflow ops_workflow { output_directory=objects_directory, subset = subset_, force = force_find_objects, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -368,7 +393,7 @@ workflow ops_workflow { output_directory=objects_directory, subset = subset_, force = force_find_objects, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -388,7 +413,7 @@ workflow ops_workflow { output_directory=objects_directory, subset = subset_, force = force_find_objects, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -417,7 +442,7 @@ workflow ops_workflow { groupby=phenotype_group_by_with_time, subset = if(sub(phenotype_image_pattern, "{t}", "")!=phenotype_image_pattern) then subset_ + "-*" else subset_, force = if(intersects_stitch_boundary_label=="cell") then force_segment_cell else force_segment_nuclei, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -445,7 +470,7 @@ workflow ops_workflow { extra_arguments=iss_registration_extra_arguments, subset = subset_, force = force_register_iss, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -478,7 +503,7 @@ workflow ops_workflow { groupby=groupby, extra_arguments=pheno_to_iss_registration_extra_arguments, force = force_register_pheno_to_iss, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -503,7 +528,7 @@ workflow ops_workflow { subset = subset_, groupby=groupby, force = force_register_pheno_to_iss_qc, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -512,37 +537,7 @@ workflow ops_workflow { cpu = register_pheno_to_iss_qc_cpu, max_retries = max_retries } - if(length(times_pheno)>1) { - scatter(phenotype_time in times_pheno) { - if(phenotype_time != reference_time_pheno) { - call tasks.register_pheno_to_pheno_qc as register_pheno_to_pheno_qc { - input: - phenotype_time=phenotype_time, - images = select_first([phenotype_url]), - image_pattern=if(sub(phenotype_image_pattern, "{t}", phenotype_time)!=phenotype_image_pattern) then sub(phenotype_image_pattern, "{t}", phenotype_time) else phenotype_image_pattern, - stacked_images=register_pheno_to_pheno.moving_output_url, # task filter this - stacked_image_pattern=groupby_pattern, - groupby=groupby, - labels=register_pheno_to_pheno.label_output_url, # task filters this - subset =subset_, - image_channel=select_first([phenotype_dapi_channel, 0]), - stacked_image_channel=0, - label_type="nuclei", - output_directory=register_pheno_to_pheno_qc_directory + "-" + phenotype_time, - force = force_register_pheno_to_pheno_qc, - docker=docker, - zones = zones, - preemptible = preemptible, - aws_queue_arn = aws_queue_arn, - disks = register_pheno_to_pheno_qc_disks, - memory = register_pheno_to_pheno_qc_memory, - cpu = register_pheno_to_pheno_qc_cpu, - max_retries = max_retries - } - } - } - } # ISS t0 to other times call tasks.register_qc as register_iss_to_iss_qc { input: @@ -557,7 +552,7 @@ workflow ops_workflow { subset = subset_, groupby=groupby, force = force_register_iss_to_iss_qc, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -584,7 +579,7 @@ workflow ops_workflow { groupby=groupby, extra_arguments=spot_detection_extra_arguments, force = force_spot_detect, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -612,7 +607,7 @@ workflow ops_workflow { subset = subset_, extra_arguments=reads_extra_arguments, force = force_reads, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -638,7 +633,7 @@ workflow ops_workflow { subset = subset_, extra_arguments=merge_extra_arguments, force = force_merge, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -677,7 +672,7 @@ workflow ops_workflow { output_directory=nuclei_features_directory + "-" + phenotype_time + "-batch" + feature_index, force = force_features, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -719,7 +714,7 @@ workflow ops_workflow { output_directory=cell_features_directory + "-" + phenotype_time + "-batch" + feature_index, subset = if(phenotype_time!="") then subset_ + "-" + phenotype_time else subset_, force = force_features, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -761,7 +756,7 @@ workflow ops_workflow { subset = if(phenotype_time!="") then subset_ + "-" + phenotype_time else subset_, force = force_features, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -784,7 +779,7 @@ workflow ops_workflow { subset = subset_, extra_arguments=merge_extra_arguments, force = force_merge, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -807,9 +802,9 @@ workflow ops_workflow { Array[String?] find_objects_nuclei_output_url = find_objects_nuclei.output_url Array[String?] find_objects_cell_output_url = find_objects_cell.output_url Array[String?] find_objects_cytosol_output_url = find_objects_cytosol.output_url - Array[Array[String]?] features_nuclei_output_url = features_nuclei.output_url - Array[Array[String]?] features_cell_output_url = features_cell.output_url - Array[Array[String]?] features_cytosol_output_url = features_cytosol.output_url + Array[Array[Array[String]]?] features_nuclei_output_url = features_nuclei.output_url + Array[Array[Array[String]]?] features_cell_output_url = features_cell.output_url + Array[Array[Array[String]]?] features_cytosol_output_url = features_cytosol.output_url Array[String?] merge_sbs_metadata_output_url = merge_sbs_metadata.output_url Array[String?] merge_features_output_url = merge_features.output_url } diff --git a/wdl/stitch_tasks.wdl b/wdl/stitch_tasks.wdl index e2f09ac..a034b57 100644 --- a/wdl/stitch_tasks.wdl +++ b/wdl/stitch_tasks.wdl @@ -1,4 +1,4 @@ -version 1.0 +version 1.1 task illumination_correction { input { @@ -46,7 +46,7 @@ task illumination_correction { } runtime { - docker:docker + container:container disks: disks zones: zones memory: memory @@ -181,7 +181,7 @@ task stitch { } runtime { - docker:docker + container:container disks: disks zones: zones memory: memory diff --git a/wdl/stitch_workflow.wdl b/wdl/stitch_workflow.wdl index f6491b0..7ea1e8f 100644 --- a/wdl/stitch_workflow.wdl +++ b/wdl/stitch_workflow.wdl @@ -1,4 +1,4 @@ -version 1.0 +version 1.1 import "utils.wdl" as utils import "stitch_tasks.wdl" as tasks @@ -46,7 +46,7 @@ workflow stitch_workflow { Boolean? force_illumination_correction Boolean? force_stitch - String docker + String container Int preemptible = 0 String zones = "us-west1-a us-west1-b us-west1-c" @@ -69,7 +69,7 @@ workflow stitch_workflow { save_group_size=true, groupby=groupby, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -100,7 +100,7 @@ workflow stitch_workflow { groupby=groupby_array, force=force_illumination_correction, output_directory=illumination_correction_output_directory, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, @@ -133,7 +133,7 @@ workflow stitch_workflow { min_overlap_fraction=stitch_min_overlap_fraction, max_shift=stitch_max_shift, blend=stitch_blend, - docker=docker, + container=container, zones = zones, preemptible = preemptible, aws_queue_arn = aws_queue_arn, diff --git a/wdl/utils.wdl b/wdl/utils.wdl index fc5c378..079ce94 100644 --- a/wdl/utils.wdl +++ b/wdl/utils.wdl @@ -1,4 +1,4 @@ -version 1.0 +version 1.1 task list_images { input { @@ -18,7 +18,7 @@ task list_images { Boolean? save_group_size Int? batch_size - String docker + String container String zones Int preemptible String aws_queue_arn @@ -82,7 +82,7 @@ task list_images { } runtime { - docker:docker +# container:container disks: disks zones: zones memory: memory From 678ae4a1f5e263ab4f8cd38498fc5f6e7a08b588 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Mon, 6 Jul 2026 17:44:11 -0400 Subject: [PATCH 48/79] features by t --- scallops/tests/test_wdl.py | 6 +-- wdl/ops_tasks.wdl | 86 +++++++++++++++++++------------------- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/scallops/tests/test_wdl.py b/scallops/tests/test_wdl.py index f24f6a4..84f6bc4 100644 --- a/scallops/tests/test_wdl.py +++ b/scallops/tests/test_wdl.py @@ -88,7 +88,7 @@ def test_stitch_wdl_z_stack(tmp_path): "z_index": "focus", "stitch_radial_correction_k": "none", "output_directory": str(tmp_path / "out"), - "docker": "", + "container": "", } with open(tmp_path / "inputs.json", "wt") as out: @@ -129,7 +129,7 @@ def test_stitch_wdl(tmp_path): "image_pattern": "{well}-{skip}.zarr", "output_directory": str(output_directory), "channel_names": ["a", "b"], - "docker": "", + "container": "", } with open(tmp_path / "inputs.json", "wt") as out: @@ -244,7 +244,7 @@ def test_ops_wdl(phenotype_rounds, tmp_path): "reads_threshold_peaks_crosstalk": "20", "barcodes": os.path.abspath("scallops/tests/data/experimentC/barcodes.csv"), "reads_labels": "cell", - "docker": "", + "container": "", } with open(tmp_path / "inputs.json", "wt") as out: diff --git a/wdl/ops_tasks.wdl b/wdl/ops_tasks.wdl index 2a0d01b..696d2b3 100644 --- a/wdl/ops_tasks.wdl +++ b/wdl/ops_tasks.wdl @@ -26,19 +26,19 @@ task segment_nuclei { command <<< set -ex - export SCALLOPS_MODEL_DIR="~{model_dir}" - - scallops segment nuclei \ - --images "~{images}" \ - ~{"--method " + method} \ - --groupby ~{sep=" " groupby} \ - ~{if defined(image_pattern) then '--image-pattern "' + image_pattern + '"' else ''} \ - ~{'--dapi-channel ' + dapi_channel} \ - ~{'--time ' + time} \ - --output "~{output_directory}" \ - --subset ~{subset} \ - ~{if defined(extra_arguments) then extra_arguments else ''} \ - ~{true="--force" false="" force} +# export SCALLOPS_MODEL_DIR="~{model_dir}" +# +# scallops segment nuclei \ +# --images "~{images}" \ +# ~{"--method " + method} \ +# --groupby ~{sep=" " groupby} \ +# ~{if defined(image_pattern) then '--image-pattern "' + image_pattern + '"' else ''} \ +# ~{'--dapi-channel ' + dapi_channel} \ +# ~{'--time ' + time} \ +# --output "~{output_directory}" \ +# --subset ~{subset} \ +# ~{if defined(extra_arguments) then extra_arguments else ''} \ +# ~{true="--force" false="" force} >>> output { @@ -47,7 +47,7 @@ task segment_nuclei { } runtime { - #container:container + container:container disks: disks zones: zones memory: memory @@ -90,24 +90,24 @@ task segment_cell { command <<< set -ex - export SCALLOPS_MODEL_DIR="~{model_dir}" - - scallops segment cell \ - --images "~{images}" \ - --groupby ~{sep=" " groupby} \ - ~{if defined(image_pattern) then '--image-pattern "' + image_pattern + '"' else ''} \ - ~{'--dapi-channel ' + dapi_channel} \ - ~{'--time ' + time} \ - --cyto-channel ~{sep=" " cyto_channel} \ - ~{"--nuclei-label " + nuclei_label} \ - ~{"--method " + method} \ - --output "~{output_directory}" \ - --subset ~{subset} \ - ~{'--threshold ' + threshold} \ - ~{'--threshold-correction-factor ' + threshold_correction_factor} \ - ~{'--chunks ' + chunks} \ - ~{if defined(extra_arguments) then extra_arguments else ''} \ - ~{true="--force" false="" force} +# export SCALLOPS_MODEL_DIR="~{model_dir}" +# +# scallops segment cell \ +# --images "~{images}" \ +# --groupby ~{sep=" " groupby} \ +# ~{if defined(image_pattern) then '--image-pattern "' + image_pattern + '"' else ''} \ +# ~{'--dapi-channel ' + dapi_channel} \ +# ~{'--time ' + time} \ +# --cyto-channel ~{sep=" " cyto_channel} \ +# ~{"--nuclei-label " + nuclei_label} \ +# ~{"--method " + method} \ +# --output "~{output_directory}" \ +# --subset ~{subset} \ +# ~{'--threshold ' + threshold} \ +# ~{'--threshold-correction-factor ' + threshold_correction_factor} \ +# ~{'--chunks ' + chunks} \ +# ~{if defined(extra_arguments) then extra_arguments else ''} \ +# ~{true="--force" false="" force} >>> output { @@ -116,7 +116,7 @@ task segment_cell { } runtime { - #container:container + container:container disks: disks zones: zones memory: memory @@ -194,7 +194,7 @@ task register_elastix { } runtime { - #container:container + container:container disks: disks zones: zones memory: memory @@ -260,7 +260,7 @@ task register_pheno_to_pheno_qc { } runtime { - #container:container + container:container disks: disks zones: zones memory: memory @@ -327,7 +327,7 @@ task register_pheno_to_iss_qc { } runtime { - #container:container + container:container disks: disks zones: zones memory: memory @@ -389,7 +389,7 @@ task register_qc { } runtime { - #container:container + container:container disks: disks zones: zones memory: memory @@ -451,7 +451,7 @@ task intersects_boundary { } runtime { - #container:container + container:container disks: disks zones: zones memory: memory @@ -504,7 +504,7 @@ task find_objects { } runtime { - #container:container + container:container disks: disks zones: zones memory: memory @@ -586,7 +586,7 @@ task features { } runtime { - #container:container + container:container disks: disks zones: zones memory: memory @@ -647,7 +647,7 @@ task spot_detect { } runtime { - #container:container + container:container disks: disks zones: zones memory: memory @@ -714,7 +714,7 @@ task reads { } runtime { - #container:container + container:container disks: disks zones: zones memory: memory @@ -788,7 +788,7 @@ task merge { } runtime { - #container:container + container:container disks: disks zones: zones memory: memory From a815cca468e9d8139510374778a76d347e986876 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 08:21:29 -0400 Subject: [PATCH 49/79] features by t --- wdl/ops_tasks.wdl | 62 +++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/wdl/ops_tasks.wdl b/wdl/ops_tasks.wdl index 696d2b3..6549051 100644 --- a/wdl/ops_tasks.wdl +++ b/wdl/ops_tasks.wdl @@ -26,19 +26,19 @@ task segment_nuclei { command <<< set -ex -# export SCALLOPS_MODEL_DIR="~{model_dir}" -# -# scallops segment nuclei \ -# --images "~{images}" \ -# ~{"--method " + method} \ -# --groupby ~{sep=" " groupby} \ -# ~{if defined(image_pattern) then '--image-pattern "' + image_pattern + '"' else ''} \ -# ~{'--dapi-channel ' + dapi_channel} \ -# ~{'--time ' + time} \ -# --output "~{output_directory}" \ -# --subset ~{subset} \ -# ~{if defined(extra_arguments) then extra_arguments else ''} \ -# ~{true="--force" false="" force} + export SCALLOPS_MODEL_DIR="~{model_dir}" + + scallops segment nuclei \ + --images "~{images}" \ + ~{"--method " + method} \ + --groupby ~{sep=" " groupby} \ + ~{if defined(image_pattern) then '--image-pattern "' + image_pattern + '"' else ''} \ + ~{'--dapi-channel ' + dapi_channel} \ + ~{'--time ' + time} \ + --output "~{output_directory}" \ + --subset ~{subset} \ + ~{if defined(extra_arguments) then extra_arguments else ''} \ + ~{true="--force" false="" force} >>> output { @@ -90,24 +90,24 @@ task segment_cell { command <<< set -ex -# export SCALLOPS_MODEL_DIR="~{model_dir}" -# -# scallops segment cell \ -# --images "~{images}" \ -# --groupby ~{sep=" " groupby} \ -# ~{if defined(image_pattern) then '--image-pattern "' + image_pattern + '"' else ''} \ -# ~{'--dapi-channel ' + dapi_channel} \ -# ~{'--time ' + time} \ -# --cyto-channel ~{sep=" " cyto_channel} \ -# ~{"--nuclei-label " + nuclei_label} \ -# ~{"--method " + method} \ -# --output "~{output_directory}" \ -# --subset ~{subset} \ -# ~{'--threshold ' + threshold} \ -# ~{'--threshold-correction-factor ' + threshold_correction_factor} \ -# ~{'--chunks ' + chunks} \ -# ~{if defined(extra_arguments) then extra_arguments else ''} \ -# ~{true="--force" false="" force} + export SCALLOPS_MODEL_DIR="~{model_dir}" + + scallops segment cell \ + --images "~{images}" \ + --groupby ~{sep=" " groupby} \ + ~{if defined(image_pattern) then '--image-pattern "' + image_pattern + '"' else ''} \ + ~{'--dapi-channel ' + dapi_channel} \ + ~{'--time ' + time} \ + --cyto-channel ~{sep=" " cyto_channel} \ + ~{"--nuclei-label " + nuclei_label} \ + ~{"--method " + method} \ + --output "~{output_directory}" \ + --subset ~{subset} \ + ~{'--threshold ' + threshold} \ + ~{'--threshold-correction-factor ' + threshold_correction_factor} \ + ~{'--chunks ' + chunks} \ + ~{if defined(extra_arguments) then extra_arguments else ''} \ + ~{true="--force" false="" force} >>> output { From 18e9121c17e2eea90f85eeed8955802a60751a7e Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 08:26:40 -0400 Subject: [PATCH 50/79] features by t --- scallops/cli/register.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scallops/cli/register.py b/scallops/cli/register.py index 6a80fd0..552f0d8 100644 --- a/scallops/cli/register.py +++ b/scallops/cli/register.py @@ -213,8 +213,7 @@ def single_registration( ) moving_label_keys = natsorted(moving_label_keys) if len(moving_label_keys) == 0: - logger.warning(f"No labels found for {image_key}.") - return image_key + raise ValueError(f"No labels found for {image_key}.") if not force and _output_exists( register_self, @@ -571,7 +570,9 @@ def get_matching_names( if labels: tokens = name.split("-") - if tokens[-1] not in label_suffixes and tokens[:-1] == image_key: + suffix = tokens[-1] + label_key = "-".join(tokens[:-1]) + if suffix not in label_suffixes or label_key != image_key: continue if not name.startswith(".") and is_ome_zarr_array(zarr.open(path, mode="r")): results.append(path) From b387d714fd03a82b47e8d3b8eae20110fa46113e Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 08:27:53 -0400 Subject: [PATCH 51/79] features by t --- wdl/utils.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl/utils.wdl b/wdl/utils.wdl index 079ce94..b55912d 100644 --- a/wdl/utils.wdl +++ b/wdl/utils.wdl @@ -82,7 +82,7 @@ task list_images { } runtime { -# container:container + container:container disks: disks zones: zones memory: memory From a0aafc1787d1f72e6dfe9ceb4ffcdac805dc907e Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 08:41:22 -0400 Subject: [PATCH 52/79] features by t --- scallops/cli/register.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scallops/cli/register.py b/scallops/cli/register.py index 552f0d8..9ba9447 100644 --- a/scallops/cli/register.py +++ b/scallops/cli/register.py @@ -212,6 +212,11 @@ def single_registration( ) ) moving_label_keys = natsorted(moving_label_keys) + if moving_timepoint_value is not None: + moving_label_keys = _filter_label_keys( + moving_label_keys, moving_timepoint_value + ) + if len(moving_label_keys) == 0: raise ValueError(f"No labels found for {image_key}.") @@ -622,6 +627,25 @@ def transform_all_images( ) +def _filter_label_keys(matching_keys, moving_timepoint: str): + results = [] + for i in range(len(matching_keys)): + key = matching_keys[i] + moving_labels = read_ome_zarr_array(key, dask=True) + if moving_labels.sizes.get("t", 0) > 0 and "t" in moving_labels.coords: + moving_times = moving_labels.coords["t"].values + time_index = -1 + for j in range(len(moving_times)): + if str(moving_times[j]) == str(moving_timepoint): + time_index = j + break + if time_index == -1: + continue + + results.append(key) + return results + + def _transform_labels( transform_parameter_object: itk.ParameterObject, matching_keys: Sequence[str], From ae5fab3e2987bf215652d04722667afd883297db Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 08:44:22 -0400 Subject: [PATCH 53/79] features by t --- scallops/cli/register.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scallops/cli/register.py b/scallops/cli/register.py index 9ba9447..7b45b2b 100644 --- a/scallops/cli/register.py +++ b/scallops/cli/register.py @@ -321,7 +321,12 @@ def single_registration( if landmarks_initialize: template_labels = None if len(moving_label_keys) > 0: - template_labels = read_ome_zarr_array(moving_label_keys[-1], dask=True) + template_label_key = None + for key in moving_label_keys: + if key.endswith("-nuclei"): + template_label_key = key + break + template_labels = read_ome_zarr_array(template_label_key, dask=True) landmarks_found = False grid_results = None for landmark_translation_attempt in range(len(landmark_initializations)): From 9f6cf225c608f12478d84c32be3fa185ff487b70 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 08:55:11 -0400 Subject: [PATCH 54/79] features by t --- wdl/ops_workflow.wdl | 1 + 1 file changed, 1 insertion(+) diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index ab5e91a..a578b82 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -307,6 +307,7 @@ workflow ops_workflow { call tasks.register_elastix as register_pheno_to_pheno { input: moving=select_all([phenotype_url]), + fixed=select_all([phenotype_url]), moving_label=segment_cell.output_url, moving_channel=phenotype_dapi_channel, moving_image_pattern=phenotype_image_pattern, From 09b8808fc537c08a338cf3ea2c1067d5f41d9a40 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 08:58:36 -0400 Subject: [PATCH 55/79] features by t --- scallops/cli/register.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/scallops/cli/register.py b/scallops/cli/register.py index 7b45b2b..42c4047 100644 --- a/scallops/cli/register.py +++ b/scallops/cli/register.py @@ -326,7 +326,20 @@ def single_registration( if key.endswith("-nuclei"): template_label_key = key break - template_labels = read_ome_zarr_array(template_label_key, dask=True) + + if template_label_key is not None: + template_labels = read_ome_zarr_array(template_label_key, dask=True) + if ( + template_labels.sizes.get("t", 0) > 0 + and "t" in template_labels.coords + ): + template_label_times = template_labels.coords["t"].values + time_index = -1 + for j in range(len(template_label_times)): + if str(template_label_times[j]) == str(moving_timepoint): + time_index = j + break + template_labels = template_labels.isel(time=time_index) landmarks_found = False grid_results = None for landmark_translation_attempt in range(len(landmark_initializations)): @@ -660,7 +673,7 @@ def _transform_labels( attrs: None | dict, moving_timepoint: str | None, output_timepoint: str | None, -): +) -> int: """Transform and save labels. This function applies a specified transformation and saves the results. @@ -675,6 +688,7 @@ def _transform_labels( """ if output_names is None: output_names = [os.path.basename(key) for key in matching_keys] + n_found = 0 for i in range(len(matching_keys)): key = matching_keys[i] name = os.path.basename(key) @@ -683,7 +697,7 @@ def _transform_labels( moving_times = moving_labels.coords["t"].values time_index = -1 for j in range(len(moving_times)): - if str(moving_times[j]) == moving_timepoint: + if str(moving_times[j]) == str(moving_timepoint): time_index = j break if time_index == -1: @@ -708,13 +722,14 @@ def _transform_labels( ) del moving_labels - + n_found += 1 _write_zarr_image( name=output_names[i], root=output_root, image=transformed_array, group="labels", ) + return n_found def single_transformix( From 007debcf8a2aa4038e57c161ed1eaafa3066fe18 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 09:07:22 -0400 Subject: [PATCH 56/79] features by t --- wdl/ops_workflow.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index a578b82..9828756 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -307,7 +307,7 @@ workflow ops_workflow { call tasks.register_elastix as register_pheno_to_pheno { input: moving=select_all([phenotype_url]), - fixed=select_all([phenotype_url]), + fixed=select_first([phenotype_url]), moving_label=segment_cell.output_url, moving_channel=phenotype_dapi_channel, moving_image_pattern=phenotype_image_pattern, From 4ab58ccb2b656bdb9b1fd3d4c0000115b6b002f1 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 09:16:09 -0400 Subject: [PATCH 57/79] features by t --- scallops/cli/register.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scallops/cli/register.py b/scallops/cli/register.py index 42c4047..745d69e 100644 --- a/scallops/cli/register.py +++ b/scallops/cli/register.py @@ -339,7 +339,7 @@ def single_registration( if str(template_label_times[j]) == str(moving_timepoint): time_index = j break - template_labels = template_labels.isel(time=time_index) + template_labels = template_labels.isel(t=time_index) landmarks_found = False grid_results = None for landmark_translation_attempt in range(len(landmark_initializations)): @@ -906,6 +906,13 @@ def run_itk_registration(arguments: argparse.Namespace) -> None: fixed_image_pattern = arguments.fixed_image_pattern group_by = arguments.groupby fixed_timepoint = arguments.fixed_time if arguments.fixed_time is not None else 0 + if ( + arguments.fixed_image_pattern is not None + or arguments.fixed_image_spacing is not None + or arguments.fixed_time is not None + ) and fixed_image_path is None: + raise ValueError("Please provide fixed image.") + moving_timepoint = arguments.moving_time if arguments.moving_time is not None else 0 unroll_channels = arguments.unroll_channels From f0ce208b4accba0920f21327f64a184c2bc83442 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 09:30:48 -0400 Subject: [PATCH 58/79] features by t --- scallops/cli/features.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scallops/cli/features.py b/scallops/cli/features.py index fdfd34b..f60ce21 100644 --- a/scallops/cli/features.py +++ b/scallops/cli/features.py @@ -324,9 +324,17 @@ def single_feature( image_ = _stack_and_rename(image_) if stacked_image is not None: stacked_image_ = _stack_and_rename(stacked_image) - + if "c" in image_.coords: + del image_.coords["c"] + if "c" in stacked_image_.coords: + del stacked_image_.coords["c"] intensity_image = ( - xr.concat((image_, stacked_image_), dim="c", join="outer") + xr.concat( + (image_, stacked_image_), + dim="c", + join="outer", + create_index_for_new_dim=False, + ) if stacked_image is not None else image_ ) From 9571b9412232ded33045ecf12d6813f388723454 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 09:34:05 -0400 Subject: [PATCH 59/79] features by t --- scallops/cli/features.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scallops/cli/features.py b/scallops/cli/features.py index f60ce21..e86e5b6 100644 --- a/scallops/cli/features.py +++ b/scallops/cli/features.py @@ -426,7 +426,7 @@ def single_feature( ): c = tokens[token_index] if c[0] == "s": - if c in channel_names: + if channel_names is not None and c in channel_names: channel_names[str(n_channels1 + int(c[1:]))] = ( channel_names.pop(c) ) From cf4beff9aae5f062f084ab470c76fda08d6004bf Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 09:46:33 -0400 Subject: [PATCH 60/79] features by t --- scallops/cli/features.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scallops/cli/features.py b/scallops/cli/features.py index e86e5b6..b510bee 100644 --- a/scallops/cli/features.py +++ b/scallops/cli/features.py @@ -350,12 +350,12 @@ def single_feature( if not force and is_parquet_file(features_path): logger.info( - f"Skipping features for {image_key} {label_name}{' at t=' + timepoint if timepoint is not None else ''}." + f"Skipping features for {image_key} {label_name}{' at t=' if timepoint is not None else ''}{timepoint if timepoint is not None else ''}." ) continue merged_df = None - print(image_key, image_key_without_t) + if merge_paths is not None and len(merge_paths) > 0: merged_df = _read_merged_or_objects( paths=merge_paths, From 816ebc951bcb3c2528ceebd6c550df95266d5b35 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 11:07:22 -0400 Subject: [PATCH 61/79] features by t --- scallops/cli/features.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scallops/cli/features.py b/scallops/cli/features.py index b510bee..a3cd2e5 100644 --- a/scallops/cli/features.py +++ b/scallops/cli/features.py @@ -314,14 +314,16 @@ def single_feature( if g is None: logger.info(f"No labels found for {image_key}") continue + if len(timepoints) != 1: + raise ValueError() labels_array = da.from_array(g[list(g.keys())[0]]) for timepoint in timepoints: - image_ = ( - image.sel(t=timepoint) - if timepoint is not None and image.sizes.get("t", 0) > 1 - else image - ) - image_ = _stack_and_rename(image_) + # image_ = ( + # image.sel(t=timepoint) + # if timepoint is not None and image.sizes.get("t", 0) > 1 + # else image + # ) + image_ = _stack_and_rename(image) if stacked_image is not None: stacked_image_ = _stack_and_rename(stacked_image) if "c" in image_.coords: From 9de85bd8aa7ad9d88370eaedb80adc484f3722ae Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 11:48:25 -0400 Subject: [PATCH 62/79] features by t --- wdl/ops_tasks.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl/ops_tasks.wdl b/wdl/ops_tasks.wdl index 6549051..6c871e8 100644 --- a/wdl/ops_tasks.wdl +++ b/wdl/ops_tasks.wdl @@ -433,7 +433,7 @@ task intersects_boundary { scallops features \ --features-~{label_type} "intersects-boundary_0" \ --labels ~{sep=" " labels} \ - {sep=" " additional_labels} \ + ~{sep=" " additional_labels} \ --groupby ~{sep=" " groupby} \ --subset "~{subset}" \ --output "~{output_directory}" \ From f46465895d896da0576fee41a35c4e3e591549fc Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 12:05:39 -0400 Subject: [PATCH 63/79] features by t --- wdl/stitch_tasks.wdl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wdl/stitch_tasks.wdl b/wdl/stitch_tasks.wdl index a034b57..eec3ae2 100644 --- a/wdl/stitch_tasks.wdl +++ b/wdl/stitch_tasks.wdl @@ -12,7 +12,7 @@ task illumination_correction { Array[String] groupby String output_directory Boolean? force - String docker + String container String zones Int preemptible String aws_queue_arn @@ -82,7 +82,7 @@ task stitch { Boolean? force String? blend - String docker + String container String zones Int preemptible String aws_queue_arn From 66b04f569d646e868c240e10476ca8bebb58998f Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 12:17:25 -0400 Subject: [PATCH 64/79] features by t --- scallops/cli/pooled_if_sbs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scallops/cli/pooled_if_sbs.py b/scallops/cli/pooled_if_sbs.py index 41b998f..dd63067 100644 --- a/scallops/cli/pooled_if_sbs.py +++ b/scallops/cli/pooled_if_sbs.py @@ -501,7 +501,7 @@ def merge_sbs_phenotype_pipeline( image_key: str, sbs_path: str | None, phenotype_paths: list[str], - phenotype_suffixes: list[str], + phenotype_suffixes: list[str | None], df_barcode: pd.DataFrame | None, output_dir: str, join_sbs: Literal["left", "right", "inner", "outer", "cross"] = "inner", From dd7ef4b8f836e7efcc5b6c41d59c0891dc395946 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 12:45:03 -0400 Subject: [PATCH 65/79] features by t --- wdl/ops_tasks.wdl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wdl/ops_tasks.wdl b/wdl/ops_tasks.wdl index 6c871e8..af6cf70 100644 --- a/wdl/ops_tasks.wdl +++ b/wdl/ops_tasks.wdl @@ -179,8 +179,8 @@ task register_elastix { --subset ~{subset} \ ~{if defined(label_output_directory) then '--label-output "' + label_output_directory + '"' else ''} \ ~{true="--unroll-channels" false="" unroll_channels} \ - ~{if defined(moving_time) then '--moving-time "' + moving_time + '"' else ''} \ - ~{if defined(fixed_time) then '--fixed-time "' + fixed_time + '"' else ''} \ + ~{if defined(moving_time) and moving_time !='' then '--moving-time "' + moving_time + '"' else ''} \ + ~{if defined(fixed_time) and fixed_time !='' then '--fixed-time "' + fixed_time + '"' else ''} \ ~{true="--force" false="" force} \ ~{true="--align-across-channels" false="" register_across_channels} \ ~{true="--output-aligned-channels-only" false="" output_aligned_channels_only} \ From f9a50a8577858f1402506236bd8b764320d520b9 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 12:50:53 -0400 Subject: [PATCH 66/79] features by t --- wdl/ops_tasks.wdl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wdl/ops_tasks.wdl b/wdl/ops_tasks.wdl index af6cf70..5f45dd5 100644 --- a/wdl/ops_tasks.wdl +++ b/wdl/ops_tasks.wdl @@ -179,8 +179,8 @@ task register_elastix { --subset ~{subset} \ ~{if defined(label_output_directory) then '--label-output "' + label_output_directory + '"' else ''} \ ~{true="--unroll-channels" false="" unroll_channels} \ - ~{if defined(moving_time) and moving_time !='' then '--moving-time "' + moving_time + '"' else ''} \ - ~{if defined(fixed_time) and fixed_time !='' then '--fixed-time "' + fixed_time + '"' else ''} \ + ~{if defined(moving_time) && moving_time !='' then '--moving-time "' + moving_time + '"' else ''} \ + ~{if defined(fixed_time) && fixed_time !='' then '--fixed-time "' + fixed_time + '"' else ''} \ ~{true="--force" false="" force} \ ~{true="--align-across-channels" false="" register_across_channels} \ ~{true="--output-aligned-channels-only" false="" output_aligned_channels_only} \ From 6d5f7885ab46d955bba6d608bd6d17c6eddd59b0 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 13:16:49 -0400 Subject: [PATCH 67/79] features by t --- scallops/cli/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scallops/cli/util.py b/scallops/cli/util.py index dd89a09..569df72 100644 --- a/scallops/cli/util.py +++ b/scallops/cli/util.py @@ -659,7 +659,7 @@ def _list_images( groupby_with_time.append("t") if reference_time is None: - reference_time = times[0] if times is not None and len(times) > 0 else "0" + reference_time = times[0] if times is not None and len(times) > 0 else "" return dict( group_size=group_size, From 62eb1aa9110b5c64a0db110f5c6fbcb58d5576df Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 13:23:21 -0400 Subject: [PATCH 68/79] features by t --- wdl/ops_tasks.wdl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wdl/ops_tasks.wdl b/wdl/ops_tasks.wdl index 5f45dd5..cd8ad61 100644 --- a/wdl/ops_tasks.wdl +++ b/wdl/ops_tasks.wdl @@ -34,7 +34,7 @@ task segment_nuclei { --groupby ~{sep=" " groupby} \ ~{if defined(image_pattern) then '--image-pattern "' + image_pattern + '"' else ''} \ ~{'--dapi-channel ' + dapi_channel} \ - ~{'--time ' + time} \ + ~{if defined(time) && time!='' then '--time "' + time + '"' else ''} \ --output "~{output_directory}" \ --subset ~{subset} \ ~{if defined(extra_arguments) then extra_arguments else ''} \ @@ -97,7 +97,7 @@ task segment_cell { --groupby ~{sep=" " groupby} \ ~{if defined(image_pattern) then '--image-pattern "' + image_pattern + '"' else ''} \ ~{'--dapi-channel ' + dapi_channel} \ - ~{'--time ' + time} \ + ~{if defined(time) && time!='' then '--time "' + time + '"' else ''} \ --cyto-channel ~{sep=" " cyto_channel} \ ~{"--nuclei-label " + nuclei_label} \ ~{"--method " + method} \ From 3ff87fa86a30ffadc28e4877fbe1f7832e9cb9c5 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 14:54:44 -0400 Subject: [PATCH 69/79] features by t --- scallops/tests/test_register_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scallops/tests/test_register_cli.py b/scallops/tests/test_register_cli.py index 14a6118..4a0f3b3 100644 --- a/scallops/tests/test_register_cli.py +++ b/scallops/tests/test_register_cli.py @@ -104,7 +104,7 @@ def test_register_itk_cli_known_shift(tmp_path): for shift in shifts: st = SimilarityTransform(translation=shift[::-1]) arrays.append(warp(image, st, preserve_range=True).astype(image.dtype)) - data = xr.DataArray(np.array(arrays), dims=["t", "y", "x"]) + data = xr.DataArray(np.array(arrays), dims=["t", "y", "x"], coords={"t": [0, 1]}) data = data.expand_dims("c", 1) data.attrs["processed"] = dict( images=[dict(pixels=dict(physical_size_x=1, physical_size_y=1))] From aa7afe057f92c63acdd9f11f7dc8318bb43bf550 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Tue, 7 Jul 2026 15:13:25 -0400 Subject: [PATCH 70/79] features by t --- scallops/cli/register_main.py | 4 ++-- scallops/cli/segment_main.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scallops/cli/register_main.py b/scallops/cli/register_main.py index 8bde07e..2354a4c 100644 --- a/scallops/cli/register_main.py +++ b/scallops/cli/register_main.py @@ -153,11 +153,11 @@ def _create_elastix_parser(subparsers: ArgumentParser, default_help: bool) -> No parser.add_argument( "--moving-time", - help="Time index (0-based) or value for moving image", + help="Time value for moving image", ) parser.add_argument( "--fixed-time", - help="Time index (0-based) or value for fixed image", + help="Time value for fixed image", ) parser.add_argument( "--unroll-channels", diff --git a/scallops/cli/segment_main.py b/scallops/cli/segment_main.py index d6b513f..ce4fc08 100644 --- a/scallops/cli/segment_main.py +++ b/scallops/cli/segment_main.py @@ -73,7 +73,7 @@ def _add_common_args(parser: ArgumentParser) -> None: ) parser.add_argument( "--time", - help="Time index (0-based) or value.", + help="Time value.", ) parser.add_argument( From 3806e85f2d3359fb63dd4b8c7d89a6a841164071 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Wed, 8 Jul 2026 13:49:15 -0400 Subject: [PATCH 71/79] features by t --- scallops/cli/register.py | 114 ++++++++++++++-------------- scallops/tests/test_register_cli.py | 52 ++++++------- 2 files changed, 80 insertions(+), 86 deletions(-) diff --git a/scallops/cli/register.py b/scallops/cli/register.py index 745d69e..3f45421 100644 --- a/scallops/cli/register.py +++ b/scallops/cli/register.py @@ -83,27 +83,27 @@ def _get_timepoint_index_and_value( ) -> tuple[int, Any]: timepoint_value = None timepoint_index = None + if isinstance(timepoint, str): if isinstance(image, Sequence): for i in range(len(image)): - if ( - "t" in image[i].coords - and str(image[i].coords["t"].values[0]) == timepoint - ): + if "t" in image[i].coords and str( + image[i].coords["t"].values[0] + ) == str(timepoint): timepoint_index = i timepoint_value = image[i].coords["t"].values[0] break elif "t" in image.coords: times = image.coords["t"].values for i in range(len(times)): - if str(times[i]) == timepoint: + if str(times[i]) == str(timepoint): timepoint_index = i timepoint_value = times[i] break + elif timepoint.isdigit(): # assume index + timepoint = int(timepoint) - if timepoint_index is None: - raise ValueError(f"Reference timepoint not found: {timepoint}.") - elif isinstance(timepoint, int): # index + if isinstance(timepoint, int): # index timepoint_index = timepoint if isinstance(image, Sequence): if "t" in image[timepoint_index].coords: @@ -111,8 +111,8 @@ def _get_timepoint_index_and_value( elif "t" in image.coords: times = image.coords["t"].values timepoint_value = times[timepoint_index] - else: - raise ValueError() + if timepoint_index is None: + raise ValueError(f"Reference timepoint not found: {timepoint}.") return timepoint_index, timepoint_value @@ -216,7 +216,6 @@ def single_registration( moving_label_keys = _filter_label_keys( moving_label_keys, moving_timepoint_value ) - if len(moving_label_keys) == 0: raise ValueError(f"No labels found for {image_key}.") @@ -469,23 +468,23 @@ def single_registration( landmark_min_count=landmark_min_count, parameter_object_across_channels=parameter_object_across_channels, ) - # moving_image_attrs = moving_image[0].attrs.copy() - # chunksize = moving_image[0].data.chunksize[-2:] - # del moving_image - # if moving_image_spacing is None: - # moving_image_spacing = get_image_spacing(moving_image_attrs) - - # if len(moving_label_keys) > 0: - # _transform_labels_t( - # transform_fs=transform_fs, - # transform_dest=transform_dest, - # label_output_root=label_output_root, - # moving_image_attrs=moving_image_attrs, - # moving_label_keys=moving_label_keys, - # moving_image_spacing=moving_image_spacing, - # moving_timepoint_value=moving_timepoint_value, - # chunksize=chunksize, - # ) + moving_image_attrs = moving_image[0].attrs.copy() + chunksize = moving_image[0].data.chunksize[-2:] + del moving_image + if moving_image_spacing is None: + moving_image_spacing = get_image_spacing(moving_image_attrs) + + if len(moving_label_keys) > 0: + _transform_labels_t( + transform_fs=transform_fs, + transform_dest=transform_dest, + label_output_root=label_output_root, + moving_image_attrs=moving_image_attrs, + moving_label_keys=moving_label_keys, + moving_image_spacing=moving_image_spacing, + moving_timepoint_value=moving_timepoint_value, + chunksize=chunksize, + ) return image_key @@ -502,6 +501,7 @@ def _transform_labels_t( ): # transform_dest structure is image_key/t=1 if len(moving_label_keys) > 0: + storage_options = {"chunks": chunksize} times = [] transform_file_paths = [] for transform_file in transform_fs.ls( @@ -518,43 +518,43 @@ def _transform_labels_t( index = index_natsorted(times) times = [times[val] for val in index] transform_file_paths = [transform_file_paths[val] for val in index] - storage_options = {"chunks": chunksize} + for moving_label_key in moving_label_keys: moving_label_key_basename = os.path.basename(moving_label_key) - transformed_labels = [] - times_ = [] - for i in range(len(times)): + tokens = moving_label_key_basename.split("-") + suffix = tokens[-1] + label_key = "-".join(tokens[:-1]) + moving_label = read_ome_zarr_array(moving_label_key).squeeze() + if moving_label.sizes.get("t", 0) > 0 and "t" in moving_label.coords: + moving_label = moving_label.sel(t=moving_timepoint_value) + for time_index in range(len(times)): transform_parameter_object = _load_itk_parameters_from_dir( - transform_file_paths[i] + transform_file_paths[time_index] ) if transform_parameter_object.GetNumberOfParameterMaps() > 0: - src = read_ome_zarr_array(moving_label_key).squeeze() - times_.append(times[i]) - if src.sizes.get("t", 0) > 0 and "t" in src.coords: - src = src.sel(t=moving_timepoint_value) - transformed_labels.append( - itk_transform_labels( - image=src, - transform_parameter_object=transform_parameter_object, - image_spacing=moving_image_spacing, - ) + t = times[time_index] + # note we can't stack the labels b/c we can't be sure non-moving times are same dimensions + transformed_label = itk_transform_labels( + image=moving_label, + transform_parameter_object=transform_parameter_object, + image_spacing=moving_image_spacing, ) - transformed_labels = xr.DataArray( - np.stack(transformed_labels), - dims=["t", "y", "x"], - coords={"t": times_}, - attrs=moving_image_attrs, - ) + transformed_label = xr.DataArray( + np.expand_dims(transformed_label, 0), + dims=["t", "y", "x"], + coords={"t": [t]}, + attrs=moving_image_attrs, + ) - _write_zarr_labels( - name=moving_label_key_basename, - root=label_output_root, - metadata=None, - group_metadata=None, - labels=transformed_labels, - storage_options=storage_options, - ) + _write_zarr_labels( + name=f"{label_key}-{t}-{suffix}", + root=label_output_root, + metadata=None, + group_metadata=None, + labels=transformed_label, + storage_options=storage_options, + ) def get_matching_names( diff --git a/scallops/tests/test_register_cli.py b/scallops/tests/test_register_cli.py index 4a0f3b3..eba38fc 100644 --- a/scallops/tests/test_register_cli.py +++ b/scallops/tests/test_register_cli.py @@ -104,7 +104,8 @@ def test_register_itk_cli_known_shift(tmp_path): for shift in shifts: st = SimilarityTransform(translation=shift[::-1]) arrays.append(warp(image, st, preserve_range=True).astype(image.dtype)) - data = xr.DataArray(np.array(arrays), dims=["t", "y", "x"], coords={"t": [0, 1]}) + + data = xr.DataArray(np.array(arrays), dims=["t", "y", "x"], coords={"t": [0, 1, 2]}) data = data.expand_dims("c", 1) data.attrs["processed"] = dict( images=[dict(pixels=dict(physical_size_x=1, physical_size_y=1))] @@ -147,9 +148,8 @@ def test_register_itk_cli_t_reference(tmp_path, array_A1_102_nuclei): tmp_path, "registration-input.zarr" ) exp = Experiment() - - reference_t_index = 2 - test_t = 10 + reference_time = "2" + test_time = "10" array_A1_102_nuclei = array_A1_102_nuclei.squeeze() exp.labels["A1-102-nuclei"] = array_A1_102_nuclei exp.save(registration_input_moving_labels_path) @@ -181,7 +181,7 @@ def test_register_itk_cli_t_reference(tmp_path, array_A1_102_nuclei): "--label-output", elastix_output_dir, "--moving-time", - str(reference_t_index), + str(reference_time), ] subprocess.check_call(cmd) result_exp = read_experiment(elastix_output_dir) @@ -196,43 +196,36 @@ def test_register_itk_cli_t_reference(tmp_path, array_A1_102_nuclei): .images["A1-102"] .squeeze() ) - times = list(original_image.t.values) - del times[reference_t_index] - times = [str(t) for t in times] - transformed_times = list(result_exp.labels["A1-102-nuclei"].coords["t"].values) - transformed_times = [str(t) for t in transformed_times] - assert times == transformed_times, ( - f"{', '.join(times)} != {', '.join(transformed_times)}" - ) - assert len(result_exp.labels.keys()) == 1 + original_image.coords["t"] = original_image.coords["t"].astype(str) + assert len(result_exp.labels.keys()) == 8 np.testing.assert_array_equal(transformed_image.t.values, original_image.t.values) np.testing.assert_array_equal(transformed_image.c.values, original_image.c.values) np.testing.assert_array_equal( - transformed_image.isel(t=reference_t_index), - original_image.isel(t=reference_t_index), + transformed_image.sel(t=reference_time), + original_image.sel(t=reference_time), err_msg="Reference t not equal via CLI", ) - for t in range(original_image.sizes["t"]): - if t != reference_t_index: + for t in original_image.coords["t"].values: + if t != reference_time: with np.testing.assert_raises(AssertionError): np.testing.assert_array_equal( - transformed_image.isel(t=t), original_image.isel(t=t) + transformed_image.sel(t=t), original_image.sel(t=t) ) transform_parameter_object = _load_itk_parameters_from_dir( - os.path.join(transform_output_dir, "A1-102", f"t={test_t}") + os.path.join(transform_output_dir, "A1-102", f"t={test_time}") ) # test load and apply saved transform for image warped = itk_transform_image( - image=original_image.sel(t=test_t, c=original_image.c.values[0]), + image=original_image.sel(t=test_time, c=original_image.c.values[0]), transform_parameter_object=transform_parameter_object, image_spacing=(1, 1), ) np.testing.assert_array_equal( - transformed_image.sel(t=test_t, c=transformed_image.c.values[0]).data, + transformed_image.sel(t=test_time, c=transformed_image.c.values[0]).data, warped.values, - err_msg=f"t {test_t} images not equal using itk_transform_image and CLI", + err_msg=f"t {test_time} images not equal using itk_transform_image and CLI", ) # test load and apply saved transform for labels warped_labels = itk_transform_labels( @@ -241,11 +234,10 @@ def test_register_itk_cli_t_reference(tmp_path, array_A1_102_nuclei): image_spacing=(1, 1), ) assert warped_labels.min() == 0 - np.testing.assert_array_equal( - result_exp.labels["A1-102-nuclei"].sel(t=str(test_t)).values, + result_exp.labels[f"A1-102-{test_time}-nuclei"].values, warped_labels, - err_msg=f"t {test_t} labels not equal using itk_transform_labels and CLI", + err_msg=f"t {test_time} labels not equal using itk_transform_labels and CLI", ) # compare results to API usage parameter_object = _load_itk_parameters([param_file]) @@ -255,7 +247,9 @@ def test_register_itk_cli_t_reference(tmp_path, array_A1_102_nuclei): moving_channel=[0], parameter_object=parameter_object, moving_image_spacing=(1, 1), - reference_timepoint=reference_t_index, + reference_timepoint=original_image.coords["t"] + .values.tolist() + .index(reference_time), ) xr.testing.assert_equal(result_np, transformed_image) @@ -368,8 +362,8 @@ def test_register_transform_labels_moving_only(tmp_path): create_itk_param_file(tmp_path), ] subprocess.check_call(cmd) - transformed_labels = read_image(output_zarr / "labels" / "plateA-A1-cell") - assert list(transformed_labels.coords["t"].values) == ["FISH"] + transformed_labels = read_image(output_zarr / "labels" / "plateA-A1-IF-cell") + assert list(transformed_labels.coords["t"].values) == ["IF"] assert transformed_labels.max() > 0 transformed_image = read_image(output_zarr / "images" / "plateA-A1") assert transformed_image.shape[0] == 2 From 868819c2454ddd4e4c5a9bda30facede1504ce77 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Wed, 8 Jul 2026 14:16:50 -0400 Subject: [PATCH 72/79] features by t --- scallops/tests/test_register_cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scallops/tests/test_register_cli.py b/scallops/tests/test_register_cli.py index eba38fc..15da1b0 100644 --- a/scallops/tests/test_register_cli.py +++ b/scallops/tests/test_register_cli.py @@ -148,8 +148,8 @@ def test_register_itk_cli_t_reference(tmp_path, array_A1_102_nuclei): tmp_path, "registration-input.zarr" ) exp = Experiment() - reference_time = "2" - test_time = "10" + reference_time = 2 + test_time = 10 array_A1_102_nuclei = array_A1_102_nuclei.squeeze() exp.labels["A1-102-nuclei"] = array_A1_102_nuclei exp.save(registration_input_moving_labels_path) @@ -196,7 +196,7 @@ def test_register_itk_cli_t_reference(tmp_path, array_A1_102_nuclei): .images["A1-102"] .squeeze() ) - original_image.coords["t"] = original_image.coords["t"].astype(str) + assert len(result_exp.labels.keys()) == 8 np.testing.assert_array_equal(transformed_image.t.values, original_image.t.values) np.testing.assert_array_equal(transformed_image.c.values, original_image.c.values) @@ -235,7 +235,7 @@ def test_register_itk_cli_t_reference(tmp_path, array_A1_102_nuclei): ) assert warped_labels.min() == 0 np.testing.assert_array_equal( - result_exp.labels[f"A1-102-{test_time}-nuclei"].values, + result_exp.labels[f"A1-102-{test_time}-nuclei"].values.squeeze(), warped_labels, err_msg=f"t {test_time} labels not equal using itk_transform_labels and CLI", ) From 5ac1301063af0cb916c2fad508e2107b6fc86018 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Thu, 9 Jul 2026 12:14:16 -0400 Subject: [PATCH 73/79] features by t --- scallops/tests/test_register_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scallops/tests/test_register_cli.py b/scallops/tests/test_register_cli.py index 15da1b0..a3e1df9 100644 --- a/scallops/tests/test_register_cli.py +++ b/scallops/tests/test_register_cli.py @@ -362,8 +362,8 @@ def test_register_transform_labels_moving_only(tmp_path): create_itk_param_file(tmp_path), ] subprocess.check_call(cmd) - transformed_labels = read_image(output_zarr / "labels" / "plateA-A1-IF-cell") - assert list(transformed_labels.coords["t"].values) == ["IF"] + transformed_labels = read_image(output_zarr / "labels" / "plateA-A1-FISH-cell") + assert list(transformed_labels.coords["t"].values) == ["FISH"] assert transformed_labels.max() > 0 transformed_image = read_image(output_zarr / "images" / "plateA-A1") assert transformed_image.shape[0] == 2 From c4aa449333fa6ff581099b48454ed3783b4f21af Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Thu, 9 Jul 2026 13:24:58 -0400 Subject: [PATCH 74/79] Don't convert time to int --- scallops/cli/segment.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scallops/cli/segment.py b/scallops/cli/segment.py index a21981e..cbde72a 100644 --- a/scallops/cli/segment.py +++ b/scallops/cli/segment.py @@ -389,11 +389,6 @@ def run_pipeline(arguments: argparse.Namespace, nuclei: bool): kwargs = dict() timepoint = arguments.time if arguments.time is not None else 0 - if isinstance(timepoint, str) and timepoint.isdigit(): - try: - timepoint = int(timepoint) - except ValueError: - pass if not nuclei: kwargs["nuclei_min_area"] = arguments.nuclei_min_area kwargs["nuclei_max_area"] = arguments.nuclei_max_area From af6a4d49ebe8265e447657b3b84e794c055be84f Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Thu, 9 Jul 2026 14:39:29 -0400 Subject: [PATCH 75/79] features by t --- scallops/cli/extract_crops.py | 53 +++- scallops/cli/extract_crops_main.py | 17 +- scallops/cli/features.py | 412 ++++++++++++++--------------- scallops/tests/test_wdl.py | 12 +- 4 files changed, 255 insertions(+), 239 deletions(-) diff --git a/scallops/cli/extract_crops.py b/scallops/cli/extract_crops.py index 0881c05..dbd92ff 100644 --- a/scallops/cli/extract_crops.py +++ b/scallops/cli/extract_crops.py @@ -8,9 +8,13 @@ import pyarrow.parquet as pq from array_api_compat import get_namespace from skimage.util import img_as_ubyte -from zarr import Group -from scallops.cli.features import _read_merged_or_objects, get_labels +from scallops.cli.features import ( + _find_labels, + _image_key_without_time_and_selected_time, + _read_merged_or_objects, +) +from scallops.cli.find_objects import get_path from scallops.cli.util import ( _get_cli_logger, cli_metadata, @@ -39,7 +43,7 @@ def single_crop( group: str, # NOT USED file_list: list[str], metadata: dict, - labels_group: Group | None, + label_paths: list[str] | None, output_dir: str, output_sep: str, merge_dir: str, @@ -57,10 +61,18 @@ def single_crop( force: bool, ): image_key = metadata["id"] - output_dir = f"{output_dir}{output_sep}{label_name}{output_sep}{image_key}" - - output_parquet_path = f"{output_dir}.parquet" + image_key_without_t, selected_timepoint = _image_key_without_time_and_selected_time( + metadata + ) + output_parquet_path = get_path( + output_dir, + output_sep, + label_name, + image_key_without_t if image_key_without_t is not None else image_key, + selected_timepoint, + ".parquet", + ) if not force and is_parquet_file(output_parquet_path): logger.info(f"Skipping features for {image_key} {label_name}") return @@ -70,16 +82,29 @@ def single_crop( image = _images2fov(file_list, metadata, dask=True).squeeze().data logger.info(f"{image_key} image shape {image.shape}") label_image = None - if labels_group is not None: - zarr_labels = get_labels( - labels_group=labels_group, - name=image_key, - suffix=label_name, # e.g. nuclei + if label_paths is not None: + g, timepoints = _find_labels( + label_paths=label_paths, + image_key=image_key, + label_name=label_name, + image_key_without_t=image_key_without_t, + selected_timepoint=selected_timepoint, ) + if g is None: + raise ValueError(f"No labels found for {image_key}") + + if len(timepoints) != 1: + raise ValueError(f"More than one timepoint found for {image_key}") + label_image = da.from_array(g[list(g.keys())[0]]) + timepoint = timepoints[0] + if timepoint is not None and label_image.ndim == 3: + index = -1 + for i in range(len(timepoints)): + if str(timepoints[i]) == str(timepoint): + index = i + break + label_image = label_image[index] - if zarr_labels is None: - raise ValueError(f"Unable to read {label_name} labels for {image_key}.") - label_image = da.from_zarr(zarr_labels) merged_df = _read_merged_or_objects( merge_dir=merge_dir, merge_dir_sep=merge_dir_sep, diff --git a/scallops/cli/extract_crops_main.py b/scallops/cli/extract_crops_main.py index a6bb070..fa86da6 100644 --- a/scallops/cli/extract_crops_main.py +++ b/scallops/cli/extract_crops_main.py @@ -1,7 +1,6 @@ import argparse import fsspec -import zarr from dask.bag import from_sequence from scallops.cli.arg_parser import _sort_groups @@ -79,16 +78,13 @@ def run_pipeline_extract_crops(arguments: argparse.Namespace): output_fs, _ = fsspec.core.url_to_fs(output_dir) output_dir = output_dir.rstrip(output_fs.sep) - labels_path = arguments.labels + label_paths = arguments.labels no_version = arguments.no_version - labels_group = None + if not mask: - labels_path = None - if labels_path is None and mask: + label_paths = None + if label_paths is None and mask: raise ValueError("Labels must be provided when `mask` is true.") - if labels_path is not None: - label_root = zarr.open(labels_path, mode="r") - labels_group = label_root["labels"] image_seq = from_sequence( _set_up_experiment( @@ -111,7 +107,7 @@ def run_pipeline_extract_crops(arguments: argparse.Namespace): output_sep=output_fs.sep, merge_dir=merge_dir, merge_dir_sep=merge_dir_sep, - labels_group=labels_group, + label_paths=label_paths, label_filter=label_filter, label_name=label_name, percentile_normalize=percentile_normalize, @@ -150,12 +146,13 @@ def _create_parser(subparsers: argparse.ArgumentParser, default_help: bool) -> N parser.add_argument( "--labels", dest="labels", + nargs="*", help="Path to zarr directory containing labels. Required when `mask` is true", ) parser.add_argument( "--mask", action="store_true", - help="Set pixels not belonging to target label to zero", + help="Set crop pixels not belonging to target label to zero", ) parser.add_argument( "--label-name", diff --git a/scallops/cli/features.py b/scallops/cli/features.py index a3cd2e5..e6d09cb 100644 --- a/scallops/cli/features.py +++ b/scallops/cli/features.py @@ -185,8 +185,12 @@ def _find_labels( if "t" in zarr_metadata: timepoints = zarr_metadata["t"] - if selected_timepoint in timepoints: - index = timepoints.index(selected_timepoint) + index = -1 + for i in range(len(timepoints)): + if str(timepoints[i]) == str(selected_timepoint): + index = i + break + if index != -1: timepoints = [timepoints[index]] return g, timepoints else: @@ -315,236 +319,226 @@ def single_feature( logger.info(f"No labels found for {image_key}") continue if len(timepoints) != 1: - raise ValueError() - labels_array = da.from_array(g[list(g.keys())[0]]) - for timepoint in timepoints: - # image_ = ( - # image.sel(t=timepoint) - # if timepoint is not None and image.sizes.get("t", 0) > 1 - # else image - # ) - image_ = _stack_and_rename(image) - if stacked_image is not None: - stacked_image_ = _stack_and_rename(stacked_image) - if "c" in image_.coords: - del image_.coords["c"] - if "c" in stacked_image_.coords: - del stacked_image_.coords["c"] - intensity_image = ( - xr.concat( - (image_, stacked_image_), - dim="c", - join="outer", - create_index_for_new_dim=False, - ) - if stacked_image is not None - else image_ + raise ValueError(f"More than one timepoint found for {image_key}") + timepoint = timepoints[0] + label_image = da.from_array(g[list(g.keys())[0]]) + features_path = get_path( + output_dir, + output_sep, + label_name, + image_key_without_t if image_key_without_t is not None else image_key, + timepoint, + ".parquet", + ) + + if not force and is_parquet_file(features_path): + logger.info( + f"Skipping features for {image_key} {label_name}{' at t=' if timepoint is not None else ''}{timepoint if timepoint is not None else ''}." ) + continue + if timepoint is not None and label_image.ndim == 3: + index = -1 + for i in range(len(timepoints)): + if str(timepoints[i]) == str(timepoint): + index = i + break + label_image = label_image[index] + + image_ = _stack_and_rename(image) + if stacked_image is not None: + stacked_image_ = _stack_and_rename(stacked_image) + if "c" in image_.coords: + del image_.coords["c"] + if "c" in stacked_image_.coords: + del stacked_image_.coords["c"] + intensity_image = ( + xr.concat( + (image_, stacked_image_), + dim="c", + join="outer", + create_index_for_new_dim=False, + ) + if stacked_image is not None + else image_ + ) - features_path = get_path( + merged_df = None + if merge_paths is not None and len(merge_paths) > 0: + merged_df = _read_merged_or_objects( + paths=merge_paths, + timepoint=timepoint, + label_name=label_name, + image_key=image_key, + image_key_without_t=image_key_without_t, + label_filter=label_filter, + add_timepoint_suffix=False, + ) + if merged_df is None: + raise ValueError(f"Metadata not found for {image_key}") + + if merged_df is None: + logger.info( + f"Find {label_name} objects for {image_key}{' at t=' if timepoint is not None else ''}{timepoint if timepoint is not None else ''}." + ) + merged_df = find_objects(label_image) + objects_path = get_path( output_dir, output_sep, label_name, - image_key_without_t if image_key_without_t is not None else image_key, + image_key, timepoint, - ".parquet", + "-objects.parquet", ) - if not force and is_parquet_file(features_path): - logger.info( - f"Skipping features for {image_key} {label_name}{' at t=' if timepoint is not None else ''}{timepoint if timepoint is not None else ''}." - ) - continue - - merged_df = None - - if merge_paths is not None and len(merge_paths) > 0: - merged_df = _read_merged_or_objects( - paths=merge_paths, - timepoint=timepoint, - label_name=label_name, - image_key=image_key, - image_key_without_t=image_key_without_t, - label_filter=label_filter, - add_timepoint_suffix=False, - ) - if merged_df is None: - raise ValueError(f"Metadata not found for {image_key}") - if timepoint is not None and labels_array.ndim == 3: - timepoint_index = timepoints.index(timepoint) - label_image = labels_array[timepoint_index] - else: - label_image = labels_array + merged_df.index.name = "label" + merged_df.columns = f"{label_prefix}_" + merged_df.columns + _to_parquet( + merged_df, + objects_path, + write_index=True, + compute=True, + custom_metadata=dict(scallops=json.dumps(cli_metadata())) + if not no_version + else None, + ) + merged_df = pd.read_parquet(objects_path) + + features = normalize_features(features) + # strip nuclei_, etc. from features_plot + features_plot_label = [] + for feature in features_plot: + tokens = feature.lower().split("_") + if tokens[0] == label_prefix: + features_plot_label.append("_".join(tokens[1:])) + + features_plot_label = normalize_features(features_plot_label) + if stacked_image_tuple is not None: + stacked_features = set() + stacked_features_plot = set() + for feature in features: # add offset for image1 + tokens = feature.lower().split("_") - if merged_df is None: - logger.info( - f"Find {label_name} objects for {image_key}{' at t=' if timepoint is not None else ''}{timepoint if timepoint is not None else ''}." - ) - merged_df = find_objects(label_image) - objects_path = get_path( - output_dir, - output_sep, - label_name, - image_key, - timepoint, - "-objects.parquet", + if ( + tokens[0] in _features_multichannel.keys() + or tokens[0] in _features_single_channel.keys() + ): + for token_index in range( + 1, + 3 if tokens[0] in _features_multichannel.keys() else 2, + ): + c = tokens[token_index] + if c[0] == "s": + if channel_names is not None and c in channel_names: + channel_names[str(n_channels1 + int(c[1:]))] = ( + channel_names.pop(c) + ) + tokens[token_index] = str(n_channels1 + int(c[1:])) + + new_feature = "_".join(tokens) + if feature in features_plot_label: + stacked_features_plot.add(new_feature) + stacked_features.add(new_feature) + + features = stacked_features + features_plot_label = stacked_features_plot + + features = list(set(natsorted(features))) + + if label_filter is not None: + merged_df = merged_df.query(label_filter) + min_max_area = label_name_to_min_max_area.get(label_name) + area_column = f"{label_prefix}_AreaShape_Area" + bounding_box_columns = [ + f"{label_prefix}_AreaShape_BoundingBoxMinimum_Y", + f"{label_prefix}_AreaShape_BoundingBoxMinimum_X", + f"{label_prefix}_AreaShape_BoundingBoxMaximum_Y", + f"{label_prefix}_AreaShape_BoundingBoxMaximum_X", + ] + if timepoint is not None: + if area_column not in merged_df.columns: + area_column = f"{area_column}_{timepoint}" + + if bounding_box_columns[0] not in merged_df.columns: + bounding_box_columns = [ + f"{c}_{timepoint}" for c in bounding_box_columns + ] + n_labels = len(merged_df) + prefix = "" + if min_max_area[0] is not None or min_max_area[1] is not None: + area_query = [] + if min_max_area[0] is not None: + area_query.append(f"{area_column}>={min_max_area[0]}") + if min_max_area[1] is not None: + area_query.append(f"{area_column}<={min_max_area[1]}") + merged_df = merged_df.query("&".join(area_query)) + n_labels_filtered = n_labels - len(merged_df) + prefix = f"Removed {n_labels_filtered:,} out of " + logger.info( + f"{prefix}{n_labels:,} {pluralize('label', n_labels)}. " + f"Area: {merged_df[area_column].min():,.0f} to {merged_df[area_column].max():,.0f}." + ) + + df = label_features( + objects_df=merged_df, + label_image=label_image, + intensity_image=intensity_image, + features=features, + normalize=normalize, + bounding_box_columns=bounding_box_columns, + channel_names=channel_names, + ) + # df will be None if only area and coordinates requested + + if df is not None: + fs = fsspec.url_to_fs(features_path)[0] + if fs.exists(features_path): + fs.rm(features_path, recursive=True) + df.index.name = "label" + df.columns = f"{label_prefix}_" + df.columns + if isinstance(df, pd.DataFrame): + table = pa.Table.from_pandas(df, preserve_index=True) + if not no_version: + table = table.replace_schema_metadata( + { + "scallops".encode(): json.dumps(cli_metadata()).encode(), + **table.schema.metadata, + } + ) + fs, output_file = fsspec.url_to_fs(features_path) + pq.write_table( + table, + features_path, + filesystem=fs, ) - merged_df.index.name = "label" - merged_df.columns = f"{label_prefix}_" + merged_df.columns + else: _to_parquet( - merged_df, - objects_path, + df, + features_path, write_index=True, compute=True, custom_metadata=dict(scallops=json.dumps(cli_metadata())) if not no_version else None, ) - merged_df = pd.read_parquet(objects_path) - features = normalize_features(features) - # strip nuclei_, etc. from features_plot - features_plot_label = [] - for feature in features_plot: - tokens = feature.lower().split("_") - if tokens[0] == label_prefix: - features_plot_label.append("_".join(tokens[1:])) - - features_plot_label = normalize_features(features_plot_label) - if stacked_image_tuple is not None: - stacked_features = set() - stacked_features_plot = set() - for feature in features: # add offset for image1 - tokens = feature.lower().split("_") - - if ( - tokens[0] in _features_multichannel.keys() - or tokens[0] in _features_single_channel.keys() - ): - for token_index in range( - 1, - 3 if tokens[0] in _features_multichannel.keys() else 2, - ): - c = tokens[token_index] - if c[0] == "s": - if channel_names is not None and c in channel_names: - channel_names[str(n_channels1 + int(c[1:]))] = ( - channel_names.pop(c) - ) - tokens[token_index] = str(n_channels1 + int(c[1:])) - - new_feature = "_".join(tokens) - if feature in features_plot_label: - stacked_features_plot.add(new_feature) - stacked_features.add(new_feature) - - features = stacked_features - features_plot_label = stacked_features_plot - - features = list(set(natsorted(features))) - - if label_filter is not None: - merged_df = merged_df.query(label_filter) - min_max_area = label_name_to_min_max_area.get(label_name) - area_column = f"{label_prefix}_AreaShape_Area" - bounding_box_columns = [ - f"{label_prefix}_AreaShape_BoundingBoxMinimum_Y", - f"{label_prefix}_AreaShape_BoundingBoxMinimum_X", - f"{label_prefix}_AreaShape_BoundingBoxMaximum_Y", - f"{label_prefix}_AreaShape_BoundingBoxMaximum_X", + if len(features_plot_label) > 0: + features_plot_label = [ + label_prefix + "_" + feature for feature in features_plot_label ] if timepoint is not None: - if area_column not in merged_df.columns: - area_column = f"{area_column}_{timepoint}" - - if bounding_box_columns[0] not in merged_df.columns: - bounding_box_columns = [ - f"{c}_{timepoint}" for c in bounding_box_columns - ] - n_labels = len(merged_df) - prefix = "" - if min_max_area[0] is not None or min_max_area[1] is not None: - area_query = [] - if min_max_area[0] is not None: - area_query.append(f"{area_column}>={min_max_area[0]}") - if min_max_area[1] is not None: - area_query.append(f"{area_column}<={min_max_area[1]}") - merged_df = merged_df.query("&".join(area_query)) - n_labels_filtered = n_labels - len(merged_df) - prefix = f"Removed {n_labels_filtered:,} out of " - logger.info( - f"{prefix}{n_labels:,} {pluralize('label', n_labels)}. " - f"Area: {merged_df[area_column].min():,.0f} to {merged_df[area_column].max():,.0f}." - ) - - df = label_features( - objects_df=merged_df, - label_image=label_image, - intensity_image=intensity_image, - features=features, - normalize=normalize, - bounding_box_columns=bounding_box_columns, - channel_names=channel_names, + features_plot_label = [f"{c}_{timepoint}" for c in features_plot_label] + df_features = pd.read_parquet(features_path, columns=features_plot_label) + centroid_columns = [ + label_prefix + "_centroid-1", + label_name + "_centroid-0", + ] + df = merged_df[centroid_columns].join(df_features) + pdf_path = get_path( + output_dir, output_sep, label_name, image_key, timepoint, ".pdf" ) - # df will be None if only area and coordinates requested - - if df is not None: - fs = fsspec.url_to_fs(features_path)[0] - if fs.exists(features_path): - fs.rm(features_path, recursive=True) - df.index.name = "label" - df.columns = f"{label_prefix}_" + df.columns - if isinstance(df, pd.DataFrame): - table = pa.Table.from_pandas(df, preserve_index=True) - if not no_version: - table = table.replace_schema_metadata( - { - "scallops".encode(): json.dumps( - cli_metadata() - ).encode(), - **table.schema.metadata, - } - ) - fs, output_file = fsspec.url_to_fs(features_path) - pq.write_table( - table, - features_path, - filesystem=fs, - ) - - else: - _to_parquet( - df, - features_path, - write_index=True, - compute=True, - custom_metadata=dict(scallops=json.dumps(cli_metadata())) - if not no_version - else None, - ) - - if len(features_plot_label) > 0: - features_plot_label = [ - label_prefix + "_" + feature for feature in features_plot_label - ] - if timepoint is not None: - features_plot_label = [ - f"{c}_{timepoint}" for c in features_plot_label - ] - df_features = pd.read_parquet( - features_path, columns=features_plot_label - ) - centroid_columns = [ - label_prefix + "_centroid-1", - label_name + "_centroid-0", - ] - df = merged_df[centroid_columns].join(df_features) - pdf_path = get_path( - output_dir, output_sep, label_name, image_key, timepoint, ".pdf" - ) - _plot_features(df, features_plot_label, pdf_path, centroid_columns) + _plot_features(df, features_plot_label, pdf_path, centroid_columns) return [] diff --git a/scallops/tests/test_wdl.py b/scallops/tests/test_wdl.py index 84f6bc4..ac57c4b 100644 --- a/scallops/tests/test_wdl.py +++ b/scallops/tests/test_wdl.py @@ -271,11 +271,11 @@ def test_ops_wdl(phenotype_rounds, tmp_path): assert ( len( merge_sbs_metadata_df.columns[ - merge_sbs_metadata_df.columns.str.contains("-qc") + merge_sbs_metadata_df.columns.str.contains("qc") ] ) > 0 - ) + ), "No QC columns found" assert ( len( merge_sbs_metadata_df.columns[ @@ -283,12 +283,12 @@ def test_ops_wdl(phenotype_rounds, tmp_path): ] ) == 0 - ) + ), "Intensity columns found in merge_sbs_metadata_df" merge_features_df = pd.read_parquet(output / "merge-features" / "plateA-A1.parquet") assert ( len(merge_features_df.columns[merge_features_df.columns.str.contains("qc")]) > 0 - ) + ), "No QC columns found" assert ( len( merge_features_df.columns[ @@ -296,10 +296,10 @@ def test_ops_wdl(phenotype_rounds, tmp_path): ] ) > 0 - ) + ), "No intensity columns found in merge_features_df" intensity_column = merge_features_df.columns[ merge_features_df.columns.str.contains("Intensity") ][0] assert len(merge_features_df.query(f"~{intensity_column}.isna()")) == len( merge_sbs_metadata_df.query("~barcode_count_0.isna()") - ) + ), "Incorrect number of recovered cells" From e09347654f941968c13e78561b59b91f13c2ebce Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Thu, 9 Jul 2026 15:37:56 -0400 Subject: [PATCH 76/79] features by t --- scallops/cli/pooled_if_sbs.py | 19 ++++++++++--------- scallops/tests/test_wdl.py | 17 +++++++++++------ wdl/ops_workflow.wdl | 2 +- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/scallops/cli/pooled_if_sbs.py b/scallops/cli/pooled_if_sbs.py index dd63067..a211d2f 100644 --- a/scallops/cli/pooled_if_sbs.py +++ b/scallops/cli/pooled_if_sbs.py @@ -580,7 +580,7 @@ def merge_sbs_phenotype_pipeline( if phenotype_suffixes[i] is not None: df.columns = df.columns + "_" + phenotype_suffixes[i] - duplicate_suffix = path.split("/")[-3] + duplicate_suffix = path.rstrip("/").split("/")[-4] if output_format == "zarr": # read index and metadata if len(_metadata_cols) > 0: @@ -693,16 +693,17 @@ def _find_phenotype_paths_and_suffixes( for x in matches: x = fs.unstrip_protocol(x) # if parent starts with t=xxx, add xxx suffix + # if name of output dir has qc, add to suffix # otherwise suffix will only be added for duplicate columns - tokens = x.split("/") - suffix_ = tokens[-2] + tokens = x.rstrip(fs.sep).split(fs.sep) + suffix1 = tokens[-2] # e.g. t=FISH or nuclei/cell/ + suffix2 = tokens[-4] # e.g. features, iss-to-iss-qc, objects suffix = None - if suffix_.startswith("t="): - suffix = suffix_[2:] - else: - suffix_ = tokens[-3] - if "qc" in suffix_: - suffix = suffix_ + + if suffix1.startswith("t="): + suffix = suffix1[2:] + if "qc" in suffix2: + suffix = suffix2 if suffix is None else suffix + "-" + suffix2 found_suffixes.append(suffix) found_paths.append(x) diff --git a/scallops/tests/test_wdl.py b/scallops/tests/test_wdl.py index ac57c4b..7d9ee83 100644 --- a/scallops/tests/test_wdl.py +++ b/scallops/tests/test_wdl.py @@ -261,13 +261,18 @@ def test_ops_wdl(phenotype_rounds, tmp_path): env["MINIWDL__SCHEDULER__CONTAINER_BACKEND"] = "miniwdl_test_local" env["SCALLOPS_TEST"] = "1" check_call(cmd, env=env) - + assert (output / "merge-sbs-metadata" / "plateA-A1.parquet").exists(), ( + "Merged metadata parquet not found" + ) + assert (output / "merge-features" / "plateA-A1.parquet").exists(), ( + "Merged features not found" + ) merge_sbs_metadata_df = pd.read_parquet( output / "merge-sbs-metadata" / "plateA-A1.parquet" ) assert len(merge_sbs_metadata_df) > len( merge_sbs_metadata_df.query("~barcode_count_0.isna()") - ) + ), "Metadata should include all cells" assert ( len( merge_sbs_metadata_df.columns[ @@ -275,7 +280,7 @@ def test_ops_wdl(phenotype_rounds, tmp_path): ] ) > 0 - ), "No QC columns found" + ), f"No QC columns found in {', '.join(merge_sbs_metadata_df.columns.tolist())}" assert ( len( merge_sbs_metadata_df.columns[ @@ -283,12 +288,12 @@ def test_ops_wdl(phenotype_rounds, tmp_path): ] ) == 0 - ), "Intensity columns found in merge_sbs_metadata_df" + ), f"Intensity columns found in {', '.join(merge_sbs_metadata_df.columns.tolist())}" merge_features_df = pd.read_parquet(output / "merge-features" / "plateA-A1.parquet") assert ( len(merge_features_df.columns[merge_features_df.columns.str.contains("qc")]) > 0 - ), "No QC columns found" + ), f"No QC columns found in {', '.join(merge_features_df.columns.tolist())}" assert ( len( merge_features_df.columns[ @@ -296,7 +301,7 @@ def test_ops_wdl(phenotype_rounds, tmp_path): ] ) > 0 - ), "No intensity columns found in merge_features_df" + ), f"No intensity columns found in {', '.join(merge_features_df.columns.tolist())}" intensity_column = merge_features_df.columns[ merge_features_df.columns.str.contains("Intensity") ][0] diff --git a/wdl/ops_workflow.wdl b/wdl/ops_workflow.wdl index 9828756..ffd6933 100644 --- a/wdl/ops_workflow.wdl +++ b/wdl/ops_workflow.wdl @@ -349,7 +349,7 @@ workflow ops_workflow { image_channel=select_first([phenotype_dapi_channel, 0]), stacked_image_channel=0, label_type="nuclei", - output_directory=register_pheno_to_pheno_qc_directory + "-" + phenotype_time, + output_directory=register_pheno_to_pheno_qc_directory, force = force_register_pheno_to_pheno_qc, container=container, zones = zones, From 52c99d829d3940caa84809ae01e71d1d90dd0a19 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Thu, 9 Jul 2026 15:41:53 -0400 Subject: [PATCH 77/79] features by t --- scallops/cli/pooled_if_sbs.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scallops/cli/pooled_if_sbs.py b/scallops/cli/pooled_if_sbs.py index a211d2f..820cabc 100644 --- a/scallops/cli/pooled_if_sbs.py +++ b/scallops/cli/pooled_if_sbs.py @@ -580,7 +580,10 @@ def merge_sbs_phenotype_pipeline( if phenotype_suffixes[i] is not None: df.columns = df.columns + "_" + phenotype_suffixes[i] - duplicate_suffix = path.rstrip("/").split("/")[-4] + path_tokens = path.rstrip("/").split("/") + duplicate_suffix = ( + path_tokens[-4] if path_tokens[-2].startswith("t=") else path_tokens[-3] + ) if output_format == "zarr": # read index and metadata if len(_metadata_cols) > 0: @@ -697,11 +700,13 @@ def _find_phenotype_paths_and_suffixes( # otherwise suffix will only be added for duplicate columns tokens = x.rstrip(fs.sep).split(fs.sep) suffix1 = tokens[-2] # e.g. t=FISH or nuclei/cell/ - suffix2 = tokens[-4] # e.g. features, iss-to-iss-qc, objects + # e.g. features, iss-to-iss-qc, objects suffix = None - if suffix1.startswith("t="): suffix = suffix1[2:] + suffix2 = tokens[-4] + else: + suffix2 = tokens[-3] if "qc" in suffix2: suffix = suffix2 if suffix is None else suffix + "-" + suffix2 found_suffixes.append(suffix) From ccf6d5537c38500ccfc2e8bdebd7e6bbec8bea20 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Fri, 10 Jul 2026 10:58:19 -0400 Subject: [PATCH 78/79] features by t --- scallops/cli/extract_crops.py | 28 +++++++++++++++------------- scallops/cli/extract_crops_main.py | 16 ++++++---------- scallops/tests/test_features.py | 12 ++++++++---- 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/scallops/cli/extract_crops.py b/scallops/cli/extract_crops.py index dbd92ff..5e26506 100644 --- a/scallops/cli/extract_crops.py +++ b/scallops/cli/extract_crops.py @@ -1,4 +1,5 @@ import json +import os from typing import Literal import dask.array as da @@ -46,8 +47,7 @@ def single_crop( label_paths: list[str] | None, output_dir: str, output_sep: str, - merge_dir: str, - merge_dir_sep: str, + merge_paths: list[str], crop_size: tuple[int, int], output_format: Literal["tiff", "npy"], label_name: str, @@ -65,6 +65,7 @@ def single_crop( image_key_without_t, selected_timepoint = _image_key_without_time_and_selected_time( metadata ) + output_parquet_path = get_path( output_dir, output_sep, @@ -76,7 +77,17 @@ def single_crop( if not force and is_parquet_file(output_parquet_path): logger.info(f"Skipping features for {image_key} {label_name}") return - + merged_df = _read_merged_or_objects( + paths=merge_paths, + timepoint=selected_timepoint, + label_name=label_name, + image_key=image_key, + image_key_without_t=image_key_without_t, + label_filter=label_filter, + add_timepoint_suffix=False, + ) + if merged_df is None: + raise ValueError(f"Unable to read merged data for {image_key}.") output_fs, _ = fsspec.core.url_to_fs(output_dir) output_fs.makedirs(output_dir, exist_ok=True) image = _images2fov(file_list, metadata, dask=True).squeeze().data @@ -105,15 +116,6 @@ def single_crop( break label_image = label_image[index] - merged_df = _read_merged_or_objects( - merge_dir=merge_dir, - merge_dir_sep=merge_dir_sep, - label_name=label_name, - image_key=image_key, - label_filter=label_filter, - ) - if merged_df is None: - raise ValueError(f"Unable to read merged data for {image_key}.") n_labels_before_filtering = len(merged_df) if label_filter is not None: merged_df = merged_df.query(label_filter) @@ -177,7 +179,6 @@ def single_crop( ) output_metadata = cli_metadata() if not no_version else dict() - table = pa.Table.from_pandas(merged_df, preserve_index=True) table = table.replace_schema_metadata( { @@ -187,6 +188,7 @@ def single_crop( ) fs, output_parquet_path = fsspec.url_to_fs(output_parquet_path) + fs.mkdirs(os.path.dirname(output_parquet_path.rstrip(fs.sep)), exist_ok=True) pq.write_table( table, output_parquet_path, diff --git a/scallops/cli/extract_crops_main.py b/scallops/cli/extract_crops_main.py index fa86da6..797bf12 100644 --- a/scallops/cli/extract_crops_main.py +++ b/scallops/cli/extract_crops_main.py @@ -38,7 +38,7 @@ def run_pipeline_extract_crops(arguments: argparse.Namespace): image_patterns = arguments.image_pattern output_dir = arguments.output - merge_dir = arguments.merge + merge_paths = arguments.merge subset = arguments.subset force = arguments.force groupby = arguments.groupby @@ -70,13 +70,9 @@ def run_pipeline_extract_crops(arguments: argparse.Namespace): if dask_server_url is None and arguments.dask_cluster is None: dask_cluster_parameters = _dask_workers_threads() - merge_dir_sep = None - if merge_dir is not None: - merge_dir_sep = fsspec.core.url_to_fs(merge_dir)[0].sep - merge_dir = merge_dir.rstrip(merge_dir_sep) - output_fs, _ = fsspec.core.url_to_fs(output_dir) output_dir = output_dir.rstrip(output_fs.sep) + output_fs.mkdirs(output_dir, exist_ok=True) label_paths = arguments.labels no_version = arguments.no_version @@ -105,8 +101,7 @@ def run_pipeline_extract_crops(arguments: argparse.Namespace): single_crop, output_dir=output_dir, output_sep=output_fs.sep, - merge_dir=merge_dir, - merge_dir_sep=merge_dir_sep, + merge_paths=merge_paths, label_paths=label_paths, label_filter=label_filter, label_name=label_name, @@ -141,7 +136,8 @@ def _create_parser(subparsers: argparse.ArgumentParser, default_help: bool) -> N required.add_argument( "--merge", - help="Path to directory containing output from `merge`", + nargs="+", + help="Path(s) to directory containing output from `find-objects`, `merge`, or `features`", ) parser.add_argument( "--labels", @@ -157,7 +153,7 @@ def _create_parser(subparsers: argparse.ArgumentParser, default_help: bool) -> N parser.add_argument( "--label-name", help="Name of labels to use. For example `nuclei` or `cell`", - default="cell", + default="nuclei", ) parser.add_argument( "--crop-size", diff --git a/scallops/tests/test_features.py b/scallops/tests/test_features.py index b2b3633..9ad1897 100644 --- a/scallops/tests/test_features.py +++ b/scallops/tests/test_features.py @@ -69,16 +69,20 @@ def test_extract_crops_cmd(tmp_path, array_A1_102_cells, array_A1_102_alnpheno, "scallops", "extract-crops", "-i", - zarr_path, - "--labels", - zarr_path, + zarr_path, # images "--merge", objects_output_path, "--output", crops_output_path, + "--image-pattern", + "{well}", + "--groupby", + "well", + "--label-name", + "cell", ] if mask: - cmd.append("--mask") + cmd += ["--mask", "--labels", zarr_path] check_call(cmd) img = read_image(crops_output_path + "/cell/test/1523.tiff") From ad4f13bdce04bae5ea792d3a28abbc0d4790d7d8 Mon Sep 17 00:00:00 2001 From: Joshua Gould Date: Fri, 10 Jul 2026 11:02:52 -0400 Subject: [PATCH 79/79] features by t --- scallops/tests/test_segmentation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scallops/tests/test_segmentation.py b/scallops/tests/test_segmentation.py index ef7bb93..f9e092a 100644 --- a/scallops/tests/test_segmentation.py +++ b/scallops/tests/test_segmentation.py @@ -280,7 +280,7 @@ def test_segment_cells_cmd(experiment_c_dask, tmp_path): "--image-pattern", "10X_c{t}-SBS-{t}/{mag}X_c{t}-{exp}-{t}_{well}_Tile-{tile}.{datatype}.tif", "--time", - "0", + "1", "--output=" + tmp_path, "--subset=A1-102", "--method",