diff --git a/slaf/ml/__init__.py b/slaf/ml/__init__.py index 866c43f..9d8befe 100644 --- a/slaf/ml/__init__.py +++ b/slaf/ml/__init__.py @@ -11,6 +11,7 @@ PrefetchBatchProcessor, SLAFIterableDataset, ) +from .expression_preprocessor import ExpressionPreprocessor from .samplers import ( RandomShuffle, Shuffle, @@ -24,6 +25,7 @@ __all__ = [ # Core DataLoaders "SLAFDataLoader", + "ExpressionPreprocessor", "TileDBDataLoader", "TileDBIterableDataset", # Dataset and Processing diff --git a/slaf/ml/aggregators.py b/slaf/ml/aggregators.py index a4cd934..f68cb4d 100644 --- a/slaf/ml/aggregators.py +++ b/slaf/ml/aggregators.py @@ -12,6 +12,10 @@ import polars as pl from slaf.core.tabular_schema import DataSchema +from slaf.ml.expression_preprocessor import ( + ExpressionPreprocessor, + apply_expression_preprocessor, +) class Window(ABC): @@ -84,6 +88,12 @@ def apply( "use_binned_expressions", True ) # Default to True for scGPT + preprocessor = kwargs.get("expression_preprocessor") + fragment_df = apply_expression_preprocessor(fragment_df, schema, preprocessor) + already_log1p = ( + isinstance(preprocessor, ExpressionPreprocessor) and preprocessor.log1p + ) + if use_binned_expressions: grouped = ( fragment_df.with_columns( @@ -93,7 +103,11 @@ def apply( .alias("gene_rank") ) .filter(pl.col("gene_rank") <= max_items) - .with_columns(pl.col(vk).log1p().alias("log_value")) + .with_columns( + (pl.col(vk) if already_log1p else pl.col(vk).log1p()).alias( + "log_value" + ) + ) .with_columns( pl.when(pl.col("log_value") > 0) .then( @@ -164,6 +178,9 @@ def apply( min_percentile = kwargs.get("min_percentile", None) + preprocessor = kwargs.get("expression_preprocessor") + fragment_df = apply_expression_preprocessor(fragment_df, schema, preprocessor) + if min_percentile is not None: grouped = ( fragment_df.with_columns( @@ -237,6 +254,9 @@ def apply( item_out = schema.item_list_key value_out = schema.value_list_key or "expr_sequence" + preprocessor = kwargs.get("expression_preprocessor") + fragment_df = apply_expression_preprocessor(fragment_df, schema, preprocessor) + result = fragment_df.with_columns( [ pl.col(vk) diff --git a/slaf/ml/dataloaders.py b/slaf/ml/dataloaders.py index ddfc89d..a1356c3 100644 --- a/slaf/ml/dataloaders.py +++ b/slaf/ml/dataloaders.py @@ -4,6 +4,7 @@ from slaf.core.slaf import SLAFArray +from .expression_preprocessor import ExpressionPreprocessor from .tokenizers import GeneformerTokenizer, ScGPTTokenizer, SLAFTokenizer # Try to import torch, but make it optional @@ -284,6 +285,7 @@ def __init__( max_queue_size: int = 5000, # Add max_queue_size parameter parallelize_fragment_reads: bool = False, # Parallelize fragment reads in MoS (cloud optimization) prefetcher_ready_timeout: float = 10.0, # Timeout for waiting for prefetcher to be ready + expression_preprocessor: ExpressionPreprocessor | None = None, ): """ Initialize the SLAF DataLoader with training configuration. @@ -339,6 +341,8 @@ def __init__( Higher values improve throughput but use more memory. Range: 1000-10000000, default: 4194304. Only used when use_mixture_of_scanners=True. + expression_preprocessor: Optional per-cell normalize_total and element-wise log1p on + COO values before rank/bin. Default None preserves the historical window behavior. parallelize_fragment_reads: Whether to parallelize fragment reads in MoS mode. Critical for cloud scenarios where network latency dominates (can improve throughput 10-30x). For local @@ -501,6 +505,7 @@ def __init__( prefetch_batch_size=prefetch_batch_size, # Pass prefetch_batch_size to dataset parallelize_fragment_reads=parallelize_fragment_reads, # Pass parallelize_fragment_reads prefetcher_ready_timeout=prefetcher_ready_timeout, # Pass prefetcher_ready_timeout + expression_preprocessor=expression_preprocessor, ) def __iter__(self): diff --git a/slaf/ml/datasets.py b/slaf/ml/datasets.py index 187faec..857e54b 100644 --- a/slaf/ml/datasets.py +++ b/slaf/ml/datasets.py @@ -55,6 +55,7 @@ from slaf.core.slaf import SLAFArray from slaf.core.tabular_schema import SLAF_LANCE_COO_SCHEMA +from slaf.ml.expression_preprocessor import ExpressionPreprocessor from slaf.ml.samplers import Shuffle from slaf.ml.tokenizers import SLAFTokenizer @@ -328,6 +329,7 @@ def __init__( n_scanners: int = 16, # Add n_scanners parameter for MoS prefetch_batch_size: int = 4194304, # Add prefetch_batch_size parameter for MoS parallelize_fragment_reads: bool = False, # Parallelize fragment reads in MoS (cloud optimization) + expression_preprocessor: ExpressionPreprocessor | None = None, ): """Initialize the PrefetchBatchProcessor.""" self.slaf_array = slaf_array @@ -347,6 +349,7 @@ def __init__( self.n_scanners = n_scanners self.prefetch_batch_size = prefetch_batch_size self.parallelize_fragment_reads = parallelize_fragment_reads + self.expression_preprocessor = expression_preprocessor # Validate MoS parameters if self.use_mixture_of_scanners: @@ -1023,13 +1026,17 @@ def load_prefetch_batch(self) -> PrefetchBatch: shuffle_time = time.time() - shuffle_start window_start = time.time() - window_params = { + window_params: dict[str, Any] = { "n_expression_bins": self.n_expression_bins, "use_binned_expressions": self.use_binned_expressions, } window_params.update( self.window_kwargs ) # Add any additional kwargs + if self.expression_preprocessor is not None: + window_params["expression_preprocessor"] = ( + self.expression_preprocessor + ) tokenizer = self.tokenizer if tokenizer is None: @@ -1539,6 +1546,7 @@ def __init__( prefetch_batch_size: int = 4194304, # Add prefetch_batch_size parameter for MoS parallelize_fragment_reads: bool = False, # Parallelize fragment reads in MoS (cloud optimization) prefetcher_ready_timeout: float = 10.0, # Timeout for waiting for prefetcher to be ready + expression_preprocessor: ExpressionPreprocessor | None = None, ): super().__init__() self.slaf_array = slaf_array @@ -1595,6 +1603,7 @@ def __init__( n_scanners=n_scanners, # Pass MoS parameters prefetch_batch_size=prefetch_batch_size, # Pass MoS parameters parallelize_fragment_reads=parallelize_fragment_reads, # Pass parallelize_fragment_reads + expression_preprocessor=expression_preprocessor, ) self.prefetcher = AsyncPrefetcher( batch_processor=self.batch_processor, diff --git a/slaf/ml/distributed.py b/slaf/ml/distributed.py index 52d3e46..accec75 100644 --- a/slaf/ml/distributed.py +++ b/slaf/ml/distributed.py @@ -20,6 +20,7 @@ from slaf.distributed.dataloader import DecompressingQueueWrapper, DistributedDataLoader # Import SLAF-specific components (for type hints and adapters) +from slaf.ml.expression_preprocessor import ExpressionPreprocessor from slaf.ml.tokenizers import GeneformerTokenizer, ScGPTTokenizer # Configure Modal image for SLAF workers. @@ -232,6 +233,7 @@ def __init__( seed: int = 42, queue_name: str | None = None, modal_queue_environment: str | None = None, + expression_preprocessor: ExpressionPreprocessor | None = None, **window_kwargs: Any, ): """ @@ -271,6 +273,8 @@ def __init__( queue_name: Name of Modal Queue (auto-generated if None) modal_queue_environment: Optional Modal Environment name for Queue/Dict (match consumer’s ``modal.Queue.from_name(..., environment_name=…)``, e.g. ``"main"`` in some apps). + expression_preprocessor: Optional COO value preprocessing before rank/bin (same as + ``SLAFDataLoader``). Merged into ``window_kwargs`` for workers. **window_kwargs: Additional window function parameters """ self.slaf_array = slaf_array @@ -303,6 +307,8 @@ def __init__( window_kwargs = dict(window_kwargs) window_kwargs.setdefault("n_expression_bins", n_expression_bins) window_kwargs.setdefault("use_binned_expressions", True) + if expression_preprocessor is not None: + window_kwargs["expression_preprocessor"] = expression_preprocessor tokenizer_factory_kwargs: dict[str, Any] | None = None tokenizer_cls: type[GeneformerTokenizer] | type[ScGPTTokenizer] | None = None diff --git a/slaf/ml/expression_preprocessor.py b/slaf/ml/expression_preprocessor.py new file mode 100644 index 0000000..54b3e61 --- /dev/null +++ b/slaf/ml/expression_preprocessor.py @@ -0,0 +1,52 @@ +"""Optional COO value transforms before window rank / bin (training pipeline).""" + +from __future__ import annotations + +from dataclasses import dataclass + +import polars as pl + +from slaf.core.tabular_schema import DataSchema + + +@dataclass(frozen=True) +class ExpressionPreprocessor: + """Per-cell and element-wise transforms on the COO ``value`` column.""" + + normalize_total_target: float | None = None + """If set, scale each cell so its values sum to this target (Scanpy-style).""" + + log1p: bool = False + """If True, apply ``log1p`` element-wise after optional normalization.""" + + +def apply_expression_preprocessor( + fragment_df: pl.DataFrame, + schema: DataSchema, + preprocessor: ExpressionPreprocessor | None, +) -> pl.DataFrame: + """Update ``schema.value_key`` when ``preprocessor`` applies ops; no-op if ``None``.""" + if preprocessor is None: + return fragment_df + if not isinstance(preprocessor, ExpressionPreprocessor): + raise TypeError( + "expression_preprocessor must be an ExpressionPreprocessor instance or None, " + f"got {type(preprocessor)!r}" + ) + if preprocessor.normalize_total_target is None and not preprocessor.log1p: + return fragment_df + + vk = schema.value_key + out = fragment_df + if preprocessor.normalize_total_target is not None: + t = float(preprocessor.normalize_total_target) + cell_sum = pl.col(vk).sum().over(schema.group_key) + out = out.with_columns( + pl.when(cell_sum > 0) + .then(pl.col(vk) * (t / cell_sum)) + .otherwise(0.0) + .alias(vk) + ) + if preprocessor.log1p: + out = out.with_columns(pl.col(vk).log1p().alias(vk)) + return out diff --git a/tests/test_aggregators.py b/tests/test_aggregators.py index ca17095..333072e 100644 --- a/tests/test_aggregators.py +++ b/tests/test_aggregators.py @@ -4,6 +4,7 @@ This module tests the window function implementations for different tokenization strategies. """ +import numpy as np import polars as pl import pytest @@ -13,6 +14,7 @@ ScGPTWindow, Window, ) +from slaf.ml.expression_preprocessor import ExpressionPreprocessor class TestWindow: @@ -185,6 +187,36 @@ def test_apply_single_cell(self): # Should contain the actual expression values assert 3.0 in expr_seq_raw or 5.0 in expr_seq_raw + def test_expression_preprocessor_normalize_log1p_raw(self): + """normalize_total + log1p before rank/raw aggregate matches numpy reference.""" + result = self.window.apply( + self.test_data, + self.schema, + 2, + use_binned_expressions=False, + expression_preprocessor=ExpressionPreprocessor( + normalize_total_target=10.0, + log1p=True, + ), + ) + cell_0 = result.filter(pl.col("cell_integer_id") == 0) + gene_seq = cell_0["gene_sequence"][0] + expr_seq = cell_0["expr_sequence"][0] + assert list(gene_seq) == [10, 20] + raw = np.array([5.0, 3.0, 1.0], dtype=np.float64) + scaled = raw * (10.0 / raw.sum()) + expected = np.log1p(scaled[:2]) + np.testing.assert_allclose(np.array(expr_seq), expected, rtol=1e-5) + + def test_expression_preprocessor_invalid_type(self): + with pytest.raises(TypeError, match="ExpressionPreprocessor"): + self.window.apply( + self.test_data, + self.schema, + 2, + expression_preprocessor="not_a_preprocess", # type: ignore[arg-type] + ) + class TestGeneformerWindow: """Test GeneformerWindow implementation""" diff --git a/uv.lock b/uv.lock index dda973a..e4a542d 100644 --- a/uv.lock +++ b/uv.lock @@ -2,7 +2,8 @@ version = 1 revision = 3 requires-python = ">=3.10, <3.14" resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.13'", + "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", "python_full_version < '3.11'", ] @@ -188,7 +189,8 @@ name = "anndata" version = "0.12.10" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.13'", + "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", ] dependencies = [ @@ -323,7 +325,8 @@ name = "bionemo-scdl" version = "0.1.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.13'", + "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", ] dependencies = [ @@ -680,7 +683,8 @@ name = "contourpy" version = "1.3.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.13'", + "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", ] dependencies = [ @@ -837,28 +841,40 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/47/93/ac8f3d5ff04d54bc814e961a43ae5b0b146154c89c61b47bb07557679b18/cryptography-46.0.7.tar.gz", hash = "sha256:e4cfd68c5f3e0bfdad0d38e023239b96a2fe84146481852dffbcca442c245aa5", size = 750652, upload-time = "2026-04-08T01:57:54.692Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/5d/4a8f770695d73be252331e60e526291e3df0c9b27556a90a6b47bccca4c2/cryptography-46.0.7-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:ea42cbe97209df307fdc3b155f1b6fa2577c0defa8f1f7d3be7d31d189108ad4", size = 7179869, upload-time = "2026-04-08T01:56:17.157Z" }, { url = "https://files.pythonhosted.org/packages/5f/45/6d80dc379b0bbc1f9d1e429f42e4cb9e1d319c7a8201beffd967c516ea01/cryptography-46.0.7-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b36a4695e29fe69215d75960b22577197aca3f7a25b9cf9d165dcfe9d80bc325", size = 4275492, upload-time = "2026-04-08T01:56:19.36Z" }, { url = "https://files.pythonhosted.org/packages/4a/9a/1765afe9f572e239c3469f2cb429f3ba7b31878c893b246b4b2994ffe2fe/cryptography-46.0.7-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5ad9ef796328c5e3c4ceed237a183f5d41d21150f972455a9d926593a1dcb308", size = 4426670, upload-time = "2026-04-08T01:56:21.415Z" }, { url = "https://files.pythonhosted.org/packages/8f/3e/af9246aaf23cd4ee060699adab1e47ced3f5f7e7a8ffdd339f817b446462/cryptography-46.0.7-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:73510b83623e080a2c35c62c15298096e2a5dc8d51c3b4e1740211839d0dea77", size = 4280275, upload-time = "2026-04-08T01:56:23.539Z" }, + { url = "https://files.pythonhosted.org/packages/0f/54/6bbbfc5efe86f9d71041827b793c24811a017c6ac0fd12883e4caa86b8ed/cryptography-46.0.7-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cbd5fb06b62bd0721e1170273d3f4d5a277044c47ca27ee257025146c34cbdd1", size = 4928402, upload-time = "2026-04-08T01:56:25.624Z" }, { url = "https://files.pythonhosted.org/packages/2d/cf/054b9d8220f81509939599c8bdbc0c408dbd2bdd41688616a20731371fe0/cryptography-46.0.7-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:420b1e4109cc95f0e5700eed79908cef9268265c773d3a66f7af1eef53d409ef", size = 4459985, upload-time = "2026-04-08T01:56:27.309Z" }, { url = "https://files.pythonhosted.org/packages/f9/46/4e4e9c6040fb01c7467d47217d2f882daddeb8828f7df800cb806d8a2288/cryptography-46.0.7-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:24402210aa54baae71d99441d15bb5a1919c195398a87b563df84468160a65de", size = 3990652, upload-time = "2026-04-08T01:56:29.095Z" }, { url = "https://files.pythonhosted.org/packages/36/5f/313586c3be5a2fbe87e4c9a254207b860155a8e1f3cca99f9910008e7d08/cryptography-46.0.7-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:8a469028a86f12eb7d2fe97162d0634026d92a21f3ae0ac87ed1c4a447886c83", size = 4279805, upload-time = "2026-04-08T01:56:30.928Z" }, + { url = "https://files.pythonhosted.org/packages/69/33/60dfc4595f334a2082749673386a4d05e4f0cf4df8248e63b2c3437585f2/cryptography-46.0.7-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9694078c5d44c157ef3162e3bf3946510b857df5a3955458381d1c7cfc143ddb", size = 4892883, upload-time = "2026-04-08T01:56:32.614Z" }, { url = "https://files.pythonhosted.org/packages/c7/0b/333ddab4270c4f5b972f980adef4faa66951a4aaf646ca067af597f15563/cryptography-46.0.7-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:42a1e5f98abb6391717978baf9f90dc28a743b7d9be7f0751a6f56a75d14065b", size = 4459756, upload-time = "2026-04-08T01:56:34.306Z" }, { url = "https://files.pythonhosted.org/packages/d2/14/633913398b43b75f1234834170947957c6b623d1701ffc7a9600da907e89/cryptography-46.0.7-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:91bbcb08347344f810cbe49065914fe048949648f6bd5c2519f34619142bbe85", size = 4410244, upload-time = "2026-04-08T01:56:35.977Z" }, { url = "https://files.pythonhosted.org/packages/10/f2/19ceb3b3dc14009373432af0c13f46aa08e3ce334ec6eff13492e1812ccd/cryptography-46.0.7-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5d1c02a14ceb9148cc7816249f64f623fbfee39e8c03b3650d842ad3f34d637e", size = 4674868, upload-time = "2026-04-08T01:56:38.034Z" }, + { url = "https://files.pythonhosted.org/packages/1a/bb/a5c213c19ee94b15dfccc48f363738633a493812687f5567addbcbba9f6f/cryptography-46.0.7-cp311-abi3-win32.whl", hash = "sha256:d23c8ca48e44ee015cd0a54aeccdf9f09004eba9fc96f38c911011d9ff1bd457", size = 3026504, upload-time = "2026-04-08T01:56:39.666Z" }, + { url = "https://files.pythonhosted.org/packages/2b/02/7788f9fefa1d060ca68717c3901ae7fffa21ee087a90b7f23c7a603c32ae/cryptography-46.0.7-cp311-abi3-win_amd64.whl", hash = "sha256:397655da831414d165029da9bc483bed2fe0e75dde6a1523ec2fe63f3c46046b", size = 3488363, upload-time = "2026-04-08T01:56:41.893Z" }, + { url = "https://files.pythonhosted.org/packages/a7/7f/cd42fc3614386bc0c12f0cb3c4ae1fc2bbca5c9662dfed031514911d513d/cryptography-46.0.7-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:462ad5cb1c148a22b2e3bcc5ad52504dff325d17daf5df8d88c17dda1f75f2a4", size = 7165618, upload-time = "2026-04-08T01:57:10.645Z" }, { url = "https://files.pythonhosted.org/packages/a5/d0/36a49f0262d2319139d2829f773f1b97ef8aef7f97e6e5bd21455e5a8fb5/cryptography-46.0.7-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:84d4cced91f0f159a7ddacad249cc077e63195c36aac40b4150e7a57e84fffe7", size = 4270628, upload-time = "2026-04-08T01:57:12.885Z" }, { url = "https://files.pythonhosted.org/packages/8a/6c/1a42450f464dda6ffbe578a911f773e54dd48c10f9895a23a7e88b3e7db5/cryptography-46.0.7-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:128c5edfe5e5938b86b03941e94fac9ee793a94452ad1365c9fc3f4f62216832", size = 4415405, upload-time = "2026-04-08T01:57:14.923Z" }, { url = "https://files.pythonhosted.org/packages/9a/92/4ed714dbe93a066dc1f4b4581a464d2d7dbec9046f7c8b7016f5286329e2/cryptography-46.0.7-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5e51be372b26ef4ba3de3c167cd3d1022934bc838ae9eaad7e644986d2a3d163", size = 4272715, upload-time = "2026-04-08T01:57:16.638Z" }, + { url = "https://files.pythonhosted.org/packages/b7/e6/a26b84096eddd51494bba19111f8fffe976f6a09f132706f8f1bf03f51f7/cryptography-46.0.7-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:cdf1a610ef82abb396451862739e3fc93b071c844399e15b90726ef7470eeaf2", size = 4918400, upload-time = "2026-04-08T01:57:19.021Z" }, { url = "https://files.pythonhosted.org/packages/c7/08/ffd537b605568a148543ac3c2b239708ae0bd635064bab41359252ef88ed/cryptography-46.0.7-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1d25aee46d0c6f1a501adcddb2d2fee4b979381346a78558ed13e50aa8a59067", size = 4450634, upload-time = "2026-04-08T01:57:21.185Z" }, { url = "https://files.pythonhosted.org/packages/16/01/0cd51dd86ab5b9befe0d031e276510491976c3a80e9f6e31810cce46c4ad/cryptography-46.0.7-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:cdfbe22376065ffcf8be74dc9a909f032df19bc58a699456a21712d6e5eabfd0", size = 3985233, upload-time = "2026-04-08T01:57:22.862Z" }, { url = "https://files.pythonhosted.org/packages/92/49/819d6ed3a7d9349c2939f81b500a738cb733ab62fbecdbc1e38e83d45e12/cryptography-46.0.7-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:abad9dac36cbf55de6eb49badd4016806b3165d396f64925bf2999bcb67837ba", size = 4271955, upload-time = "2026-04-08T01:57:24.814Z" }, + { url = "https://files.pythonhosted.org/packages/80/07/ad9b3c56ebb95ed2473d46df0847357e01583f4c52a85754d1a55e29e4d0/cryptography-46.0.7-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:935ce7e3cfdb53e3536119a542b839bb94ec1ad081013e9ab9b7cfd478b05006", size = 4879888, upload-time = "2026-04-08T01:57:26.88Z" }, { url = "https://files.pythonhosted.org/packages/b8/c7/201d3d58f30c4c2bdbe9b03844c291feb77c20511cc3586daf7edc12a47b/cryptography-46.0.7-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:35719dc79d4730d30f1c2b6474bd6acda36ae2dfae1e3c16f2051f215df33ce0", size = 4449961, upload-time = "2026-04-08T01:57:29.068Z" }, { url = "https://files.pythonhosted.org/packages/a5/ef/649750cbf96f3033c3c976e112265c33906f8e462291a33d77f90356548c/cryptography-46.0.7-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7bbc6ccf49d05ac8f7d7b5e2e2c33830d4fe2061def88210a126d130d7f71a85", size = 4401696, upload-time = "2026-04-08T01:57:31.029Z" }, { url = "https://files.pythonhosted.org/packages/41/52/a8908dcb1a389a459a29008c29966c1d552588d4ae6d43f3a1a4512e0ebe/cryptography-46.0.7-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a1529d614f44b863a7b480c6d000fe93b59acee9c82ffa027cfadc77521a9f5e", size = 4664256, upload-time = "2026-04-08T01:57:33.144Z" }, + { url = "https://files.pythonhosted.org/packages/4b/fa/f0ab06238e899cc3fb332623f337a7364f36f4bb3f2534c2bb95a35b132c/cryptography-46.0.7-cp38-abi3-win32.whl", hash = "sha256:f247c8c1a1fb45e12586afbb436ef21ff1e80670b2861a90353d9b025583d246", size = 3013001, upload-time = "2026-04-08T01:57:34.933Z" }, + { url = "https://files.pythonhosted.org/packages/d2/f1/00ce3bde3ca542d1acd8f8cfa38e446840945aa6363f9b74746394b14127/cryptography-46.0.7-cp38-abi3-win_amd64.whl", hash = "sha256:506c4ff91eff4f82bdac7633318a526b1d1309fc07ca76a3ad182cb5b686d6d3", size = 3472985, upload-time = "2026-04-08T01:57:36.714Z" }, + { url = "https://files.pythonhosted.org/packages/63/0c/dca8abb64e7ca4f6b2978769f6fea5ad06686a190cec381f0a796fdcaaba/cryptography-46.0.7-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fc9ab8856ae6cf7c9358430e49b368f3108f050031442eaeb6b9d87e4dcf4e4f", size = 3476879, upload-time = "2026-04-08T01:57:38.664Z" }, { url = "https://files.pythonhosted.org/packages/3a/ea/075aac6a84b7c271578d81a2f9968acb6e273002408729f2ddff517fed4a/cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d3b99c535a9de0adced13d159c5a9cf65c325601aa30f4be08afd680643e9c15", size = 4219700, upload-time = "2026-04-08T01:57:40.625Z" }, { url = "https://files.pythonhosted.org/packages/6c/7b/1c55db7242b5e5612b29fc7a630e91ee7a6e3c8e7bf5406d22e206875fbd/cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d02c738dacda7dc2a74d1b2b3177042009d5cab7c7079db74afc19e56ca1b455", size = 4385982, upload-time = "2026-04-08T01:57:42.725Z" }, { url = "https://files.pythonhosted.org/packages/cb/da/9870eec4b69c63ef5925bf7d8342b7e13bc2ee3d47791461c4e49ca212f4/cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:04959522f938493042d595a736e7dbdff6eb6cc2339c11465b3ff89343b65f65", size = 4219115, upload-time = "2026-04-08T01:57:44.939Z" }, { url = "https://files.pythonhosted.org/packages/f4/72/05aa5832b82dd341969e9a734d1812a6aadb088d9eb6f0430fc337cc5a8f/cryptography-46.0.7-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:3986ac1dee6def53797289999eabe84798ad7817f3e97779b5061a95b0ee4968", size = 4385479, upload-time = "2026-04-08T01:57:46.86Z" }, + { url = "https://files.pythonhosted.org/packages/20/2a/1b016902351a523aa2bd446b50a5bc1175d7a7d1cf90fe2ef904f9b84ebc/cryptography-46.0.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:258514877e15963bd43b558917bc9f54cf7cf866c38aa576ebf47a77ddbc43a4", size = 3412829, upload-time = "2026-04-08T01:57:48.874Z" }, ] [[package]] @@ -1314,6 +1330,65 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/09/e21df6aef1e1ffc0c816f0522ddc3f6dcded766c3261813131c78a704470/gitpython-3.1.46-py3-none-any.whl", hash = "sha256:79812ed143d9d25b6d176a10bb511de0f9c67b1fa641d82097b0ab90398a2058", size = 208620, upload-time = "2026-01-01T15:37:30.574Z" }, ] +[[package]] +name = "google-api-core" +version = "2.30.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "googleapis-common-protos" }, + { name = "proto-plus" }, + { name = "protobuf" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/16/ce/502a57fb0ec752026d24df1280b162294b22a0afb98a326084f9a979138b/google_api_core-2.30.3.tar.gz", hash = "sha256:e601a37f148585319b26db36e219df68c5d07b6382cff2d580e83404e44d641b", size = 177001, upload-time = "2026-04-10T00:41:28.035Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/15/e56f351cf6ef1cfea58e6ac226a7318ed1deb2218c4b3cc9bd9e4b786c5a/google_api_core-2.30.3-py3-none-any.whl", hash = "sha256:a85761ba72c444dad5d611c2220633480b2b6be2521eca69cca2dbb3ffd6bfe8", size = 173274, upload-time = "2026-04-09T22:57:16.198Z" }, +] + +[[package]] +name = "google-auth" +version = "2.49.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyasn1-modules" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/fc/e925290a1ad95c975c459e2df070fac2b90954e13a0370ac505dff78cb99/google_auth-2.49.2.tar.gz", hash = "sha256:c1ae38500e73065dcae57355adb6278cf8b5c8e391994ae9cbadbcb9631ab409", size = 333958, upload-time = "2026-04-10T00:41:21.888Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/76/d241a5c927433420507215df6cac1b1fa4ac0ba7a794df42a84326c68da8/google_auth-2.49.2-py3-none-any.whl", hash = "sha256:c2720924dfc82dedb962c9f52cabb2ab16714fd0a6a707e40561d217574ed6d5", size = 240638, upload-time = "2026-04-10T00:41:14.501Z" }, +] + +[[package]] +name = "google-cloud-core" +version = "2.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dc/24/6ca08b0a03c7b0c620427503ab00353a4ae806b848b93bcea18b6b76fde6/google_cloud_core-2.5.1.tar.gz", hash = "sha256:3dc94bdec9d05a31d9f355045ed0f369fbc0d8c665076c734f065d729800f811", size = 36078, upload-time = "2026-03-30T22:50:08.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/d9/5bb050cb32826466aa9b25f79e2ca2879fe66cb76782d4ed798dd7506151/google_cloud_core-2.5.1-py3-none-any.whl", hash = "sha256:ea62cdf502c20e3e14be8a32c05ed02113d7bef454e40ff3fab6fe1ec9f1f4e7", size = 29452, upload-time = "2026-03-30T22:48:31.567Z" }, +] + +[[package]] +name = "google-cloud-storage" +version = "3.10.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, + { name = "google-cloud-core" }, + { name = "google-crc32c" }, + { name = "google-resumable-media" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4c/47/205eb8e9a1739b5345843e5a425775cbdc472cc38e7eda082ba5b8d02450/google_cloud_storage-3.10.1.tar.gz", hash = "sha256:97db9aa4460727982040edd2bd13ff3d5e2260b5331ad22895802da1fc2a5286", size = 17309950, upload-time = "2026-03-23T09:35:23.409Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/ff/ca9ab2417fa913d75aae38bf40bf856bb2749a604b2e0f701b37cfcd23cc/google_cloud_storage-3.10.1-py3-none-any.whl", hash = "sha256:a72f656759b7b99bda700f901adcb3425a828d4a29f911bc26b3ea79c5b1217f", size = 324453, upload-time = "2026-03-23T09:35:21.368Z" }, +] + [[package]] name = "google-crc32c" version = "1.8.0" @@ -1344,6 +1419,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9c/97/7d75fe37a7a6ed171a2cf17117177e7aab7e6e0d115858741b41e9dd4254/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c", size = 28800, upload-time = "2025-12-16T00:40:30.322Z" }, ] +[[package]] +name = "google-resumable-media" +version = "2.8.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-crc32c" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3f/d1/b1ea14b93b6b78f57fc580125de44e9f593ab88dd2460f1a8a8d18f74754/google_resumable_media-2.8.2.tar.gz", hash = "sha256:f3354a182ebd193ae3f42e3ef95e6c9b10f128320de23ac7637236713b1acd70", size = 2164510, upload-time = "2026-03-30T23:34:25.369Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/f8/50bfaf4658431ff9de45c5c3935af7ab01157a4903c603cd0eee6e78e087/google_resumable_media-2.8.2-py3-none-any.whl", hash = "sha256:82b6d8ccd11765268cdd2a2123f417ec806b8eef3000a9a38dfe3033da5fb220", size = 81511, upload-time = "2026-03-30T23:34:09.671Z" }, +] + +[[package]] +name = "googleapis-common-protos" +version = "1.74.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/20/18/a746c8344152d368a5aac738d4c857012f2c5d1fd2eac7e17b647a7861bd/googleapis_common_protos-1.74.0.tar.gz", hash = "sha256:57971e4eeeba6aad1163c1f0fc88543f965bb49129b8bb55b2b7b26ecab084f1", size = 151254, upload-time = "2026-04-02T21:23:26.679Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/b0/be5d3329badb9230b765de6eea66b73abd5944bdeb5afb3562ddcd80ae84/googleapis_common_protos-1.74.0-py3-none-any.whl", hash = "sha256:702216f78610bb510e3f12ac3cafd281b7ac45cc5d86e90ad87e4d301a3426b5", size = 300743, upload-time = "2026-04-02T21:22:49.108Z" }, +] + [[package]] name = "griffelib" version = "2.0.2" @@ -3128,7 +3227,8 @@ name = "networkx" version = "3.6.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.13'", + "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } @@ -3295,7 +3395,8 @@ name = "numpy" version = "2.4.4" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.13'", + "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/d7/9f/b8cef5bffa569759033adda9481211426f12f53299629b410340795c2514/numpy-2.4.4.tar.gz", hash = "sha256:2d390634c5182175533585cc89f3608a4682ccb173cc9bb940b2881c8d6f8fa0", size = 20731587, upload-time = "2026-03-29T13:22:01.298Z" } @@ -3930,6 +4031,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bd/4d/fc923f5c85318ee8cc903566dc4e0ebe41b2dfc1d2ecf5546db232397ed6/properdocs-1.6.7-py3-none-any.whl", hash = "sha256:6fa0cfa2e01bf338f684892c8a506cf70ea88ae7f3479c933b6fa20168101cbd", size = 225406, upload-time = "2026-03-20T20:07:46.875Z" }, ] +[[package]] +name = "proto-plus" +version = "1.27.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/0d/94dfe80193e79d55258345901acd2917523d56e8381bc4dee7fd38e3868a/proto_plus-1.27.2.tar.gz", hash = "sha256:b2adde53adadf75737c44d3dcb0104fde65250dfc83ad59168b4aa3e574b6a24", size = 57204, upload-time = "2026-03-26T22:18:57.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/f3/1fba73eeffafc998a25d59703b63f8be4fe8a5cb12eaff7386a0ba0f7125/proto_plus-1.27.2-py3-none-any.whl", hash = "sha256:6432f75893d3b9e70b9c412f1d2f03f65b11fb164b793d14ae2ca01821d22718", size = 50450, upload-time = "2026-03-26T22:13:42.927Z" }, +] + [[package]] name = "protobuf" version = "6.33.6" @@ -4019,6 +4132,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2e/c3/94ade4906a2f88bc935772f59c934013b4205e773bcb4239db114a6da136/pyarrow_hotfix-0.7-py3-none-any.whl", hash = "sha256:3236f3b5f1260f0e2ac070a55c1a7b339c4bb7267839bd2015e283234e758100", size = 7923, upload-time = "2025-04-25T10:17:05.224Z" }, ] +[[package]] +name = "pyasn1" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, +] + [[package]] name = "pycparser" version = "3.0" @@ -4768,7 +4902,8 @@ name = "scanpy" version = "1.12.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.13'", + "python_full_version == '3.12.*'", ] dependencies = [ { name = "anndata", version = "0.12.10", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, @@ -4866,7 +5001,8 @@ name = "scikit-learn" version = "1.8.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.13'", + "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", ] dependencies = [ @@ -4994,7 +5130,8 @@ name = "scipy" version = "1.16.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.13'", + "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", ] dependencies = [ @@ -5099,7 +5236,8 @@ name = "scvi-tools" version = "1.4.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.13'", + "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", ] dependencies = [ @@ -5332,7 +5470,7 @@ dependencies = [ { name = "scanpy", version = "1.12.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "scipy", version = "1.16.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "smart-open", extra = ["s3"] }, + { name = "smart-open", extra = ["gcs", "s3"] }, { name = "tqdm" }, { name = "typer" }, ] @@ -5465,7 +5603,7 @@ requires-dist = [ { name = "scvi-tools", marker = "extra == 'dev'", specifier = ">=1.3.3" }, { name = "seaborn", marker = "extra == 'dev'", specifier = ">=0.13.0" }, { name = "slafdb", extras = ["ml", "advanced"], marker = "extra == 'full'" }, - { name = "smart-open", extras = ["s3"], specifier = ">=7.4.1" }, + { name = "smart-open", extras = ["s3", "gcs"], specifier = ">=7.4.1" }, { name = "tiledb", marker = "extra == 'ml'", specifier = ">=0.34.2" }, { name = "tiledbsoma", marker = "extra == 'ml'", specifier = ">=1.17.1" }, { name = "tiledbsoma", marker = "extra == 'test'", specifier = ">=1.17.1" }, @@ -5498,6 +5636,9 @@ wheels = [ ] [package.optional-dependencies] +gcs = [ + { name = "google-cloud-storage" }, +] s3 = [ { name = "boto3" }, ] @@ -5555,7 +5696,8 @@ name = "sparse" version = "0.18.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.13'", + "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", ] dependencies = [ @@ -6207,7 +6349,8 @@ name = "vortex-data" version = "0.68.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.13'", + "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", ] dependencies = [ @@ -6503,7 +6646,8 @@ name = "xarray" version = "2026.4.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.13'", + "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", ] dependencies = [