Cellposeindocker

.dockerignore

@@ -1,2 +1,38 @@
+# Version control
+.git
+.github
+
+# IDE / editor
+.idea
+.vscode
+*.swp
+*.swo
+
+# Python build artifacts
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+*.egg-info
+dist
+build
+.tox
+.nox
+
+# Tests and coverage
 scallops/tests
+.pytest_cache
+.coverage
+.coverage.*
+htmlcov
+
+# Documentation (not needed for install)
 docs
+requirements.doc.txt
+
+# macOS
+.DS_Store
+
+# _version.py is injected via --build-arg SCM_VERSION by the Makefile (make docker)
+# so it does not need to be in the build context
+scallops/_version.py

.github/workflows/docker.yml

@@ -18,6 +18,14 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v6
+        with:
+          fetch-depth: 0  # full history so setuptools_scm can resolve the version from tags
+
+      - name: Generate package version
+        id: scm
+        run: |
+          pip install --quiet setuptools_scm
+          echo "version=$(python -m setuptools_scm)" >> "$GITHUB_OUTPUT"
 
       - name: Log in to the GitHub Container Registry
         uses: docker/login-action@v3
@@ -51,5 +59,6 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          build-args: SCM_VERSION=${{ steps.scm.outputs.version }}
           cache-from: type=gha
           cache-to: type=gha,mode=max

Dockerfile

@@ -1,18 +1,115 @@
-FROM python:3.13-slim
+# syntax=docker/dockerfile:1
+# SCM_VERSION is passed by:
+#   - CI:      .github/workflows/docker.yml  (resolved via python -m setuptools_scm)
+#   - locally: docker.mk                     (make -f docker.mk docker / docker-gpu)
+# Falls back to 0.0.0+unknown when building directly with docker/podman build .
+#
+# GPU build (TF-GPU + RAPIDS):
+#   make -f docker.mk docker-gpu RAPIDS_VERSION=25.06
+# CPU build (default):
+#   make -f docker.mk docker
+ARG TF_VERSION=2.21.0
+ARG SCM_VERSION=0.0.0+unknown
+# RAPIDS: required when IS_GPU=1, ignored when IS_GPU=0.
+# See https://docs.rapids.ai/install for current release versions.
+ARG RAPIDS_VERSION=
+ARG RAPIDS_CUDA=cu12
 
-ENV AWS_RETRY_MODE=adaptive \
-    AWS_MAX_ATTEMPTS=10
-RUN apt-get update -qq && \
-    apt-get install -qq --no-install-recommends -y \
+FROM tensorflow/tensorflow:${TF_VERSION}
+ARG SCM_VERSION
+ARG RAPIDS_VERSION
+ARG RAPIDS_CUDA
+# IS_GPU is derived from TF_VERSION by docker.mk (1 when TF_VERSION contains
+# -gpu, 0 otherwise). Docker ENV cannot evaluate shell expressions, so ARG is
+# the necessary bridge: docker.mk computes it, ARG captures it, ENV persists it.
+# Default 0 (CPU) when building directly without docker.mk.
+ARG IS_GPU=0
+ENV IS_GPU=${IS_GPU}
+
+COPY --from=docker.io/astral/uv:latest /uv /uvx /bin/
+
+# CPU builds: remove NVIDIA and deadsnakes apt repos — they fail SSL
+# verification on restricted networks and are unused (CUDA comes from PyPI
+# wheels, not apt).
+# GPU builds: keep the NVIDIA repos (the GPU base image uses them) but still
+# remove deadsnakes which is never needed.
+# build-essential: needed for mahotas/centrosome (requirements.txt) and Cython.
+# git: needed to clone ufish from its pinned tag.
+RUN if [ "$IS_GPU" = "0" ]; then \
+      rm -f /etc/apt/sources.list.d/cuda.list \
+            /etc/apt/sources.list.d/nvidia-ml.list \
+            /etc/apt/trusted.gpg.d/cuda-keyring.gpg; \
+    fi && \
+    rm -f /etc/apt/sources.list.d/deadsnakes*.list && \
+    apt-get update -qq && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -qq --no-install-recommends -y \
       build-essential \
-      git \
-      ca-certificates
-WORKDIR /app
-COPY requirements.txt requirements.ufish.txt ./
-RUN pip install -q --no-cache-dir --upgrade pip && pip install -q --no-cache-dir -r requirements.txt
-RUN pip install -q --no-cache-dir -r requirements.ufish.txt
-COPY . ./
-RUN pip install .
-RUN apt-get remove -qq -y build-essential git && \
-    apt-get autoremove -qq -y && apt-get clean -qq && \
-    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /app/
+      git && \
+    rm -rf /var/lib/apt/lists/*
+
+# SETUPTOOLS_SCM_PRETEND_VERSION set as Docker ENV so it propagates into the
+# PEP 517 build subprocess that uv spawns (inline shell variables do not
+# reliably cross that boundary). Tested: ENV propagation works, inline does not.
+ENV UV_SYSTEM_PYTHON=1 \
+    UV_NO_CACHE=1 \
+    UV_HTTP_TIMEOUT=300 \
+    SETUPTOOLS_SCM_PRETEND_VERSION=${SCM_VERSION}
+
+WORKDIR /build
+
+# Print the Python version in use so it's visible in build logs
+RUN python3 --version
+
+# Each file is its own layer ordered most→least stable for cache efficiency.
+
+# ufish: git-pinned tag, rarely bumped
+COPY requirements.ufish.txt ./
+RUN uv pip install -r requirements.ufish.txt
+
+# cellpose 3.x: declares numpy<2.1 but is runtime-compatible with numpy 2.x;
+# separate step so uv resolves against already-installed numpy
+COPY requirements.cellpose.txt ./
+RUN uv pip install -r requirements.cellpose.txt
+
+# extra optional deps: only change when this Dockerfile changes
+# napari/pytest excluded — napari is a GUI tool (useless headless) and test
+# deps have no place in a production image
+RUN uv pip install pysam dask-ml miniwdl
+
+# core deps: TF already in base image so strip it to avoid re-downloading
+COPY requirements.txt ./
+RUN grep -v '^tensorflow' requirements.txt | uv pip install -r /dev/stdin
+
+# RAPIDS: GPU-accelerated drop-in replacements for pandas (cuDF), scikit-learn
+# (cuML), and dask (dask-cudf). Only installed when IS_GPU=1.
+# For CPU builds this step is a no-op.
+RUN if [ "$IS_GPU" = "1" ]; then \
+      if [ -z "${RAPIDS_VERSION}" ]; then \
+        echo "ERROR: RAPIDS_VERSION is required for GPU builds." \
+             "Pass --build-arg RAPIDS_VERSION=<version>" \
+             "(see https://docs.rapids.ai/install)" && exit 1; \
+      fi && \
+      pip install --no-cache-dir \
+        --extra-index-url https://pypi.nvidia.com \
+        cudf-${RAPIDS_CUDA}==${RAPIDS_VERSION} \
+        cuml-${RAPIDS_CUDA}==${RAPIDS_VERSION} \
+        dask-cudf-${RAPIDS_CUDA}==${RAPIDS_VERSION}; \
+    fi
+
+# --no-deps: all deps already installed above; avoids re-downloading tensorflow.
+# SETUPTOOLS_SCM_PRETEND_VERSION (set via ENV above) ensures setuptools_scm
+# writes the correct version into _version.py and the wheel metadata.
+# PYTHON_VERSION is inferred dynamically from the interpreter so it always
+# matches whatever Python the TF base image ships — no ARG to keep in sync.
+COPY . .
+RUN [ "${SCM_VERSION}" = "0.0.0+unknown" ] && \
+      echo "WARNING: SCM_VERSION not set — version will be 0.0.0+unknown. Use 'make -f docker.mk docker' to stamp the correct version." || true && \
+    uv pip install --no-deps . && \
+    PYVER=$(python3 -c "import sys; v=sys.version_info; print(f'{v.major}.{v.minor}')") && \
+    python3 -m compileall -q /usr/local/lib/python${PYVER} >/dev/null 2>&1 || true && \
+    rm -rf /build
+
+ENV AWS_RETRY_MODE=adaptive \
+    AWS_MAX_ATTEMPTS=10 \
+    TF_CPP_MIN_LOG_LEVEL=2 \
+    PYTHONUNBUFFERED=1

docker.mk

@@ -0,0 +1,113 @@
+# docker.mk — local mirror of the CI/CD Docker workflow (.github/workflows/docker.yml)
+#
+# The CI workflow (docker/build-push-action) does the following that this file replicates:
+#   1. Resolves the package version via setuptools_scm and passes it as SCM_VERSION build-arg
+#   2. Attaches OCI standard labels (revision, source, version, created, …)
+#   3. Tags the image with the semver version AND a short git-SHA tag
+#   4. Uses docker buildx (not plain docker build) for consistency with CI
+#
+# What CI does that this file intentionally does NOT replicate:
+#   - GitHub Actions layer cache (type=gha) — use a local registry cache if needed
+#   - Automatic push to ghcr.io — use `make -f docker.mk docker-push` explicitly
+#
+# Usage:
+#   make -f docker.mk docker                                    # CPU image (default)
+#   make -f docker.mk docker-gpu RAPIDS_VERSION=25.06           # GPU image + RAPIDS
+#   make -f docker.mk docker-push                               # build CPU + push to GHCR
+#   make -f docker.mk docker-push TF_VERSION=2.21.0-gpu \
+#                                 RAPIDS_VERSION=25.06          # build GPU + push
+
+# ── configurable knobs ────────────────────────────────────────────────────────
+REGISTRY      ?= ghcr.io/genentech
+IMAGE         ?= scallops
+TF_VERSION    ?= 2.21.0
+# RAPIDS_VERSION: required for GPU builds, ignored for CPU builds.
+# See https://docs.rapids.ai/install for current release versions.
+RAPIDS_VERSION ?=
+RAPIDS_CUDA   ?= cu12
+
+# ── derived flags ─────────────────────────────────────────────────────────────
+# IS_GPU is computed from TF_VERSION so the Dockerfile never needs it passed
+# explicitly by the user. Docker ENV cannot evaluate shell expressions, so
+# docker.mk is the place to derive it.
+IS_GPU := $(if $(findstring -gpu,$(TF_VERSION)),1,0)
+
+FULL_IMAGE    := $(REGISTRY)/$(IMAGE)
+
+# ── uv: prefer PATH, fall back to the default install location ────────────────
+UV := $(shell which uv 2>/dev/null || echo "$(HOME)/.local/bin/uv")
+
+# ── values computed at parse time (none depend on setuptools_scm) ─────────────
+GIT_SHA       := $(shell git rev-parse HEAD 2>/dev/null || echo "unknown")
+GIT_SHA_SHORT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown")
+GIT_SOURCE    := $(shell git remote get-url origin 2>/dev/null \
+                   | sed 's|git@github.com:|https://github.com/|; s|\.git$$||')
+BUILD_DATE    := $(shell date -u +%Y-%m-%dT%H:%M:%SZ)
+
+# ── internal helpers ──────────────────────────────────────────────────────────
+
+# Installs missing prerequisites at shell execution time (not Make parse time).
+# uv is used instead of pip to avoid PEP-668 "externally managed environment"
+# errors on Homebrew/system Python installs.
+define _check_prereqs
+	@"$(UV)" --version >/dev/null 2>&1 \
+	    || (echo "[docker.mk] uv not found at $(UV) — installing via official installer..." \
+	        && curl -LsSf https://astral.sh/uv/install.sh | sh \
+	        && "$(UV)" --version >/dev/null 2>&1) \
+	    || (echo "[docker.mk] ERROR: uv not found and installation failed." \
+	            "Install manually: https://docs.astral.sh/uv/getting-started/installation/" \
+	        && exit 1)
+	@docker buildx version >/dev/null 2>&1 \
+	    || (echo "[docker.mk] ERROR: docker buildx required (Docker 20.10+ or the buildx plugin)" \
+	        && exit 1)
+endef
+
+# SCM_VERSION is a shell variable ($$) so it is evaluated at shell execution
+# time, after _check_prereqs has ensured uv is present.
+# `uv run --with setuptools_scm` creates a throwaway venv — no system Python
+# pollution, no pip permission issues.
+# IS_GPU and RAPIDS args are always passed; the Dockerfile ignores RAPIDS when
+# IS_GPU=0 so there is no penalty for always forwarding them.
+define _build
+	@SCM_VERSION=$$("$(UV)" run --with setuptools_scm --no-project --quiet \
+	                  python3 -m setuptools_scm) \
+	 && DOCKER_TAG=$$(echo "$$SCM_VERSION" | tr '+' '-') \
+	 && echo "[docker.mk] Building $(FULL_IMAGE):$$DOCKER_TAG (IS_GPU=$(IS_GPU), package version $$SCM_VERSION)" \
+	 && docker buildx build \
+	      --build-arg SCM_VERSION=$$SCM_VERSION \
+	      --build-arg TF_VERSION=$(TF_VERSION) \
+	      --build-arg IS_GPU=$(IS_GPU) \
+	      --build-arg RAPIDS_VERSION=$(RAPIDS_VERSION) \
+	      --build-arg RAPIDS_CUDA=$(RAPIDS_CUDA) \
+	      --label "org.opencontainers.image.created=$(BUILD_DATE)" \
+	      --label "org.opencontainers.image.revision=$(GIT_SHA)" \
+	      --label "org.opencontainers.image.source=$(GIT_SOURCE)" \
+	      --label "org.opencontainers.image.version=$$SCM_VERSION" \
+	      --label "org.opencontainers.image.title=$(IMAGE)" \
+	      --label "org.opencontainers.image.url=$(GIT_SOURCE)" \
+	      -t "$(FULL_IMAGE):$$DOCKER_TAG" \
+	      -t "$(FULL_IMAGE):sha-$(GIT_SHA_SHORT)" \
+	      -t "$(FULL_IMAGE):latest"
+endef
+
+.PHONY: docker docker-gpu docker-push
+
+## Build the CPU image locally (default)
+docker:
+	$(call _check_prereqs)
+	$(call _build) .
+
+## Build the GPU image (TF-GPU base + RAPIDS). RAPIDS_VERSION is required.
+## Example: make -f docker.mk docker-gpu RAPIDS_VERSION=25.06
+docker-gpu:
+	@[ -n "$(RAPIDS_VERSION)" ] \
+	    || (echo "[docker.mk] ERROR: RAPIDS_VERSION is required for GPU builds." \
+	            "Example: make -f docker.mk docker-gpu RAPIDS_VERSION=25.06" \
+	            "(see https://docs.rapids.ai/install)" && exit 1)
+	$(call _check_prereqs)
+	$(call _build) .
+
+## Build and push to GHCR (requires `docker login ghcr.io` first)
+docker-push:
+	$(call _check_prereqs)
+	$(call _build) --push .

docs/install.rst

@@ -37,5 +37,87 @@ Developer Instructions
 
     pip install -r requirements.txt -e .
 
+Docker
+======
+
+Pre-built images are published to the GitHub Container Registry after every
+push to ``main`` and on every release tag::
+
+    docker pull ghcr.io/genentech/scallops:latest
+
+Run a command::
+
+    docker run --rm ghcr.io/genentech/scallops:latest scallops --help
+
+Mount local data into the container::
+
+    docker run --rm -v /path/to/data:/data ghcr.io/genentech/scallops:latest \
+        scallops <command> ...
+
+Building locally
+----------------
+
+A ``docker.mk`` file is provided that acts as the local equivalent of the CI
+workflow.  It runs ``setuptools_scm`` as a preflight step to stamp the correct
+version into the image, attaches OCI labels, and tags the image with the
+current version::
+
+    make -f docker.mk docker
+
+GPU support
+-----------
+
+The default image uses ``tensorflow/tensorflow:2.21.0`` (CPU variant) as its
+base.  The table below summarises which libraries use the GPU and which
+commands are affected:
+
+.. list-table::
+   :header-rows: 1
+   :widths: 20 20 35 25
+
+   * - Library
+     - GPU in default image
+     - Affected commands
+     - Notes
+   * - PyTorch
+     - **Yes** (auto-detected)
+     - ``segment`` (cellpose backend), ``dialout`` (U-FISH)
+     - PyPI wheels are CUDA-capable; GPU is used automatically when available
+   * - TensorFlow
+     - No (CPU only)
+     - ``segment`` (StarDist backend)
+     - Requires the GPU image (see below)
+   * - RAPIDS
+     - No (CPU only)
+     - All commands using pandas / scikit-learn / dask internally
+     - Requires the GPU image (see below)
+
+**GPU image** — enables TensorFlow GPU and installs `RAPIDS`_ (cuDF, cuML,
+dask-cudf), which accelerate the data-science operations used throughout the
+pipeline.  ``RAPIDS_VERSION`` must match a `current RAPIDS release`_::
+
+    make -f docker.mk docker-gpu RAPIDS_VERSION=25.06
+
+The build automatically sets the ``IS_GPU`` environment variable to ``1``
+inside the container.  You can inspect it at runtime to confirm GPU support
+was compiled in::
+
+    docker run --rm ghcr.io/genentech/scallops:latest printenv IS_GPU
+
+At runtime, pass ``--gpus all`` (Docker) or the equivalent Podman flag and
+ensure the `NVIDIA Container Toolkit`_ is installed on the host::
+
+    docker run --gpus all --rm <gpu-image> scallops segment ...
+
+.. note::
+
+   NVIDIA drivers must be installed on the host — the container image itself
+   does not include drivers.  PyTorch selects the GPU backend automatically;
+   TensorFlow and RAPIDS require the GPU image built with
+   ``TF_VERSION=2.21.0-gpu``.
+
 .. _Mamba: https://mamba.readthedocs.io/en/latest/installation.html
 .. _Conda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html
+.. _NVIDIA Container Toolkit: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html
+.. _RAPIDS: https://rapids.ai
+.. _current RAPIDS release: https://docs.rapids.ai/install