diff --git a/.dockerignore b/.dockerignore index 6ab18dc..5f64a64 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,38 @@ +# Version control +.git +.github + +# IDE / editor +.idea +.vscode +*.swp +*.swo + +# Python build artifacts +__pycache__ +*.pyc +*.pyo +*.pyd +*.egg-info +dist +build +.tox +.nox + +# Tests and coverage scallops/tests +.pytest_cache +.coverage +.coverage.* +htmlcov + +# Documentation (not needed for install) docs +requirements.doc.txt + +# macOS +.DS_Store + +# _version.py is injected via --build-arg SCM_VERSION by the Makefile (make docker) +# so it does not need to be in the build context +scallops/_version.py diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 386e20b..6304aed 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -18,6 +18,14 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v6 + with: + fetch-depth: 0 # full history so setuptools_scm can resolve the version from tags + + - name: Generate package version + id: scm + run: | + pip install --quiet setuptools_scm + echo "version=$(python -m setuptools_scm)" >> "$GITHUB_OUTPUT" - name: Log in to the GitHub Container Registry uses: docker/login-action@v3 @@ -51,5 +59,6 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + build-args: SCM_VERSION=${{ steps.scm.outputs.version }} cache-from: type=gha cache-to: type=gha,mode=max diff --git a/Dockerfile b/Dockerfile index 6d8bbd5..4b455eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,115 @@ -FROM python:3.13-slim +# syntax=docker/dockerfile:1 +# SCM_VERSION is passed by: +# - CI: .github/workflows/docker.yml (resolved via python -m setuptools_scm) +# - locally: docker.mk (make -f docker.mk docker / docker-gpu) +# Falls back to 0.0.0+unknown when building directly with docker/podman build . +# +# GPU build (TF-GPU + RAPIDS): +# make -f docker.mk docker-gpu RAPIDS_VERSION=25.06 +# CPU build (default): +# make -f docker.mk docker +ARG TF_VERSION=2.21.0 +ARG SCM_VERSION=0.0.0+unknown +# RAPIDS: required when IS_GPU=1, ignored when IS_GPU=0. +# See https://docs.rapids.ai/install for current release versions. +ARG RAPIDS_VERSION= +ARG RAPIDS_CUDA=cu12 -ENV AWS_RETRY_MODE=adaptive \ - AWS_MAX_ATTEMPTS=10 -RUN apt-get update -qq && \ - apt-get install -qq --no-install-recommends -y \ +FROM tensorflow/tensorflow:${TF_VERSION} +ARG SCM_VERSION +ARG RAPIDS_VERSION +ARG RAPIDS_CUDA +# IS_GPU is derived from TF_VERSION by docker.mk (1 when TF_VERSION contains +# -gpu, 0 otherwise). Docker ENV cannot evaluate shell expressions, so ARG is +# the necessary bridge: docker.mk computes it, ARG captures it, ENV persists it. +# Default 0 (CPU) when building directly without docker.mk. +ARG IS_GPU=0 +ENV IS_GPU=${IS_GPU} + +COPY --from=docker.io/astral/uv:latest /uv /uvx /bin/ + +# CPU builds: remove NVIDIA and deadsnakes apt repos — they fail SSL +# verification on restricted networks and are unused (CUDA comes from PyPI +# wheels, not apt). +# GPU builds: keep the NVIDIA repos (the GPU base image uses them) but still +# remove deadsnakes which is never needed. +# build-essential: needed for mahotas/centrosome (requirements.txt) and Cython. +# git: needed to clone ufish from its pinned tag. +RUN if [ "$IS_GPU" = "0" ]; then \ + rm -f /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list \ + /etc/apt/trusted.gpg.d/cuda-keyring.gpg; \ + fi && \ + rm -f /etc/apt/sources.list.d/deadsnakes*.list && \ + apt-get update -qq && \ + DEBIAN_FRONTEND=noninteractive apt-get install -qq --no-install-recommends -y \ build-essential \ - git \ - ca-certificates -WORKDIR /app -COPY requirements.txt requirements.ufish.txt ./ -RUN pip install -q --no-cache-dir --upgrade pip && pip install -q --no-cache-dir -r requirements.txt -RUN pip install -q --no-cache-dir -r requirements.ufish.txt -COPY . ./ -RUN pip install . -RUN apt-get remove -qq -y build-essential git && \ - apt-get autoremove -qq -y && apt-get clean -qq && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /app/ + git && \ + rm -rf /var/lib/apt/lists/* + +# SETUPTOOLS_SCM_PRETEND_VERSION set as Docker ENV so it propagates into the +# PEP 517 build subprocess that uv spawns (inline shell variables do not +# reliably cross that boundary). Tested: ENV propagation works, inline does not. +ENV UV_SYSTEM_PYTHON=1 \ + UV_NO_CACHE=1 \ + UV_HTTP_TIMEOUT=300 \ + SETUPTOOLS_SCM_PRETEND_VERSION=${SCM_VERSION} + +WORKDIR /build + +# Print the Python version in use so it's visible in build logs +RUN python3 --version + +# Each file is its own layer ordered most→least stable for cache efficiency. + +# ufish: git-pinned tag, rarely bumped +COPY requirements.ufish.txt ./ +RUN uv pip install -r requirements.ufish.txt + +# cellpose 3.x: declares numpy<2.1 but is runtime-compatible with numpy 2.x; +# separate step so uv resolves against already-installed numpy +COPY requirements.cellpose.txt ./ +RUN uv pip install -r requirements.cellpose.txt + +# extra optional deps: only change when this Dockerfile changes +# napari/pytest excluded — napari is a GUI tool (useless headless) and test +# deps have no place in a production image +RUN uv pip install pysam dask-ml miniwdl + +# core deps: TF already in base image so strip it to avoid re-downloading +COPY requirements.txt ./ +RUN grep -v '^tensorflow' requirements.txt | uv pip install -r /dev/stdin + +# RAPIDS: GPU-accelerated drop-in replacements for pandas (cuDF), scikit-learn +# (cuML), and dask (dask-cudf). Only installed when IS_GPU=1. +# For CPU builds this step is a no-op. +RUN if [ "$IS_GPU" = "1" ]; then \ + if [ -z "${RAPIDS_VERSION}" ]; then \ + echo "ERROR: RAPIDS_VERSION is required for GPU builds." \ + "Pass --build-arg RAPIDS_VERSION=" \ + "(see https://docs.rapids.ai/install)" && exit 1; \ + fi && \ + pip install --no-cache-dir \ + --extra-index-url https://pypi.nvidia.com \ + cudf-${RAPIDS_CUDA}==${RAPIDS_VERSION} \ + cuml-${RAPIDS_CUDA}==${RAPIDS_VERSION} \ + dask-cudf-${RAPIDS_CUDA}==${RAPIDS_VERSION}; \ + fi + +# --no-deps: all deps already installed above; avoids re-downloading tensorflow. +# SETUPTOOLS_SCM_PRETEND_VERSION (set via ENV above) ensures setuptools_scm +# writes the correct version into _version.py and the wheel metadata. +# PYTHON_VERSION is inferred dynamically from the interpreter so it always +# matches whatever Python the TF base image ships — no ARG to keep in sync. +COPY . . +RUN [ "${SCM_VERSION}" = "0.0.0+unknown" ] && \ + echo "WARNING: SCM_VERSION not set — version will be 0.0.0+unknown. Use 'make -f docker.mk docker' to stamp the correct version." || true && \ + uv pip install --no-deps . && \ + PYVER=$(python3 -c "import sys; v=sys.version_info; print(f'{v.major}.{v.minor}')") && \ + python3 -m compileall -q /usr/local/lib/python${PYVER} >/dev/null 2>&1 || true && \ + rm -rf /build + +ENV AWS_RETRY_MODE=adaptive \ + AWS_MAX_ATTEMPTS=10 \ + TF_CPP_MIN_LOG_LEVEL=2 \ + PYTHONUNBUFFERED=1 diff --git a/docker.mk b/docker.mk new file mode 100644 index 0000000..32955f6 --- /dev/null +++ b/docker.mk @@ -0,0 +1,113 @@ +# docker.mk — local mirror of the CI/CD Docker workflow (.github/workflows/docker.yml) +# +# The CI workflow (docker/build-push-action) does the following that this file replicates: +# 1. Resolves the package version via setuptools_scm and passes it as SCM_VERSION build-arg +# 2. Attaches OCI standard labels (revision, source, version, created, …) +# 3. Tags the image with the semver version AND a short git-SHA tag +# 4. Uses docker buildx (not plain docker build) for consistency with CI +# +# What CI does that this file intentionally does NOT replicate: +# - GitHub Actions layer cache (type=gha) — use a local registry cache if needed +# - Automatic push to ghcr.io — use `make -f docker.mk docker-push` explicitly +# +# Usage: +# make -f docker.mk docker # CPU image (default) +# make -f docker.mk docker-gpu RAPIDS_VERSION=25.06 # GPU image + RAPIDS +# make -f docker.mk docker-push # build CPU + push to GHCR +# make -f docker.mk docker-push TF_VERSION=2.21.0-gpu \ +# RAPIDS_VERSION=25.06 # build GPU + push + +# ── configurable knobs ──────────────────────────────────────────────────────── +REGISTRY ?= ghcr.io/genentech +IMAGE ?= scallops +TF_VERSION ?= 2.21.0 +# RAPIDS_VERSION: required for GPU builds, ignored for CPU builds. +# See https://docs.rapids.ai/install for current release versions. +RAPIDS_VERSION ?= +RAPIDS_CUDA ?= cu12 + +# ── derived flags ───────────────────────────────────────────────────────────── +# IS_GPU is computed from TF_VERSION so the Dockerfile never needs it passed +# explicitly by the user. Docker ENV cannot evaluate shell expressions, so +# docker.mk is the place to derive it. +IS_GPU := $(if $(findstring -gpu,$(TF_VERSION)),1,0) + +FULL_IMAGE := $(REGISTRY)/$(IMAGE) + +# ── uv: prefer PATH, fall back to the default install location ──────────────── +UV := $(shell which uv 2>/dev/null || echo "$(HOME)/.local/bin/uv") + +# ── values computed at parse time (none depend on setuptools_scm) ───────────── +GIT_SHA := $(shell git rev-parse HEAD 2>/dev/null || echo "unknown") +GIT_SHA_SHORT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown") +GIT_SOURCE := $(shell git remote get-url origin 2>/dev/null \ + | sed 's|git@github.com:|https://github.com/|; s|\.git$$||') +BUILD_DATE := $(shell date -u +%Y-%m-%dT%H:%M:%SZ) + +# ── internal helpers ────────────────────────────────────────────────────────── + +# Installs missing prerequisites at shell execution time (not Make parse time). +# uv is used instead of pip to avoid PEP-668 "externally managed environment" +# errors on Homebrew/system Python installs. +define _check_prereqs + @"$(UV)" --version >/dev/null 2>&1 \ + || (echo "[docker.mk] uv not found at $(UV) — installing via official installer..." \ + && curl -LsSf https://astral.sh/uv/install.sh | sh \ + && "$(UV)" --version >/dev/null 2>&1) \ + || (echo "[docker.mk] ERROR: uv not found and installation failed." \ + "Install manually: https://docs.astral.sh/uv/getting-started/installation/" \ + && exit 1) + @docker buildx version >/dev/null 2>&1 \ + || (echo "[docker.mk] ERROR: docker buildx required (Docker 20.10+ or the buildx plugin)" \ + && exit 1) +endef + +# SCM_VERSION is a shell variable ($$) so it is evaluated at shell execution +# time, after _check_prereqs has ensured uv is present. +# `uv run --with setuptools_scm` creates a throwaway venv — no system Python +# pollution, no pip permission issues. +# IS_GPU and RAPIDS args are always passed; the Dockerfile ignores RAPIDS when +# IS_GPU=0 so there is no penalty for always forwarding them. +define _build + @SCM_VERSION=$$("$(UV)" run --with setuptools_scm --no-project --quiet \ + python3 -m setuptools_scm) \ + && DOCKER_TAG=$$(echo "$$SCM_VERSION" | tr '+' '-') \ + && echo "[docker.mk] Building $(FULL_IMAGE):$$DOCKER_TAG (IS_GPU=$(IS_GPU), package version $$SCM_VERSION)" \ + && docker buildx build \ + --build-arg SCM_VERSION=$$SCM_VERSION \ + --build-arg TF_VERSION=$(TF_VERSION) \ + --build-arg IS_GPU=$(IS_GPU) \ + --build-arg RAPIDS_VERSION=$(RAPIDS_VERSION) \ + --build-arg RAPIDS_CUDA=$(RAPIDS_CUDA) \ + --label "org.opencontainers.image.created=$(BUILD_DATE)" \ + --label "org.opencontainers.image.revision=$(GIT_SHA)" \ + --label "org.opencontainers.image.source=$(GIT_SOURCE)" \ + --label "org.opencontainers.image.version=$$SCM_VERSION" \ + --label "org.opencontainers.image.title=$(IMAGE)" \ + --label "org.opencontainers.image.url=$(GIT_SOURCE)" \ + -t "$(FULL_IMAGE):$$DOCKER_TAG" \ + -t "$(FULL_IMAGE):sha-$(GIT_SHA_SHORT)" \ + -t "$(FULL_IMAGE):latest" +endef + +.PHONY: docker docker-gpu docker-push + +## Build the CPU image locally (default) +docker: + $(call _check_prereqs) + $(call _build) . + +## Build the GPU image (TF-GPU base + RAPIDS). RAPIDS_VERSION is required. +## Example: make -f docker.mk docker-gpu RAPIDS_VERSION=25.06 +docker-gpu: + @[ -n "$(RAPIDS_VERSION)" ] \ + || (echo "[docker.mk] ERROR: RAPIDS_VERSION is required for GPU builds." \ + "Example: make -f docker.mk docker-gpu RAPIDS_VERSION=25.06" \ + "(see https://docs.rapids.ai/install)" && exit 1) + $(call _check_prereqs) + $(call _build) . + +## Build and push to GHCR (requires `docker login ghcr.io` first) +docker-push: + $(call _check_prereqs) + $(call _build) --push . diff --git a/docs/install.rst b/docs/install.rst index 6220c25..36d6e34 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -37,5 +37,87 @@ Developer Instructions pip install -r requirements.txt -e . +Docker +====== + +Pre-built images are published to the GitHub Container Registry after every +push to ``main`` and on every release tag:: + + docker pull ghcr.io/genentech/scallops:latest + +Run a command:: + + docker run --rm ghcr.io/genentech/scallops:latest scallops --help + +Mount local data into the container:: + + docker run --rm -v /path/to/data:/data ghcr.io/genentech/scallops:latest \ + scallops ... + +Building locally +---------------- + +A ``docker.mk`` file is provided that acts as the local equivalent of the CI +workflow. It runs ``setuptools_scm`` as a preflight step to stamp the correct +version into the image, attaches OCI labels, and tags the image with the +current version:: + + make -f docker.mk docker + +GPU support +----------- + +The default image uses ``tensorflow/tensorflow:2.21.0`` (CPU variant) as its +base. The table below summarises which libraries use the GPU and which +commands are affected: + +.. list-table:: + :header-rows: 1 + :widths: 20 20 35 25 + + * - Library + - GPU in default image + - Affected commands + - Notes + * - PyTorch + - **Yes** (auto-detected) + - ``segment`` (cellpose backend), ``dialout`` (U-FISH) + - PyPI wheels are CUDA-capable; GPU is used automatically when available + * - TensorFlow + - No (CPU only) + - ``segment`` (StarDist backend) + - Requires the GPU image (see below) + * - RAPIDS + - No (CPU only) + - All commands using pandas / scikit-learn / dask internally + - Requires the GPU image (see below) + +**GPU image** — enables TensorFlow GPU and installs `RAPIDS`_ (cuDF, cuML, +dask-cudf), which accelerate the data-science operations used throughout the +pipeline. ``RAPIDS_VERSION`` must match a `current RAPIDS release`_:: + + make -f docker.mk docker-gpu RAPIDS_VERSION=25.06 + +The build automatically sets the ``IS_GPU`` environment variable to ``1`` +inside the container. You can inspect it at runtime to confirm GPU support +was compiled in:: + + docker run --rm ghcr.io/genentech/scallops:latest printenv IS_GPU + +At runtime, pass ``--gpus all`` (Docker) or the equivalent Podman flag and +ensure the `NVIDIA Container Toolkit`_ is installed on the host:: + + docker run --gpus all --rm scallops segment ... + +.. note:: + + NVIDIA drivers must be installed on the host — the container image itself + does not include drivers. PyTorch selects the GPU backend automatically; + TensorFlow and RAPIDS require the GPU image built with + ``TF_VERSION=2.21.0-gpu``. + .. _Mamba: https://mamba.readthedocs.io/en/latest/installation.html .. _Conda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html +.. _NVIDIA Container Toolkit: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html +.. _RAPIDS: https://rapids.ai +.. _current RAPIDS release: https://docs.rapids.ai/install diff --git a/scallops/__main__.py b/scallops/__main__.py index 83f924e..6147be4 100644 --- a/scallops/__main__.py +++ b/scallops/__main__.py @@ -56,6 +56,9 @@ def create_parsers(default_help: bool = False) -> argparse.ArgumentParser: root_parser = ScallopsArgumentParser( prog="scallops", description=f"Version {version('scallops')}" ) + root_parser.add_argument( + "--version", action="version", version=f"%(prog)s {version('scallops')}" + ) subparsers = root_parser.add_subparsers(help="Command help") pooled_if_sbs_main._create_parser(subparsers, default_help) features_main._create_parser(subparsers, default_help)