From 47f7fe1bef1efcb5acdb4ccf55324c8d00311f13 Mon Sep 17 00:00:00 2001 From: Ryan S <267728323+ironcommit@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:14:31 -0700 Subject: [PATCH] chore: update source distributions in containers Signed-off-by: Ryan S <267728323+ironcommit@users.noreply.github.com> --- .github/workflows/release.yaml | 4 + NOTICE | 2 + docker-bake.hcl | 49 ++- docker/Dockerfile.auditor-tasks | 29 ++ docker/Dockerfile.nmp-api | 20 ++ docker/Dockerfile.nmp-core | 19 ++ docker/Dockerfile.nmp-cpu-tasks | 19 ++ docker/Dockerfile.nmp-customizer-tasks | 23 +- docker/Dockerfile.nmp-unsloth-training | 27 +- docker/Dockerfile.safe-synthesizer-tasks | 21 ++ .../automodel/Dockerfile.nmp-automodel-base | 36 ++- .../Dockerfile.nmp-automodel-training | 11 + docker/base/Dockerfile.nmp-python-base | 10 +- docker/rl/Dockerfile.nmp-rl-base | 31 +- docker/rl/Dockerfile.nmp-rl-training | 13 + docker/scripts/collect-apt-sources.sh | 224 ++++++++++++++ docker/scripts/collect-cpython-source.py | 168 +++++++++++ docker/scripts/collect-python-sdists.py | 281 ++++++++++++++++++ docker/scripts/collect-workspace-sdists.py | 120 ++++++++ tests/unit/test_collect_apt_sources.py | 91 ++++++ tests/unit/test_collect_cpython_source.py | 92 ++++++ tests/unit/test_collect_workspace_sdists.py | 111 +++++++ third_party/licenses.jsonl | 14 +- .../license/overrides.yaml | 11 +- .../tests/license/test_license_utils.py | 31 ++ 25 files changed, 1423 insertions(+), 34 deletions(-) create mode 100644 docker/scripts/collect-apt-sources.sh create mode 100644 docker/scripts/collect-cpython-source.py create mode 100644 docker/scripts/collect-python-sdists.py create mode 100644 docker/scripts/collect-workspace-sdists.py create mode 100644 tests/unit/test_collect_apt_sources.py create mode 100644 tests/unit/test_collect_cpython_source.py create mode 100644 tests/unit/test_collect_workspace_sdists.py diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2a2eb0e8b4..63aa520e97 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -721,6 +721,10 @@ jobs: cadence: process.env.RELEASE_TYPE, version: process.env.RELEASE_LABEL, containers: JSON.parse(process.env.CONTAINER_IDS), + collect_sources: true, + bake_env: { + NMP_COLLECT_SOURCES: "1", + }, }, }); diff --git a/NOTICE b/NOTICE index 7a00ce2ae6..7da0027251 100644 --- a/NOTICE +++ b/NOTICE @@ -3,6 +3,8 @@ Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. Licensed under the Apache License, Version 2.0. +NOTICE AND DISCLAIMER: This software automatically retrieves, accesses or interacts with external materials. Those retrieved materials are not distributed with this software and are governed solely by separate terms, conditions and licenses. You are solely responsible for finding, reviewing and complying with all applicable terms, conditions, and licenses, and for verifying the security, integrity and suitability of any retrieved materials for your specific use case. This software is provided "AS IS", without warranty of any kind. The author makes no representations or warranties regarding any retrieved materials, and assumes no liability for any losses, damages, liabilities or legal consequences from your use or inability to use this software or any retrieved materials. Use this software and the retrieved materials at your own risk. + ================================================================================ Third-Party / Vendored and Distributed Components ================================================================================ diff --git a/docker-bake.hcl b/docker-bake.hcl index 1ea7a11246..ff8f0175c8 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -28,6 +28,10 @@ variable "USE_PREBUILT_BASES" { default = "" } +variable "NMP_COLLECT_SOURCES" { + default = "0" +} + variable "NMP_PYTHON_IMAGE" { default = "python:3.13.14-slim-trixie" } @@ -387,6 +391,9 @@ target "nmp-customizer-tasks" { causal-conv1d-wheel-src = causal_conv1d_wheel_context() mamba-ssm-wheel-src = mamba_ssm_wheel_context() } + args = { + NMP_COLLECT_SOURCES = NMP_COLLECT_SOURCES + } cache-to = maybe_registry_cache_to("nmp-customizer-tasks") cache-from = maybe_registry_cache_from("nmp-customizer-tasks") tags = sha_and_maybe_latest_tags("nmp-customizer-tasks") @@ -404,7 +411,8 @@ target "nmp-customizer-tasks-smoke-test" { mamba-ssm-wheel-src = mamba_ssm_wheel_context() } args = { - SMOKE_MARKER = "smoke_nmp_customizer_tasks" + NMP_COLLECT_SOURCES = NMP_COLLECT_SOURCES + SMOKE_MARKER = "smoke_nmp_customizer_tasks" } cache-from = maybe_registry_cache_from("nmp-customizer-tasks") output = ["type=cacheonly"] @@ -428,8 +436,9 @@ target "nmp-rl-base-builder" { context = "." dockerfile = "docker/rl/Dockerfile.nmp-rl-base" args = { - NEMO_RL_REPO = NEMO_RL_REPO - NEMO_RL_REF = NEMO_RL_REF + NEMO_RL_REPO = NEMO_RL_REPO + NEMO_RL_REF = NEMO_RL_REF + NMP_COLLECT_SOURCES = NMP_COLLECT_SOURCES } cache-to = maybe_registry_cache_to("nmp-rl-base") cache-from = maybe_registry_cache_from("nmp-rl-base") @@ -448,6 +457,9 @@ target "nmp-rl-training" { platform-workspace = "target:rl-platform-workspace" nmp-rl-base = rl_base_context() } + args = { + NMP_COLLECT_SOURCES = NMP_COLLECT_SOURCES + } cache-to = maybe_registry_cache_to("nmp-rl-training") cache-from = maybe_registry_cache_from("nmp-rl-training") tags = sha_and_maybe_latest_tags("nmp-rl-training") @@ -465,7 +477,8 @@ target "nmp-rl-training-smoke-test" { nmp-rl-base = rl_base_context() } args = { - SMOKE_MARKER = "smoke_nmp_rl_training" + NMP_COLLECT_SOURCES = NMP_COLLECT_SOURCES + SMOKE_MARKER = "smoke_nmp_rl_training" } cache-from = maybe_registry_cache_from("nmp-rl-training") output = ["type=cacheonly"] @@ -615,8 +628,9 @@ target "nmp-api-docker" { } args = { NMP_PLATFORM_VERSION = notequal(BAKE_TAG, "") ? BAKE_TAG : "dev" - NMP_CODE_REVISION = notequal(CI_COMMIT_SHA, "") ? CI_COMMIT_SHA : "dev" + NMP_CODE_REVISION = notequal(CI_COMMIT_SHA, "") ? CI_COMMIT_SHA : "dev" NMP_API_RUNTIME_BASE = NMP_API_RUNTIME_BASE + NMP_COLLECT_SOURCES = NMP_COLLECT_SOURCES } cache-to = maybe_registry_cache_to("nmp-api") cache-from = maybe_registry_cache_from("nmp-api") @@ -640,6 +654,7 @@ target "nmp-core-docker" { } args = { NMP_CORE_RUNTIME_BASE = NMP_CORE_RUNTIME_BASE + NMP_COLLECT_SOURCES = NMP_COLLECT_SOURCES } cache-to = maybe_registry_cache_to("nmp-core") cache-from = maybe_registry_cache_from("nmp-core") @@ -657,6 +672,9 @@ target "nmp-cpu-tasks-docker" { nmp-python-base = "target:nmp-python-base" nmp-workspace = "target:nmp-workspace" } + args = { + NMP_COLLECT_SOURCES = NMP_COLLECT_SOURCES + } cache-to = maybe_registry_cache_to("nmp-cpu-tasks") cache-from = maybe_registry_cache_from("nmp-cpu-tasks") tags = sha_and_maybe_latest_tags("nmp-cpu-tasks") @@ -715,7 +733,8 @@ target "safe-synthesizer-tasks-docker" { context = "." dockerfile = "docker/Dockerfile.safe-synthesizer-tasks" args = { - CONTAINER_VARIANT = "${SAFE_SYNTHESIZER_CONTAINER_VARIANT}" + CONTAINER_VARIANT = "${SAFE_SYNTHESIZER_CONTAINER_VARIANT}" + NMP_COLLECT_SOURCES = NMP_COLLECT_SOURCES } cache-to = maybe_registry_cache_to("safe-synthesizer-tasks") cache-from = maybe_registry_cache_from("safe-synthesizer-tasks") @@ -732,7 +751,8 @@ target "safe-synthesizer-tasks-smoke-test" { context = "." dockerfile = "docker/Dockerfile.safe-synthesizer-tasks" args = { - CONTAINER_VARIANT = "${SAFE_SYNTHESIZER_CONTAINER_VARIANT}" + CONTAINER_VARIANT = "${SAFE_SYNTHESIZER_CONTAINER_VARIANT}" + NMP_COLLECT_SOURCES = NMP_COLLECT_SOURCES } cache-from = maybe_registry_cache_from("safe-synthesizer-tasks") output = ["type=cacheonly"] @@ -891,6 +911,9 @@ target "nmp-automodel-base-builder" { causal-conv1d-wheel-image = causal_conv1d_wheel_context() mamba-ssm-wheel-image = mamba_ssm_wheel_context() } + args = { + NMP_COLLECT_SOURCES = NMP_COLLECT_SOURCES + } platforms = get_platforms() } @@ -902,6 +925,9 @@ target "nmp-automodel-training-docker" { platform-workspace = "target:automodel-platform-workspace" nmp-automodel-base = automodel_base_context() } + args = { + NMP_COLLECT_SOURCES = NMP_COLLECT_SOURCES + } cache-to = maybe_registry_cache_to("nmp-automodel-training") cache-from = maybe_registry_cache_from("nmp-automodel-training") tags = sha_and_maybe_latest_tags("nmp-automodel-training") @@ -918,7 +944,8 @@ target "nmp-automodel-training-smoke-test" { nmp-automodel-base = automodel_base_context() } args = { - SMOKE_MARKER = "smoke_nmp_automodel_training" + NMP_COLLECT_SOURCES = NMP_COLLECT_SOURCES + SMOKE_MARKER = "smoke_nmp_automodel_training" } cache-from = maybe_registry_cache_from("nmp-automodel-training") output = ["type=cacheonly"] @@ -942,6 +969,9 @@ target "nmp-unsloth-training" { causal-conv1d-wheel-image = causal_conv1d_wheel_context() mamba-ssm-wheel-image = mamba_ssm_wheel_context() } + args = { + NMP_COLLECT_SOURCES = NMP_COLLECT_SOURCES + } cache-to = maybe_registry_cache_to("nmp-unsloth-training") cache-from = maybe_registry_cache_from("nmp-unsloth-training") tags = sha_and_maybe_latest_tags("nmp-unsloth-training") @@ -999,6 +1029,9 @@ target "auditor-tasks-docker" { root-busybox = "target:root-busybox" } dockerfile = "docker/Dockerfile.auditor-tasks" + args = { + NMP_COLLECT_SOURCES = NMP_COLLECT_SOURCES + } cache-to = maybe_registry_cache_to("auditor-tasks") cache-from = maybe_registry_cache_from("auditor-tasks") tags = sha_and_maybe_latest_tags("auditor-tasks") diff --git a/docker/Dockerfile.auditor-tasks b/docker/Dockerfile.auditor-tasks index 5eb62896b7..28095e3d91 100644 --- a/docker/Dockerfile.auditor-tasks +++ b/docker/Dockerfile.auditor-tasks @@ -22,6 +22,7 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-reco && rm -rf /var/lib/apt/lists/* COPY --from=ghcr.io/astral-sh/uv:0.9.14 /uv /bin/uv +COPY docker/scripts/collect-python-sdists.py /usr/local/bin/ ENV UV_COMPILE_BYTECODE=1 \ UV_LINK_MODE=copy @@ -88,11 +89,31 @@ RUN --mount=type=cache,target=/root/.cache/uv \ "cryptography>=50.0.0,<51" \ "nltk>=3.10.0" +ARG NMP_COLLECT_SOURCES=0 +ENV NMP_COLLECT_SOURCES=${NMP_COLLECT_SOURCES} +RUN --mount=type=cache,target=/root/.cache/uv \ + uv run --no-project --python /app/.venv/bin/python \ + /usr/local/bin/collect-python-sdists.py \ + --python /app/.venv/bin/python \ + --output /source-distributions/auditor-tasks \ + --lock-file /app/uv.lock \ + --label app-venv && \ + uv run --no-project --python /app/.garak_venv/bin/python \ + /usr/local/bin/collect-python-sdists.py \ + --python /app/.garak_venv/bin/python \ + --output /source-distributions/auditor-tasks \ + --lock-file /app/uv.lock \ + --label garak-venv + FROM ${AUDITOR_PYTHON_IMAGE} AS base ARG USERNAME=nvs ARG USER_UID=1000 ARG USER_GID=1000 WORKDIR /app +COPY --from=ghcr.io/astral-sh/uv:0.9.14 /uv /bin/uv +COPY docker/scripts/collect-apt-sources.sh \ + docker/scripts/collect-cpython-source.py \ + /usr/local/bin/ RUN apt-get update && apt-get upgrade -y && \ list_installed_perl_cve_packages() { \ dpkg-query -W -f='${binary:Package} ${db:Status-Abbrev}\n' \ @@ -150,8 +171,16 @@ ENV PATH="/app/.venv/bin:$PATH" # Copy installed packages and application COPY --chown=nvs:nvs --from=py-builder /app /app +COPY --chown=nvs:nvs --from=py-builder /source-distributions/auditor-tasks/ /usr/share/nemo-platform/source-distributions/auditor-tasks/ ARG CACHE_HOME COPY --chown=nvs:nvs --from=py-builder ${CACHE_HOME} ${CACHE_HOME} +ARG NMP_COLLECT_SOURCES=0 +ENV NMP_COLLECT_SOURCES=${NMP_COLLECT_SOURCES} +RUN bash /usr/local/bin/collect-apt-sources.sh /usr/share/nemo-platform/source-distributions/auditor-tasks/apt --installed && \ + uv run --no-project --python /usr/local/bin/python \ + /usr/local/bin/collect-cpython-source.py \ + --python /usr/local/bin/python \ + --output /usr/share/nemo-platform/source-distributions/auditor-tasks ENV USERNAME=nvs \ HOME=/home/nvs diff --git a/docker/Dockerfile.nmp-api b/docker/Dockerfile.nmp-api index 48a3cb8ddc..ce2fe0c55b 100644 --- a/docker/Dockerfile.nmp-api +++ b/docker/Dockerfile.nmp-api @@ -55,6 +55,25 @@ RUN if find /tmp/fastembed_cache/models--qdrant--all-MiniLM-L6-v2-onnx/snapshots uv run --no-sync python -c 'import os, tempfile; from huggingface_hub import snapshot_download; snapshot_download(repo_id="qdrant/all-MiniLM-L6-v2-onnx", cache_dir=os.path.join(tempfile.gettempdir(), "fastembed_cache"))'; \ fi +ARG NMP_COLLECT_SOURCES=0 +ENV NMP_COLLECT_SOURCES=${NMP_COLLECT_SOURCES} +RUN --mount=type=cache,target=/root/.cache/uv \ + uv run --no-project --python /app/.venv/bin/python \ + /usr/local/bin/collect-workspace-sdists.py \ + --python /app/.venv/bin/python \ + --output /app/.source-distributions/nmp-api && \ + uv run --no-project --python /app/.venv/bin/python \ + /usr/local/bin/collect-python-sdists.py \ + --python /app/.venv/bin/python \ + --output /app/.source-distributions/nmp-api \ + --lock-file /app/uv.lock \ + --label app-venv && \ + bash /usr/local/bin/collect-apt-sources.sh /app/.source-distributions/nmp-api/apt --installed && \ + uv run --no-project --python /app/.venv/bin/python \ + /usr/local/bin/collect-cpython-source.py \ + --python /app/.venv/bin/python \ + --output /app/.source-distributions/nmp-api + FROM ${NMP_API_RUNTIME_BASE} AS runtime WORKDIR /app @@ -83,6 +102,7 @@ ENV USERNAME=nvs \ # DuckDB extension cache (aws, httpfs) for S3 log storage COPY --chown=1000:1000 --from=builder /root/.duckdb /home/nvs/.duckdb COPY --chown=1000:1000 --from=builder /app/.venv /app/.venv +COPY --chown=1000:1000 --from=builder /app/.source-distributions/nmp-api/ /usr/share/nemo-platform/source-distributions/nmp-api/ ENV VIRTUAL_ENV=/app/.venv \ PATH="/app/.venv/bin:$PATH" \ PYTHONPATH="/app/.venv/lib/python${PY_VERSION}/site-packages" diff --git a/docker/Dockerfile.nmp-core b/docker/Dockerfile.nmp-core index 3f8f876d03..c703c484d8 100644 --- a/docker/Dockerfile.nmp-core +++ b/docker/Dockerfile.nmp-core @@ -18,6 +18,24 @@ RUN /app/.venv/bin/python -c 'from nmp.core.auth.app.embedded_pdp.policy_wasm im # Download the extension artifacts directly over HTTPS and install from local files # because DuckDB's default remote INSTALL path can be blocked or flaky in some build environments. RUN sh /app/script/install_duckdb_extensions.sh +ARG NMP_COLLECT_SOURCES=0 +ENV NMP_COLLECT_SOURCES=${NMP_COLLECT_SOURCES} +RUN --mount=type=cache,target=/root/.cache/uv \ + uv run --no-project --python /app/.venv/bin/python \ + /usr/local/bin/collect-workspace-sdists.py \ + --python /app/.venv/bin/python \ + --output /app/.source-distributions/nmp-core && \ + uv run --no-project --python /app/.venv/bin/python \ + /usr/local/bin/collect-python-sdists.py \ + --python /app/.venv/bin/python \ + --output /app/.source-distributions/nmp-core \ + --lock-file /app/uv.lock \ + --label app-venv && \ + bash /usr/local/bin/collect-apt-sources.sh /app/.source-distributions/nmp-core/apt --installed && \ + uv run --no-project --python /app/.venv/bin/python \ + /usr/local/bin/collect-cpython-source.py \ + --python /app/.venv/bin/python \ + --output /app/.source-distributions/nmp-core FROM ${NMP_CORE_RUNTIME_BASE} AS runtime WORKDIR /app @@ -48,6 +66,7 @@ ENV USERNAME=nvs \ COPY --chown=1000:1000 --from=builder /root/.duckdb /root/.duckdb COPY --chown=1000:1000 --from=builder /root/.duckdb /home/nvs/.duckdb COPY --chown=1000:1000 --from=builder /app/.venv /app/.venv +COPY --chown=1000:1000 --from=builder /app/.source-distributions/nmp-core/ /usr/share/nemo-platform/source-distributions/nmp-core/ ENV VIRTUAL_ENV=/app/.venv \ PATH="/app/.venv/bin:$PATH" \ PYTHONPATH="/app/.venv/lib/python${PY_VERSION}/site-packages" diff --git a/docker/Dockerfile.nmp-cpu-tasks b/docker/Dockerfile.nmp-cpu-tasks index 3c3f46e14a..9837c901b3 100644 --- a/docker/Dockerfile.nmp-cpu-tasks +++ b/docker/Dockerfile.nmp-cpu-tasks @@ -11,11 +11,30 @@ RUN --mount=type=cache,target=/root/.cache/uv \ # upstream cpu-tasks group includes nemo-evaluator-plugin directly. RUN --mount=type=cache,target=/root/.cache/uv \ uv sync --frozen --inexact --package nemo-evaluator-plugin --no-editable +ARG NMP_COLLECT_SOURCES=0 +ENV NMP_COLLECT_SOURCES=${NMP_COLLECT_SOURCES} +RUN --mount=type=cache,target=/root/.cache/uv \ + uv run --no-project --python /app/.venv/bin/python \ + /usr/local/bin/collect-workspace-sdists.py \ + --python /app/.venv/bin/python \ + --output /app/.source-distributions/nmp-cpu-tasks && \ + uv run --no-project --python /app/.venv/bin/python \ + /usr/local/bin/collect-python-sdists.py \ + --python /app/.venv/bin/python \ + --output /app/.source-distributions/nmp-cpu-tasks \ + --lock-file /app/uv.lock \ + --label app-venv && \ + bash /usr/local/bin/collect-apt-sources.sh /app/.source-distributions/nmp-cpu-tasks/apt --installed && \ + uv run --no-project --python /app/.venv/bin/python \ + /usr/local/bin/collect-cpython-source.py \ + --python /app/.venv/bin/python \ + --output /app/.source-distributions/nmp-cpu-tasks FROM ${NMP_PYTHON_BASE} AS runtime ENV USERNAME=nvs \ HOME=/home/nvs COPY --chown=nvs:nvs --from=builder /app/.venv /app/.venv +COPY --chown=nvs:nvs --from=builder /app/.source-distributions/nmp-cpu-tasks/ /usr/share/nemo-platform/source-distributions/nmp-cpu-tasks/ COPY --chown=nvs:nvs --from=builder /app/plugins/nemo-data-designer/tiktoken-cache /app/tiktoken-cache ENV TIKTOKEN_CACHE_DIR=/app/tiktoken-cache ENV PATH="/app/.venv/bin:$PATH" diff --git a/docker/Dockerfile.nmp-customizer-tasks b/docker/Dockerfile.nmp-customizer-tasks index 9fc44860f7..a8a75a68dd 100644 --- a/docker/Dockerfile.nmp-customizer-tasks +++ b/docker/Dockerfile.nmp-customizer-tasks @@ -3,19 +3,24 @@ # # Hosts file_io / model_entity (nmp.customization_common.tasks.*), model_spec # analysis (nmp.core.models.tasks.model_spec), and the LoRA adapter sidecar. -# Built on NGC PyTorch 26.05 + transformers/accelerate + Nemotron wheels +# Built on NGC PyTorch 26.07 + transformers/accelerate + Nemotron wheels # (mamba-ssm, causal-conv1d) + platform glue + nmp-models. ARG SMOKE_MARKER=smoke_nmp_customizer_tasks -ARG PYTORCH_BASE=nvcr.io/nvidia/pytorch:26.05-py3 +ARG PYTORCH_BASE=nvcr.io/nvidia/pytorch:26.07-py3 FROM ${PYTORCH_BASE} AS base WORKDIR /app COPY --from=ghcr.io/astral-sh/uv:0.9.14 /uv /bin/uv +COPY docker/scripts/collect-apt-sources.sh \ + docker/scripts/collect-python-sdists.py \ + /usr/local/bin/ ENV PATH="/bin:${PATH}" +RUN dpkg-query -W -f='${binary:Package}\t${Version}\n' > /tmp/base-dpkg-packages.txt + ENV VIRTUAL_ENV=/opt/venv \ UV_PROJECT_ENVIRONMENT=/opt/venv \ UV_LINK_MODE=copy \ @@ -68,7 +73,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ -e /app/services/core/models # Pin scanner-tracked runtime packages into /opt/venv before removing stale -# NGC system-site copies inherited from nvcr.io/nvidia/pytorch:26.05-py3. +# NGC system-site copies inherited from nvcr.io/nvidia/pytorch:26.07-py3. RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --python ${VIRTUAL_ENV}/bin/python --no-cache \ --overrides /app/docker/customizer/preserve_base_torch.txt \ @@ -121,6 +126,18 @@ RUN apt-get update && \ /usr/local/bin/nsys \ /usr/local/cuda/bin/nsys +ARG NMP_COLLECT_SOURCES=0 +ENV NMP_COLLECT_SOURCES=${NMP_COLLECT_SOURCES} +RUN bash /usr/local/bin/collect-apt-sources.sh \ + /usr/share/nemo-platform/source-distributions/nmp-customizer-tasks/apt \ + --changed-from /tmp/base-dpkg-packages.txt + +RUN --mount=type=cache,target=/root/.cache/uv \ + python /usr/local/bin/collect-python-sdists.py \ + --python ${VIRTUAL_ENV}/bin/python \ + --output /usr/share/nemo-platform/source-distributions/nmp-customizer-tasks \ + --label app-venv + ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" ENTRYPOINT ["/opt/venv/bin/python"] CMD ["-m", "nmp.customization_common.tasks.file_io", "--help"] diff --git a/docker/Dockerfile.nmp-unsloth-training b/docker/Dockerfile.nmp-unsloth-training index f432ac6050..53ae5c7998 100644 --- a/docker/Dockerfile.nmp-unsloth-training +++ b/docker/Dockerfile.nmp-unsloth-training @@ -13,7 +13,7 @@ # bitsandbytes, xformers, etc.). `--overrides preserve_base_torch.txt` # blocks uv from installing/upgrading torch into the venv so the NGC # base's PyTorch + CUDA remain the runtime stack. -# 1b. bitsandbytes — compiled from source against NGC CUDA 13.2 (same pattern +# 1b. bitsandbytes — compiled from source against the NGC base CUDA (same pattern # as docker/automodel/Dockerfile.nmp-automodel-base). PyPI wheels only ship through # cuda130; source build replaces the wheel from step 1. # 1c. mamba-ssm + causal-conv1d — prebuilt cu13.2 / cp312 wheels (shared with @@ -28,10 +28,10 @@ # Publish target: nmp-unsloth-training # Default tag: `local` (override via BAKE_TAG at build time). -# NGC PyTorch base. 26.05-py3 ships PyTorch 2.12 + CUDA 13.2 + Python 3.12 (aligned with +# NGC PyTorch base. 26.07-py3 ships PyTorch + CUDA 13.3 + Python 3.12 (aligned with # docker/automodel/Dockerfile.nmp-automodel-base). Override at build time: # --set nmp-unsloth-training.args.PYTORCH_BASE=... -ARG PYTORCH_BASE=nvcr.io/nvidia/pytorch:26.05-py3 +ARG PYTORCH_BASE=nvcr.io/nvidia/pytorch:26.07-py3 # Prebuilt CUDA-extension wheels (mamba-ssm + causal-conv1d), shared with # docker/automodel/Dockerfile.nmp-automodel-base. The bake `nmp-unsloth-training` target @@ -46,8 +46,13 @@ FROM ${PYTORCH_BASE} AS base WORKDIR /opt COPY --from=ghcr.io/astral-sh/uv:0.9.14 /uv /bin/uv +COPY docker/scripts/collect-apt-sources.sh \ + docker/scripts/collect-python-sdists.py \ + /usr/local/bin/ ENV PATH="/bin:${PATH}" +RUN dpkg-query -W -f='${binary:Package}\t${Version}\n' > /tmp/base-dpkg-packages.txt + ENV VIRTUAL_ENV=/opt/venv \ UV_PROJECT_ENVIRONMENT=/opt/venv \ UV_LINK_MODE=copy \ @@ -105,11 +110,13 @@ RUN --mount=type=cache,target=/root/.cache/uv \ transformers==${TRANSFORMERS_VERSION} \ huggingface-hub==${HF_HUB_VERSION} -# Step 1b: bitsandbytes from source — matches automodel base (CUDA 13.2 nvcc). +# Step 1b: bitsandbytes from source — matches automodel base CUDA. RUN --mount=type=cache,target=/root/.cache/uv \ git clone https://github.com/bitsandbytes-foundation/bitsandbytes.git /tmp/bitsandbytes && \ cd /tmp/bitsandbytes && \ git checkout ${BITSANDBYTES_VERSION} && \ + mkdir -p /usr/share/nemo-platform/source-distributions/nmp-unsloth-training/git && \ + git archive --format=tar.gz --output /usr/share/nemo-platform/source-distributions/nmp-unsloth-training/git/bitsandbytes-${BITSANDBYTES_VERSION}.tar.gz HEAD && \ cmake -DCOMPUTE_CAPABILITY="75;80;86;87;89;90;100;103;110;120;121" -DCOMPUTE_BACKEND=cuda -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -S . && \ make -j${BNB_MAX_JOBS} && \ uv pip install --python ${VIRTUAL_ENV}/bin/python --no-cache scikit-build-core --no-deps && \ @@ -182,7 +189,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ "soupsieve>=2.8.4,<3" \ "mlflow-skinny>=3.11.1,<3.12.0" -# Stale NGC system site-packages (verified on nvcr.io/nvidia/pytorch:26.05-py3). +# Stale NGC system site-packages (verified on nvcr.io/nvidia/pytorch:26.07-py3). RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --only-upgrade openssl libssl3t64 && \ rm -rf /var/lib/apt/lists/* @@ -226,6 +233,16 @@ RUN rm -rf \ /usr/local/bin/nsys \ /usr/local/cuda/bin/nsys +ARG NMP_COLLECT_SOURCES=0 +ENV NMP_COLLECT_SOURCES=${NMP_COLLECT_SOURCES} +RUN --mount=type=cache,target=/root/.cache/uv \ + bash /usr/local/bin/collect-apt-sources.sh \ + /usr/share/nemo-platform/source-distributions/nmp-unsloth-training/apt \ + --changed-from /tmp/base-dpkg-packages.txt && \ + python /usr/local/bin/collect-python-sdists.py \ + --python ${VIRTUAL_ENV}/bin/python \ + --output /usr/share/nemo-platform/source-distributions/nmp-unsloth-training \ + --label app-venv ENTRYPOINT ["/opt/venv/bin/python"] CMD ["-m", "nmp.unsloth.tasks.training", "--help"] diff --git a/docker/Dockerfile.safe-synthesizer-tasks b/docker/Dockerfile.safe-synthesizer-tasks index 8c59fa3d71..dc78035fa5 100644 --- a/docker/Dockerfile.safe-synthesizer-tasks +++ b/docker/Dockerfile.safe-synthesizer-tasks @@ -42,6 +42,10 @@ ENV UV_HTTP_TIMEOUT=120 \ UV_HTTP_RETRIES=8 COPY --from=uv /uv /uvx /usr/local/bin/ +COPY docker/scripts/collect-apt-sources.sh \ + docker/scripts/collect-cpython-source.py \ + docker/scripts/collect-python-sdists.py \ + /usr/local/bin/ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ @@ -191,6 +195,23 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --no-install-package cryptography \ --no-install-package pyarrow +ARG NMP_COLLECT_SOURCES=0 +ENV NMP_COLLECT_SOURCES=${NMP_COLLECT_SOURCES} +RUN --mount=type=cache,target=/root/.cache/uv \ + python /usr/local/bin/collect-python-sdists.py \ + --python /opt/venv/bin/python \ + --output /usr/share/nemo-platform/source-distributions/safe-synthesizer-tasks \ + --lock-file /build/uv.lock \ + --skip-package nemo-safe-synthesizer-plugin \ + --skip-package nemo-platform-sdk \ + --skip-package nemo-platform-plugin \ + --label app-venv && \ + bash /usr/local/bin/collect-apt-sources.sh /usr/share/nemo-platform/source-distributions/safe-synthesizer-tasks/apt --installed && \ + uv run --no-project --python /opt/venv/bin/python \ + /usr/local/bin/collect-cpython-source.py \ + --python /opt/venv/bin/python \ + --output /usr/share/nemo-platform/source-distributions/safe-synthesizer-tasks + COPY docker/scripts/cve-cleanup.sh /bin/ # Perl CVE cleanup removes dpkg-dev/build-essential; keep the non-Perl # compiler toolchain available for runtime JIT/native extension builds. diff --git a/docker/automodel/Dockerfile.nmp-automodel-base b/docker/automodel/Dockerfile.nmp-automodel-base index ecb37c0ba5..49271e636d 100644 --- a/docker/automodel/Dockerfile.nmp-automodel-base +++ b/docker/automodel/Dockerfile.nmp-automodel-base @@ -7,7 +7,7 @@ # Pin to the tip of the Automodel release branch (origin/r0.5.0), not main. # Re-pin AUTOMODEL_COMMIT when cutting to a new r0.x.y. ARG AUTOMODEL_COMMIT=84e85792e34a240f09194a919c1e62c9d3e47938 -ARG PYTORCH_BASE=nvcr.io/nvidia/pytorch:26.05-py3 +ARG PYTORCH_BASE=nvcr.io/nvidia/pytorch:26.07-py3 FROM causal-conv1d-wheel-image AS causal-conv1d-wheel-src FROM mamba-ssm-wheel-image AS mamba-ssm-wheel-src @@ -25,6 +25,7 @@ ARG PYTORCH_BASE WORKDIR /opt COPY --from=ghcr.io/astral-sh/uv:0.9.14 /uv /bin/uv +COPY docker/scripts/collect-python-sdists.py /usr/local/bin/ ENV VIRTUAL_ENV=/opt/venv \ UV_PROJECT_ENVIRONMENT=/opt/venv \ @@ -69,12 +70,20 @@ RUN --mount=from=causal-conv1d-wheel-src,target=/tmp/causal-conv1d-wheel-src,rea /tmp/mamba-ssm-wheel-src/wheels/cu13.2/mamba_ssm-2.3.0-cp312*.whl RUN --mount=type=cache,target=/root/.cache/uv \ - uv pip install --no-build-isolation --no-deps git+https://github.com/fanshiqing/grouped_gemm@v1.1.4 + git clone https://github.com/fanshiqing/grouped_gemm.git /tmp/grouped_gemm && \ + cd /tmp/grouped_gemm && \ + git checkout v1.1.4 && \ + mkdir -p /source-distributions/nmp-automodel-base/git && \ + git archive --format=tar.gz --output /source-distributions/nmp-automodel-base/git/grouped_gemm-v1.1.4.tar.gz HEAD && \ + uv pip install --no-build-isolation --no-deps . && \ + rm -rf /tmp/grouped_gemm RUN --mount=type=cache,target=/root/.cache/uv \ git clone https://github.com/bitsandbytes-foundation/bitsandbytes.git && \ cd bitsandbytes && \ git checkout 0.49.2 && \ + mkdir -p /source-distributions/nmp-automodel-base/git && \ + git archive --format=tar.gz --output /source-distributions/nmp-automodel-base/git/bitsandbytes-0.49.2.tar.gz HEAD && \ cmake -DCOMPUTE_CAPABILITY="75;80;86;87;89;90;100;103;110;120;121" -DCOMPUTE_BACKEND=cuda -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -S . && \ make -j"$(nproc)" && \ uv pip install scikit-build-core --no-deps && \ @@ -113,15 +122,30 @@ RUN --mount=type=cache,target=/root/.cache/uv \ "grpcio>=1.81.1,<2" \ "wandb==0.28.1" +ARG NMP_COLLECT_SOURCES=0 +ENV NMP_COLLECT_SOURCES=${NMP_COLLECT_SOURCES} +RUN --mount=type=cache,target=/root/.cache/uv \ + python /usr/local/bin/collect-python-sdists.py \ + --python /opt/venv/bin/python \ + --output /source-distributions/nmp-automodel-base \ + --lock-file /opt/Automodel/uv.lock \ + --label base-venv + # Published base image (same filesystem as builder). FROM ${PYTORCH_BASE} AS nmp-automodel-base ARG PYTORCH_BASE COPY --from=nmp-automodel-base-builder /opt/venv /opt/venv COPY --from=nmp-automodel-base-builder /opt/Automodel /opt/Automodel +COPY --from=nmp-automodel-base-builder /source-distributions/nmp-automodel-base/ /usr/share/nemo-platform/source-distributions/nmp-automodel-base/ # Builder pins uv 0.9.14 but does not ship it in the venv layer; PyTorch base may ship 0.10.x. COPY --from=ghcr.io/astral-sh/uv:0.9.14 /uv /bin/uv +COPY docker/scripts/collect-apt-sources.sh \ + docker/scripts/collect-python-sdists.py \ + /usr/local/bin/ -# Stale copies under NGC system site-packages (verified on nvcr.io/nvidia/pytorch:26.05-py3). +RUN dpkg-query -W -f='${binary:Package}\t${Version}\n' > /tmp/base-dpkg-packages.txt + +# Stale copies under NGC system site-packages (verified on nvcr.io/nvidia/pytorch:26.07-py3). # CVE scanners read dist-packages even when --system-site-packages resolves imports from /opt/venv. # rm -rf (not pip uninstall) avoids corrupted dist-info RECORDs on the NGC base image. RUN apt-get update && \ @@ -166,6 +190,12 @@ RUN rm -rf \ /usr/local/bin/nsys \ /usr/local/cuda/bin/nsys +ARG NMP_COLLECT_SOURCES=0 +ENV NMP_COLLECT_SOURCES=${NMP_COLLECT_SOURCES} +RUN bash /usr/local/bin/collect-apt-sources.sh \ + /usr/share/nemo-platform/source-distributions/nmp-automodel-base/apt \ + --changed-from /tmp/base-dpkg-packages.txt + ENV VIRTUAL_ENV=/opt/venv \ UV_PROJECT_ENVIRONMENT=/opt/venv \ HF_HUB_ENABLE_HF_TRANSFER=1 diff --git a/docker/automodel/Dockerfile.nmp-automodel-training b/docker/automodel/Dockerfile.nmp-automodel-training index f5a08cbe9d..19feda8097 100644 --- a/docker/automodel/Dockerfile.nmp-automodel-training +++ b/docker/automodel/Dockerfile.nmp-automodel-training @@ -26,6 +26,8 @@ WORKDIR /app RUN mkdir -p /home/${USERNAME}/.cache && \ chown -R ${USER_UID}:${USER_GID} /home/${USERNAME} /app/services/automodel +RUN uv pip freeze --python ${VIRTUAL_ENV}/bin/python --exclude-editable > /tmp/base-python-freeze.txt + RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --python ${VIRTUAL_ENV}/bin/python --no-cache \ --overrides /app/docker/automodel/no_override_requirements.txt \ @@ -40,6 +42,15 @@ RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --python ${VIRTUAL_ENV}/bin/python --no-cache --no-deps \ -e /opt/Automodel +ARG NMP_COLLECT_SOURCES=0 +ENV NMP_COLLECT_SOURCES=${NMP_COLLECT_SOURCES} +RUN --mount=type=cache,target=/root/.cache/uv \ + python /usr/local/bin/collect-python-sdists.py \ + --python ${VIRTUAL_ENV}/bin/python \ + --output /usr/share/nemo-platform/source-distributions/nmp-automodel-training \ + --baseline-freeze /tmp/base-python-freeze.txt \ + --label platform-glue + ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" ENTRYPOINT ["/opt/venv/bin/python"] CMD ["-m", "nmp.automodel.tasks.training", "--help"] diff --git a/docker/base/Dockerfile.nmp-python-base b/docker/base/Dockerfile.nmp-python-base index 4032caf471..4191c64cc2 100644 --- a/docker/base/Dockerfile.nmp-python-base +++ b/docker/base/Dockerfile.nmp-python-base @@ -11,6 +11,12 @@ ARG NMP_PYTHON_IMAGE=python:3.13.14-slim-trixie FROM ${NMP_PYTHON_IMAGE} AS nmp-python-base-builder +COPY docker/scripts/collect-apt-sources.sh \ + docker/scripts/collect-cpython-source.py \ + docker/scripts/collect-python-sdists.py \ + docker/scripts/collect-workspace-sdists.py \ + /usr/local/bin/ + RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \ build-essential \ ca-certificates \ @@ -28,10 +34,10 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-reco && rm -rf /var/lib/apt/lists/* \ && pip uninstall --yes setuptools wheel -WORKDIR /app - COPY --from=ghcr.io/astral-sh/uv:0.9.14 /uv /bin/uv +WORKDIR /app + RUN uv venv --seed --python=3.13 /app/.venv ENV PYTHONDONTWRITEBYTECODE=1 \ diff --git a/docker/rl/Dockerfile.nmp-rl-base b/docker/rl/Dockerfile.nmp-rl-base index 7d418c25f2..f6ac851cdf 100644 --- a/docker/rl/Dockerfile.nmp-rl-base +++ b/docker/rl/Dockerfile.nmp-rl-base @@ -17,7 +17,7 @@ # Kept in lockstep with the pinned NEMO_RL_REF: RL's own Dockerfile builds on this tag, and its lock # overrides nvidia-nccl-cu13 to the version that ships here. Bump both together. -ARG BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:26.05-cuda13.2-devel-ubuntu24.04 +ARG BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:26.07-cuda13.3-devel-ubuntu24.04 # Source. Default clones the soluwalana/RL fork at the given ref; override with a local checkout via # `--build-context nemo-rl=` or point elsewhere via the NEMO_RL_{REPO,REF} build-args. Gym is @@ -47,6 +47,7 @@ ADD ${NEMO_RL_REPO}#${NEMO_RL_REF} / FROM ${BASE_IMAGE} AS base USER root ENV DEBIAN_FRONTEND=noninteractive +RUN dpkg-query -W -f='${binary:Package}\t${Version}\n' > /tmp/base-dpkg-packages.txt # openssl is the only dpkg package here with an out-of-SLA CVE. Upgraded explicitly rather than via # a blanket `apt-get upgrade`, which would also pull newer CUDA packages from the NVIDIA repos. @@ -85,6 +86,9 @@ ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python \ RUN curl -LsSf "https://astral.sh/uv/${UV_VERSION}/install.sh" | sh && \ cp /root/.local/bin/uv /usr/local/bin/uv && \ uv python install "${PYTHON_VERSION}" +COPY docker/scripts/collect-apt-sources.sh \ + docker/scripts/collect-python-sdists.py \ + /usr/local/bin/ # Ray / NeMo-RL runtime behavior (affects the running container, not the build): # RAY_USAGE_STATS_ENABLED=0 - no telemetry phone-home. @@ -373,7 +377,6 @@ RUN apt-get purge -y ccache vim vim-common less >/dev/null 2>&1 || true; \ /opt/nvidia/nsight-systems-cli /opt/nvidia/nsight-compute \ /usr/local/bin/nsys /usr/local/bin/nsys-ui /usr/local/bin/ncu /usr/local/bin/ncu-ui \ || true - # NO_VCS_VERSION=1: with no .git present, nemo_rl/package_info.py's `git rev-parse` would fail on every # `import nemo_rl` (harmlessly, but it forks a subprocess each time, and this image starts many # worker processes). The flag is package_info.py's own opt-out, so the lookup is skipped entirely. @@ -426,6 +429,30 @@ fi echo "privatized vllm/ in ${privatized} venv(s)" EOF +ARG NMP_COLLECT_SOURCES=0 +ENV NMP_COLLECT_SOURCES=${NMP_COLLECT_SOURCES} +RUN --mount=type=cache,target=/root/.cache/uv <<'EOF' bash -eu +source_dir=/usr/share/nemo-platform/source-distributions/nmp-rl-base +bash /usr/local/bin/collect-apt-sources.sh "${source_dir}/apt" \ + --changed-from /tmp/base-dpkg-packages.txt +uv run --no-project --python /opt/nemo_rl_venv/bin/python \ + /usr/local/bin/collect-python-sdists.py \ + --python /opt/nemo_rl_venv/bin/python \ + --output "${source_dir}" \ + --lock-file /opt/nemo-rl/uv.lock \ + --label nemo-rl-venv +for python_path in /opt/ray_venvs/*/bin/python /opt/gym_venvs/*/bin/python; do + [ -x "${python_path}" ] || continue + label="$(printf '%s' "${python_path%/bin/python}" | sed -E 's#^/opt/##; s#[^A-Za-z0-9_.-]+#-#g')" + uv run --no-project --python "${python_path}" \ + /usr/local/bin/collect-python-sdists.py \ + --python "${python_path}" \ + --output "${source_dir}" \ + --lock-file /opt/nemo-rl/uv.lock \ + --label "${label}" +done +EOF + # Gym falls back to this in-tree cache when a process does not inherit NRL_CONTAINER / UV_CACHE_DIR # (the sandboxed Gym host's per-app servers), and fails with EACCES on a root-owned tree. Keeping # the fallback writable is the safety net; the real fix is env propagation, which lives in NeMo-RL. diff --git a/docker/rl/Dockerfile.nmp-rl-training b/docker/rl/Dockerfile.nmp-rl-training index c2b8b1c99c..793bb6c2c2 100644 --- a/docker/rl/Dockerfile.nmp-rl-training +++ b/docker/rl/Dockerfile.nmp-rl-training @@ -36,6 +36,9 @@ RUN mkdir -p /home/${USERNAME}/.cache && \ # Single RL_PYTHON_ENTRYPOINT the compiler stamps onto every step: /opt/venv/bin/python. RUN ln -sfn /opt/nemo_rl_venv /opt/venv +RUN UV_CACHE_DIR=/root/.cache/uv \ + uv-glue pip freeze --python /opt/nemo_rl_venv/bin/python --exclude-editable > /tmp/base-python-freeze.txt + # Editable platform glue WITH dependency resolution. The from-source base ships clean package # metadata, so uv can resolve the glue's real deps (typer, sniffio, ngcsdk, opentelemetry-*, # sqlalchemy, kubernetes, fastapi, ...) instead of us pinning them by hand. @@ -59,6 +62,16 @@ RUN --mount=type=cache,target=/root/.cache/uv \ -e /app/packages/nmp_customization_common \ -e /app/services/rl +ARG NMP_COLLECT_SOURCES=0 +ENV NMP_COLLECT_SOURCES=${NMP_COLLECT_SOURCES} +RUN --mount=type=cache,target=/root/.cache/uv \ + UV_BIN=uv-glue uv-glue run --no-project --python /opt/nemo_rl_venv/bin/python \ + /usr/local/bin/collect-python-sdists.py \ + --python /opt/nemo_rl_venv/bin/python \ + --output /usr/share/nemo-platform/source-distributions/nmp-rl-training \ + --baseline-freeze /tmp/base-python-freeze.txt \ + --label platform-glue + # Runtime uv cache: the base's /opt/uv_cache ships READ-ONLY on purpose - the prefetched venvs # symlink their package files into it, so a writable cache would let code running in this container # (including user-authored NeMo-Gym environment code) rewrite the training stack's own packages. diff --git a/docker/scripts/collect-apt-sources.sh b/docker/scripts/collect-apt-sources.sh new file mode 100644 index 0000000000..bd9918670d --- /dev/null +++ b/docker/scripts/collect-apt-sources.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +set -u + +usage() { + echo "usage: $0 OUTPUT_DIR [--installed] [--changed-from DPKG_MANIFEST] [PACKAGE ...]" >&2 +} + +source_collection_enabled() { + case "${NMP_COLLECT_SOURCES:-0}" in + 1 | true | TRUE | True | yes | YES | Yes | on | ON | On) + return 0 + ;; + *) + return 1 + ;; + esac +} + +if [ "$#" -lt 1 ]; then + usage + exit 2 +fi + +output_dir="$1" +shift + +include_installed=0 +changed_from_file="" +while [ "$#" -gt 0 ]; do + case "${1}" in + --installed) + include_installed=1 + shift + ;; + --changed-from) + if [ "$#" -lt 2 ]; then + usage + exit 2 + fi + changed_from_file="$2" + shift 2 + ;; + --) + shift + break + ;; + --*) + usage + exit 2 + ;; + *) + break + ;; + esac +done + +if ! source_collection_enabled; then + mkdir -p "${output_dir}/manifests" + echo "source collection disabled; set NMP_COLLECT_SOURCES=1 to enable" > \ + "${output_dir}/manifests/source-collection-disabled.txt" + exit 0 +fi + +mkdir -p "${output_dir}/sources" "${output_dir}/manifests" +missing_file="${output_dir}/manifests/missing-apt-sources.txt" +downloaded_file="${output_dir}/manifests/downloaded-apt-sources.txt" +log_file="${output_dir}/manifests/apt-source.log" +: > "${missing_file}" +: > "${downloaded_file}" +: > "${log_file}" + +if ! command -v apt-get >/dev/null 2>&1; then + echo "apt-get is not available" >> "${missing_file}" + exit 0 +fi + +packages_file="$(mktemp)" +source_packages_file="$(mktemp)" +trap 'rm -f "${packages_file}" "${source_packages_file}"' EXIT +: > "${packages_file}" +: > "${source_packages_file}" + +if command -v dpkg-query >/dev/null 2>&1; then + # shellcheck disable=SC2016 + dpkg_query_format='${binary:Package}\t${Version}\t${source:Package}\t${source:Version}\n' + current_packages_file="${output_dir}/manifests/installed-dpkg-packages.txt" + dpkg-query -W -f="${dpkg_query_format}" > "${current_packages_file}" 2>/dev/null || true + if [ "${include_installed}" -eq 1 ]; then + awk -F '\t' ' + length($1) > 0 { + source_package = $3 + if (source_package == "") { + source_package = $1 + sub(/:[^:]+$/, "", source_package) + } + source_version = $4 + if (source_version == "") { + source_version = $2 + } + if (source_version != "") { + printf "%s=%s\n", source_package, source_version + } else { + print source_package + } + } + ' "${current_packages_file}" >> "${source_packages_file}" 2>/dev/null || true + fi + if [ -n "${changed_from_file}" ]; then + if [ ! -f "${changed_from_file}" ]; then + echo "baseline dpkg manifest not found: ${changed_from_file}" >> "${missing_file}" + exit 2 + fi + cp "${changed_from_file}" "${output_dir}/manifests/baseline-dpkg-packages.txt" + awk -F '\t' ' + function print_source_package() { + source_package = $3 + if (source_package == "") { + source_package = $1 + sub(/:[^:]+$/, "", source_package) + } + source_version = $4 + if (source_version == "") { + source_version = $2 + } + if (source_version != "") { + printf "%s=%s\n", source_package, source_version + } else { + print source_package + } + } + NR == FNR { + baseline[$1] = $2 + next + } + length($1) > 0 && (!($1 in baseline) || baseline[$1] != $2) { + print_source_package() + } + ' "${changed_from_file}" "${current_packages_file}" >> "${source_packages_file}" + fi +fi + +for package in "$@"; do + printf '%s\n' "${package}" >> "${packages_file}" +done + +if [ ! -s "${packages_file}" ] && [ ! -s "${source_packages_file}" ]; then + echo "no apt packages selected for source collection" >> "${log_file}" + exit 0 +fi + +enable_deb_src_for_official_repos() { + local file tmp + + for file in /etc/apt/sources.list /etc/apt/sources.list.d/*.list; do + [ -f "${file}" ] || continue + tmp="${file}.src-tmp" + awk ' + { + print $0 + line = $0 + if (line ~ /^[[:space:]]*deb[[:space:]]+/ && + line !~ /^[[:space:]]*deb-src[[:space:]]+/ && + line ~ /(deb\.debian\.org|security\.debian\.org|archive\.ubuntu\.com|security\.ubuntu\.com|ports\.ubuntu\.com)/) { + sub(/^[[:space:]]*deb[[:space:]]+/, "deb-src ", line) + print line + } + } + ' "${file}" > "${tmp}" && mv "${tmp}" "${file}" + done + + for file in /etc/apt/sources.list.d/*.sources; do + [ -f "${file}" ] || continue + if grep -Eq 'URIs:.*(deb\.debian\.org|security\.debian\.org|archive\.ubuntu\.com|security\.ubuntu\.com|ports\.ubuntu\.com)' "${file}"; then + sed -i -E '/^Types:/ {/deb-src/! s/$/ deb-src/}' "${file}" || true + fi + done +} + +enable_deb_src_for_official_repos + +if [ -d /etc/apt ]; then + tar -C /etc/apt -czf "${output_dir}/manifests/apt-sources-config.tar.gz" sources.list sources.list.d 2>/dev/null || true +fi + +if ! apt-get update >> "${log_file}" 2>&1; then + echo "apt-get update failed after enabling deb-src; skipping apt source collection" >> "${missing_file}" + exit 0 +fi + +while IFS= read -r package; do + [ -n "${package}" ] || continue + source_package="$( + apt-cache show --no-all-versions "${package}" 2>/dev/null | + awk -F': ' ' + /^Source:/ { + value = $2 + sub(/[[:space:]]*\(.*/, "", value) + print value + found = 1 + exit + } + END { if (!found) exit 1 } + ' + )" + if [ -z "${source_package}" ]; then + source_package="${package}" + fi + printf '%s\n' "${source_package}" >> "${source_packages_file}" +done < "${packages_file}" + +sort -u "${source_packages_file}" | while IFS= read -r source_package; do + [ -n "${source_package}" ] || continue + if ( + cd "${output_dir}/sources" && + apt-get source --download-only --only-source "${source_package}" >> "${log_file}" 2>&1 + ); then + printf '%s\n' "${source_package}" >> "${downloaded_file}" + else + printf '%s\n' "${source_package}" >> "${missing_file}" + fi +done + +apt-get clean >/dev/null 2>&1 || true +rm -rf /var/lib/apt/lists/* 2>/dev/null || true diff --git a/docker/scripts/collect-cpython-source.py b/docker/scripts/collect-cpython-source.py new file mode 100644 index 0000000000..d55d0a55fb --- /dev/null +++ b/docker/scripts/collect-cpython-source.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Best-effort CPython source archive collector for official Python base images.""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import re +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from html.parser import HTMLParser +from pathlib import Path, PurePosixPath + +SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") + + +class ReleaseFilesParser(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.rows: list[tuple[list[str], list[str]]] = [] + self._in_row = False + self._in_code = False + self._links: list[str] = [] + self._checksums: list[str] = [] + self._code_chunks: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag == "tr": + self._in_row = True + self._links = [] + self._checksums = [] + return + if not self._in_row: + return + if tag == "a": + href = dict(attrs).get("href") + if href: + self._links.append(href) + elif tag == "code": + self._in_code = True + self._code_chunks = [] + + def handle_data(self, data: str) -> None: + if self._in_code: + self._code_chunks.append(data) + + def handle_endtag(self, tag: str) -> None: + if tag == "code" and self._in_code: + self._checksums.append("".join(self._code_chunks)) + self._in_code = False + self._code_chunks = [] + elif tag == "tr" and self._in_row: + self.rows.append((self._links, self._checksums)) + self._in_row = False + + +def url_filename(url: str) -> str: + path = urllib.parse.urlparse(url).path + return PurePosixPath(urllib.parse.unquote(path)).name + + +def cpython_version_from_filename(filename: str) -> str: + for suffix in (".tar.xz", ".tgz"): + if filename.startswith("Python-") and filename.endswith(suffix): + return filename.removeprefix("Python-").removesuffix(suffix) + raise RuntimeError(f"could not infer CPython version from archive filename: {filename}") + + +def release_page_url(version: str) -> str: + return f"https://www.python.org/downloads/release/python-{version.replace('.', '')}/" + + +def release_page_sha256(version: str, filename: str) -> str: + metadata_url = release_page_url(version) + with urllib.request.urlopen(metadata_url, timeout=30) as response: + parser = ReleaseFilesParser() + parser.feed(response.read().decode("utf-8", errors="replace")) + + for links, checksums in parser.rows: + if not any(url_filename(link) == filename for link in links): + continue + for checksum in checksums: + normalized = "".join(checksum.split()).lower() + if SHA256_PATTERN.fullmatch(normalized): + return normalized + raise RuntimeError(f"could not find SHA-256 checksum for {filename} on {metadata_url}") + + +def verify_sha256(data: bytes, expected: str) -> None: + actual = hashlib.sha256(data).hexdigest() + if actual != expected.lower(): + raise RuntimeError(f"sha256 mismatch: expected {expected}, got {actual}") + + +def source_collection_enabled() -> bool: + return os.environ.get("NMP_COLLECT_SOURCES", "0").strip().lower() in {"1", "true", "yes", "on"} + + +def record_source_collection_disabled(output_dir: Path) -> None: + manifests = output_dir / "manifests" + manifests.mkdir(parents=True, exist_ok=True) + (manifests / "source-collection-disabled.txt").write_text( + "source collection disabled; set NMP_COLLECT_SOURCES=1 to enable\n", + encoding="utf-8", + ) + + +def python_version(python: str) -> str: + return subprocess.check_output( + [python, "-c", "import platform; print(platform.python_version())"], + text=True, + stderr=subprocess.STDOUT, + ).strip() + + +def download(url: str, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + filename = url_filename(url) + expected_sha256 = release_page_sha256(cpython_version_from_filename(filename), filename) + if destination.is_file() and destination.stat().st_size > 0: + verify_sha256(destination.read_bytes(), expected_sha256) + return + last_error: Exception | None = None + for attempt in range(3): + try: + with urllib.request.urlopen(url, timeout=60) as response: + data = response.read() + verify_sha256(data, expected_sha256) + destination.write_bytes(data) + return + except (OSError, RuntimeError, urllib.error.URLError) as error: + last_error = error + time.sleep(2**attempt) + raise RuntimeError(str(last_error)) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--python", default=sys.executable, help="Python interpreter to identify") + parser.add_argument("--output", required=True, help="Source distribution output directory") + args = parser.parse_args() + + output_dir = Path(args.output) + manifests = output_dir / "manifests" + manifests.mkdir(parents=True, exist_ok=True) + if not source_collection_enabled(): + record_source_collection_disabled(output_dir) + return 0 + + version = python_version(args.python) + filename = f"Python-{version}.tgz" + url = f"https://www.python.org/ftp/python/{version}/{filename}" + download(url, output_dir / "cpython" / filename) + + (manifests / "downloaded-cpython-source.txt").write_text(f"{filename}\t{url}\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/scripts/collect-python-sdists.py b/docker/scripts/collect-python-sdists.py new file mode 100644 index 0000000000..ee09987d73 --- /dev/null +++ b/docker/scripts/collect-python-sdists.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Best-effort PyPI source distribution collector for an installed environment.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +import time +import tomllib +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from pathlib import Path + +FREEZE_RE = re.compile(r"^([A-Za-z0-9_.-]+)==([^;\s]+)$") + + +@dataclass(frozen=True) +class Sdist: + url: str + hash_name: str | None = None + hash_value: str | None = None + + +def source_collection_enabled() -> bool: + return os.environ.get("NMP_COLLECT_SOURCES", "0").strip().lower() in {"1", "true", "yes", "on"} + + +def record_source_collection_disabled(output_dir: Path) -> None: + manifests = output_dir / "manifests" + manifests.mkdir(parents=True, exist_ok=True) + (manifests / "source-collection-disabled.txt").write_text( + "source collection disabled; set NMP_COLLECT_SOURCES=1 to enable\n", + encoding="utf-8", + ) + + +def normalize_name(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def run_text(command: list[str]) -> str: + return subprocess.check_output(command, text=True, stderr=subprocess.STDOUT) + + +def uv_bin() -> str: + return os.environ.get("UV_BIN", "uv") + + +def purelib_for(python: str) -> Path: + script = "import sysconfig; print(sysconfig.get_paths()['purelib'])" + return Path(run_text([python, "-c", script]).strip()) + + +def local_direct_url_packages(purelib: Path) -> set[str]: + packages: set[str] = set() + for dist_info in purelib.glob("*.dist-info"): + direct_url = dist_info / "direct_url.json" + if not direct_url.is_file(): + continue + try: + data = json.loads(direct_url.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + url = str(data.get("url", "")) + if url.startswith("file://"): + packages.add(normalize_name(dist_info.name.split("-")[0])) + return packages + + +def freeze_packages(python: str, purelib: Path) -> dict[str, tuple[str, str]]: + output = run_text([uv_bin(), "pip", "freeze", "--python", python, "--path", str(purelib), "--exclude-editable"]) + return parse_freeze(output) + + +def parse_freeze(output: str) -> dict[str, tuple[str, str]]: + packages: dict[str, tuple[str, str]] = {} + for line in output.splitlines(): + line = line.strip() + match = FREEZE_RE.match(line) + if match is None: + continue + name, version = match.groups() + packages[normalize_name(name)] = (name, version) + return packages + + +def baseline_packages(path: str | None) -> dict[str, tuple[str, str]]: + if path is None: + return {} + try: + return parse_freeze(Path(path).read_text(encoding="utf-8")) + except OSError: + return {} + + +def manifest_path(manifests: Path, stem: str, label: str | None) -> Path: + if not label: + return manifests / f"{stem}.txt" + safe_label = re.sub(r"[^A-Za-z0-9_.-]+", "-", label).strip("-") + return manifests / f"{stem}-{safe_label}.txt" + + +def sdists_from_uv_lock(lock_file: Path) -> dict[tuple[str, str], Sdist]: + try: + data = tomllib.loads(lock_file.read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError): + return {} + + sdists: dict[tuple[str, str], Sdist] = {} + for package in data.get("package", []): + name = package.get("name") + version = package.get("version") + sdist = package.get("sdist") + if not isinstance(name, str) or not isinstance(version, str) or not isinstance(sdist, dict): + continue + url = sdist.get("url") + if not isinstance(url, str): + continue + hash_name = None + hash_value = None + raw_hash = sdist.get("hash") + if isinstance(raw_hash, str) and ":" in raw_hash: + hash_name, hash_value = raw_hash.split(":", 1) + sdists[(normalize_name(name), version)] = Sdist(url=url, hash_name=hash_name, hash_value=hash_value) + return sdists + + +def pypi_sdist(name: str, version: str) -> Sdist | None: + metadata_url = ( + f"https://pypi.org/pypi/{urllib.parse.quote(normalize_name(name))}/{urllib.parse.quote(version)}/json" + ) + with urllib.request.urlopen(metadata_url, timeout=30) as response: + payload = json.loads(response.read().decode("utf-8")) + for file_info in payload.get("urls", []): + if file_info.get("packagetype") != "sdist": + continue + url = file_info.get("url") + if not isinstance(url, str): + continue + digests = file_info.get("digests") or {} + sha256 = digests.get("sha256") + return Sdist(url=url, hash_name="sha256" if sha256 else None, hash_value=sha256) + return None + + +def download(url: str, destination: Path, hash_name: str | None, hash_value: str | None) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.is_file() and destination.stat().st_size > 0: + return + last_error: Exception | None = None + for attempt in range(3): + try: + with urllib.request.urlopen(url, timeout=60) as response: + data = response.read() + if hash_name and hash_value: + digest = hashlib.new(hash_name) + digest.update(data) + actual = digest.hexdigest() + if actual.lower() != hash_value.lower(): + raise RuntimeError(f"{hash_name} mismatch: expected {hash_value}, got {actual}") + destination.write_bytes(data) + return + except (OSError, RuntimeError, urllib.error.URLError) as error: + last_error = error + time.sleep(2**attempt) + raise RuntimeError(str(last_error)) + + +def unique_filename(output_dir: Path, url: str, name: str, version: str) -> Path: + parsed = urllib.parse.urlparse(url) + filename = Path(urllib.parse.unquote(parsed.path)).name + if not filename: + filename = f"{normalize_name(name)}-{version}.tar.gz" + destination = output_dir / "pypi" / filename + if not destination.exists(): + return destination + if destination.stat().st_size > 0: + return destination + return ( + output_dir / "pypi" / f"{normalize_name(name)}-{version}-{hashlib.sha256(url.encode()).hexdigest()[:12]}.tar.gz" + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--python", required=True, help="Python interpreter for the environment to inspect") + parser.add_argument("--output", required=True, help="Source distribution output directory") + parser.add_argument("--lock-file", action="append", default=[], help="uv.lock file to use for exact sdist URLs") + parser.add_argument( + "--baseline-freeze", help="Freeze output whose matching name==version entries should be skipped" + ) + parser.add_argument("--label", help="Label to suffix manifest filenames when collecting multiple environments") + parser.add_argument( + "--skip-package", action="append", default=[], help="Package name to exclude from PyPI sdist collection" + ) + args = parser.parse_args() + + output_dir = Path(args.output) + manifests = output_dir / "manifests" + manifests.mkdir(parents=True, exist_ok=True) + if not source_collection_enabled(): + record_source_collection_disabled(output_dir) + return 0 + + (output_dir / "pypi").mkdir(parents=True, exist_ok=True) + + try: + purelib = purelib_for(args.python) + packages = freeze_packages(args.python, purelib) + except subprocess.CalledProcessError as error: + (manifests / "missing-pypi-sdists.txt").write_text( + f"failed to inspect Python environment {args.python}: {error.output}\n", + encoding="utf-8", + ) + return 0 + + local_packages = local_direct_url_packages(purelib) + for package_name in local_packages: + packages.pop(package_name, None) + for package_name in args.skip_package: + packages.pop(normalize_name(package_name), None) + baseline = baseline_packages(args.baseline_freeze) + for package_name, package in list(packages.items()): + if baseline.get(package_name) == package: + packages.pop(package_name) + + lock_sdists: dict[tuple[str, str], Sdist] = {} + for lock_file in args.lock_file: + lock_sdists.update(sdists_from_uv_lock(Path(lock_file))) + + downloaded: list[str] = [] + missing: list[str] = [] + + for normalized, (name, version) in sorted(packages.items()): + sdist = lock_sdists.get((normalized, version)) + source = "uv.lock" + if sdist is None: + source = "pypi-json" + try: + sdist = pypi_sdist(name, version) + except (OSError, RuntimeError, urllib.error.URLError, json.JSONDecodeError) as error: + missing.append(f"{name}=={version}\tmetadata lookup failed: {error}") + continue + if sdist is None: + missing.append(f"{name}=={version}\tno sdist found") + continue + destination = unique_filename(output_dir, sdist.url, name, version) + try: + download(sdist.url, destination, sdist.hash_name, sdist.hash_value) + except RuntimeError as error: + missing.append(f"{name}=={version}\tdownload failed from {sdist.url}: {error}") + continue + downloaded.append(f"{name}=={version}\t{destination.name}\t{source}\t{sdist.url}") + + manifest_path(manifests, "installed-python-packages", args.label).write_text( + "\n".join(f"{name}=={version}" for name, version in sorted(packages.values())) + "\n", + encoding="utf-8", + ) + manifest_path(manifests, "downloaded-pypi-sdists", args.label).write_text( + "\n".join(downloaded) + ("\n" if downloaded else ""), + encoding="utf-8", + ) + manifest_path(manifests, "missing-pypi-sdists", args.label).write_text( + "\n".join(missing) + ("\n" if missing else ""), + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/scripts/collect-workspace-sdists.py b/docker/scripts/collect-workspace-sdists.py new file mode 100644 index 0000000000..7e4817a943 --- /dev/null +++ b/docker/scripts/collect-workspace-sdists.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Build source distributions for local packages installed into a Python environment.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import urllib.parse +from pathlib import Path + + +def source_collection_enabled() -> bool: + return os.environ.get("NMP_COLLECT_SOURCES", "0").strip().lower() in {"1", "true", "yes", "on"} + + +def record_source_collection_disabled(output_dir: Path) -> None: + manifests = output_dir / "manifests" + manifests.mkdir(parents=True, exist_ok=True) + (manifests / "source-collection-disabled.txt").write_text( + "source collection disabled; set NMP_COLLECT_SOURCES=1 to enable\n", + encoding="utf-8", + ) + + +def run_text(command: list[str]) -> str: + return subprocess.check_output(command, text=True, stderr=subprocess.STDOUT) + + +def uv_bin() -> str: + return os.environ.get("UV_BIN", "uv") + + +def purelib_for(python: str) -> Path: + script = "import sysconfig; print(sysconfig.get_paths()['purelib'])" + return Path(run_text([python, "-c", script]).strip()) + + +def direct_url_projects(purelib: Path) -> set[Path]: + projects: set[Path] = set() + for dist_info in purelib.glob("*.dist-info"): + direct_url = dist_info / "direct_url.json" + if not direct_url.is_file(): + continue + try: + data = json.loads(direct_url.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + url = str(data.get("url", "")) + if not url.startswith("file://"): + continue + path = Path(urllib.parse.unquote(urllib.parse.urlparse(url).path)) + if (path / "pyproject.toml").is_file(): + projects.add(path) + return projects + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--output", required=True, help="Source distribution output directory") + parser.add_argument("--python", help="Python interpreter whose installed local packages should be inspected") + parser.add_argument("--project", action="append", default=[], help="Additional local project path to build") + args = parser.parse_args() + + output_dir = Path(args.output) + workspace_dir = output_dir / "workspace" + manifests = output_dir / "manifests" + manifests.mkdir(parents=True, exist_ok=True) + if not source_collection_enabled(): + record_source_collection_disabled(output_dir) + return 0 + + workspace_dir.mkdir(parents=True, exist_ok=True) + + projects: set[Path] = set() + missing: list[str] = [] + if args.python: + try: + projects.update(direct_url_projects(purelib_for(args.python))) + except subprocess.CalledProcessError as error: + (manifests / "workspace-sdist-errors.txt").write_text( + f"failed to inspect Python environment {args.python}: {error.output}\n", + encoding="utf-8", + ) + for project in args.project: + path = Path(project) + if not path.exists(): + missing.append(f"{path}\tproject path does not exist") + elif not (path / "pyproject.toml").is_file(): + missing.append(f"{path}\tmissing pyproject.toml") + else: + projects.add(path) + + built: list[str] = [] + for project in sorted(projects): + try: + output = run_text([uv_bin(), "build", "--sdist", "--out-dir", str(workspace_dir), str(project)]) + except (OSError, subprocess.CalledProcessError) as error: + if isinstance(error, subprocess.CalledProcessError) and error.output: + message = error.output.strip() + else: + message = str(error) + missing.append(f"{project}\t{message}") + continue + built.append(f"{project}\t{output.strip()}") + + (manifests / "built-workspace-sdists.txt").write_text("\n".join(built) + ("\n" if built else ""), encoding="utf-8") + (manifests / "missing-workspace-sdists.txt").write_text( + "\n".join(missing) + ("\n" if missing else ""), encoding="utf-8" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/unit/test_collect_apt_sources.py b/tests/unit/test_collect_apt_sources.py new file mode 100644 index 0000000000..cd21601902 --- /dev/null +++ b/tests/unit/test_collect_apt_sources.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import stat +import subprocess +import textwrap +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "docker" / "scripts" / "collect-apt-sources.sh" + + +def write_executable(path: Path, content: str) -> None: + path.write_text(textwrap.dedent(content).lstrip()) + path.chmod(path.stat().st_mode | stat.S_IXUSR) + + +def test_installed_sources_use_installed_source_version_and_explicit_sources_stay_unversioned( + tmp_path: Path, +) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + apt_get_log = tmp_path / "apt-get.log" + dpkg_query_log = tmp_path / "dpkg-query.log" + + write_executable( + fake_bin / "dpkg-query", + f""" + #!/usr/bin/env bash + printf '%s\\n' "$*" >> "{dpkg_query_log}" + printf 'libssl3:amd64\\t3.0.2-0ubuntu1.20\\topenssl\\t3.0.2-0ubuntu1.20\\n' + """, + ) + write_executable( + fake_bin / "apt-cache", + """ + #!/usr/bin/env bash + package="${@: -1}" + case "${package}" in + libssl3:amd64) + printf 'Package: libssl3\\nSource: openssl (3.0.2-0ubuntu1.20)\\n' + ;; + explicit-bin) + printf 'Package: explicit-bin\\nSource: explicit-src (9.9-1)\\n' + ;; + esac + """, + ) + write_executable( + fake_bin / "apt-get", + f""" + #!/usr/bin/env bash + printf '%s\\n' "$*" >> "{apt_get_log}" + case "$1" in + update|source|clean) + exit 0 + ;; + esac + exit 1 + """, + ) + + env = os.environ.copy() + env["PATH"] = f"{fake_bin}{os.pathsep}{env['PATH']}" + env["NMP_COLLECT_SOURCES"] = "1" + output_dir = tmp_path / "out" + + result = subprocess.run( + ["bash", str(SCRIPT), str(output_dir), "--installed", "explicit-bin"], + check=False, + cwd=tmp_path, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + assert result.returncode == 0, result.stderr + dpkg_query_args = dpkg_query_log.read_text() + assert "${source:Package}" in dpkg_query_args + assert "${source:Version}" in dpkg_query_args + + apt_get_calls = apt_get_log.read_text().splitlines() + assert "source --download-only --only-source openssl=3.0.2-0ubuntu1.20" in apt_get_calls + assert "source --download-only --only-source explicit-src" in apt_get_calls + assert not any("explicit-src=" in call for call in apt_get_calls) + + downloaded_sources = (output_dir / "manifests" / "downloaded-apt-sources.txt").read_text().splitlines() + assert "openssl=3.0.2-0ubuntu1.20" in downloaded_sources + assert "explicit-src" in downloaded_sources diff --git a/tests/unit/test_collect_cpython_source.py b/tests/unit/test_collect_cpython_source.py new file mode 100644 index 0000000000..be947f7b5a --- /dev/null +++ b/tests/unit/test_collect_cpython_source.py @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import hashlib +import sys +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "docker" / "scripts" / "collect-cpython-source.py" + + +def load_module(): + spec = spec_from_file_location("collect_cpython_source", SCRIPT) + assert spec is not None + assert spec.loader is not None + module = module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_download_failures_are_not_swallowed(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + module = load_module() + + def fail_download(url: str, destination: Path) -> None: + raise RuntimeError("download failed") + + monkeypatch.setattr(module, "python_version", lambda python: "3.12.11") + monkeypatch.setattr(module, "download", fail_download) + monkeypatch.setenv("NMP_COLLECT_SOURCES", "1") + monkeypatch.setattr( + sys, + "argv", + [str(SCRIPT), "--python", "/fake/python", "--output", str(tmp_path)], + ) + + with pytest.raises(RuntimeError, match="download failed"): + module.main() + + assert not (tmp_path / "manifests" / "missing-cpython-source.txt").exists() + + +def test_download_verifies_release_checksum_before_writing(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + module = load_module() + archive_url = "https://www.python.org/ftp/python/3.12.13/Python-3.12.13.tgz" + destination = tmp_path / "Python-3.12.13.tgz" + valid_archive = b"valid archive" + tampered_archive = b"tampered archive" + checksum = hashlib.sha256(valid_archive).hexdigest() + release_page = f""" + + + + + +
Download Gzipped source tarball{checksum[:32]} {checksum[32:]}
+ """.encode() + archive_attempts = 0 + + class Response: + def __init__(self, body: bytes) -> None: + self.body = body + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def read(self) -> bytes: + return self.body + + def fake_urlopen(url: str, timeout: int) -> Response: + nonlocal archive_attempts + if url == "https://www.python.org/downloads/release/python-31213/": + return Response(release_page) + if url == archive_url: + archive_attempts += 1 + if archive_attempts == 2: + assert not destination.exists() + return Response(tampered_archive if archive_attempts == 1 else valid_archive) + raise AssertionError(f"unexpected URL: {url}") + + monkeypatch.setattr(module.urllib.request, "urlopen", fake_urlopen) + monkeypatch.setattr(module.time, "sleep", lambda seconds: None) + + module.download(archive_url, destination) + + assert archive_attempts == 2 + assert destination.read_bytes() == valid_archive diff --git a/tests/unit/test_collect_workspace_sdists.py b/tests/unit/test_collect_workspace_sdists.py new file mode 100644 index 0000000000..81c1f059e8 --- /dev/null +++ b/tests/unit/test_collect_workspace_sdists.py @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import sys +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "docker" / "scripts" / "collect-workspace-sdists.py" + + +def load_module(): + spec = spec_from_file_location("collect_workspace_sdists", SCRIPT) + assert spec is not None + assert spec.loader is not None + module = module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_explicit_invalid_projects_are_recorded_as_missing( + monkeypatch, + tmp_path: Path, +) -> None: + module = load_module() + valid_project = tmp_path / "valid-project" + valid_project.mkdir() + (valid_project / "pyproject.toml").write_text("[project]\nname = 'valid-project'\n", encoding="utf-8") + directory_without_pyproject = tmp_path / "not-a-project" + directory_without_pyproject.mkdir() + nonexistent_project = tmp_path / "missing-project" + output_dir = tmp_path / "out" + build_commands: list[list[str]] = [] + + def fake_run_text(command: list[str]) -> str: + build_commands.append(command) + return "built valid-project" + + monkeypatch.setattr(module, "run_text", fake_run_text) + monkeypatch.setenv("NMP_COLLECT_SOURCES", "1") + monkeypatch.delenv("UV_BIN", raising=False) + monkeypatch.setattr( + sys, + "argv", + [ + str(SCRIPT), + "--output", + str(output_dir), + "--project", + str(valid_project), + "--project", + str(directory_without_pyproject), + "--project", + str(nonexistent_project), + ], + ) + + assert module.main() == 0 + + assert build_commands == [ + [ + "uv", + "build", + "--sdist", + "--out-dir", + str(output_dir / "workspace"), + str(valid_project), + ] + ] + assert (output_dir / "manifests" / "built-workspace-sdists.txt").read_text(encoding="utf-8") == ( + f"{valid_project}\tbuilt valid-project\n" + ) + assert (output_dir / "manifests" / "missing-workspace-sdists.txt").read_text(encoding="utf-8") == ( + f"{directory_without_pyproject}\tmissing pyproject.toml\n{nonexistent_project}\tproject path does not exist\n" + ) + + +def test_uv_startup_errors_are_recorded_as_missing( + monkeypatch, + tmp_path: Path, +) -> None: + module = load_module() + valid_project = tmp_path / "valid-project" + valid_project.mkdir() + (valid_project / "pyproject.toml").write_text("[project]\nname = 'valid-project'\n", encoding="utf-8") + output_dir = tmp_path / "out" + + def fake_run_text(command: list[str]) -> str: + raise OSError("uv unavailable") + + monkeypatch.setattr(module, "run_text", fake_run_text) + monkeypatch.setenv("NMP_COLLECT_SOURCES", "1") + monkeypatch.delenv("UV_BIN", raising=False) + monkeypatch.setattr( + sys, + "argv", + [ + str(SCRIPT), + "--output", + str(output_dir), + "--project", + str(valid_project), + ], + ) + + assert module.main() == 0 + + assert (output_dir / "manifests" / "built-workspace-sdists.txt").read_text(encoding="utf-8") == "" + assert (output_dir / "manifests" / "missing-workspace-sdists.txt").read_text(encoding="utf-8") == ( + f"{valid_project}\tuv unavailable\n" + ) diff --git a/third_party/licenses.jsonl b/third_party/licenses.jsonl index 4960d1b3e1..ec35c2da3b 100644 --- a/third_party/licenses.jsonl +++ b/third_party/licenses.jsonl @@ -30,7 +30,7 @@ {"name": "bracex", "license": "MIT", "compatible": true} {"name": "cachetools", "license": "MIT", "compatible": true} {"name": "caio", "license": "APACHE-2.0", "compatible": true} -{"name": "certifi", "license": "LGPL", "compatible": true} +{"name": "certifi", "license": "MPL-2.0", "compatible": true} {"name": "cffi", "license": "MIT", "compatible": true} {"name": "chardet", "license": "LGPL-2.1-OR-LATER", "compatible": true} {"name": "charset-normalizer", "license": "MIT", "compatible": true} @@ -141,7 +141,7 @@ {"name": "google.golang.org/protobuf", "license": "BSD-3-CLAUSE", "compatible": true} {"name": "googleapis-common-protos", "license": "APACHE-2.0", "compatible": true} {"name": "gopkg.in/yaml.v3", "license": "MIT", "compatible": true} -{"name": "greenlet", "license": "MIT", "compatible": true} +{"name": "greenlet", "license": "MIT AND PYTHON-2.0", "compatible": true} {"name": "griffelib", "license": "ISC", "compatible": true} {"name": "grpcio", "license": "APACHE-2.0", "compatible": true} {"name": "gunicorn", "license": "MIT", "compatible": true} @@ -196,7 +196,7 @@ {"name": "langchain-mcp-adapters", "license": "MIT", "compatible": true} {"name": "langchain-milvus", "license": "MIT", "compatible": true} {"name": "langchain-nvidia-ai-endpoints", "license": "MIT", "compatible": true} -{"name": "langchain-oci", "license": "MIT", "compatible": true} +{"name": "langchain-oci", "license": "UPL-1.0", "compatible": true} {"name": "langchain-openai", "license": "MIT", "compatible": true} {"name": "langchain-protocol", "license": "MIT", "compatible": true} {"name": "langchain-text-splitters", "license": "MIT", "compatible": true} @@ -281,7 +281,7 @@ {"name": "packaging", "license": "APACHE-2.0", "compatible": true} {"name": "pandas", "license": "BSD-3-CLAUSE", "compatible": true} {"name": "pathable", "license": "APACHE-2.0", "compatible": true} -{"name": "pathspec", "license": "LGPL", "compatible": true} +{"name": "pathspec", "license": "MPL-2.0", "compatible": true} {"name": "pillow", "license": "MIT-CMU", "compatible": true} {"name": "pip", "license": "MIT", "compatible": true} {"name": "pkce", "license": "MIT", "compatible": true} @@ -298,7 +298,7 @@ {"name": "propcache", "license": "APACHE-2.0", "compatible": true} {"name": "protobuf", "license": "BSD-3-CLAUSE", "compatible": true} {"name": "psutil", "license": "BSD-3-CLAUSE", "compatible": true} -{"name": "psycopg2-binary", "license": "LGPL", "compatible": true} +{"name": "psycopg2-binary", "license": "LGPL-3.0-OR-LATER WITH OPENVPN-OPENSSL-EXCEPTION", "compatible": true} {"name": "py-key-value-aio", "license": "APACHE-2.0", "compatible": true} {"name": "py-rust-stemmers", "license": "MIT", "compatible": true} {"name": "pyarrow", "license": "APACHE-2.0", "compatible": true} @@ -325,7 +325,7 @@ {"name": "ragas", "license": "APACHE-2.0", "compatible": true} {"name": "realtime", "license": "MIT", "compatible": true} {"name": "referencing", "license": "MIT", "compatible": true} -{"name": "regex", "license": "APACHE-2.0", "compatible": true} +{"name": "regex", "license": "APACHE-2.0 AND CNRI-PYTHON", "compatible": true} {"name": "requests", "license": "APACHE-2.0", "compatible": true} {"name": "requests-oauthlib", "license": "ISC", "compatible": true} {"name": "requests-toolbelt", "license": "APACHE-2.0", "compatible": true} @@ -379,7 +379,7 @@ {"name": "tomli-w", "license": "MIT", "compatible": true} {"name": "tomlkit", "license": "MIT", "compatible": true} {"name": "tornado", "license": "APACHE-2.0", "compatible": true} -{"name": "tqdm", "license": "MIT", "compatible": true} +{"name": "tqdm", "license": "MIT AND MPL-2.0", "compatible": true} {"name": "transformers", "license": "APACHE-2.0", "compatible": true} {"name": "typer", "license": "MIT", "compatible": true} {"name": "types-aioboto3", "license": "MIT", "compatible": true} diff --git a/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/license/overrides.yaml b/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/license/overrides.yaml index 2aaf3b1ea7..3a601829c5 100644 --- a/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/license/overrides.yaml +++ b/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/license/overrides.yaml @@ -103,7 +103,7 @@ overrides: amqp: BSD-3-Clause # AMQP protocol implementation billiard: BSD-3-Clause # Multiprocessing pool (used by Celery) flower: BSD-3-Clause # Celery monitoring tool - psycopg2-binary: LGPL # PostgreSQL adapter + psycopg2-binary: LGPL-3.0-or-later WITH openvpn-openssl-exception # PostgreSQL adapter pyarrow: Apache-2.0 # Apache Arrow Python bindings vine: BSD-3-Clause # Promises/futures library for Celery @@ -169,7 +169,7 @@ overrides: # NLP / AI Evaluation fastembed: Apache-2.0 # Fast text embedding - langchain-oci: MIT # LangChain Oracle Cloud Infrastructure integration + langchain-oci: UPL-1.0 # LangChain Oracle Cloud Infrastructure integration ragas: Apache-2.0 # RAG evaluation framework range-regex: BSD-3-Clause # Regex for ranges ratelimit: MIT # API rate limiting @@ -228,12 +228,12 @@ overrides: # Licenses not automatically approved aiohappyeyeballs: PSF-2.0 - certifi: LGPL + certifi: MPL-2.0 datetime: ZPL-2.0 email-validator: Unlicense filelock: Unlicense nvidia-cutlass-dsl: NVIDIA-Software-End-User-License-Agreement - pathspec: LGPL + pathspec: MPL-2.0 pillow: MIT-CMU pycountry: LGPL-2.1 python-gitlab: LGPL-3.0 @@ -247,9 +247,12 @@ overrides: backports-zstd: PSF-2.0 dspy: MIT + greenlet: MIT AND Python-2.0 narwhals: MIT + regex: Apache-2.0 AND CNRI-Python rpds-py: MIT sqlite-vec: MIT # https://pypi.org/project/sqlite-vec/ + tqdm: MIT AND MPL-2.0 uuid-utils: BSD-3-Clause # Auth service diff --git a/tools/nemo-platform-sdk-tools/tests/license/test_license_utils.py b/tools/nemo-platform-sdk-tools/tests/license/test_license_utils.py index 78db988fab..b714747eed 100644 --- a/tools/nemo-platform-sdk-tools/tests/license/test_license_utils.py +++ b/tools/nemo-platform-sdk-tools/tests/license/test_license_utils.py @@ -233,6 +233,37 @@ def test_format_licenses_table_applies_override_for_cu129_version(self): class TestFormatLicenses: """Tests for license report formatting.""" + def test_reviewed_license_overrides_cover_corrected_inventory_values(self): + """Reviewed overrides keep generated license inventory from losing corrected SPDX expressions.""" + import yaml + + repo_root = Path(__file__).parents[4] + overrides_file = Path(__file__).parents[2] / "src" / "nemo_platform_sdk_tools" / "license" / "overrides.yaml" + inventory_file = repo_root / "third_party" / "licenses.jsonl" + overrides = yaml.safe_load(overrides_file.read_text(encoding="utf-8"))["overrides"] + normalized_overrides = {normalize_package_name(name): license for name, license in overrides.items()} + inventory_licenses = { + normalize_package_name(row["name"]): row["license"] + for row in (json.loads(line) for line in inventory_file.read_text(encoding="utf-8").splitlines()) + } + expected_licenses = { + "certifi": "MPL-2.0", + "greenlet": "MIT AND Python-2.0", + "langchain-oci": "UPL-1.0", + "pathspec": "MPL-2.0", + "psycopg2-binary": "LGPL-3.0-or-later WITH openvpn-openssl-exception", + "regex": "Apache-2.0 AND CNRI-Python", + "tqdm": "MIT AND MPL-2.0", + } + normalized_expected = { + normalize_package_name(name): license_expression for name, license_expression in expected_licenses.items() + } + + assert {name: normalized_overrides[name] for name in normalized_expected} == normalized_expected + assert {name: inventory_licenses[name] for name in normalized_expected} == { + name: license_expression.upper() for name, license_expression in normalized_expected.items() + } + def test_format_licenses_fills_missing_osv_package_from_overrides(self, tmp_path): """A reviewed override fills an exported requirement omitted by OSV.""" from nemo_platform_sdk_tools.license.generator import format_licenses