From 38c16ba9d26fccdf706e3bcef8e5006f5053eee1 Mon Sep 17 00:00:00 2001 From: Matt Kornfield Date: Wed, 12 Aug 2026 15:21:26 +0000 Subject: [PATCH 1/5] chore: CVE upgrades for wandb, gitpython and cleanups Signed-off-by: Matt Kornfield --- docker/Dockerfile.nmp-api | 2 ++ docker/Dockerfile.nmp-cpu-tasks | 2 +- docker/Dockerfile.nmp-customizer-tasks | 9 ++++++++- docker/Dockerfile.nmp-unsloth-training | 3 +++ docker/Dockerfile.safe-synthesizer-tasks | 4 ++-- .../automodel/Dockerfile.nmp-automodel-base | 8 ++++++-- docker/automodel/no_override_requirements.txt | 2 +- docker/rl/Dockerfile.nmp-rl-base | 19 +++++++++++++++++++ docker/scripts/cve-cleanup.sh | 12 +++++++++--- docker/unsloth/no_override_requirements.txt | 2 +- pyproject.toml | 4 ++-- third_party/requirements-main.txt | 14 +++++++------- uv.lock | 19 ++++++++++--------- 13 files changed, 71 insertions(+), 29 deletions(-) diff --git a/docker/Dockerfile.nmp-api b/docker/Dockerfile.nmp-api index 48a3cb8ddc..1855fa5016 100644 --- a/docker/Dockerfile.nmp-api +++ b/docker/Dockerfile.nmp-api @@ -91,6 +91,8 @@ COPY --chown=1000:1000 --from=nmp-studio-ui /artifacts /static/studio COPY --chown=1000:1000 --from=builder /app/plugins/nemo-data-designer/tiktoken-cache /app/tiktoken-cache ENV TIKTOKEN_CACHE_DIR=/app/tiktoken-cache COPY --chown=1000:1000 --from=builder /tmp/fastembed_cache /tmp/fastembed_cache +COPY docker/scripts/cve-cleanup.sh /bin/ +RUN sh /bin/cve-cleanup.sh ARG NMP_PLATFORM_VERSION=dev ARG NMP_CODE_REVISION= ENV NMP_PLATFORM_VERSION=${NMP_PLATFORM_VERSION} diff --git a/docker/Dockerfile.nmp-cpu-tasks b/docker/Dockerfile.nmp-cpu-tasks index 3c3f46e14a..022210bf54 100644 --- a/docker/Dockerfile.nmp-cpu-tasks +++ b/docker/Dockerfile.nmp-cpu-tasks @@ -21,7 +21,7 @@ ENV TIKTOKEN_CACHE_DIR=/app/tiktoken-cache ENV PATH="/app/.venv/bin:$PATH" USER 0 COPY docker/scripts/cve-cleanup.sh /bin/ -RUN bash /bin/cve-cleanup.sh +RUN sh /bin/cve-cleanup.sh ENTRYPOINT ["nemo-platform"] # Override CMD with "run" in orchestration (e.g., Helm, docker-compose) CMD ["--help"] diff --git a/docker/Dockerfile.nmp-customizer-tasks b/docker/Dockerfile.nmp-customizer-tasks index 9fc44860f7..178e61074f 100644 --- a/docker/Dockerfile.nmp-customizer-tasks +++ b/docker/Dockerfile.nmp-customizer-tasks @@ -80,7 +80,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \ "soupsieve>=2.8.4,<3" \ "tornado>=6.5.7,<7" \ "urllib3>=2.7.0,<3" \ - "wandb==0.28.1" + "GitPython>=3.1.58,<4" \ + "wandb==0.28.2" # CVE cleanup for unused/stale packages from the NGC base image. The task entry # points import from /opt/venv; scanners still report the shadowed system copies. @@ -103,11 +104,15 @@ RUN apt-get update && \ /usr/local/lib/python3.12/dist-packages/grpc \ /usr/local/lib/python3.12/dist-packages/grpcio \ /usr/local/lib/python3.12/dist-packages/grpcio-*.dist-info \ + /usr/local/lib/python3.12/dist-packages/flash_attn \ + /usr/local/lib/python3.12/dist-packages/flash_attn-*.dist-info \ /usr/local/lib/python3.12/dist-packages/wandb \ /usr/local/lib/python3.12/dist-packages/wandb-*.dist-info \ /usr/local/lib/python3.12/dist-packages/mlflow \ /usr/local/lib/python3.12/dist-packages/mlflow-*.dist-info \ /usr/local/lib/python3.12/dist-packages/mlflow_skinny-*.dist-info \ + /usr/local/lib/python3.12/dist-packages/git \ + /usr/local/lib/python3.12/dist-packages/gitpython-*.dist-info \ /usr/local/lib/python3.12/dist-packages/mistune \ /usr/local/lib/python3.12/dist-packages/mistune-*.dist-info \ /usr/local/lib/python3.12/dist-packages/PIL \ @@ -117,6 +122,8 @@ RUN apt-get update && \ /usr/local/lib/python3.12/dist-packages/soupsieve-*.dist-info \ /usr/local/lib/python3.12/dist-packages/jupyter_server \ /usr/local/lib/python3.12/dist-packages/jupyter_server-*.dist-info \ + /usr/local/lib/python3.12/dist-packages/jupyterlab \ + /usr/local/lib/python3.12/dist-packages/jupyterlab-*.dist-info \ /usr/local/cuda/NsightSystems-cli-* \ /usr/local/bin/nsys \ /usr/local/cuda/bin/nsys diff --git a/docker/Dockerfile.nmp-unsloth-training b/docker/Dockerfile.nmp-unsloth-training index f432ac6050..38a07327f7 100644 --- a/docker/Dockerfile.nmp-unsloth-training +++ b/docker/Dockerfile.nmp-unsloth-training @@ -180,6 +180,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ "mistune>=3.3.3,<4" \ "pillow>=12.3.0,<13" \ "soupsieve>=2.8.4,<3" \ + "GitPython>=3.1.58,<4" \ "mlflow-skinny>=3.11.1,<3.12.0" # Stale NGC system site-packages (verified on nvcr.io/nvidia/pytorch:26.05-py3). @@ -222,6 +223,8 @@ RUN rm -rf \ /usr/local/lib/python3.12/dist-packages/soupsieve-*.dist-info \ /usr/local/lib/python3.12/dist-packages/jupyter_server \ /usr/local/lib/python3.12/dist-packages/jupyter_server-*.dist-info \ + /usr/local/lib/python3.12/dist-packages/jupyterlab \ + /usr/local/lib/python3.12/dist-packages/jupyterlab-*.dist-info \ /usr/local/cuda/NsightSystems-cli-* \ /usr/local/bin/nsys \ /usr/local/cuda/bin/nsys diff --git a/docker/Dockerfile.safe-synthesizer-tasks b/docker/Dockerfile.safe-synthesizer-tasks index 8c59fa3d71..28c9d09844 100644 --- a/docker/Dockerfile.safe-synthesizer-tasks +++ b/docker/Dockerfile.safe-synthesizer-tasks @@ -115,7 +115,7 @@ RUN printf '%s\n' \ "${SAFE_SYNTHESIZER_RUNTIME_PACKAGE}" \ > /tmp/safe-synthesizer-runtime.txt && \ printf '%s\n' \ - wandb==0.28.1 \ + wandb==0.28.2 \ 'aiohttp>=3.14.3,<4' \ 'cryptography>=50.0.0,<51' \ 'pyarrow>=23.0.1,<24' \ @@ -195,7 +195,7 @@ COPY docker/scripts/cve-cleanup.sh /bin/ # Perl CVE cleanup removes dpkg-dev/build-essential; keep the non-Perl # compiler toolchain available for runtime JIT/native extension builds. RUN apt-mark manual gcc g++ make libc6-dev && \ - bash /bin/cve-cleanup.sh && \ + sh /bin/cve-cleanup.sh && \ printf 'int main(void) { return 0; }\n' | gcc -x c - -o /tmp/gcc-check && \ rm -f /tmp/gcc-check RUN mkdir -p "${XDG_CACHE_HOME}" "${MPLCONFIGDIR}" "${OUTLINES_CACHE_DIR}" && \ diff --git a/docker/automodel/Dockerfile.nmp-automodel-base b/docker/automodel/Dockerfile.nmp-automodel-base index ecb37c0ba5..3873c4e496 100644 --- a/docker/automodel/Dockerfile.nmp-automodel-base +++ b/docker/automodel/Dockerfile.nmp-automodel-base @@ -94,7 +94,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install \ "aiohttp>=3.13.3,<4" \ "black>=26.3.1,<27" \ - "GitPython>=3.1.57,<4" \ + "GitPython>=3.1.58,<4" \ "jaraco-context>=6.1.0,<7" \ "jupyter-server>=2.20.0,<3" \ "jupyterlab>=4.6.1,<5" \ @@ -111,7 +111,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ "urllib3>=2.7.0,<3" \ "mlflow-skinny>=3.11.1,<3.12.0" \ "grpcio>=1.81.1,<2" \ - "wandb==0.28.1" + "wandb==0.28.2" # Published base image (same filesystem as builder). FROM ${PYTORCH_BASE} AS nmp-automodel-base @@ -153,6 +153,8 @@ RUN rm -rf \ /usr/local/lib/python3.12/dist-packages/mlflow \ /usr/local/lib/python3.12/dist-packages/mlflow-*.dist-info \ /usr/local/lib/python3.12/dist-packages/mlflow_skinny-*.dist-info \ + /usr/local/lib/python3.12/dist-packages/git \ + /usr/local/lib/python3.12/dist-packages/gitpython-*.dist-info \ /usr/local/lib/python3.12/dist-packages/PIL \ /usr/local/lib/python3.12/dist-packages/pillow.libs \ /usr/local/lib/python3.12/dist-packages/pillow-*.dist-info \ @@ -162,6 +164,8 @@ RUN rm -rf \ /usr/local/lib/python3.12/dist-packages/soupsieve-*.dist-info \ /usr/local/lib/python3.12/dist-packages/jupyter_server \ /usr/local/lib/python3.12/dist-packages/jupyter_server-*.dist-info \ + /usr/local/lib/python3.12/dist-packages/jupyterlab \ + /usr/local/lib/python3.12/dist-packages/jupyterlab-*.dist-info \ /usr/local/cuda/NsightSystems-cli-* \ /usr/local/bin/nsys \ /usr/local/cuda/bin/nsys diff --git a/docker/automodel/no_override_requirements.txt b/docker/automodel/no_override_requirements.txt index b50dec921a..858f5301f9 100644 --- a/docker/automodel/no_override_requirements.txt +++ b/docker/automodel/no_override_requirements.txt @@ -10,4 +10,4 @@ safetensors; sys_platform == 'never' numpy; sys_platform == 'never' # Keep W&B's bundled wandb-core binary on a build with patched go-git/go-billy. -wandb==0.28.1 +wandb==0.28.2 diff --git a/docker/rl/Dockerfile.nmp-rl-base b/docker/rl/Dockerfile.nmp-rl-base index 7d418c25f2..d1cb4542dd 100644 --- a/docker/rl/Dockerfile.nmp-rl-base +++ b/docker/rl/Dockerfile.nmp-rl-base @@ -194,6 +194,10 @@ uv sync ${UV_SYNC_MODE} --all-groups --no-install-project # build/test/et # in this cached compile layer (the editable ROOT install moves below, after the full-source COPY). uv pip install --python /opt/nemo_rl_venv/bin/python "opensandbox>=0.1.9" "tenacity>=9.1.4" +# W&B ships a scanner-visible Go binary. Keep it above the fixed go-git/grpc +# and Go stdlib floors even when NeMo-RL's lock has an older wheel. +uv pip install --python /opt/nemo_rl_venv/bin/python "wandb==0.28.2" + # CVE GHSA-mqqc-3gqh-h2x8: drop ray's bundled old aiohttp from the uv cache. find "${UV_CACHE_DIR}" -type d -path "*ray/_private/runtime_env/agent/thirdparty_files/aiohttp*" -exec rm -rf {} + || true @@ -325,6 +329,21 @@ if [ -d "${GYM_CACHE}" ]; then fi echo "in-tree Gym cache is ${GYM_CACHE_MB} MB (metadata only) - ok" fi + +# Some actor/env venvs may contain W&B depending on the selected NeMo-RL extras. +# Upgrade those in place, then drop any now-unreferenced older W&B cache archive +# so scanners do not report stale wandb-core binaries under /opt/uv_cache. +for py in /opt/ray_venvs/*/bin/python /opt/gym_venvs/*/bin/python; do + [ -x "${py}" ] || continue + if "${py}" -c 'import importlib.util, sys; sys.exit(0 if importlib.util.find_spec("wandb") else 1)'; then + uv pip install --python "${py}" "wandb==0.28.2" + fi +done +for d in "${UV_CACHE_DIR}"/archive-v0/*/; do + ls "${d}"wandb-*.dist-info >/dev/null 2>&1 || continue + find /opt/nemo_rl_venv /opt/ray_venvs /opt/gym_venvs -lname "*$(basename "${d}")*" -print -quit 2>/dev/null | grep -q . && continue + rm -rf "${d}" +done EOF # Container fingerprint (matches NeMo-RL's release stage). With NRL_CONTAINER=1 set, nemo_rl's diff --git a/docker/scripts/cve-cleanup.sh b/docker/scripts/cve-cleanup.sh index 2e66bb368d..debc9fc61a 100644 --- a/docker/scripts/cve-cleanup.sh +++ b/docker/scripts/cve-cleanup.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/sh set -e @@ -14,7 +14,7 @@ set -e # binaries does not remove the scanner finding. for venv in /app/.venv /opt/venv; do if [ -d "${venv}" ]; then - if [ -x "${venv}/bin/python" ]; then + if [ -x "${venv}/bin/python" ] && command -v uv >/dev/null 2>&1; then uv pip uninstall --python "${venv}/bin/python" \ opencv-contrib-python \ opencv-contrib-python-headless \ @@ -23,7 +23,9 @@ for venv in /app/.venv /opt/venv; do fi # Fallback for other packages that ship standalone ffmpeg binaries. - find "${venv}" -type f \( -name ffmpeg -o -name 'ffmpeg-*' \) -delete + if command -v find >/dev/null 2>&1; then + find "${venv}" -type f \( -name ffmpeg -o -name 'ffmpeg-*' \) -delete + fi fi done @@ -87,6 +89,10 @@ rm -rf /usr/local/lib/python3.12/dist-packages/urllib3 \ rm -rf /usr/local/lib/python3.12/dist-packages/mlflow \ /usr/local/lib/python3.12/dist-packages/mlflow-*.dist-info \ /usr/local/lib/python3.12/dist-packages/mlflow_skinny-*.dist-info +rm -rf /usr/local/lib/python3.12/dist-packages/git \ + /usr/local/lib/python3.12/dist-packages/gitpython-*.dist-info +rm -rf /usr/local/lib/python3.12/dist-packages/jupyterlab \ + /usr/local/lib/python3.12/dist-packages/jupyterlab-*.dist-info rm -rf /usr/local/lib/python3.12/dist-packages/PIL \ /usr/local/lib/python3.12/dist-packages/pillow.libs \ /usr/local/lib/python3.12/dist-packages/pillow-*.dist-info diff --git a/docker/unsloth/no_override_requirements.txt b/docker/unsloth/no_override_requirements.txt index 157ff66714..ee090a12f7 100644 --- a/docker/unsloth/no_override_requirements.txt +++ b/docker/unsloth/no_override_requirements.txt @@ -11,4 +11,4 @@ accelerate; sys_platform == 'never' safetensors; sys_platform == 'never' # Keep W&B's bundled wandb-core binary on a build with patched go-git/go-billy. -wandb==0.28.1 +wandb==0.28.2 diff --git a/pyproject.toml b/pyproject.toml index 84452d8779..03832977c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -272,7 +272,7 @@ environments = [ ] constraint-dependencies = [ - "GitPython>=3.1.57", + "GitPython>=3.1.58", "Mako>=1.3.12", "Pygments>=2.20.0", "aiohttp>=3.14.1", # High/Medium/Low OOS – auditor-tasks + customizer @@ -340,7 +340,7 @@ override-dependencies = [ "ujson>=5.12.0", # High – nmp-api/cpu-tasks/gpu-tasks "xgrammar>=0.1.32", # High – customizer + nmp-gpu-tasks "fastmcp>=3.2.0", # GHSA-vv7q-7jx5-f767 (Critical) + GHSA-rww4-4w9c-7733 (High); overrides vendored sdk/python/nemo-platform <3 constraint - "wandb>=0.28.1", # wandb-core Go CVEs (nspect); unsloth + training images + "wandb>=0.28.2", # wandb-core Go CVEs (nspect); unsloth + training images "cryptography>=50.0.0,<51", # GHSA-g6cj-pr64-35w5; overrides nvidia-nat-core's <47 cap after changelog review "click>=8.2.0", # Below CVEs are based on constraints that garak has "litellm>=1.83.10", # CVE-2026-42208 (Critical SQL injection), CVE-2026-40217, and 5 other High CVEs diff --git a/third_party/requirements-main.txt b/third_party/requirements-main.txt index 3d29c50b3a..63fb346690 100644 --- a/third_party/requirements-main.txt +++ b/third_party/requirements-main.txt @@ -3505,13 +3505,13 @@ validators==0.35.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') --hash=sha256:992d6c48a4e77c81f1b4daba10d16c3a9bb0dbb79b3a19ea847ff0928e70497a \ --hash=sha256:e8c947097eae7892cb3d26868d637f79f47b4a0554bc6b80065dfe5aac3705dd # via ngcsdk -wandb==0.28.1 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ - --hash=sha256:7233061080507a4b4098bed1ccb381ce6f890c60397cd4153d060285bcb267bd \ - --hash=sha256:870ccb1a01238b0ac07c6fd96a0810a1f79090aba04ea29f4ee012ac8327705d \ - --hash=sha256:8cfb898b6a6c884d9c9294b02764e88bce65049f027a124d6bee53fe722469b6 \ - --hash=sha256:ae9ae6fb29e2e2b1d097ed8b75c0c0240c778c2a8cad1d996dee870a1e401c2c \ - --hash=sha256:cf2b1533945395e4fdbe6182b272bb0ca8a02c10b3086a395e2d57686ae3ed0d \ - --hash=sha256:da909a76e65c64c0d93acc485d2a19f66e336f1e3f725f1c98a070883e084943 +wandb==0.28.2 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ + --hash=sha256:0c475c6b93d57e0455f3d1323e323927696502e6af7ee15e389a552114edacff \ + --hash=sha256:1db698d107871c66b2dcbb0cf4dc2af1ddb159ba94e957e890158ec60ab2de54 \ + --hash=sha256:81468f5c00b5453a2a1d3c7ca7a18056c52a85fa4df60299052cb96829d81b11 \ + --hash=sha256:86e017d3f38bb99374de4d991a2ea0e6e71722164918585b3dea66190b28d0eb \ + --hash=sha256:9dba560ce076f96a0033a0d2898d97cba651fc95fec7274fd09920bae8aec5cf \ + --hash=sha256:f9aa4037d18168dd4e804ff8213100c4a3ebe3e52b5f29463bbc7fadcaadff75 # via nmp-unsloth wasmtime==47.0.1 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ --hash=sha256:0aeb53d4c8e682cccebbaac48882939b34e71a1243b622129de3e6834728044d \ diff --git a/uv.lock b/uv.lock index 69fd80b99c..99aace453a 100644 --- a/uv.lock +++ b/uv.lock @@ -84,7 +84,7 @@ constraints = [ { name = "datamodel-code-generator", specifier = ">=0.71.0" }, { name = "diffusers", specifier = ">=0.38.0" }, { name = "filelock", specifier = ">=3.20.3" }, - { name = "gitpython", specifier = ">=3.1.57" }, + { name = "gitpython", specifier = ">=3.1.58" }, { name = "joserfc", specifier = ">=1.7.2" }, { name = "json-repair", specifier = ">=0.60.1" }, { name = "jupyter-server", specifier = ">=2.20.0" }, @@ -144,7 +144,7 @@ overrides = [ { name = "simpleeval", specifier = ">=1.0.5" }, { name = "ujson", specifier = ">=5.12.0" }, { name = "urllib3", specifier = ">=2.7.0" }, - { name = "wandb", specifier = ">=0.28.1" }, + { name = "wandb", specifier = ">=0.28.2" }, { name = "wheel", specifier = ">=0.46.2" }, { name = "xgrammar", specifier = ">=0.1.32" }, ] @@ -11417,10 +11417,11 @@ wheels = [ [[package]] name = "wandb" -version = "0.28.1" +version = "0.28.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "opentelemetry-api", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "packaging", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "platformdirs", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "protobuf", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -11430,13 +11431,13 @@ dependencies = [ { name = "sentry-sdk", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/92/fb/8d3f96a8b143060d6fa145462d0785981373e04694e4152555ccb5d23939/wandb-0.28.1.tar.gz", hash = "sha256:870ccb1a01238b0ac07c6fd96a0810a1f79090aba04ea29f4ee012ac8327705d", size = 40578119, upload-time = "2026-07-16T18:47:05.413Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/41/c2eb0e25e0504287442f0829a5dc380669b683f78068c79220acc626383f/wandb-0.28.2.tar.gz", hash = "sha256:9dba560ce076f96a0033a0d2898d97cba651fc95fec7274fd09920bae8aec5cf", size = 41011367, upload-time = "2026-08-12T01:34:20.484Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/06/21/8df50164d07623cfcefec19bbf9327d9be84b637a827cea1f0c06db005fd/wandb-0.28.1-py3-none-macosx_12_0_arm64.whl", hash = "sha256:da909a76e65c64c0d93acc485d2a19f66e336f1e3f725f1c98a070883e084943", size = 24277925, upload-time = "2026-07-16T18:46:42.383Z" }, - { url = "https://files.pythonhosted.org/packages/e2/1a/d15bcfb4417fa69edcaa33db8ea012db733da1057e193b047e3f69fdd671/wandb-0.28.1-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:ae9ae6fb29e2e2b1d097ed8b75c0c0240c778c2a8cad1d996dee870a1e401c2c", size = 24832138, upload-time = "2026-07-16T18:46:47.433Z" }, - { url = "https://files.pythonhosted.org/packages/b3/da/49924c7df2952dfd82c86c3779c339c0c3d6f6439387c03d97d0470c3658/wandb-0.28.1-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:8cfb898b6a6c884d9c9294b02764e88bce65049f027a124d6bee53fe722469b6", size = 26486533, upload-time = "2026-07-16T18:46:49.839Z" }, - { url = "https://files.pythonhosted.org/packages/11/c0/06b23518e29690784f1b3081e39c7679ca076cb0af094cb9b4bb309150f5/wandb-0.28.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:cf2b1533945395e4fdbe6182b272bb0ca8a02c10b3086a395e2d57686ae3ed0d", size = 25022635, upload-time = "2026-07-16T18:46:52.376Z" }, - { url = "https://files.pythonhosted.org/packages/23/30/6de2f7995a8a6eecbd03d24c79a139a734c0168f5520cf4c7ccb43c1dbbc/wandb-0.28.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:7233061080507a4b4098bed1ccb381ce6f890c60397cd4153d060285bcb267bd", size = 27008895, upload-time = "2026-07-16T18:46:55.025Z" }, + { url = "https://files.pythonhosted.org/packages/47/d9/8fb47c6c0b6e70378fc2b5da6dfba5d1d3178366a06bd7e672c3249964c3/wandb-0.28.2-py3-none-macosx_12_0_arm64.whl", hash = "sha256:f9aa4037d18168dd4e804ff8213100c4a3ebe3e52b5f29463bbc7fadcaadff75", size = 26716653, upload-time = "2026-08-12T01:33:58.338Z" }, + { url = "https://files.pythonhosted.org/packages/03/46/e69c813d88f37f77da92cfd9c63e1e9b5ce70834b0f420f876c66590353f/wandb-0.28.2-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:0c475c6b93d57e0455f3d1323e323927696502e6af7ee15e389a552114edacff", size = 27385453, upload-time = "2026-08-12T01:34:03.876Z" }, + { url = "https://files.pythonhosted.org/packages/3c/7f/f3e23a2ff01848d7a5adab9f307992b92ef99c5e6bff2bd89cd46d651e3a/wandb-0.28.2-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:1db698d107871c66b2dcbb0cf4dc2af1ddb159ba94e957e890158ec60ab2de54", size = 29622780, upload-time = "2026-08-12T01:34:06.621Z" }, + { url = "https://files.pythonhosted.org/packages/9e/1d/e087f71898ebfa86f68264e91f44ee5b8106823f09e2644043e3f1d5fe49/wandb-0.28.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:86e017d3f38bb99374de4d991a2ea0e6e71722164918585b3dea66190b28d0eb", size = 27425107, upload-time = "2026-08-12T01:34:09.359Z" }, + { url = "https://files.pythonhosted.org/packages/f2/58/3f7a624c880d7b9e60b4a000b0bb751c20244d0bcc002d3ff683cf828651/wandb-0.28.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:81468f5c00b5453a2a1d3c7ca7a18056c52a85fa4df60299052cb96829d81b11", size = 29851453, upload-time = "2026-08-12T01:34:12.133Z" }, ] [[package]] From d208961e29dbfdaf4414f0cfaaf7edbe12563c6f Mon Sep 17 00:00:00 2001 From: Matt Kornfield Date: Wed, 12 Aug 2026 17:56:58 +0000 Subject: [PATCH 2/5] chore: harden docker artifact downloads Signed-off-by: Matt Kornfield --- docker/base/Dockerfile.policy-wasm | 8 ++++++-- docker/rl/Dockerfile.nmp-rl-base | 5 +++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docker/base/Dockerfile.policy-wasm b/docker/base/Dockerfile.policy-wasm index fb006c0fc4..a57db9889a 100644 --- a/docker/base/Dockerfile.policy-wasm +++ b/docker/base/Dockerfile.policy-wasm @@ -11,8 +11,12 @@ RUN set -eux; \ esac; \ opa_asset="opa_linux_${opa_arch}_static"; \ opa_url="https://github.com/open-policy-agent/opa/releases/download/${OPA_VERSION}/${opa_asset}"; \ - curl -fsSL -o "/tmp/${opa_asset}" "${opa_url}"; \ - curl -fsSL -o "/tmp/${opa_asset}.sha256" "${opa_url}.sha256"; \ + curl -fsSL --retry 5 --retry-delay 2 --retry-all-errors \ + --connect-timeout 10 --max-time 60 --retry-max-time 180 \ + -o "/tmp/${opa_asset}" "${opa_url}"; \ + curl -fsSL --retry 5 --retry-delay 2 --retry-all-errors \ + --connect-timeout 10 --max-time 20 --retry-max-time 90 \ + -o "/tmp/${opa_asset}.sha256" "${opa_url}.sha256"; \ (cd /tmp && sha256sum -c "${opa_asset}.sha256"); \ chmod +x "/tmp/${opa_asset}"; \ mv "/tmp/${opa_asset}" /usr/local/bin/opa; \ diff --git a/docker/rl/Dockerfile.nmp-rl-base b/docker/rl/Dockerfile.nmp-rl-base index d1cb4542dd..12ae4f3da9 100644 --- a/docker/rl/Dockerfile.nmp-rl-base +++ b/docker/rl/Dockerfile.nmp-rl-base @@ -79,8 +79,13 @@ ARG PYTHON_VERSION # UV_PYTHON pins the interpreter for every uv call, at build and at runtime (node-built venvs too). # Required, not cosmetic: RL ships a `.python-version` that otherwise wins, so bumping PYTHON_VERSION # alone leaves every venv on RL's patch release and the requested one unused on disk. +# Raise uv's HTTP retry budget and cap concurrent downloads for large GitHub-hosted wheels like vLLM, +# which can hit transient HTTP/2 refused-stream failures under BuildKit. ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python \ UV_PYTHON=${PYTHON_VERSION} \ + UV_HTTP_TIMEOUT=120 \ + UV_HTTP_RETRIES=8 \ + UV_CONCURRENT_DOWNLOADS=4 \ PATH="/usr/local/bin:/root/.local/bin:${PATH}" RUN curl -LsSf "https://astral.sh/uv/${UV_VERSION}/install.sh" | sh && \ cp /root/.local/bin/uv /usr/local/bin/uv && \ From 2476aa308709530ff3c6cf683bd5aa6e1e30bef6 Mon Sep 17 00:00:00 2001 From: Matt Kornfield Date: Wed, 12 Aug 2026 18:17:15 +0000 Subject: [PATCH 3/5] chore: reuse OPA fallback downloader in docker builds Signed-off-by: Matt Kornfield --- docker/base/Dockerfile.policy-wasm | 21 +----------- script/build_policy_wasm.sh | 53 ++++++++++++++++++++---------- 2 files changed, 36 insertions(+), 38 deletions(-) diff --git a/docker/base/Dockerfile.policy-wasm b/docker/base/Dockerfile.policy-wasm index a57db9889a..6f61012773 100644 --- a/docker/base/Dockerfile.policy-wasm +++ b/docker/base/Dockerfile.policy-wasm @@ -2,29 +2,10 @@ FROM alpine:3.19 AS root-policy-wasm-builder RUN apk add --no-cache curl tar ARG OPA_VERSION=v1.8.0 -ARG TARGETARCH -RUN set -eux; \ - case "${TARGETARCH}" in \ - amd64|arm64) opa_arch="${TARGETARCH}" ;; \ - "") opa_arch="$(uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/')" ;; \ - *) echo "Unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; \ - esac; \ - opa_asset="opa_linux_${opa_arch}_static"; \ - opa_url="https://github.com/open-policy-agent/opa/releases/download/${OPA_VERSION}/${opa_asset}"; \ - curl -fsSL --retry 5 --retry-delay 2 --retry-all-errors \ - --connect-timeout 10 --max-time 60 --retry-max-time 180 \ - -o "/tmp/${opa_asset}" "${opa_url}"; \ - curl -fsSL --retry 5 --retry-delay 2 --retry-all-errors \ - --connect-timeout 10 --max-time 20 --retry-max-time 90 \ - -o "/tmp/${opa_asset}.sha256" "${opa_url}.sha256"; \ - (cd /tmp && sha256sum -c "${opa_asset}.sha256"); \ - chmod +x "/tmp/${opa_asset}"; \ - mv "/tmp/${opa_asset}" /usr/local/bin/opa; \ - rm -f "/tmp/${opa_asset}.sha256" WORKDIR /build COPY script/build_policy_wasm.sh /build/script/ COPY services/core/auth/src/nmp/core/auth/app/policies /build/services/core/auth/src/nmp/core/auth/app/policies -RUN REPO_ROOT=/build OUTPUT_DIR=/artifacts sh /build/script/build_policy_wasm.sh +RUN OPA_VERSION="${OPA_VERSION}" REPO_ROOT=/build OUTPUT_DIR=/artifacts sh /build/script/build_policy_wasm.sh FROM scratch AS root-policy-wasm-artifacts COPY --from=root-policy-wasm-builder /artifacts/policy.wasm /artifacts/policy.wasm diff --git a/script/build_policy_wasm.sh b/script/build_policy_wasm.sh index dd93be78e5..f4f5756520 100755 --- a/script/build_policy_wasm.sh +++ b/script/build_policy_wasm.sh @@ -12,6 +12,11 @@ # OPA_VERSION - OPA release to use. Default: v1.8.0 # OPA_BIN - Optional explicit OPA binary path. Must match OPA_VERSION. # OPA_CACHE_DIR - Directory for downloaded OPA binaries. Default: .cache/opa +# OPA_DOWNLOAD_BASE_URLS +# - Space-separated OPA download roots to try in order. +# Default: official OPA downloads, then GitHub releases. +# OPA_DOWNLOAD_BASE_URL +# - Legacy single download root override. set -eu # --- Configuration --- @@ -19,7 +24,13 @@ REPO_ROOT="${REPO_ROOT:-$(git rev-parse --show-toplevel)}" OPA_VERSION="${OPA_VERSION:-v1.8.0}" OPA_VERSION_NO_V="${OPA_VERSION#v}" OPA_CACHE_DIR="${OPA_CACHE_DIR:-${REPO_ROOT}/.cache/opa}" -OPA_DOWNLOAD_BASE_URL="${OPA_DOWNLOAD_BASE_URL:-https://openpolicyagent.org/downloads}" +if [ -n "${OPA_DOWNLOAD_BASE_URLS:-}" ]; then + RESOLVED_OPA_DOWNLOAD_BASE_URLS="${OPA_DOWNLOAD_BASE_URLS}" +elif [ -n "${OPA_DOWNLOAD_BASE_URL:-}" ]; then + RESOLVED_OPA_DOWNLOAD_BASE_URLS="${OPA_DOWNLOAD_BASE_URL}" +else + RESOLVED_OPA_DOWNLOAD_BASE_URLS="https://openpolicyagent.org/downloads https://github.com/open-policy-agent/opa/releases/download" +fi POLICY_DIR="${REPO_ROOT}/services/core/auth/src/nmp/core/auth/app/policies" OUTPUT_DIR="${OUTPUT_DIR:-${REPO_ROOT}/services/core/auth/src/nmp/core/auth/assets}" @@ -109,24 +120,30 @@ download_opa() { } trap cleanup_download EXIT - url="${OPA_DOWNLOAD_BASE_URL}/${OPA_VERSION}/${asset}" - sha_url="${url}.sha256" - echo "Downloading OPA ${OPA_VERSION} from ${url}..." >&2 # Bound both a single attempt (--max-time) and the whole retry window (--retry-max-time). - # --max-time RESETS on each retry, so without --retry-max-time the aggregate is unbounded: 4 - # attempts could run for minutes. That matters because callers impose their own ceiling — - # embedded_pdp/policy_wasm.py runs this script with DEFAULT_BUILD_TIMEOUT_SECONDS=120 — and would - # kill the build mid-retry. Budget: <=45s aggregate + <=30s final attempt here, <=15s + <=10s for - # the checksum, leaving headroom for the `opa build` itself inside 120s. - if ! curl -fsSL --retry 3 --retry-delay 2 --retry-all-errors \ - --connect-timeout 10 --max-time 30 --retry-max-time 45 "${url}" -o "${tmp_bin}"; then - echo "Failed to download OPA binary from ${url}." >&2 - print_opa_help "${asset}" - exit 1 - fi - if ! curl -fsSL --retry 3 --retry-delay 2 --retry-all-errors \ - --connect-timeout 10 --max-time 10 --retry-max-time 15 "${sha_url}" -o "${tmp_sha}"; then - echo "Failed to download OPA checksum from ${sha_url}." >&2 + # --max-time RESETS on each retry, so without --retry-max-time the aggregate is unbounded. That + # matters because callers impose their own ceiling — embedded_pdp/policy_wasm.py runs this script + # with DEFAULT_BUILD_TIMEOUT_SECONDS=120 — and would kill the build mid-retry. Keep each download + # root bounded so the default official+GitHub fallback still leaves headroom for `opa build`. + downloaded=0 + for base_url in ${RESOLVED_OPA_DOWNLOAD_BASE_URLS}; do + url="${base_url%/}/${OPA_VERSION}/${asset}" + sha_url="${url}.sha256" + echo "Downloading OPA ${OPA_VERSION} from ${url}..." >&2 + + if curl -fsSL --retry 3 --retry-delay 2 --retry-all-errors \ + --connect-timeout 10 --max-time 30 --retry-max-time 45 "${url}" -o "${tmp_bin}" && \ + curl -fsSL --retry 3 --retry-delay 2 --retry-all-errors \ + --connect-timeout 10 --max-time 10 --retry-max-time 15 "${sha_url}" -o "${tmp_sha}"; then + downloaded=1 + break + fi + + echo "Failed to download OPA from ${base_url}; trying next source if configured." >&2 + rm -f "${tmp_bin}" "${tmp_sha}" + done + + if [ "${downloaded}" != "1" ]; then print_opa_help "${asset}" exit 1 fi From 1d9b99097cb165861d5a553eb75a1f28e5dc40d4 Mon Sep 17 00:00:00 2001 From: Matt Kornfield Date: Wed, 12 Aug 2026 20:00:23 +0000 Subject: [PATCH 4/5] fix: safe-synthesizer main fix and retry fix Signed-off-by: Matt Kornfield --- docker/rl/Dockerfile.nmp-rl-base | 31 +++++++++--- .../tasks/safe_synthesizer/__main__.py | 8 +++- .../tests/unit/test_local_run.py | 48 +++++++++++++++++++ 3 files changed, 79 insertions(+), 8 deletions(-) diff --git a/docker/rl/Dockerfile.nmp-rl-base b/docker/rl/Dockerfile.nmp-rl-base index 12ae4f3da9..76c6d6c3a3 100644 --- a/docker/rl/Dockerfile.nmp-rl-base +++ b/docker/rl/Dockerfile.nmp-rl-base @@ -187,12 +187,31 @@ RUN <<"EOF" bash -exu export UV_LINK_MODE=symlink uv venv --seed -uv sync ${UV_SYNC_MODE} --no-install-project -uv sync ${UV_SYNC_MODE} --extra vllm --no-install-project # GRPO generation (vllm 0.20 cu130 + flashinfer + deep_gemm/deep_ep) -uv sync ${UV_SYNC_MODE} --extra fsdp --no-install-project # DPO/GRPO policy training (flash-attn, mamba-ssm, causal-conv1d) -uv sync ${UV_SYNC_MODE} --extra modelopt --no-install-project # quantization / model-opt -uv sync ${UV_SYNC_MODE} --extra nemo_gym --no-install-project # NeMo-Gym (uv workspace member) -uv sync ${UV_SYNC_MODE} --all-groups --no-install-project # build/test/etc groups +uv_sync() { + local attempt + local sleep_seconds + + for attempt in 1 2 3; do + if uv sync "$@"; then + return 0 + fi + + if [ "${attempt}" -eq 3 ]; then + return 1 + fi + + sleep_seconds=$((attempt * 20)) + echo "uv sync failed; retrying in ${sleep_seconds}s (attempt $((attempt + 1))/3)" >&2 + sleep "${sleep_seconds}" + done +} + +uv_sync ${UV_SYNC_MODE} --no-install-project +uv_sync ${UV_SYNC_MODE} --extra vllm --no-install-project # GRPO generation (vllm 0.20 cu130 + flashinfer + deep_gemm/deep_ep) +uv_sync ${UV_SYNC_MODE} --extra fsdp --no-install-project # DPO/GRPO policy training (flash-attn, mamba-ssm, causal-conv1d) +uv_sync ${UV_SYNC_MODE} --extra modelopt --no-install-project # quantization / model-opt +uv_sync ${UV_SYNC_MODE} --extra nemo_gym --no-install-project # NeMo-Gym (uv workspace member) +uv_sync ${UV_SYNC_MODE} --all-groups --no-install-project # build/test/etc groups # Gym sandbox CLIENT SDK (OpenSandbox provider). The OpenSandbox SERVER is deployed separately on # the cluster; this is the client the Gym stack imports to call it. Source-independent, so it stays diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py index 8a34fc6f18..d3e03b51e1 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py @@ -354,9 +354,13 @@ def run_config( else: ss.process_data() total_time = time.monotonic() - (ss._total_start or time.monotonic()) - pii_replaced_df = getattr(ss, "_train_df", None) + pii_replaced_df = getattr(ss, "_training_df", None) if pii_replaced_df is None: - raise RuntimeError("process_data() completed but _train_df is None") + # Older upstream builds used _train_df; keep a fallback so + # plugin-local runs do not depend on exactly one NSS internals name. + pii_replaced_df = getattr(ss, "_train_df", None) + if pii_replaced_df is None: + raise RuntimeError("process_data() completed but processed training data is unavailable") from nemo_safe_synthesizer.results import make_nss_results ss.results = make_nss_results(generate_results=pii_replaced_df, total_time=total_time) diff --git a/plugins/nemo-safe-synthesizer/tests/unit/test_local_run.py b/plugins/nemo-safe-synthesizer/tests/unit/test_local_run.py index ae213eba6c..718e3a9e00 100644 --- a/plugins/nemo-safe-synthesizer/tests/unit/test_local_run.py +++ b/plugins/nemo-safe-synthesizer/tests/unit/test_local_run.py @@ -89,6 +89,54 @@ def test_validate_flat_tabular_data_allows_flat_columns(monkeypatch): task_main._validate_flat_tabular_data(data) +def test_run_config_pii_only_uses_processed_training_df(tmp_path, monkeypatch): + task_main = import_task_main_without_heavy_runtime(monkeypatch) + processed = pd.DataFrame({"name": ["REDACTED"], "age": [1]}) + + class FakeSafeSynthesizer: + def __init__(self, config, save_path): + self._total_start = None + self._training_df = None + self._workdir = None + + def with_data_source(self, data_source): + return self + + def process_data(self): + self._training_df = processed + + monkeypatch.setattr(task_main, "SafeSynthesizer", FakeSafeSynthesizer) + results_module = ModuleType("nemo_safe_synthesizer.results") + setattr( + results_module, + "make_nss_results", + lambda generate_results, total_time=None: SimpleNamespace( + synthetic_data=generate_results, + summary=SimpleNamespace(model_dump=lambda: {"row_count": len(generate_results)}), + evaluation_report_html=None, + ), + ) + monkeypatch.setitem(sys.modules, "nemo_safe_synthesizer.results", results_module) + job_config = task_main.SafeSynthesizerJobConfig.model_validate( + { + "data_source": "default/data#input.csv", + "config": { + "enable_synthesis": False, + "enable_replace_pii": False, + }, + } + ) + + result, adapter_path = task_main.run_config( + job_config, + pd.DataFrame({"name": ["Alice"], "age": [1]}), + tmp_path, + ) + + assert adapter_path is None + pd.testing.assert_frame_equal(result.synthetic_data, processed) + + def test_run_from_env_reports_missing_config_path(monkeypatch): task_main = import_task_main_without_heavy_runtime(monkeypatch) monkeypatch.setattr(task_main, "initialize_observability", lambda: None) From b67b648e28ac50f04c657574689f8bc7ebdbaf4a Mon Sep 17 00:00:00 2001 From: Matt Kornfield Date: Thu, 13 Aug 2026 15:10:59 +0000 Subject: [PATCH 5/5] chore: notice file update Signed-off-by: Matt Kornfield --- NOTICE | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/NOTICE b/NOTICE index 7a40515ade..0a719da4be 100644 --- a/NOTICE +++ b/NOTICE @@ -123,13 +123,13 @@ Ray-bundled JAXB runtime dependencies wandb-core MPL-2.0 Go dependencies Scope: Unmodified Go module dependencies linked into the wandb-core executable distributed with the wandb Python package in container images. - wandb package version: 0.28.1 + wandb package version: 0.28.2 wandb source distribution: - https://files.pythonhosted.org/packages/92/fb/8d3f96a8b143060d6fa145462d0785981373e04694e4152555ccb5d23939/wandb-0.28.1.tar.gz + https://files.pythonhosted.org/packages/fd/41/c2eb0e25e0504287442f0829a5dc380669b683f78068c79220acc626383f/wandb-0.28.2.tar.gz wandb upstream source: https://github.com/wandb/wandb wandb-core source at reported build commit: - https://github.com/wandb/wandb/tree/v0.28.1 + https://github.com/wandb/wandb/tree/v0.28.2 License: MPL-2.0 License text: LICENSES/MPL-2.0.txt