From f406dbdc17481ea11590894597dec23c9060c3a2 Mon Sep 17 00:00:00 2001 From: Johannes Rausch Date: Wed, 12 Aug 2026 13:54:33 +0200 Subject: [PATCH 1/4] Automate Puzzletron GPU lifecycle tests Pin the execution image and dependency contract so the focused lifecycle workflow runs against a reproducible one-GPU environment. Signed-off-by: Johannes Rausch --- .github/workflows/puzzletron_gpu_tests.yml | 105 ++++++++++++ examples/puzzletron/ci/Dockerfile | 31 ++++ examples/puzzletron/ci/README.md | 40 +++++ examples/puzzletron/ci/resolve_ci_image.py | 77 +++++++++ examples/puzzletron/ci/setup_env.sh | 159 ++++++++++++++++++ examples/puzzletron/ci_environment.json | 6 + noxfile.py | 27 ++- .../puzzletron/test_ci_image_contract.py | 105 ++++++++++++ 8 files changed, 541 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/puzzletron_gpu_tests.yml create mode 100644 examples/puzzletron/ci/Dockerfile create mode 100644 examples/puzzletron/ci/README.md create mode 100644 examples/puzzletron/ci/resolve_ci_image.py create mode 100644 examples/puzzletron/ci/setup_env.sh create mode 100644 tests/unit/torch/puzzletron/test_ci_image_contract.py diff --git a/.github/workflows/puzzletron_gpu_tests.yml b/.github/workflows/puzzletron_gpu_tests.yml new file mode 100644 index 00000000000..b99fe568ba4 --- /dev/null +++ b/.github/workflows/puzzletron_gpu_tests.yml @@ -0,0 +1,105 @@ +name: Puzzletron GPU tests + +on: + push: + branches: ["pull-request/[0-9]+"] + schedule: + - cron: "30 1 * * *" + workflow_dispatch: + # On-demand + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ startsWith(github.ref, 'refs/heads/pull-request/') && github.ref || github.sha }} + cancel-in-progress: true + +jobs: + pr-gate: + uses: ./.github/workflows/_pr_gate.yml + permissions: + checks: read + secrets: inherit + with: + files: | + .github/workflows/_pr_gate.yml + .github/workflows/puzzletron_gpu_tests.yml + .github/actions/cache-extensions/** + examples/puzzletron/**/*.py + examples/puzzletron/**/*.sh + examples/puzzletron/**/*.yaml + examples/puzzletron/ci/** + examples/puzzletron/ci_environment.json + examples/puzzletron/requirements.txt + modelopt/torch/puzzletron/** + noxfile.py + puzzletron_orchestrator/** + puzzletron_setup/** + pyproject.toml + tests/conftest.py + tests/_test_utils/torch/puzzletron/** + tests/_test_utils/torch/transformers_models.py + tests/gpu/torch/puzzletron/** + + resolve-image: + needs: [pr-gate] + if: needs.pr-gate.outputs.run_tests == 'true' + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + image: ${{ steps.image.outputs.image }} + cache_key: ${{ steps.image.outputs.cache_key }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Resolve immutable Puzzletron image + id: image + env: + PUZZLETRON_GPU_CI_IMAGE: ${{ vars.PUZZLETRON_GPU_CI_IMAGE }} + run: python examples/puzzletron/ci/resolve_ci_image.py >> "${GITHUB_OUTPUT}" + + gpu-puzzletron: + needs: [resolve-image] + runs-on: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} + timeout-minutes: 50 + container: + image: ${{ needs.resolve-image.outputs.image }} + credentials: + username: "$oauthtoken" + password: ${{ secrets.NGC_API_KEY }} + options: --shm-size=16gb + env: + GIT_DEPTH: 1000 + PIP_CONSTRAINT: "" + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: nv-gha-runners/setup-proxy-cache@main + - uses: ./.github/actions/cache-extensions + with: + cache-key: rtxpro6000-puzzletron-${{ needs.resolve-image.outputs.cache_key }} + - name: Run the Puzzletron lifecycle gate + run: nox -s gpu_puzzletron + + gpu-puzzletron-required-check: + if: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && always() }} + needs: [pr-gate, resolve-image, gpu-puzzletron] + runs-on: ubuntu-latest + steps: + - name: Report intentionally scoped Puzzletron GPU tests + if: needs.pr-gate.outputs.run_tests != 'true' + run: | + echo "## Puzzletron GPU tests were not required" >> "${GITHUB_STEP_SUMMARY}" + echo >> "${GITHUB_STEP_SUMMARY}" + echo "No Puzzletron lifecycle path changed in this pull request." >> "${GITHUB_STEP_SUMMARY}" + - name: Required Puzzletron GPU tests did not succeed + if: >- + ${{ needs.pr-gate.result != 'success' || + (needs.pr-gate.outputs.run_tests == 'true' && + (needs.resolve-image.result != 'success' || + needs.gpu-puzzletron.result != 'success')) }} + run: exit 1 diff --git a/examples/puzzletron/ci/Dockerfile b/examples/puzzletron/ci/Dockerfile new file mode 100644 index 00000000000..9fc75152848 --- /dev/null +++ b/examples/puzzletron/ci/Dockerfile @@ -0,0 +1,31 @@ +FROM nvidia/cuda:12.9.2-cudnn-devel-ubuntu24.04@sha256:b4db213759eb86d55a7271909bdad891fab300ec5700fb4f4656463b2f51980f + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ARG DEBIAN_FRONTEND=noninteractive + +ENV VIRTUAL_ENV=/venv +ENV PATH=/venv/bin/:$PATH +ENV PIP_NO_CACHE_DIR=1 +ENV PUZZLETRON_CI_ENVIRONMENT=/opt/puzzletron/ci_environment.json +ENV PUZZLETRON_REQUIREMENTS=/opt/puzzletron/requirements.txt +ENV PYTHONUNBUFFERED=1 + +COPY examples/puzzletron/ci_environment.json /opt/puzzletron/ci_environment.json +COPY examples/puzzletron/requirements.txt /opt/puzzletron/requirements.txt +COPY examples/puzzletron/ci/setup_env.sh /opt/puzzletron/setup_env.sh +COPY pyproject.toml LICENSE_HEADER /opt/modelopt-dependencies/ + +RUN bash /opt/puzzletron/setup_env.sh --deps +RUN mkdir -p /opt/modelopt-dependencies/modelopt && \ + touch /opt/modelopt-dependencies/modelopt/__init__.py && \ + python -m pip install "/opt/modelopt-dependencies[hf,puzzletron,dev-test]" && \ + python -m pip uninstall -y nvidia-modelopt && \ + python -m pip check && \ + bash /opt/puzzletron/setup_env.sh --verify + +LABEL org.opencontainers.image.source="https://github.com/NVIDIA/Model-Optimizer" \ + com.nvidia.modelopt.puzzletron.environment="examples/puzzletron/ci_environment.json" \ + com.nvidia.modelopt.puzzletron.environment-recipe="examples/puzzletron/ci/setup_env.sh" + +WORKDIR /workspace/modelopt diff --git a/examples/puzzletron/ci/README.md b/examples/puzzletron/ci/README.md new file mode 100644 index 00000000000..89fc3361635 --- /dev/null +++ b/examples/puzzletron/ci/README.md @@ -0,0 +1,40 @@ +# Puzzletron GPU lifecycle CI image + +This directory defines the execution image for the hermetic one-GPU Puzzletron +lifecycle gate. The image supplies the pinned CUDA and Python dependencies; +the `gpu_puzzletron` Nox session installs the checked-out ModelOpt revision and +runs the sole reusable tiny-Qwen lifecycle test. + +The base image is pinned by OCI digest in both +[`Dockerfile`](Dockerfile) and [`ci_environment.json`](../ci_environment.json). +The environment file also owns the exact Torch, Transformers, LMMS-Eval, +AutoModel, AIPerf, and Nox versions. The image preinstalls ModelOpt's declared +runtime and test dependencies. The setup script verifies the pins when the +image is built and again after installing only the checked-out ModelOpt source, +without resolving dependencies or using build isolation at runtime. + +Build the image from the repository root: + +```bash +docker build \ + --platform linux/amd64 \ + --file examples/puzzletron/ci/Dockerfile \ + --tag modelopt-puzzletron-ci:local \ + . +``` + +Run the same entry point used by CI with one visible GPU: + +```bash +docker run --gpus device=0 --ipc=host --rm \ + --volume "${PWD}:/workspace/modelopt" \ + --workdir /workspace/modelopt \ + modelopt-puzzletron-ci:local \ + nox -s gpu_puzzletron +``` + +Publishing the reviewed image is a separate registry operation. After +publication, configure `PUZZLETRON_GPU_CI_IMAGE` with the complete immutable +`nvcr.io/...@sha256:...` reference and provide the pull-only `NGC_API_KEY` +secret. The workflow rejects tags and non-NVCR references before allocating a +GPU runner. diff --git a/examples/puzzletron/ci/resolve_ci_image.py b/examples/puzzletron/ci/resolve_ci_image.py new file mode 100644 index 00000000000..4f04ad1f5df --- /dev/null +++ b/examples/puzzletron/ci/resolve_ci_image.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Validate and resolve the immutable image used by Puzzletron GPU CI.""" + +import json +import os +import re +import sys +from pathlib import Path + +_NVCR_IMAGE = re.compile( + r"nvcr\.io/(?:[A-Za-z0-9._-]+/)*[A-Za-z0-9._-]+@sha256:(?P[0-9a-f]{64})" +) +_CUDA_BASE_IMAGE = re.compile(r"nvidia/cuda:[A-Za-z0-9._-]+@sha256:[0-9a-f]{64}") + + +def resolve_image_reference(image: str) -> tuple[str, str]: + """Return an immutable nvcr.io image and its digest cache key.""" + match = _NVCR_IMAGE.fullmatch(image) + if match is None: + raise ValueError("PUZZLETRON_GPU_CI_IMAGE must be an immutable nvcr.io digest") + return image, match.group("digest") + + +def validate_repository_contract(repository_root: Path) -> None: + """Verify the checked-out image recipe agrees with its recorded environment.""" + ci_root = repository_root / "examples/puzzletron" + environment = json.loads((ci_root / "ci_environment.json").read_text()) + dockerfile = (ci_root / "ci/Dockerfile").read_text() + base_image = environment["gpu_image"]["base_image"] + + if _CUDA_BASE_IMAGE.fullmatch(base_image) is None: + raise ValueError("gpu_image.base_image must use a full lowercase SHA-256 digest") + + required_lines = ( + f"FROM {base_image}", + "ENV PUZZLETRON_CI_ENVIRONMENT=/opt/puzzletron/ci_environment.json", + "ENV PUZZLETRON_REQUIREMENTS=/opt/puzzletron/requirements.txt", + "COPY pyproject.toml LICENSE_HEADER /opt/modelopt-dependencies/", + "RUN bash /opt/puzzletron/setup_env.sh --deps", + 'python -m pip install "/opt/modelopt-dependencies[hf,puzzletron,dev-test]"', + "bash /opt/puzzletron/setup_env.sh --verify", + ) + missing = [line for line in required_lines if line not in dockerfile] + if missing: + raise ValueError(f"Dockerfile is missing recorded contract lines: {missing}") + + +def main() -> int: + """Write validated values in GitHub output format.""" + try: + validate_repository_contract(Path.cwd()) + image, cache_key = resolve_image_reference(os.environ.get("PUZZLETRON_GPU_CI_IMAGE", "")) + except (KeyError, OSError, ValueError, json.JSONDecodeError) as error: + print(f"::error::{error}", file=sys.stderr) + return 1 + + print(f"image={image}") + print(f"cache_key={cache_key}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/puzzletron/ci/setup_env.sh b/examples/puzzletron/ci/setup_env.sh new file mode 100644 index 00000000000..e22aa5b27c0 --- /dev/null +++ b/examples/puzzletron/ci/setup_env.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +MODE="${1:---deps}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MODEL_OPT_ROOT="${MODEL_OPT_ROOT:-$(cd "${SCRIPT_DIR}/../../.." && pwd)}" +VIRTUAL_ENV="${VIRTUAL_ENV:-/venv}" +PUZZLETRON_CI_ENVIRONMENT="${PUZZLETRON_CI_ENVIRONMENT:-${MODEL_OPT_ROOT}/examples/puzzletron/ci_environment.json}" +PUZZLETRON_REQUIREMENTS="${PUZZLETRON_REQUIREMENTS:-${MODEL_OPT_ROOT}/examples/puzzletron/requirements.txt}" + +export GIT_TERMINAL_PROMPT="${GIT_TERMINAL_PROMPT:-0}" +export PIP_NO_CACHE_DIR="${PIP_NO_CACHE_DIR:-1}" +export PYTHONUNBUFFERED="${PYTHONUNBUFFERED:-1}" +export VIRTUAL_ENV +export PATH="${VIRTUAL_ENV}/bin:${PATH}" + +environment_value() { + python3 - "${PUZZLETRON_CI_ENVIRONMENT}" "$1" <<'PY' +import json +import sys + +value = json.load(open(sys.argv[1], encoding="utf-8")) +for part in sys.argv[2].split("."): + value = value[part] +print(value) +PY +} + +verify_environment() { + python - "${PUZZLETRON_CI_ENVIRONMENT}" <<'PY' +import json +import sys +from importlib import metadata + +import torch +from packaging.version import Version + +environment = json.load(open(sys.argv[1], encoding="utf-8")) +expected = { + "python": environment["python"], + "torch": environment["torch"], + "torchvision": environment["torchvision"], + "transformers": environment["transformers"], + "lmms-eval": environment["lmms_eval"]["base_version"], + "nemo-automodel": environment["nemo_automodel"]["base_version"], + "aiperf": environment["gpu_image"]["aiperf"], + "nox": environment["gpu_image"]["nox"], +} +actual = { + "python": f"{sys.version_info.major}.{sys.version_info.minor}", + **{ + package: Version(metadata.version(package)).base_version + for package in expected + if package != "python" + }, +} +mismatches = { + package: (actual[package], expected_version) + for package, expected_version in expected.items() + if actual[package] != expected_version +} +if mismatches: + raise RuntimeError(f"Pinned Puzzletron image mismatch: {mismatches}") + +for package, key in (("lmms-eval", "lmms_eval"), ("nemo-automodel", "nemo_automodel")): + direct_url = json.loads(metadata.distribution(package).read_text("direct_url.json") or "{}") + vcs_info = direct_url.get("vcs_info") or {} + actual_source = ( + str(direct_url.get("url") or "").removesuffix(".git").rstrip("/"), + vcs_info.get("commit_id"), + ) + expected_source = ( + environment[key]["repository"].removesuffix(".git").rstrip("/"), + environment[key]["commit"], + ) + if actual_source != expected_source: + raise RuntimeError( + f"Pinned Puzzletron dependency {package!r} source mismatch: " + f"actual={actual_source!r}, expected={expected_source!r}" + ) + +if torch.version.cuda != environment["gpu_image"]["torch_cuda"]: + raise RuntimeError( + "Pinned Puzzletron CUDA mismatch: " + f"actual={torch.version.cuda!r}, expected={environment['gpu_image']['torch_cuda']!r}" + ) +PY +} + +install_dependencies() { + apt-get update + apt-get install -y --no-install-recommends \ + build-essential ca-certificates cmake git ninja-build \ + python3 python3-dev python3-pip python3-venv + apt-get clean + + python3 -m venv "${VIRTUAL_ENV}" + python -m pip install --upgrade \ + pip "setuptools>=80,<81" "setuptools-scm>=8,<10" setuptools-rust wheel \ + "packaging>=24.2" "cmake>=3.26.1" ninja jinja2 + + python -m pip install \ + "torch==$(environment_value torch)" \ + "torchvision==$(environment_value torchvision)" \ + "torchaudio==$(environment_value torch)" \ + --index-url https://download.pytorch.org/whl/cu129 + python -m pip install \ + -r "${PUZZLETRON_REQUIREMENTS}" \ + "nemo-automodel @ git+$(environment_value nemo_automodel.repository)@$(environment_value nemo_automodel.commit)" \ + "aiperf==$(environment_value gpu_image.aiperf)" \ + "nox==$(environment_value gpu_image.nox)" + python -m pip install "transformers==$(environment_value transformers)" + python -m pip check + verify_environment +} + +install_modelopt() { + if [[ ! -x "${VIRTUAL_ENV}/bin/python" ]] || \ + [[ "$(python -c 'import sys; print(sys.prefix)')" != "${VIRTUAL_ENV}" ]]; then + echo "Puzzletron CI must install ModelOpt inside ${VIRTUAL_ENV}." >&2 + exit 1 + fi + python -m pip uninstall -y nvidia-modelopt + python -m pip install --no-build-isolation --no-deps -e \ + "${MODEL_OPT_ROOT}[hf,puzzletron,dev-test]" + python -m pip check + verify_environment +} + +case "${MODE}" in + --deps) + install_dependencies + ;; + --modelopt) + install_modelopt + ;; + --verify) + verify_environment + ;; + *) + echo "Usage: $0 [--deps | --modelopt | --verify]" >&2 + exit 2 + ;; +esac diff --git a/examples/puzzletron/ci_environment.json b/examples/puzzletron/ci_environment.json index 9e39fd5d6b8..c0d79e5c23e 100644 --- a/examples/puzzletron/ci_environment.json +++ b/examples/puzzletron/ci_environment.json @@ -5,6 +5,12 @@ "torch": "2.11.0", "torchvision": "0.26.0", "transformers": "5.8.1", + "gpu_image": { + "base_image": "nvidia/cuda:12.9.2-cudnn-devel-ubuntu24.04@sha256:b4db213759eb86d55a7271909bdad891fab300ec5700fb4f4656463b2f51980f", + "torch_cuda": "12.9", + "aiperf": "0.11.0", + "nox": "2026.4.10" + }, "lmms_eval": { "base_version": "0.7.0", "repository": "https://github.com/EvolvingLMMs-Lab/lmms-eval.git", diff --git a/noxfile.py b/noxfile.py index 81771d49ade..be97c614cdb 100644 --- a/noxfile.py +++ b/noxfile.py @@ -66,7 +66,7 @@ ) -def _verify_puzzletron_v2_environment(session): +def _verify_puzzletron_v2_environment(session, *, gpu_image=False): """Fail before collection when the dedicated Puzzletron runtime drifts.""" expected_versions = { "python": PUZZLETRON_V2_CI_ENVIRONMENT["python"], @@ -76,6 +76,13 @@ def _verify_puzzletron_v2_environment(session): "lmms-eval": PUZZLETRON_V2_LMMS_SOURCE["base_version"], "nemo-automodel": PUZZLETRON_V2_AUTOMODEL_SOURCE["base_version"], } + if gpu_image: + expected_versions.update( + { + "aiperf": PUZZLETRON_V2_CI_ENVIRONMENT["gpu_image"]["aiperf"], + "nox": PUZZLETRON_V2_CI_ENVIRONMENT["gpu_image"]["nox"], + } + ) expected_vcs = { "lmms-eval": PUZZLETRON_V2_LMMS_SOURCE, "nemo-automodel": PUZZLETRON_V2_AUTOMODEL_SOURCE, @@ -96,12 +103,12 @@ def _verify_puzzletron_v2_environment(session): expected_vcs = {expected_vcs!r} actual = {{ "python": f"{{sys.version_info.major}}.{{sys.version_info.minor}}", - "torch": Version(version("torch")).base_version, - "torchvision": Version(version("torchvision")).base_version, - "transformers": Version(version("transformers")).base_version, - "lmms-eval": Version(version("lmms-eval")).base_version, - "nemo-automodel": Version(version("nemo-automodel")).base_version, }} +actual.update({{ + name: Version(version(name)).base_version + for name in expected + if name != "python" +}}) mismatches = {{ name: (actual[name], expected_version) for name, expected_version in expected.items() @@ -239,7 +246,8 @@ def gpu(session): def gpu_puzzletron(session): """Verify the pinned runtime, then run the focused Puzzletron lifecycle GPU test.""" session.env["CUDA_VISIBLE_DEVICES"] = os.environ.get("CUDA_VISIBLE_DEVICES", "0") - _verify_puzzletron_v2_environment(session) + session.run("bash", "examples/puzzletron/ci/setup_env.sh", "--modelopt", external=True) + _verify_puzzletron_v2_environment(session, gpu_image=True) session.run( "python", "-c", @@ -248,8 +256,9 @@ def gpu_puzzletron(session): "assert torch.cuda.is_available(), 'Puzzletron GPU CI requires CUDA'; " "assert torch.cuda.device_count() == 1, " "f'Puzzletron GPU CI requires exactly one visible GPU, got {torch.cuda.device_count()}'; " - "assert torch.version.cuda == '12.9', " - "f'Puzzletron GPU CI requires CUDA 12.9, got {torch.version.cuda}'" + f"assert torch.version.cuda == " + f"{PUZZLETRON_V2_CI_ENVIRONMENT['gpu_image']['torch_cuda']!r}, " + "f'Puzzletron GPU CI requires the pinned CUDA runtime, got {torch.version.cuda}'" ), ) session.run( diff --git a/tests/unit/torch/puzzletron/test_ci_image_contract.py b/tests/unit/torch/puzzletron/test_ci_image_contract.py new file mode 100644 index 00000000000..30f9bbea544 --- /dev/null +++ b/tests/unit/torch/puzzletron/test_ci_image_contract.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the repository-owned Puzzletron GPU image and workflow contract.""" + +import importlib.util +import json +import re + +import pytest +import yaml + + +def _load_image_resolver(project_root_path): + resolver_path = project_root_path / "examples/puzzletron/ci/resolve_ci_image.py" + spec = importlib.util.spec_from_file_location("puzzletron_ci_image", resolver_path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_gpu_image_uses_the_recorded_immutable_base(project_root_path): + environment_path = project_root_path / "examples/puzzletron/ci_environment.json" + dockerfile_path = project_root_path / "examples/puzzletron/ci/Dockerfile" + environment = json.loads(environment_path.read_text()) + dockerfile = dockerfile_path.read_text() + + base_image = environment["gpu_image"]["base_image"] + assert re.fullmatch(r"nvidia/cuda:[A-Za-z0-9._-]+@sha256:[0-9a-f]{64}", base_image) + assert f"FROM {base_image}" in dockerfile + assert "ENV PUZZLETRON_REQUIREMENTS=/opt/puzzletron/requirements.txt" in dockerfile + assert "RUN bash /opt/puzzletron/setup_env.sh --deps" in dockerfile + assert 'python -m pip install "/opt/modelopt-dependencies[hf,puzzletron,dev-test]"' in ( + dockerfile + ) + assert "bash /opt/puzzletron/setup_env.sh --verify" in dockerfile + + resolver = _load_image_resolver(project_root_path) + resolver.validate_repository_contract(project_root_path) + + +@pytest.mark.parametrize( + "image", + [ + "nvcr.io/nvidia/modelopt/puzzletron:latest", + "docker.io/nvidia/modelopt/puzzletron@sha256:" + "a" * 64, + "nvcr.io/nvidia/modelopt/puzzletron@sha256:" + "A" * 64, + "nvcr.io/nvidia/modelopt/puzzletron@sha256:" + "a" * 63, + "nvcr.io/nvidia//puzzletron@sha256:" + "a" * 64, + ], +) +def test_gpu_image_resolver_rejects_mutable_or_malformed_references(project_root_path, image): + resolver = _load_image_resolver(project_root_path) + with pytest.raises(ValueError, match="immutable nvcr.io digest"): + resolver.resolve_image_reference(image) + + +def test_gpu_image_resolver_returns_the_digest_cache_key(project_root_path): + resolver = _load_image_resolver(project_root_path) + digest = "a" * 64 + image = f"nvcr.io/nvidia/modelopt/puzzletron@sha256:{digest}" + + assert resolver.resolve_image_reference(image) == (image, digest) + + +def test_runtime_modelopt_install_cannot_resolve_dependencies(project_root_path): + setup_script = (project_root_path / "examples/puzzletron/ci/setup_env.sh").read_text() + + assert "python -m pip install --no-build-isolation --no-deps -e" in setup_script + + +def test_gpu_workflow_routes_the_pinned_image_to_the_existing_nox_session(project_root_path): + workflow_path = project_root_path / ".github/workflows/puzzletron_gpu_tests.yml" + workflow = yaml.load(workflow_path.read_text(), Loader=yaml.BaseLoader) + + assert workflow["on"]["push"]["branches"] == ["pull-request/[0-9]+"] + jobs = workflow["jobs"] + assert jobs["gpu-puzzletron"]["container"]["image"] == ( + "${{ needs.resolve-image.outputs.image }}" + ) + assert jobs["gpu-puzzletron"]["steps"][-1]["run"] == "nox -s gpu_puzzletron" + + assert jobs["resolve-image"]["permissions"]["contents"] == "read" + resolve_steps = jobs["resolve-image"]["steps"] + assert resolve_steps[0]["uses"] == "actions/checkout@v6" + assert resolve_steps[0]["with"]["persist-credentials"] == "false" + resolve_step = resolve_steps[1] + assert resolve_step["run"] == ( + 'python examples/puzzletron/ci/resolve_ci_image.py >> "${GITHUB_OUTPUT}"' + ) + assert "PUZZLETRON_GPU_CI_IMAGE" in resolve_step["env"] + assert jobs["gpu-puzzletron"]["timeout-minutes"] == "50" From ab440c37bc2120f54d2ae4ff76d75c303e661edb Mon Sep 17 00:00:00 2001 From: Johannes Rausch Date: Wed, 12 Aug 2026 14:37:00 +0200 Subject: [PATCH 2/4] Harden Puzzletron CI workflow contracts Signed-off-by: Johannes Rausch --- .github/workflows/puzzletron_gpu_tests.yml | 3 +-- examples/puzzletron/ci/resolve_ci_image.py | 2 ++ tests/unit/torch/puzzletron/test_ci_image_contract.py | 8 +++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/puzzletron_gpu_tests.yml b/.github/workflows/puzzletron_gpu_tests.yml index b99fe568ba4..84764151910 100644 --- a/.github/workflows/puzzletron_gpu_tests.yml +++ b/.github/workflows/puzzletron_gpu_tests.yml @@ -1,6 +1,6 @@ name: Puzzletron GPU tests -on: +"on": push: branches: ["pull-request/[0-9]+"] schedule: @@ -20,7 +20,6 @@ jobs: uses: ./.github/workflows/_pr_gate.yml permissions: checks: read - secrets: inherit with: files: | .github/workflows/_pr_gate.yml diff --git a/examples/puzzletron/ci/resolve_ci_image.py b/examples/puzzletron/ci/resolve_ci_image.py index 4f04ad1f5df..4be3904203e 100644 --- a/examples/puzzletron/ci/resolve_ci_image.py +++ b/examples/puzzletron/ci/resolve_ci_image.py @@ -21,6 +21,8 @@ import sys from pathlib import Path +__all__ = ["resolve_image_reference", "validate_repository_contract"] + _NVCR_IMAGE = re.compile( r"nvcr\.io/(?:[A-Za-z0-9._-]+/)*[A-Za-z0-9._-]+@sha256:(?P[0-9a-f]{64})" ) diff --git a/tests/unit/torch/puzzletron/test_ci_image_contract.py b/tests/unit/torch/puzzletron/test_ci_image_contract.py index 30f9bbea544..a054d377722 100644 --- a/tests/unit/torch/puzzletron/test_ci_image_contract.py +++ b/tests/unit/torch/puzzletron/test_ci_image_contract.py @@ -49,6 +49,7 @@ def test_gpu_image_uses_the_recorded_immutable_base(project_root_path): assert "bash /opt/puzzletron/setup_env.sh --verify" in dockerfile resolver = _load_image_resolver(project_root_path) + assert resolver.__all__ == ["resolve_image_reference", "validate_repository_contract"] resolver.validate_repository_contract(project_root_path) @@ -84,10 +85,11 @@ def test_runtime_modelopt_install_cannot_resolve_dependencies(project_root_path) def test_gpu_workflow_routes_the_pinned_image_to_the_existing_nox_session(project_root_path): workflow_path = project_root_path / ".github/workflows/puzzletron_gpu_tests.yml" - workflow = yaml.load(workflow_path.read_text(), Loader=yaml.BaseLoader) + workflow = yaml.safe_load(workflow_path.read_text()) assert workflow["on"]["push"]["branches"] == ["pull-request/[0-9]+"] jobs = workflow["jobs"] + assert "secrets" not in jobs["pr-gate"] assert jobs["gpu-puzzletron"]["container"]["image"] == ( "${{ needs.resolve-image.outputs.image }}" ) @@ -96,10 +98,10 @@ def test_gpu_workflow_routes_the_pinned_image_to_the_existing_nox_session(projec assert jobs["resolve-image"]["permissions"]["contents"] == "read" resolve_steps = jobs["resolve-image"]["steps"] assert resolve_steps[0]["uses"] == "actions/checkout@v6" - assert resolve_steps[0]["with"]["persist-credentials"] == "false" + assert resolve_steps[0]["with"]["persist-credentials"] is False resolve_step = resolve_steps[1] assert resolve_step["run"] == ( 'python examples/puzzletron/ci/resolve_ci_image.py >> "${GITHUB_OUTPUT}"' ) assert "PUZZLETRON_GPU_CI_IMAGE" in resolve_step["env"] - assert jobs["gpu-puzzletron"]["timeout-minutes"] == "50" + assert jobs["gpu-puzzletron"]["timeout-minutes"] == 50 From a1fb57b46a639f61ffdbe4f1731632584afc9337 Mon Sep 17 00:00:00 2001 From: Johannes Rausch Date: Wed, 12 Aug 2026 18:16:30 +0200 Subject: [PATCH 3/4] Remove registry secrets from Puzzletron PR CI Require the immutable execution image to support anonymous pulls so copied pull-request branches never receive registry credentials. Signed-off-by: Johannes Rausch --- .github/workflows/puzzletron_gpu_tests.yml | 3 --- .github/workflows/unit_tests.yml | 2 ++ examples/puzzletron/ci/README.md | 11 ++++++----- tests/unit/torch/puzzletron/test_ci_image_contract.py | 5 ++++- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/puzzletron_gpu_tests.yml b/.github/workflows/puzzletron_gpu_tests.yml index 84764151910..3ab3d927bda 100644 --- a/.github/workflows/puzzletron_gpu_tests.yml +++ b/.github/workflows/puzzletron_gpu_tests.yml @@ -66,9 +66,6 @@ jobs: timeout-minutes: 50 container: image: ${{ needs.resolve-image.outputs.image }} - credentials: - username: "$oauthtoken" - password: ${{ secrets.NGC_API_KEY }} options: --shm-size=16gb env: GIT_DEPTH: 1000 diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index e4dd7560776..24882951b01 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -7,6 +7,7 @@ on: branches: [main, release/*, feature/*] paths: - ".github/workflows/unit_tests.yml" + - ".github/workflows/puzzletron_gpu_tests.yml" - "examples/puzzletron/**" - "modelopt/**" - "noxfile.py" @@ -83,6 +84,7 @@ jobs: with: files: | .github/workflows/unit_tests.yml + .github/workflows/puzzletron_gpu_tests.yml examples/puzzletron/**/*.html examples/puzzletron/**/*.json examples/puzzletron/**/*.md diff --git a/examples/puzzletron/ci/README.md b/examples/puzzletron/ci/README.md index 89fc3361635..eb3c6173e03 100644 --- a/examples/puzzletron/ci/README.md +++ b/examples/puzzletron/ci/README.md @@ -33,8 +33,9 @@ docker run --gpus device=0 --ipc=host --rm \ nox -s gpu_puzzletron ``` -Publishing the reviewed image is a separate registry operation. After -publication, configure `PUZZLETRON_GPU_CI_IMAGE` with the complete immutable -`nvcr.io/...@sha256:...` reference and provide the pull-only `NGC_API_KEY` -secret. The workflow rejects tags and non-NVCR references before allocating a -GPU runner. +Publishing the reviewed image is a separate registry operation. Publish it with +anonymous pull access, then configure `PUZZLETRON_GPU_CI_IMAGE` with the +complete immutable `nvcr.io/...@sha256:...` reference. The workflow +intentionally does not expose registry credentials to copied pull-request +branches, and rejects tags and non-NVCR references before allocating a GPU +runner. diff --git a/tests/unit/torch/puzzletron/test_ci_image_contract.py b/tests/unit/torch/puzzletron/test_ci_image_contract.py index a054d377722..8fd42ea9e43 100644 --- a/tests/unit/torch/puzzletron/test_ci_image_contract.py +++ b/tests/unit/torch/puzzletron/test_ci_image_contract.py @@ -85,7 +85,8 @@ def test_runtime_modelopt_install_cannot_resolve_dependencies(project_root_path) def test_gpu_workflow_routes_the_pinned_image_to_the_existing_nox_session(project_root_path): workflow_path = project_root_path / ".github/workflows/puzzletron_gpu_tests.yml" - workflow = yaml.safe_load(workflow_path.read_text()) + workflow_text = workflow_path.read_text() + workflow = yaml.safe_load(workflow_text) assert workflow["on"]["push"]["branches"] == ["pull-request/[0-9]+"] jobs = workflow["jobs"] @@ -93,6 +94,8 @@ def test_gpu_workflow_routes_the_pinned_image_to_the_existing_nox_session(projec assert jobs["gpu-puzzletron"]["container"]["image"] == ( "${{ needs.resolve-image.outputs.image }}" ) + assert "credentials" not in jobs["gpu-puzzletron"]["container"] + assert "NGC_API_KEY" not in workflow_text assert jobs["gpu-puzzletron"]["steps"][-1]["run"] == "nox -s gpu_puzzletron" assert jobs["resolve-image"]["permissions"]["contents"] == "read" From 9abfd3f5557cd9fd2b8b38dc62707141db9db7b9 Mon Sep 17 00:00:00 2001 From: Johannes Rausch Date: Thu, 13 Aug 2026 10:25:18 +0200 Subject: [PATCH 4/4] Harden Puzzletron GPU workflow activation Signed-off-by: Johannes Rausch --- .github/workflows/puzzletron_gpu_tests.yml | 18 ++- .github/workflows/unit_tests.yml | 1 + examples/puzzletron/ci/README.md | 9 +- examples/puzzletron/ci/resolve_ci_image.py | 12 +- noxfile.py | 2 +- .../puzzletron/test_ci_image_contract.py | 118 ++++++++++++++++++ 6 files changed, 153 insertions(+), 7 deletions(-) diff --git a/.github/workflows/puzzletron_gpu_tests.yml b/.github/workflows/puzzletron_gpu_tests.yml index 3ab3d927bda..6df20770015 100644 --- a/.github/workflows/puzzletron_gpu_tests.yml +++ b/.github/workflows/puzzletron_gpu_tests.yml @@ -20,6 +20,8 @@ jobs: uses: ./.github/workflows/_pr_gate.yml permissions: checks: read + # Cross-module Torch changes are covered by the CPU and nightly Puzzletron runs; + # keep pull-request GPU allocation limited to Puzzletron-owned paths. with: files: | .github/workflows/_pr_gate.yml @@ -50,6 +52,7 @@ jobs: outputs: image: ${{ steps.image.outputs.image }} cache_key: ${{ steps.image.outputs.cache_key }} + configured: ${{ steps.image.outputs.configured }} steps: - uses: actions/checkout@v6 with: @@ -62,6 +65,7 @@ jobs: gpu-puzzletron: needs: [resolve-image] + if: needs.resolve-image.outputs.configured == 'true' runs-on: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} timeout-minutes: 50 container: @@ -92,10 +96,22 @@ jobs: echo "## Puzzletron GPU tests were not required" >> "${GITHUB_STEP_SUMMARY}" echo >> "${GITHUB_STEP_SUMMARY}" echo "No Puzzletron lifecycle path changed in this pull request." >> "${GITHUB_STEP_SUMMARY}" + - name: Report unconfigured Puzzletron GPU image + if: >- + ${{ needs.pr-gate.outputs.run_tests == 'true' && + needs.resolve-image.result == 'success' && + needs.resolve-image.outputs.configured == 'false' }} + run: | + echo "## Puzzletron GPU tests are awaiting an image" >> "${GITHUB_STEP_SUMMARY}" + echo >> "${GITHUB_STEP_SUMMARY}" + echo "Configure PUZZLETRON_GPU_CI_IMAGE with an immutable nvcr.io digest to enable this gate." >> "${GITHUB_STEP_SUMMARY}" - name: Required Puzzletron GPU tests did not succeed if: >- ${{ needs.pr-gate.result != 'success' || (needs.pr-gate.outputs.run_tests == 'true' && (needs.resolve-image.result != 'success' || - needs.gpu-puzzletron.result != 'success')) }} + (needs.resolve-image.outputs.configured != 'false' && + needs.resolve-image.outputs.configured != 'true') || + (needs.resolve-image.outputs.configured == 'true' && + needs.gpu-puzzletron.result != 'success'))) }} run: exit 1 diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 24882951b01..aeadc642ac4 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -85,6 +85,7 @@ jobs: files: | .github/workflows/unit_tests.yml .github/workflows/puzzletron_gpu_tests.yml + examples/puzzletron/ci/** examples/puzzletron/**/*.html examples/puzzletron/**/*.json examples/puzzletron/**/*.md diff --git a/examples/puzzletron/ci/README.md b/examples/puzzletron/ci/README.md index eb3c6173e03..f9e1332f3d5 100644 --- a/examples/puzzletron/ci/README.md +++ b/examples/puzzletron/ci/README.md @@ -35,7 +35,8 @@ docker run --gpus device=0 --ipc=host --rm \ Publishing the reviewed image is a separate registry operation. Publish it with anonymous pull access, then configure `PUZZLETRON_GPU_CI_IMAGE` with the -complete immutable `nvcr.io/...@sha256:...` reference. The workflow -intentionally does not expose registry credentials to copied pull-request -branches, and rejects tags and non-NVCR references before allocating a GPU -runner. +complete immutable `nvcr.io/...@sha256:...` reference. Until the variable is +configured, the workflow reports an explicit non-error skip and does not +allocate a GPU runner. Once configured, invalid tags, mutable references, and +non-NVCR images fail before GPU allocation. The workflow intentionally does not +expose registry credentials to copied pull-request branches. diff --git a/examples/puzzletron/ci/resolve_ci_image.py b/examples/puzzletron/ci/resolve_ci_image.py index 4be3904203e..a08232ffaad 100644 --- a/examples/puzzletron/ci/resolve_ci_image.py +++ b/examples/puzzletron/ci/resolve_ci_image.py @@ -65,11 +65,21 @@ def main() -> int: """Write validated values in GitHub output format.""" try: validate_repository_contract(Path.cwd()) - image, cache_key = resolve_image_reference(os.environ.get("PUZZLETRON_GPU_CI_IMAGE", "")) + configured_image = os.environ.get("PUZZLETRON_GPU_CI_IMAGE", "") + if not configured_image: + print( + "::warning::PUZZLETRON_GPU_CI_IMAGE is not configured; " + "skipping Puzzletron GPU tests until an immutable image is published.", + file=sys.stderr, + ) + print("configured=false") + return 0 + image, cache_key = resolve_image_reference(configured_image) except (KeyError, OSError, ValueError, json.JSONDecodeError) as error: print(f"::error::{error}", file=sys.stderr) return 1 + print("configured=true") print(f"image={image}") print(f"cache_key={cache_key}") return 0 diff --git a/noxfile.py b/noxfile.py index be97c614cdb..679a445eb74 100644 --- a/noxfile.py +++ b/noxfile.py @@ -246,8 +246,8 @@ def gpu(session): def gpu_puzzletron(session): """Verify the pinned runtime, then run the focused Puzzletron lifecycle GPU test.""" session.env["CUDA_VISIBLE_DEVICES"] = os.environ.get("CUDA_VISIBLE_DEVICES", "0") + session.env["PUZZLETRON_CI_ENVIRONMENT"] = str(PUZZLETRON_V2_CI_ENVIRONMENT_PATH) session.run("bash", "examples/puzzletron/ci/setup_env.sh", "--modelopt", external=True) - _verify_puzzletron_v2_environment(session, gpu_image=True) session.run( "python", "-c", diff --git a/tests/unit/torch/puzzletron/test_ci_image_contract.py b/tests/unit/torch/puzzletron/test_ci_image_contract.py index 8fd42ea9e43..1c11e4eff08 100644 --- a/tests/unit/torch/puzzletron/test_ci_image_contract.py +++ b/tests/unit/torch/puzzletron/test_ci_image_contract.py @@ -22,6 +22,17 @@ import pytest import yaml +import noxfile + + +class _RecordingSession: + def __init__(self): + self.env = {} + self.calls = [] + + def run(self, *args, **kwargs): + self.calls.append((args, kwargs)) + def _load_image_resolver(project_root_path): resolver_path = project_root_path / "examples/puzzletron/ci/resolve_ci_image.py" @@ -77,6 +88,53 @@ def test_gpu_image_resolver_returns_the_digest_cache_key(project_root_path): assert resolver.resolve_image_reference(image) == (image, digest) +def test_gpu_image_resolver_reports_an_unconfigured_repository_variable( + project_root_path, monkeypatch, capsys +): + resolver = _load_image_resolver(project_root_path) + monkeypatch.chdir(project_root_path) + monkeypatch.delenv("PUZZLETRON_GPU_CI_IMAGE", raising=False) + + assert resolver.main() == 0 + captured = capsys.readouterr() + assert captured.out == "configured=false\n" + assert "::warning::PUZZLETRON_GPU_CI_IMAGE is not configured" in captured.err + + +def test_gpu_image_resolver_reports_a_configured_immutable_image( + project_root_path, monkeypatch, capsys +): + resolver = _load_image_resolver(project_root_path) + monkeypatch.chdir(project_root_path) + digest = "a" * 64 + image = f"nvcr.io/nvidia/modelopt/puzzletron@sha256:{digest}" + monkeypatch.setenv("PUZZLETRON_GPU_CI_IMAGE", image) + + assert resolver.main() == 0 + captured = capsys.readouterr() + assert captured.out.splitlines() == [ + "configured=true", + f"image={image}", + f"cache_key={digest}", + ] + assert captured.err == "" + + +def test_gpu_image_resolver_rejects_a_malformed_configured_image( + project_root_path, monkeypatch, capsys +): + resolver = _load_image_resolver(project_root_path) + monkeypatch.chdir(project_root_path) + monkeypatch.setenv("PUZZLETRON_GPU_CI_IMAGE", "nvcr.io/nvidia/puzzletron:latest") + + assert resolver.main() == 1 + captured = capsys.readouterr() + assert captured.out == "" + assert captured.err == ( + "::error::PUZZLETRON_GPU_CI_IMAGE must be an immutable nvcr.io digest\n" + ) + + def test_runtime_modelopt_install_cannot_resolve_dependencies(project_root_path): setup_script = (project_root_path / "examples/puzzletron/ci/setup_env.sh").read_text() @@ -91,6 +149,10 @@ def test_gpu_workflow_routes_the_pinned_image_to_the_existing_nox_session(projec assert workflow["on"]["push"]["branches"] == ["pull-request/[0-9]+"] jobs = workflow["jobs"] assert "secrets" not in jobs["pr-gate"] + assert jobs["resolve-image"]["outputs"]["configured"] == ( + "${{ steps.image.outputs.configured }}" + ) + assert jobs["gpu-puzzletron"]["if"] == "needs.resolve-image.outputs.configured == 'true'" assert jobs["gpu-puzzletron"]["container"]["image"] == ( "${{ needs.resolve-image.outputs.image }}" ) @@ -108,3 +170,59 @@ def test_gpu_workflow_routes_the_pinned_image_to_the_existing_nox_session(projec ) assert "PUZZLETRON_GPU_CI_IMAGE" in resolve_step["env"] assert jobs["gpu-puzzletron"]["timeout-minutes"] == 50 + + required_steps = jobs["gpu-puzzletron-required-check"]["steps"] + unconfigured_step = next( + step + for step in required_steps + if step["name"] == "Report unconfigured Puzzletron GPU image" + ) + assert " ".join(unconfigured_step["if"].split()) == ( + "${{ needs.pr-gate.outputs.run_tests == 'true' && " + "needs.resolve-image.result == 'success' && " + "needs.resolve-image.outputs.configured == 'false' }}" + ) + failure_step = next( + step + for step in required_steps + if step["name"] == "Required Puzzletron GPU tests did not succeed" + ) + assert " ".join(failure_step["if"].split()) == ( + "${{ needs.pr-gate.result != 'success' || " + "(needs.pr-gate.outputs.run_tests == 'true' && " + "(needs.resolve-image.result != 'success' || " + "(needs.resolve-image.outputs.configured != 'false' && " + "needs.resolve-image.outputs.configured != 'true') || " + "(needs.resolve-image.outputs.configured == 'true' && " + "needs.gpu-puzzletron.result != 'success'))) }}" + ) + + +def test_gpu_contract_routes_all_ci_recipe_changes_through_cpu_tests(project_root_path): + unit_workflow = yaml.safe_load( + (project_root_path / ".github/workflows/unit_tests.yml").read_text() + ) + steps = unit_workflow["jobs"]["check-file-changes"]["steps"] + puzzletron_step = next(step for step in steps if step.get("id") == "puzzletron_changed") + puzzletron_paths = puzzletron_step["with"]["files"] + + assert "examples/puzzletron/ci/**" in puzzletron_paths.splitlines() + + +def test_gpu_nox_session_uses_the_checked_out_environment_contract(): + session = _RecordingSession() + + noxfile.gpu_puzzletron.func(session) + + assert session.env["PUZZLETRON_CI_ENVIRONMENT"] == str( + noxfile.PUZZLETRON_V2_CI_ENVIRONMENT_PATH + ) + assert session.calls[0] == ( + ("bash", "examples/puzzletron/ci/setup_env.sh", "--modelopt"), + {"external": True}, + ) + assert [args[:2] for args, _kwargs in session.calls] == [ + ("bash", "examples/puzzletron/ci/setup_env.sh"), + ("python", "-c"), + ("python", "-m"), + ]