Add Puzzletron v2 GPU quality baseline

examples/puzzletron/README.md

@@ -85,6 +85,21 @@ The defaults file is loaded only when passed explicitly and takes precedence
 over the selected profile. To expose every per-section and nested setting, use
 the advanced flow explicitly:
 
+Automation can use the same setup entry point without answering prompts. The
+defaults file must provide every required value that has no resolved default:
+
+```bash
+python examples/puzzletron/puzzletron_setup_v2.py \
+  --defaults /path/to/setup-v2-defaults.yaml \
+  --campaign-dir /path/to/campaign \
+  --profile smoke \
+  --non-interactive
+```
+
+Non-interactive setup fails instead of guessing when a required answer has no
+resolved default. It generates and validates the same smoke and production
+bundles as the interactive wizard.
+
 ```bash
 python examples/puzzletron/puzzletron_setup_v2.py --full
 ```
@@ -269,13 +284,15 @@ git -C "${AUTOMODEL_ROOT}" rev-parse HEAD
 ```
 
 ```bash
-python - <<'PY'
+PYTHONPATH="${MODEL_OPT_ROOT}" python - <<'PY'
 import importlib.metadata as metadata
 import json
 import os
 
 from packaging.version import Version
 
+from examples.puzzletron.ci_environment import verify_installed_vcs_source
+
 import aiperf
 import lmms_eval
 import modelopt
@@ -307,10 +324,17 @@ assert Version(metadata.version("torchvision")).release == Version(
     ci_environment["torchvision"]
 ).release
 assert transformers.__version__ == ci_environment["transformers"]
-assert metadata.version("lmms-eval") == ci_environment["lmms_eval"]
+assert Version(metadata.version("lmms-eval")).base_version == (
+    ci_environment["lmms_eval"]["base_version"]
+)
 assert Version(metadata.version("nemo-automodel")).base_version == (
     ci_environment["nemo_automodel"]["base_version"]
 )
+for package, source in (
+    ("lmms-eval", ci_environment["lmms_eval"]),
+    ("nemo-automodel", ci_environment["nemo_automodel"]),
+):
+    verify_installed_vcs_source(package, source)
 assert torch.version.cuda == "12.9"
 assert torch.cuda.is_available()
 PY
@@ -492,6 +516,11 @@ The setup wizard can also add downstream evaluation for materialized campaign
 candidates. See [post-MIP pipelines](docs/post_mip_pipeline.md) to configure it
 or add it to an existing campaign.
 
+Remote model code and AIPerf v0.11 online tokenizer resolution are disabled by
+default. Enable remote code only for a trusted model source. The tokenizer
+compatibility option permits the AIPerf child process to resolve its tokenizer
+online even when the surrounding campaign is configured for offline loading.
+
 ### Legacy checked-in Nano campaign
 
 The checked-in Nano experiment uses the legacy `zero_shot_evaluation`,

examples/puzzletron/ci_environment.json

@@ -1,11 +1,15 @@
 {
   "schema_version": 1,
-  "scope": "puzzletron_v2_cpu_ci",
+  "scope": "puzzletron_v2_ci",
   "python": "3.12",
   "torch": "2.11.0",
   "torchvision": "0.26.0",
   "transformers": "5.8.1",
-  "lmms_eval": "0.7.0",
+  "lmms_eval": {
+    "base_version": "0.7.0",
+    "repository": "https://github.com/EvolvingLMMs-Lab/lmms-eval.git",
+    "commit": "15c32bfec165df13c269ddd3cda03b2ed9137825"
+  },
   "nemo_automodel": {
     "base_version": "0.5.0",
     "repository": "https://github.com/Separius/Automodel.git",

examples/puzzletron/ci_environment.py

@@ -0,0 +1,68 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Verification helpers for the pinned Puzzletron CI environment."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from importlib import metadata
+from typing import Any
+from urllib.parse import unquote, urlparse
+
+__all__ = ["verify_installed_vcs_source"]
+
+
+def _normalized_repository(url: object) -> str:
+    return str(url or "").removesuffix(".git").rstrip("/")
+
+
+def _installed_vcs_source(package: str) -> tuple[str | None, str | None]:
+    payload = json.loads(metadata.distribution(package).read_text("direct_url.json") or "{}")
+    vcs_info = payload.get("vcs_info") or {}
+    if vcs_info.get("commit_id"):
+        return payload.get("url"), vcs_info["commit_id"]
+    if (payload.get("dir_info") or {}).get("editable") and str(payload.get("url", "")).startswith(
+        "file:"
+    ):
+        root = unquote(urlparse(payload["url"]).path)
+        repository = subprocess.check_output(
+            ["git", "-C", root, "remote", "get-url", "origin"], text=True
+        ).strip()
+        commit = subprocess.check_output(
+            ["git", "-C", root, "rev-parse", "HEAD"], text=True
+        ).strip()
+        dirty = subprocess.check_output(
+            ["git", "-C", root, "status", "--porcelain", "--untracked-files=all"],
+            text=True,
+        ).strip()
+        if dirty:
+            raise RuntimeError(f"Pinned Puzzletron dependency {package!r} is dirty: {dirty}")
+        return repository, commit
+    return payload.get("url"), vcs_info.get("commit_id")
+
+
+def verify_installed_vcs_source(package: str, expected: dict[str, Any]) -> None:
+    """Require an installed VCS dependency to match its repository and commit."""
+
+    repository, commit = _installed_vcs_source(package)
+    expected_source = (_normalized_repository(expected["repository"]), expected["commit"])
+    actual_source = (_normalized_repository(repository), commit)
+    if actual_source != expected_source:
+        raise RuntimeError(
+            f"Pinned Puzzletron dependency {package!r} source mismatch: "
+            f"actual={actual_source!r}, expected={expected_source!r}"
+        )

examples/puzzletron/distributed_eval/run_coordinator.sh

@@ -1,4 +1,19 @@
 #!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 set -Eeuo pipefail
 
 : "${CAMPAIGN_DIR:?set CAMPAIGN_DIR}"
@@ -88,6 +103,11 @@ from pathlib import Path
 import subprocess
 import sys
 
+from examples.puzzletron.finalize_replacement_scoring import (
+    finalization_marker_is_current,
+    write_finalization_marker,
+)
+
 (
     completion_dir_text,
     marker_name,
@@ -104,8 +124,12 @@ completion_dir.mkdir(parents=True, exist_ok=True)
 with (completion_dir / ".finalize.lock").open("a+") as lock:
     fcntl.flock(lock, fcntl.LOCK_EX)
     finalized = completion_dir / "finalized"
-    if finalized.is_file():
+    root = Path(puzzle_dir)
+    root_summary = root / "artifacts" / "replacement_scoring" / "summary.json"
+    root_manifest = root / "manifests" / "replacement_scoring.json"
+    if finalization_marker_is_current(finalized, root_manifest, root_summary):
         raise SystemExit(0)
+    finalized.unlink(missing_ok=True)
     completed = tuple(completion_dir.glob("*.done"))
     expected = int(expected_text)
     if len(completed) < expected:
@@ -126,7 +150,7 @@ with (completion_dir / ".finalize.lock").open("a+") as lock:
         ],
         check=True,
     )
-    finalized.touch()
+    write_finalization_marker(finalized, root_manifest)
 PY
   else
     "${PYTHON_BIN}" "${SCRIPT_DIR}/../finalize_replacement_scoring.py" \

examples/puzzletron/distributed_eval/run_depth_pool.sh

@@ -1,6 +1,18 @@
 #!/usr/bin/env bash
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 set -Eeuo pipefail
 
@@ -10,10 +22,12 @@ set -Eeuo pipefail
 : "${WORKER_COUNT:?set WORKER_COUNT to the number of worker groups}"
 : "${PUZZLETRON_GROUP_INDEX:=${PUZZLETRON_TASK_INDEX:-${SLURM_PROCID:-}}}"
 : "${PUZZLETRON_GROUP_INDEX:?run this script as one orchestrator worker-group task}"
+: "${PUZZLETRON_GROUP_RANK:=0}"
 
 PYTHON_BIN="${PYTHON_BIN:-python}"
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
 GROUP_INDEX="${PUZZLETRON_GROUP_INDEX}"
+GROUP_RANK="${PUZZLETRON_GROUP_RANK}"
 JOB_ID="${SLURM_JOB_ID:-local}"
 WORKER_PREFIX="${JOB_ID}-depth-"
 MANIFEST_PATH="${CAMPAIGN_DIR}/manifest.json"
@@ -38,7 +52,7 @@ cleanup() {
   local rc=$?
   trap - EXIT INT TERM
   set +e
-  if [[ "${GROUP_INDEX}" == "0" ]]; then
+  if [[ "${GROUP_INDEX}" == "0" && "${GROUP_RANK}" == "0" ]]; then
     drain_workers
   fi
   if [[ -n "${worker_pid}" ]] && kill -0 "${worker_pid}" 2>/dev/null; then
@@ -50,7 +64,7 @@ cleanup() {
 trap cleanup EXIT INT TERM
 
 # Rank 0 creates the shared campaign before any worker attempts to open it.
-if [[ "${GROUP_INDEX}" == "0" && ! -f "${MANIFEST_PATH}" ]]; then
+if [[ "${GROUP_INDEX}" == "0" && "${GROUP_RANK}" == "0" && ! -f "${MANIFEST_PATH}" ]]; then
   CUDA_VISIBLE_DEVICES="" "${PYTHON_BIN}" \
     -m modelopt.torch.puzzletron.distributed_eval.cli init \
     --campaign-dir "${CAMPAIGN_DIR}" \
@@ -72,21 +86,17 @@ done
 
 # Every scheduler task owns one GPU slice and starts one worker group. Multiple
 # independent worker groups may share a node.
-export NNODES=1
-export NODE_RANK=0
 export NPROC_PER_NODE="${NPROC_PER_NODE:-${WORLD_SIZE}}"
 export WORKER_GROUP_INDEX="${GROUP_INDEX}"
 export WORKER_ID="${WORKER_PREFIX}${GROUP_INDEX}"
 export WORKER_HOST="${WORKER_HOST:-$(hostname -f)}"
 export WORKER_PORT="${WORKER_PORT:-$((5010 + GROUP_INDEX))}"
-export RDZV_ENDPOINT="127.0.0.1:$((29500 + GROUP_INDEX))"
-export RDZV_ID="depth-${JOB_ID}-${GROUP_INDEX}"
 
 bash "${SCRIPT_DIR}/run_worker.sh" &
 worker_pid=$!
 
 coordinator_rc=0
-if [[ "${GROUP_INDEX}" == "0" ]]; then
+if [[ "${GROUP_INDEX}" == "0" && "${GROUP_RANK}" == "0" ]]; then
   # Do not start depth iteration zero until every resident model is ready.
   CUDA_VISIBLE_DEVICES="" "${PYTHON_BIN}" - \
       "${CAMPAIGN_DIR}" \

examples/puzzletron/distributed_eval/run_replacement_pool.sh

@@ -1,6 +1,18 @@
 #!/usr/bin/env bash
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 set -Eeuo pipefail
 
@@ -10,10 +22,12 @@ set -Eeuo pipefail
 : "${WORKER_COUNT:?set WORKER_COUNT to the number of worker groups}"
 : "${PUZZLETRON_GROUP_INDEX:=${PUZZLETRON_TASK_INDEX:-${SLURM_PROCID:-}}}"
 : "${PUZZLETRON_GROUP_INDEX:?run this script as one orchestrator worker-group task}"
+: "${PUZZLETRON_GROUP_RANK:=0}"
 
 PYTHON_BIN="${PYTHON_BIN:-python}"
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
 GROUP_INDEX="${PUZZLETRON_GROUP_INDEX}"
+GROUP_RANK="${PUZZLETRON_GROUP_RANK}"
 JOB_ID="${SLURM_JOB_ID:-local}"
 WORKER_PREFIX="${JOB_ID}-replacement-"
 MANIFEST_PATH="${CAMPAIGN_DIR}/manifest.json"
@@ -38,7 +52,7 @@ cleanup() {
   local rc=$?
   trap - EXIT INT TERM
   set +e
-  if [[ "${GROUP_INDEX}" == "0" ]]; then
+  if [[ "${GROUP_INDEX}" == "0" && "${GROUP_RANK}" == "0" ]]; then
     drain_workers
   fi
   if [[ -n "${worker_pid}" ]] && kill -0 "${worker_pid}" 2>/dev/null; then
@@ -49,7 +63,7 @@ cleanup() {
 }
 trap cleanup EXIT INT TERM
 
-if [[ "${GROUP_INDEX}" == "0" && ! -f "${MANIFEST_PATH}" ]]; then
+if [[ "${GROUP_INDEX}" == "0" && "${GROUP_RANK}" == "0" && ! -f "${MANIFEST_PATH}" ]]; then
   CUDA_VISIBLE_DEVICES="" "${PYTHON_BIN}" \
     -m modelopt.torch.puzzletron.distributed_eval.cli init \
     --campaign-dir "${CAMPAIGN_DIR}" \
@@ -69,21 +83,17 @@ while [[ ! -f "${MANIFEST_PATH}" ]]; do
   sleep 1
 done
 
-export NNODES=1
-export NODE_RANK=0
 export NPROC_PER_NODE="${NPROC_PER_NODE:-${WORLD_SIZE}}"
 export WORKER_GROUP_INDEX="${GROUP_INDEX}"
 export WORKER_ID="${WORKER_PREFIX}${GROUP_INDEX}"
 export WORKER_HOST="${WORKER_HOST:-$(hostname -f)}"
 export WORKER_PORT="${WORKER_PORT:-$((5010 + GROUP_INDEX))}"
-export RDZV_ENDPOINT="127.0.0.1:$((29500 + GROUP_INDEX))"
-export RDZV_ID="replacement-${JOB_ID}-${GROUP_INDEX}"
 
 bash "${SCRIPT_DIR}/run_worker.sh" &
 worker_pid=$!
 
 coordinator_rc=0
-if [[ "${GROUP_INDEX}" == "0" ]]; then
+if [[ "${GROUP_INDEX}" == "0" && "${GROUP_RANK}" == "0" ]]; then
   CUDA_VISIBLE_DEVICES="" "${PYTHON_BIN}" - \
       "${CAMPAIGN_DIR}" \
       "${WORKER_COUNT}" \