diff --git a/docs/JOURNAL.md b/docs/JOURNAL.md index 117bb79..40a5a71 100644 --- a/docs/JOURNAL.md +++ b/docs/JOURNAL.md @@ -18,6 +18,16 @@ protocol. **Newest entries at the top.** Tag each entry with one or more of: --- +## 2026-07-20 — Remote helper scripts defaulted to GPU 0 instead of allocated GPU 5 #mistake #decision +**Context:** issue #33 — running `scripts/setup_remote.sh` / `scripts/run_remote.sh` without +`TRINITY_GPU_INDEX` set. +**Expected:** remote workflows use GPU 5, matching `DEFAULT_TRINITY_GPU_INDEX` and `secrets.env.example`. +**Actual:** shell fallbacks used `${TRINITY_GPU_INDEX:-0}`, pinning jobs to GPU 0 on shared boxes. +**Root cause:** scripts were written before the project standardized on GPU 5 as the allocated device. +**Fix / decision:** change the shell fallbacks to `:-5` in `setup_remote.sh`, `run_remote.sh`, and +`remote_env.sh`; add a regression test that `setup_remote.sh` logs GPU 5 when unset. +**Follow-up:** none. + ## 2026-07-12 — Validator Postgres tests no longer silently skip in CI #decision #repro **Context:** issue #118 flagged that validator DB-backed tests could ``pytest.skip`` whenever Postgres was unreachable, including on CI where no database service was provisioned. diff --git a/scripts/remote_env.sh b/scripts/remote_env.sh index 3eea33a..dc0ba07 100755 --- a/scripts/remote_env.sh +++ b/scripts/remote_env.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash # Shared environment for all remote-GPU commands. -# Default to GPU 0 on a normal single-GPU VPS, but allow overrides. +# Default to the allocated GPU index (5) unless overridden. set -euo pipefail # We are allocated exactly one physical GPU on the remote box. -export CUDA_VISIBLE_DEVICES="${TRINITY_GPU_INDEX:-0}" +export CUDA_VISIBLE_DEVICES="${TRINITY_GPU_INDEX:-5}" # Project location on the remote box (overridable). export TRINITY_REMOTE_DIR="${TRINITY_REMOTE_DIR:-$HOME/trinity}" diff --git a/scripts/run_remote.sh b/scripts/run_remote.sh index 34b082e..9c37e8b 100755 --- a/scripts/run_remote.sh +++ b/scripts/run_remote.sh @@ -6,7 +6,7 @@ set -euo pipefail HOST="${TRINITY_GPU_HOST:-trinity-gpu}" REMOTE_DIR="${TRINITY_REMOTE_DIR:-trinity}" -GPU_INDEX="${TRINITY_GPU_INDEX:-0}" +GPU_INDEX="${TRINITY_GPU_INDEX:-5}" LOCAL_DIR="$(cd "$(dirname "$0")/.." && pwd)" SYNC_DIR="${TRINITY_SYNC_DIR:-$LOCAL_DIR}" SYNC_ENABLED="${TRINITY_SYNC_ENABLED:-1}" diff --git a/scripts/setup_remote.sh b/scripts/setup_remote.sh index 833b16c..38542b7 100755 --- a/scripts/setup_remote.sh +++ b/scripts/setup_remote.sh @@ -6,7 +6,7 @@ set -euo pipefail HOST="${TRINITY_GPU_HOST:-trinity-gpu}" REMOTE_DIR="${TRINITY_REMOTE_DIR:-trinity}" -GPU_INDEX="${TRINITY_GPU_INDEX:-0}" +GPU_INDEX="${TRINITY_GPU_INDEX:-5}" LOCAL_DIR="$(cd "$(dirname "$0")/.." && pwd)" if [[ -n "${TRINITY_SECRETS_FILE:-}" ]]; then @@ -45,7 +45,7 @@ export PATH="$HOME/.local/bin:$PATH" uv venv --python 3.12 .venv 2>/dev/null || true source .venv/bin/activate uv pip install -e . >/dev/null -echo "--- GPU ${TRINITY_GPU_INDEX:-0} ---" -CUDA_VISIBLE_DEVICES="${TRINITY_GPU_INDEX:-0}" python -c "import torch; print('torch', torch.__version__, 'cuda', torch.cuda.is_available(), torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'n/a')" +echo "--- GPU ${TRINITY_GPU_INDEX:-5} ---" +CUDA_VISIBLE_DEVICES="${TRINITY_GPU_INDEX:-5}" python -c "import torch; print('torch', torch.__version__, 'cuda', torch.cuda.is_available(), torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'n/a')" REMOTE echo "[setup_remote] done." diff --git a/tests/test_setup_remote.py b/tests/test_setup_remote.py index b93135e..d550135 100644 --- a/tests/test_setup_remote.py +++ b/tests/test_setup_remote.py @@ -96,6 +96,39 @@ def test_setup_remote_copies_secrets_file(tmp_path): assert "bash -s" in ssh_log.read_text(encoding="utf-8") +def test_setup_remote_defaults_to_gpu_five_when_unset(tmp_path): + bin_dir, ssh_log, rsync_log = _make_fake_tools(tmp_path) + remote_root = tmp_path / "remote_root" + remote_root.mkdir() + local_secrets = tmp_path / "secrets.env" + local_secrets.write_text("API_KEY=local-secret\n", encoding="utf-8") + + env = os.environ.copy() + env.update( + { + "PATH": f"{bin_dir}{os.pathsep}{env.get('PATH', '')}", + "FAKE_REMOTE_ROOT": str(remote_root), + "TRINITY_GPU_HOST": "fake-host", + "TRINITY_REMOTE_DIR": "trinity", + "TRINITY_SECRETS_FILE": str(local_secrets), + } + ) + env.pop("TRINITY_GPU_INDEX", None) + + result = subprocess.run( + ["bash", str(REPO / "scripts" / "setup_remote.sh")], + cwd=REPO, + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert "GPU 5" in result.stdout + assert "TRINITY_GPU_INDEX=5" in ssh_log.read_text(encoding="utf-8") + + def test_setup_remote_missing_secrets_file_fails(tmp_path): bin_dir, ssh_log, rsync_log = _make_fake_tools(tmp_path) remote_root = tmp_path / "remote_root"