fix(scripts): default remote GPU helpers to index 5 (#33)

docs/JOURNAL.md

@@ -18,6 +18,16 @@ protocol. **Newest entries at the top.** Tag each entry with one or more of:
 
 ---
 
+## 2026-07-20 — Remote helper scripts defaulted to GPU 0 instead of allocated GPU 5  #mistake #decision
+**Context:** issue #33 — running `scripts/setup_remote.sh` / `scripts/run_remote.sh` without
+`TRINITY_GPU_INDEX` set.
+**Expected:** remote workflows use GPU 5, matching `DEFAULT_TRINITY_GPU_INDEX` and `secrets.env.example`.
+**Actual:** shell fallbacks used `${TRINITY_GPU_INDEX:-0}`, pinning jobs to GPU 0 on shared boxes.
+**Root cause:** scripts were written before the project standardized on GPU 5 as the allocated device.
+**Fix / decision:** change the shell fallbacks to `:-5` in `setup_remote.sh`, `run_remote.sh`, and
+`remote_env.sh`; add a regression test that `setup_remote.sh` logs GPU 5 when unset.
+**Follow-up:** none.
+
 ## 2026-07-12 — Validator Postgres tests no longer silently skip in CI  #decision #repro
 **Context:** issue #118 flagged that validator DB-backed tests could ``pytest.skip`` whenever Postgres
 was unreachable, including on CI where no database service was provisioned.

scripts/remote_env.sh

@@ -1,10 +1,10 @@
 #!/usr/bin/env bash
 # Shared environment for all remote-GPU commands.
-# Default to GPU 0 on a normal single-GPU VPS, but allow overrides.
+# Default to the allocated GPU index (5) unless overridden.
 set -euo pipefail
 
 # We are allocated exactly one physical GPU on the remote box.
-export CUDA_VISIBLE_DEVICES="${TRINITY_GPU_INDEX:-0}"
+export CUDA_VISIBLE_DEVICES="${TRINITY_GPU_INDEX:-5}"
 
 # Project location on the remote box (overridable).
 export TRINITY_REMOTE_DIR="${TRINITY_REMOTE_DIR:-$HOME/trinity}"

scripts/run_remote.sh

@@ -6,7 +6,7 @@ set -euo pipefail
 
 HOST="${TRINITY_GPU_HOST:-trinity-gpu}"
 REMOTE_DIR="${TRINITY_REMOTE_DIR:-trinity}"
-GPU_INDEX="${TRINITY_GPU_INDEX:-0}"
+GPU_INDEX="${TRINITY_GPU_INDEX:-5}"
 LOCAL_DIR="$(cd "$(dirname "$0")/.." && pwd)"
 SYNC_DIR="${TRINITY_SYNC_DIR:-$LOCAL_DIR}"
 SYNC_ENABLED="${TRINITY_SYNC_ENABLED:-1}"

scripts/setup_remote.sh

@@ -6,7 +6,7 @@ set -euo pipefail
 
 HOST="${TRINITY_GPU_HOST:-trinity-gpu}"
 REMOTE_DIR="${TRINITY_REMOTE_DIR:-trinity}"
-GPU_INDEX="${TRINITY_GPU_INDEX:-0}"
+GPU_INDEX="${TRINITY_GPU_INDEX:-5}"
 LOCAL_DIR="$(cd "$(dirname "$0")/.." && pwd)"
 
 if [[ -n "${TRINITY_SECRETS_FILE:-}" ]]; then
@@ -45,7 +45,7 @@ export PATH="$HOME/.local/bin:$PATH"
 uv venv --python 3.12 .venv 2>/dev/null || true
 source .venv/bin/activate
 uv pip install -e . >/dev/null
-echo "--- GPU ${TRINITY_GPU_INDEX:-0} ---"
-CUDA_VISIBLE_DEVICES="${TRINITY_GPU_INDEX:-0}" python -c "import torch; print('torch', torch.__version__, 'cuda', torch.cuda.is_available(), torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'n/a')"
+echo "--- GPU ${TRINITY_GPU_INDEX:-5} ---"
+CUDA_VISIBLE_DEVICES="${TRINITY_GPU_INDEX:-5}" python -c "import torch; print('torch', torch.__version__, 'cuda', torch.cuda.is_available(), torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'n/a')"
 REMOTE
 echo "[setup_remote] done."

tests/test_setup_remote.py

@@ -96,6 +96,39 @@ def test_setup_remote_copies_secrets_file(tmp_path):
     assert "bash -s" in ssh_log.read_text(encoding="utf-8")
 
 
+def test_setup_remote_defaults_to_gpu_five_when_unset(tmp_path):
+    bin_dir, ssh_log, rsync_log = _make_fake_tools(tmp_path)
+    remote_root = tmp_path / "remote_root"
+    remote_root.mkdir()
+    local_secrets = tmp_path / "secrets.env"
+    local_secrets.write_text("API_KEY=local-secret\n", encoding="utf-8")
+
+    env = os.environ.copy()
+    env.update(
+        {
+            "PATH": f"{bin_dir}{os.pathsep}{env.get('PATH', '')}",
+            "FAKE_REMOTE_ROOT": str(remote_root),
+            "TRINITY_GPU_HOST": "fake-host",
+            "TRINITY_REMOTE_DIR": "trinity",
+            "TRINITY_SECRETS_FILE": str(local_secrets),
+        }
+    )
+    env.pop("TRINITY_GPU_INDEX", None)
+
+    result = subprocess.run(
+        ["bash", str(REPO / "scripts" / "setup_remote.sh")],
+        cwd=REPO,
+        env=env,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert "GPU 5" in result.stdout
+    assert "TRINITY_GPU_INDEX=5" in ssh_log.read_text(encoding="utf-8")
+
+
 def test_setup_remote_missing_secrets_file_fails(tmp_path):
     bin_dir, ssh_log, rsync_log = _make_fake_tools(tmp_path)
     remote_root = tmp_path / "remote_root"