From 1f9c816082218c3b2d8ddb05637162cb2ad55cf1 Mon Sep 17 00:00:00 2001 From: David Shoen Date: Tue, 23 Jun 2026 12:24:59 +0300 Subject: [PATCH 01/63] feat(opal-server): gated /internal git-fetcher cache stats endpoint Add an off-by-default diagnostics endpoint so tests can observe the in-memory GitPolicyFetcher cache sizes (repo_locks/repos/repos_last_fetched) and process RSS that the upcoming memory-leak fix eliminates. - debug_stats.py: read-only git_fetcher_cache_stats() helper + a register_internal_stats_route() registrar that mounts GET /internal/git-fetcher-cache-stats only when enabled. - config.py: new OPAL_DEBUG_INTERNAL_STATS flag, default False. - server.py: register the route, gated by the flag, beside /healthcheck. No production behavior change when the flag is off (the default). Also ignore .claude/ so private planning artifacts are never committed. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 3 ++ packages/opal-server/opal_server/config.py | 9 ++++ .../opal-server/opal_server/debug_stats.py | 41 +++++++++++++++++++ packages/opal-server/opal_server/server.py | 5 +++ .../tests/debug_stats_endpoint_test.py | 23 +++++++++++ .../opal_server/tests/debug_stats_test.py | 21 ++++++++++ 6 files changed, 102 insertions(+) create mode 100644 packages/opal-server/opal_server/debug_stats.py create mode 100644 packages/opal-server/opal_server/tests/debug_stats_endpoint_test.py create mode 100644 packages/opal-server/opal_server/tests/debug_stats_test.py diff --git a/.gitignore b/.gitignore index 77faef34a..c3224ec1c 100644 --- a/.gitignore +++ b/.gitignore @@ -137,3 +137,6 @@ dmypy.json *.iml .DS_Store + +# Private Claude Code working artifacts (plans/specs) — never commit +.claude/ diff --git a/packages/opal-server/opal_server/config.py b/packages/opal-server/opal_server/config.py index 9faac9be4..aaf3d6f75 100644 --- a/packages/opal-server/opal_server/config.py +++ b/packages/opal-server/opal_server/config.py @@ -358,6 +358,15 @@ class OpalServerConfig(Confi): description="Set if OPAL server should enable tracing with datadog APM", ) + DEBUG_INTERNAL_STATS = confi.bool( + "DEBUG_INTERNAL_STATS", + False, + description=( + "Expose GET /internal/git-fetcher-cache-stats with in-memory cache " + "sizes and process RSS. For diagnostics/tests only; keep off in production." + ), + ) + SCOPES = confi.bool("SCOPES", default=False, description="Enable scopes") SCOPES_REPO_CLONES_SHARDS = confi.int( diff --git a/packages/opal-server/opal_server/debug_stats.py b/packages/opal-server/opal_server/debug_stats.py new file mode 100644 index 000000000..20ea6b341 --- /dev/null +++ b/packages/opal-server/opal_server/debug_stats.py @@ -0,0 +1,41 @@ +"""Read-only introspection of the git-fetcher in-memory caches. + +Used only by the off-by-default /internal stats endpoint so tests can +observe the cache growth that the memory-leak fix (PR2) eliminates. +""" +from pathlib import Path +from typing import Dict + +from fastapi import FastAPI +from opal_server.git_fetcher import GitPolicyFetcher + + +def _read_rss_kb() -> int: + """Resident set size of this process in kilobytes (Linux), else 0.""" + try: + for line in Path("/proc/self/status").read_text().splitlines(): + if line.startswith("VmRSS:"): + return int(line.split()[1]) + except (OSError, ValueError, IndexError): + return 0 + return 0 + + +def git_fetcher_cache_stats() -> Dict[str, int]: + """Sizes of the three process-global GitPolicyFetcher caches + RSS.""" + return { + "repo_locks": len(GitPolicyFetcher.repo_locks), + "repos": len(GitPolicyFetcher.repos), + "repos_last_fetched": len(GitPolicyFetcher.repos_last_fetched), + "rss_kb": _read_rss_kb(), + } + + +def register_internal_stats_route(app: FastAPI, enabled: bool) -> None: + """Mount GET /internal/git-fetcher-cache-stats only when enabled.""" + if not enabled: + return + + @app.get("/internal/git-fetcher-cache-stats", include_in_schema=False) + def _git_fetcher_cache_stats() -> Dict[str, int]: + return git_fetcher_cache_stats() diff --git a/packages/opal-server/opal_server/server.py b/packages/opal-server/opal_server/server.py index a846e5a80..2b88b3db8 100644 --- a/packages/opal-server/opal_server/server.py +++ b/packages/opal-server/opal_server/server.py @@ -26,6 +26,7 @@ ) from opal_server.config import opal_server_config from opal_server.data.api import init_data_updates_router +from opal_server.debug_stats import register_internal_stats_route from opal_server.data.data_update_publisher import DataUpdatePublisher from opal_server.loadlimiting import init_loadlimit_router from opal_server.policy.bundles.api import router as bundles_router @@ -293,6 +294,10 @@ async def redoc_html(req: Request) -> HTMLResponse: def healthcheck(): return {"status": "ok"} + register_internal_stats_route( + app, enabled=opal_server_config.DEBUG_INTERNAL_STATS + ) + return app def _configure_lifecycle_callbacks(self, app: FastAPI): diff --git a/packages/opal-server/opal_server/tests/debug_stats_endpoint_test.py b/packages/opal-server/opal_server/tests/debug_stats_endpoint_test.py new file mode 100644 index 000000000..8ba0f678a --- /dev/null +++ b/packages/opal-server/opal_server/tests/debug_stats_endpoint_test.py @@ -0,0 +1,23 @@ +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from opal_server.debug_stats import register_internal_stats_route + + +def _app_with_flag(enabled: bool) -> FastAPI: + app = FastAPI() + register_internal_stats_route(app, enabled=enabled) + return app + + +def test_endpoint_absent_when_disabled(): + client = TestClient(_app_with_flag(False)) + assert client.get("/internal/git-fetcher-cache-stats").status_code == 404 + + +def test_endpoint_present_when_enabled(): + client = TestClient(_app_with_flag(True)) + resp = client.get("/internal/git-fetcher-cache-stats") + assert resp.status_code == 200 + body = resp.json() + assert set(body) == {"repo_locks", "repos", "repos_last_fetched", "rss_kb"} diff --git a/packages/opal-server/opal_server/tests/debug_stats_test.py b/packages/opal-server/opal_server/tests/debug_stats_test.py new file mode 100644 index 000000000..b2d7935db --- /dev/null +++ b/packages/opal-server/opal_server/tests/debug_stats_test.py @@ -0,0 +1,21 @@ +from opal_server.config import opal_server_config +from opal_server.debug_stats import git_fetcher_cache_stats +from opal_server.git_fetcher import GitPolicyFetcher + + +def test_stats_report_dict_sizes(monkeypatch): + monkeypatch.setattr(GitPolicyFetcher, "repo_locks", {"a": object()}) + monkeypatch.setattr(GitPolicyFetcher, "repos", {"p1": object(), "p2": object()}) + monkeypatch.setattr(GitPolicyFetcher, "repos_last_fetched", {}) + + stats = git_fetcher_cache_stats() + + assert stats["repo_locks"] == 1 + assert stats["repos"] == 2 + assert stats["repos_last_fetched"] == 0 + assert isinstance(stats["rss_kb"], int) + assert stats["rss_kb"] >= 0 + + +def test_internal_stats_flag_defaults_off(): + assert opal_server_config.DEBUG_INTERNAL_STATS is False From c176ce9dbbf89bd92b70c3aad5ee486259868a4a Mon Sep 17 00:00:00 2001 From: David Shoen Date: Tue, 23 Jun 2026 12:24:59 +0300 Subject: [PATCH 02/63] test(git-leak): add OPAL git leak/resilience test bed A self-contained docker-compose stack (opal-server x2 workers + Redis + Postgres broadcaster + Gitea) plus a pytest harness that reproduces, as tests that fail on master, the git-fetcher memory leak, the offline-repo hang, the slow serial boot, and the broadcaster-disconnect gap. These become the regression gates for the follow-up fixes. - seed/: idempotent Gitea seeding sidecar (N policy repos) + Dockerfile. - docker-compose.yml: 4-service stack, opal-server built from the repo's own docker/Dockerfile (server target), scopes on, Postgres broadcaster. - helpers.py / conftest.py: HTTP + infra helpers and stack fixtures. - test_leak.py / test_resilience.py / test_boot.py: the flagship tests. - README.md: how to run and expected fail-on-master behavior. Co-Authored-By: Claude Opus 4.8 (1M context) --- app-tests/git-leak/README.md | 33 +++++++ app-tests/git-leak/conftest.py | 44 +++++++++ app-tests/git-leak/docker-compose.yml | 103 ++++++++++++++++++++ app-tests/git-leak/helpers.py | 94 +++++++++++++++++++ app-tests/git-leak/seed/Dockerfile | 6 ++ app-tests/git-leak/seed/seed_gitea.py | 130 ++++++++++++++++++++++++++ app-tests/git-leak/test_boot.py | 37 ++++++++ app-tests/git-leak/test_leak.py | 56 +++++++++++ app-tests/git-leak/test_resilience.py | 54 +++++++++++ 9 files changed, 557 insertions(+) create mode 100644 app-tests/git-leak/README.md create mode 100644 app-tests/git-leak/conftest.py create mode 100644 app-tests/git-leak/docker-compose.yml create mode 100644 app-tests/git-leak/helpers.py create mode 100644 app-tests/git-leak/seed/Dockerfile create mode 100644 app-tests/git-leak/seed/seed_gitea.py create mode 100644 app-tests/git-leak/test_boot.py create mode 100644 app-tests/git-leak/test_leak.py create mode 100644 app-tests/git-leak/test_resilience.py diff --git a/app-tests/git-leak/README.md b/app-tests/git-leak/README.md new file mode 100644 index 000000000..33a291f86 --- /dev/null +++ b/app-tests/git-leak/README.md @@ -0,0 +1,33 @@ +# OPAL git-leak / resilience test bed + +Reproduces (as failing tests on `master`) the four issues fixed by PR2–PR5: +memory leak, offline-repo hang, slow serial boot, broadcaster no-reconnect. + +## Stack +- `opal_server` (2 workers, scopes on, Postgres broadcaster, built from `docker/Dockerfile`) +- `redis`, `postgres`, `gitea` (+ one-shot `gitea-admin` and `seed` sidecars) + +## Run +```bash +cd app-tests/git-leak +python -m pytest -v --boot-scopes=50 # full set +python -m pytest test_leak.py -v --boot-scopes=20 # just the leak gates +``` +Useful flags: `--boot-scopes=N` (any N), `--keep-stack` (skip teardown), +env `BOOT_TARGET_SECONDS=120` (tighten the boot gate). + +## Expected on master +All five flagship tests FAIL (except the boot test, which only fails when +`BOOT_TARGET_SECONDS` is set low). They become the regression gates for the fixes. + +## Requires +Docker + docker compose v2, plus host Python with `pytest pytest-timeout requests GitPython`. + +## Notes +- Auth is disabled in the stack: `OPAL_AUTH_PUBLIC_KEY` is left unset so the JWT + verifier is disabled and the harness can call scope routes without minting JWTs. + Local test bed only; never a production setting. +- The server runs 2 uvicorn workers with a Postgres broadcaster, mirroring a + realistic multi-worker deployment. The `GitPolicyFetcher` caches read by the + `/internal/git-fetcher-cache-stats` endpoint are per-process, so the harness + polls with generous timeouts to let the leader worker converge. diff --git a/app-tests/git-leak/conftest.py b/app-tests/git-leak/conftest.py new file mode 100644 index 000000000..8c538aa06 --- /dev/null +++ b/app-tests/git-leak/conftest.py @@ -0,0 +1,44 @@ +import os + +import pytest + +from helpers import OpalServerClient, compose + + +def pytest_addoption(parser): + parser.addoption( + "--boot-scopes", + action="store", + default="50", + help="number of repos to seed/boot (default 50)", + ) + parser.addoption( + "--keep-stack", + action="store_true", + default=False, + help="do not tear the compose stack down after the run", + ) + + +@pytest.fixture(scope="session") +def repo_count(request) -> int: + return int(request.config.getoption("--boot-scopes")) + + +@pytest.fixture(scope="session") +def stack(request, repo_count): + os.environ["REPO_COUNT"] = str(repo_count) + # build + start infra; seed runs to completion then exits + compose("up", "-d", "--build") + # block until seeding sidecar has finished creating repos + compose("wait", "seed") + client = OpalServerClient() + client.wait_healthy() + yield client + if not request.config.getoption("--keep-stack"): + compose("down", "-v") + + +@pytest.fixture() +def opal(stack) -> OpalServerClient: + return stack diff --git a/app-tests/git-leak/docker-compose.yml b/app-tests/git-leak/docker-compose.yml new file mode 100644 index 000000000..8f9f474f9 --- /dev/null +++ b/app-tests/git-leak/docker-compose.yml @@ -0,0 +1,103 @@ +name: opal-git-leak-test + +services: + redis: + image: redis:7-alpine + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 3s + retries: 30 + + postgres: + image: postgres:16-alpine + environment: + POSTGRES_USER: opal + POSTGRES_PASSWORD: opal + POSTGRES_DB: opal + # not published to the host: only opal_server reaches it over the compose + # network, and bounce_postgres() uses `docker compose stop/start`. Publishing + # 5432 would collide with any Postgres already running on the host. + healthcheck: + test: ["CMD-SHELL", "pg_isready -U opal"] + interval: 2s + timeout: 3s + retries: 30 + + gitea: + image: gitea/gitea:1.21 + environment: + GITEA__security__INSTALL_LOCK: "true" + GITEA__server__ROOT_URL: "http://gitea:3000/" + GITEA__database__DB_TYPE: "sqlite3" + # not published to the host: only the seed sidecar and opal_server reach it + # over the compose network (via http://gitea:3000). Avoids host port clashes. + volumes: + - gitea-data:/data + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:3000/api/v1/version || exit 1"] + interval: 3s + timeout: 5s + retries: 40 + + gitea-admin: + # creates the admin user once gitea is healthy + image: gitea/gitea:1.21 + depends_on: + gitea: + condition: service_healthy + user: git + entrypoint: ["/bin/sh", "-c"] + command: + - > + gitea admin user create --username opaladmin --password opaladmin + --email admin@example.com --admin --must-change-password=false + --config /data/gitea/conf/app.ini || true + volumes: + - gitea-data:/data + restart: "no" + + seed: + build: ./seed + depends_on: + gitea: + condition: service_healthy + gitea-admin: + condition: service_completed_successfully + environment: + GITEA_URL: "http://gitea:3000" + GITEA_ADMIN_USER: "opaladmin" + GITEA_ADMIN_PASSWORD: "opaladmin" + REPO_COUNT: "${REPO_COUNT:-50}" + volumes: + - seed-output:/seed-output + restart: "no" + + opal_server: + build: + context: ../.. + dockerfile: docker/Dockerfile + target: server + environment: + UVICORN_NUM_WORKERS: "2" + OPAL_SCOPES: "1" + OPAL_REDIS_URL: "redis://redis:6379" + OPAL_BROADCAST_URI: "postgres://opal:opal@postgres:5432/opal" + OPAL_BASE_DIR: "/opal" + OPAL_POLICY_REFRESH_INTERVAL: "0" + OPAL_DEBUG_INTERNAL_STATS: "1" + # OPAL_AUTH_PUBLIC_KEY is intentionally left unset: with no public key the + # JWT verifier is disabled, so the harness can call scope routes without + # minting JWTs. Local test bed only; never a production setting. + OPAL_LOG_FORMAT_INCLUDE_PID: "true" + ports: + - "7002:7002" + depends_on: + redis: + condition: service_healthy + postgres: + condition: service_healthy + +volumes: + gitea-data: + seed-output: diff --git a/app-tests/git-leak/helpers.py b/app-tests/git-leak/helpers.py new file mode 100644 index 000000000..93871f194 --- /dev/null +++ b/app-tests/git-leak/helpers.py @@ -0,0 +1,94 @@ +"""HTTP + infra helpers for the git-leak test bed.""" +import subprocess +import time +from pathlib import Path +from typing import Dict, List + +import requests + +OPAL_URL = "http://localhost:7002" +GITEA_INTERNAL_URL = "http://gitea:3000" +GITEA_USER = "opaladmin" + +# the compose project lives next to this file; compose() runs from here +_COMPOSE_DIR = str(Path(__file__).resolve().parent) + + +class OpalServerClient: + def __init__(self, base_url: str = OPAL_URL): + self.base_url = base_url.rstrip("/") + + def wait_healthy(self, timeout: int = 180) -> None: + deadline = time.time() + timeout + last = None + while time.time() < deadline: + try: + if requests.get(f"{self.base_url}/healthcheck", timeout=5).status_code == 200: + return + except requests.RequestException as exc: + last = exc + time.sleep(2) + raise RuntimeError(f"opal-server not healthy in {timeout}s (last: {last})") + + def stats(self) -> Dict[str, int]: + resp = requests.get( + f"{self.base_url}/internal/git-fetcher-cache-stats", timeout=10 + ) + resp.raise_for_status() + return resp.json() + + def put_scope(self, scope_id: str, repo_url: str, branch: str = "main") -> None: + body = { + "scope_id": scope_id, + "policy": { + "source_type": "git", + "url": repo_url, + "auth": {"auth_type": "none"}, + "branch": branch, + "directories": ["."], + "extensions": [".rego", ".json"], + "manifest": ".manifest", + "poll_updates": False, + }, + "data": {"entries": []}, + } + # the scope router mounts at prefix="/scopes" with @router.put("") + resp = requests.put(f"{self.base_url}/scopes", json=body, timeout=30) + resp.raise_for_status() + + def delete_scope(self, scope_id: str) -> None: + resp = requests.delete(f"{self.base_url}/scopes/{scope_id}", timeout=30) + if resp.status_code not in (200, 204, 404): + resp.raise_for_status() + + def refresh_all(self) -> None: + # publishes a refresh on the webhook topic; leader pulls all scopes + resp = requests.post(f"{self.base_url}/scopes/refresh", timeout=30) + if resp.status_code == 404: + return # endpoint name differs across versions; caller falls back to poll + resp.raise_for_status() + + +def gitea_repo_url(name: str) -> str: + # url reachable from inside the opal_server container + return f"{GITEA_INTERNAL_URL}/{GITEA_USER}/{name}.git" + + +def compose(*args: str) -> subprocess.CompletedProcess: + return subprocess.run( + ["docker", "compose", *args], + cwd=_COMPOSE_DIR, + capture_output=True, + text=True, + check=True, + ) + + +def bounce_postgres(down_seconds: int = 5) -> None: + compose("stop", "postgres") + time.sleep(down_seconds) + compose("start", "postgres") + + +def list_seeded_repos(count: int) -> List[str]: + return [f"policy-repo-{i:04d}" for i in range(count)] diff --git a/app-tests/git-leak/seed/Dockerfile b/app-tests/git-leak/seed/Dockerfile new file mode 100644 index 000000000..ee9d776d2 --- /dev/null +++ b/app-tests/git-leak/seed/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.11-slim +RUN apt-get update && apt-get install -y --no-install-recommends git \ + && rm -rf /var/lib/apt/lists/* +RUN pip install --no-cache-dir requests GitPython +COPY seed_gitea.py /seed_gitea.py +ENTRYPOINT ["python", "/seed_gitea.py"] diff --git a/app-tests/git-leak/seed/seed_gitea.py b/app-tests/git-leak/seed/seed_gitea.py new file mode 100644 index 000000000..fb41dbc2a --- /dev/null +++ b/app-tests/git-leak/seed/seed_gitea.py @@ -0,0 +1,130 @@ +"""Seed a Gitea instance with N policy repos for the OPAL git-leak test bed. + +Idempotent: re-running creates only the missing repos. Each repo gets a +single commit containing a minimal OPA policy tree. + +Env: + GITEA_URL e.g. http://gitea:3000 + GITEA_ADMIN_USER admin username (created out-of-band by compose) + GITEA_ADMIN_PASSWORD admin password + REPO_COUNT how many repos to ensure exist (default 50) +""" +import os +import sys +import time +from pathlib import Path + +import requests +from git import Actor, Repo + +POLICY_REGO = """package example + +default allow = false + +allow { + input.user == "admin" +} +""" + +DATA_JSON = '{"roles": {"admin": ["read", "write"]}}\n' + + +def _wait_for_gitea(base_url: str, timeout: int = 120) -> None: + deadline = time.time() + timeout + while time.time() < deadline: + try: + if requests.get(f"{base_url}/api/v1/version", timeout=5).status_code == 200: + return + except requests.RequestException: + pass + time.sleep(2) + raise RuntimeError(f"Gitea not reachable at {base_url} within {timeout}s") + + +def _ensure_token(base_url: str, user: str, password: str) -> str: + name = "seed-token" + resp = requests.post( + f"{base_url}/api/v1/users/{user}/tokens", + auth=(user, password), + json={"name": name, "scopes": ["write:repository", "write:user"]}, + timeout=10, + ) + if resp.status_code == 201: + return resp.json()["sha1"] + # token already exists -> delete then recreate (Gitea won't reveal an existing secret) + requests.delete( + f"{base_url}/api/v1/users/{user}/tokens/{name}", auth=(user, password), timeout=10 + ) + resp = requests.post( + f"{base_url}/api/v1/users/{user}/tokens", + auth=(user, password), + json={"name": name, "scopes": ["write:repository", "write:user"]}, + timeout=10, + ) + resp.raise_for_status() + return resp.json()["sha1"] + + +def _ensure_repo(base_url: str, token: str, user: str, name: str) -> None: + headers = {"Authorization": f"token {token}"} + exists = requests.get( + f"{base_url}/api/v1/repos/{user}/{name}", headers=headers, timeout=10 + ) + if exists.status_code == 200: + return + created = requests.post( + f"{base_url}/api/v1/user/repos", + headers=headers, + json={"name": name, "private": False, "auto_init": False}, + timeout=10, + ) + created.raise_for_status() + + +def _push_policy(base_url: str, token: str, user: str, name: str, workdir: Path) -> None: + repo_dir = workdir / name + repo_dir.mkdir(parents=True, exist_ok=True) + (repo_dir / "example.rego").write_text(POLICY_REGO) + (repo_dir / "data.json").write_text(DATA_JSON) + + repo = Repo.init(repo_dir, initial_branch="main") + repo.index.add(["example.rego", "data.json"]) + author = Actor("seed", "seed@example.com") + repo.index.commit("seed policy", author=author, committer=author) + + push_url = base_url.replace("http://", f"http://{user}:{token}@") + f"/{user}/{name}.git" + origin = repo.create_remote("origin", push_url) + origin.push(refspec="main:main") + + +def main() -> int: + base_url = os.environ["GITEA_URL"].rstrip("/") + user = os.environ["GITEA_ADMIN_USER"] + password = os.environ["GITEA_ADMIN_PASSWORD"] + count = int(os.environ.get("REPO_COUNT", "50")) + + _wait_for_gitea(base_url) + token = _ensure_token(base_url, user, password) + + workdir = Path("/tmp/seed-work") + for i in range(count): + name = f"policy-repo-{i:04d}" + _ensure_repo(base_url, token, user, name) + # only push if the repo is empty (freshly created) + head = requests.get( + f"{base_url}/api/v1/repos/{user}/{name}/branches/main", + headers={"Authorization": f"token {token}"}, + timeout=10, + ) + if head.status_code != 200: + _push_policy(base_url, token, user, name, workdir) + print(f"seeded {name}", flush=True) + + # write the token where the test harness can read it + Path("/seed-output/token").write_text(token) + print(f"DONE: ensured {count} repos", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/app-tests/git-leak/test_boot.py b/app-tests/git-leak/test_boot.py new file mode 100644 index 000000000..b9a80b5d7 --- /dev/null +++ b/app-tests/git-leak/test_boot.py @@ -0,0 +1,37 @@ +import os +import time + +import pytest + +from helpers import compose, gitea_repo_url, list_seeded_repos + + +@pytest.mark.timeout(2400) +def test_boot_loads_all_scopes(opal, repo_count): + """Measure how long a fresh boot takes to load all scope repos. + + On master this is serial and slow (the ~20-min problem at scale). + PR4 tightens BOOT_TARGET_SECONDS to assert the parallel speedup. + """ + n = repo_count + repos = list_seeded_repos(n) + for i, name in enumerate(repos): + opal.put_scope(f"boot-{i}", gitea_repo_url(name)) + + # restart only the server so it re-runs sync_scopes on boot + compose("restart", "opal_server") + opal.wait_healthy(timeout=600) + + start = time.time() + deadline = start + 2000 + while time.time() < deadline: + if opal.stats()["repos"] >= n: + break + time.sleep(2) + elapsed = time.time() - start + + # PR1 records the baseline (loose). PR4 will set BOOT_TARGET_SECONDS low. + BOOT_TARGET_SECONDS = int(os.environ.get("BOOT_TARGET_SECONDS", "2000")) + print(f"boot loaded {n} scopes in {elapsed:.1f}s (target {BOOT_TARGET_SECONDS}s)") + assert opal.stats()["repos"] >= n, "not all scopes loaded after boot" + assert elapsed < BOOT_TARGET_SECONDS, f"boot too slow: {elapsed:.1f}s" diff --git a/app-tests/git-leak/test_leak.py b/app-tests/git-leak/test_leak.py new file mode 100644 index 000000000..76db3729c --- /dev/null +++ b/app-tests/git-leak/test_leak.py @@ -0,0 +1,56 @@ +import time + +import pytest + +from helpers import gitea_repo_url, list_seeded_repos + + +def _wait_until(predicate, timeout=30, interval=0.5): + deadline = time.time() + timeout + while time.time() < deadline: + if predicate(): + return True + time.sleep(interval) + return False + + +@pytest.mark.timeout(900) +def test_churn_releases_caches(opal, repo_count): + """Create then delete many scopes; the three caches must return to empty. + + FAILS on master: delete_scope never purges GitPolicyFetcher caches. + """ + n = min(repo_count, 100) + repos = list_seeded_repos(n) + for i, name in enumerate(repos): + opal.put_scope(f"churn-{i}", gitea_repo_url(name)) + _wait_until(lambda: opal.stats()["repos"] >= n, timeout=600) + + for i in range(n): + opal.delete_scope(f"churn-{i}") + + released = _wait_until(lambda: opal.stats()["repos"] == 0, timeout=30) + stats = opal.stats() + assert released, f"repos cache did not drain: {stats}" + assert stats["repos_last_fetched"] == 0, stats + + +@pytest.mark.timeout(900) +def test_repeat_sync_does_not_grow(opal, repo_count): + """Re-syncing the same scopes must not grow the caches unboundedly. + + FAILS on master: repos cache only ever grows. + """ + n = min(repo_count, 50) + repos = list_seeded_repos(n) + for i, name in enumerate(repos): + opal.put_scope(f"stable-{i}", gitea_repo_url(name)) + _wait_until(lambda: opal.stats()["repos"] >= n, timeout=600) + + baseline = opal.stats()["repos"] + for _ in range(10): + opal.refresh_all() + time.sleep(2) + + grown = opal.stats()["repos"] + assert grown <= baseline, f"repos cache grew on repeat sync: {baseline} -> {grown}" diff --git a/app-tests/git-leak/test_resilience.py b/app-tests/git-leak/test_resilience.py new file mode 100644 index 000000000..64598d741 --- /dev/null +++ b/app-tests/git-leak/test_resilience.py @@ -0,0 +1,54 @@ +import time + +import pytest + +from helpers import bounce_postgres, gitea_repo_url, list_seeded_repos + + +@pytest.mark.timeout(300) +def test_offline_repo_does_not_block_healthy_scopes(opal, repo_count): + """An unreachable repo must not stop healthy scopes from serving. + + FAILS on master: the scopes path has no fetch timeout, so a hung + clone occupies the shared executor and the server stalls. + """ + # a routable-but-dead address: TEST-NET-1 (RFC 5737), no git server there + opal.put_scope("offline", "http://192.0.2.1/dead/repo.git", branch="main") + + healthy = list_seeded_repos(1)[0] + opal.put_scope("healthy", gitea_repo_url(healthy)) + + # the healthy scope's repo must appear in the cache within a bounded time, + # even though the offline scope is hanging + deadline = time.time() + 60 + served = False + while time.time() < deadline: + if opal.stats()["repos"] >= 1: + served = True + break + time.sleep(2) + assert served, "healthy scope never loaded while an offline repo was hanging" + + +@pytest.mark.timeout(300) +def test_server_recovers_after_postgres_bounce(opal): + """A transient Postgres outage must not leave the server permanently down. + + FAILS on master: broadcaster disconnect SIGTERMs the worker with no + in-process reconnect; after a bounce the /internal stats endpoint does + not come back within the window without an external supervisor restart. + """ + assert opal.stats() # healthy before + bounce_postgres(down_seconds=5) + + deadline = time.time() + 60 + recovered = False + while time.time() < deadline: + try: + opal.wait_healthy(timeout=5) + opal.stats() + recovered = True + break + except Exception: + time.sleep(2) + assert recovered, "server did not recover within 60s of a postgres bounce" From bd49676013111ac92fe2612d81e642d8d6509d9c Mon Sep 17 00:00:00 2001 From: David Shoen Date: Tue, 23 Jun 2026 12:37:24 +0300 Subject: [PATCH 03/63] test(git-leak): add GiteaAdmin and make_repo_unreachable helpers Complete the helpers.py surface promised in the plan's file-structure table. Both are now functional and used, not dead code: - make_repo_unreachable(name): returns a git URL on a routable-but-dead TEST-NET-1 host (RFC 5737). test_offline_repo now uses it instead of an inlined literal. - GiteaAdmin: host-side Gitea admin client (list_repos / repo_exists / create_repo / delete_repo), exposed as the `gitea_admin` pytest fixture for tests that need to inspect or stage repos beyond the seed sidecar. Gitea is published on host port 13000 (uncommon, to avoid the usual :3000 clash) so GiteaAdmin can reach it; opal_server and the seed sidecar still use the internal http://gitea:3000. README updated with the helper and port notes. Verified live: GiteaAdmin lists the seeded repos and round-trips create/exists/delete against Gitea over the published port. Co-Authored-By: Claude Opus 4.8 (1M context) --- app-tests/git-leak/README.md | 10 +++ app-tests/git-leak/conftest.py | 8 ++- app-tests/git-leak/docker-compose.yml | 7 ++- app-tests/git-leak/helpers.py | 88 +++++++++++++++++++++++++++ app-tests/git-leak/test_resilience.py | 11 +++- 5 files changed, 118 insertions(+), 6 deletions(-) diff --git a/app-tests/git-leak/README.md b/app-tests/git-leak/README.md index 33a291f86..bf4060b33 100644 --- a/app-tests/git-leak/README.md +++ b/app-tests/git-leak/README.md @@ -7,6 +7,16 @@ memory leak, offline-repo hang, slow serial boot, broadcaster no-reconnect. - `opal_server` (2 workers, scopes on, Postgres broadcaster, built from `docker/Dockerfile`) - `redis`, `postgres`, `gitea` (+ one-shot `gitea-admin` and `seed` sidecars) +Only `opal_server` (`:7002`) and `gitea` (`:13000` on the host) are published; +Postgres is internal to the compose network. + +## Helpers (`helpers.py`) +- `OpalServerClient` — drive opal over HTTP (`stats`, `put_scope`, `delete_scope`, `refresh_all`). +- `GiteaAdmin` — host-side Gitea admin client (`list_repos`, `repo_exists`, + `create_repo`, `delete_repo`); also exposed as the `gitea_admin` pytest fixture. +- `make_repo_unreachable(name)` — git URL on a routable-but-dead host (TEST-NET-1) for the offline-repo test. +- `bounce_postgres(down_seconds)` — stop/start Postgres to simulate a broadcaster outage. + ## Run ```bash cd app-tests/git-leak diff --git a/app-tests/git-leak/conftest.py b/app-tests/git-leak/conftest.py index 8c538aa06..ba7052079 100644 --- a/app-tests/git-leak/conftest.py +++ b/app-tests/git-leak/conftest.py @@ -2,7 +2,7 @@ import pytest -from helpers import OpalServerClient, compose +from helpers import GiteaAdmin, OpalServerClient, compose def pytest_addoption(parser): @@ -42,3 +42,9 @@ def stack(request, repo_count): @pytest.fixture() def opal(stack) -> OpalServerClient: return stack + + +@pytest.fixture(scope="session") +def gitea_admin(stack) -> GiteaAdmin: + """Host-side Gitea admin client (depends on `stack` so Gitea is up).""" + return GiteaAdmin() diff --git a/app-tests/git-leak/docker-compose.yml b/app-tests/git-leak/docker-compose.yml index 8f9f474f9..3704e5c83 100644 --- a/app-tests/git-leak/docker-compose.yml +++ b/app-tests/git-leak/docker-compose.yml @@ -30,8 +30,11 @@ services: GITEA__security__INSTALL_LOCK: "true" GITEA__server__ROOT_URL: "http://gitea:3000/" GITEA__database__DB_TYPE: "sqlite3" - # not published to the host: only the seed sidecar and opal_server reach it - # over the compose network (via http://gitea:3000). Avoids host port clashes. + # published on 13000 (not 3000) for the host-side GiteaAdmin helper; the + # uncommon port avoids the usual :3000 clash. opal_server and the seed + # sidecar still reach it over the compose network via http://gitea:3000. + ports: + - "13000:3000" volumes: - gitea-data:/data healthcheck: diff --git a/app-tests/git-leak/helpers.py b/app-tests/git-leak/helpers.py index 93871f194..936a7fc61 100644 --- a/app-tests/git-leak/helpers.py +++ b/app-tests/git-leak/helpers.py @@ -7,8 +7,16 @@ import requests OPAL_URL = "http://localhost:7002" +# reachable from inside the opal_server container (compose network) GITEA_INTERNAL_URL = "http://gitea:3000" +# reachable from the host-side test harness (published port, see docker-compose.yml) +GITEA_HOST_URL = "http://localhost:13000" GITEA_USER = "opaladmin" +GITEA_PASSWORD = "opaladmin" + +# TEST-NET-1 (RFC 5737): routable but runs no git server, so a clone hangs +# instead of failing fast — used to simulate an offline/unreachable repo. +UNREACHABLE_HOST = "192.0.2.1" # the compose project lives next to this file; compose() runs from here _COMPOSE_DIR = str(Path(__file__).resolve().parent) @@ -69,11 +77,91 @@ def refresh_all(self) -> None: resp.raise_for_status() +class GiteaAdmin: + """Host-side admin client for the test bed's Gitea. + + The ``seed`` sidecar does the bulk repo creation from inside the compose + network; this class lets a test inspect or mutate Gitea repos directly + from the host (e.g. assert seeding happened, or add/remove a single repo + for a specific scenario). It authenticates with the admin user that the + ``gitea-admin`` sidecar created, over the published host port. + """ + + def __init__( + self, + base_url: str = GITEA_HOST_URL, + user: str = GITEA_USER, + password: str = GITEA_PASSWORD, + ): + self.base_url = base_url.rstrip("/") + self._user = user + self._auth = (user, password) + + def repo_exists(self, name: str) -> bool: + resp = requests.get( + f"{self.base_url}/api/v1/repos/{self._user}/{name}", + auth=self._auth, + timeout=10, + ) + return resp.status_code == 200 + + def list_repos(self) -> List[str]: + names: List[str] = [] + page = 1 + while True: + resp = requests.get( + f"{self.base_url}/api/v1/users/{self._user}/repos", + params={"page": page, "limit": 50}, + auth=self._auth, + timeout=10, + ) + resp.raise_for_status() + batch = resp.json() + if not batch: + break + names.extend(r["name"] for r in batch) + page += 1 + return names + + def create_repo(self, name: str) -> None: + if self.repo_exists(name): + return + resp = requests.post( + f"{self.base_url}/api/v1/user/repos", + json={"name": name, "private": False, "auto_init": True}, + auth=self._auth, + timeout=10, + ) + resp.raise_for_status() + + def delete_repo(self, name: str) -> None: + resp = requests.delete( + f"{self.base_url}/api/v1/repos/{self._user}/{name}", + auth=self._auth, + timeout=10, + ) + if resp.status_code not in (204, 404): + resp.raise_for_status() + + def gitea_repo_url(name: str) -> str: # url reachable from inside the opal_server container return f"{GITEA_INTERNAL_URL}/{GITEA_USER}/{name}.git" +def make_repo_unreachable(name: str) -> str: + """Return a git URL for ``name`` pointing at a routable-but-dead host. + + Simulates an offline/unreachable policy repo: the address is in + TEST-NET-1 (RFC 5737), which is routable but runs no git server, so a + clone hangs rather than failing fast — exercising the missing fetch + timeout on the scopes path (the bug PR3 fixes). The URL keeps the same + ``/{user}/{name}.git`` shape as a real Gitea repo so the scope looks + ordinary apart from the unreachable host. + """ + return f"http://{UNREACHABLE_HOST}/{GITEA_USER}/{name}.git" + + def compose(*args: str) -> subprocess.CompletedProcess: return subprocess.run( ["docker", "compose", *args], diff --git a/app-tests/git-leak/test_resilience.py b/app-tests/git-leak/test_resilience.py index 64598d741..7d9427903 100644 --- a/app-tests/git-leak/test_resilience.py +++ b/app-tests/git-leak/test_resilience.py @@ -2,7 +2,12 @@ import pytest -from helpers import bounce_postgres, gitea_repo_url, list_seeded_repos +from helpers import ( + bounce_postgres, + gitea_repo_url, + list_seeded_repos, + make_repo_unreachable, +) @pytest.mark.timeout(300) @@ -12,8 +17,8 @@ def test_offline_repo_does_not_block_healthy_scopes(opal, repo_count): FAILS on master: the scopes path has no fetch timeout, so a hung clone occupies the shared executor and the server stalls. """ - # a routable-but-dead address: TEST-NET-1 (RFC 5737), no git server there - opal.put_scope("offline", "http://192.0.2.1/dead/repo.git", branch="main") + # a routable-but-dead address (TEST-NET-1, RFC 5737): the clone hangs + opal.put_scope("offline", make_repo_unreachable("dead-repo"), branch="main") healthy = list_seeded_repos(1)[0] opal.put_scope("healthy", gitea_repo_url(healthy)) From afeb9698ae65c3d4e373cc8468b75ae87149f68b Mon Sep 17 00:00:00 2001 From: David Shoen Date: Tue, 23 Jun 2026 12:55:54 +0300 Subject: [PATCH 04/63] test(git-leak): correct postgres-bounce framing (passes on master) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified test_server_recovers_after_postgres_bounce against the stack: it PASSES on master (~14-19s). On a broadcaster drop the affected worker triggers a graceful shutdown, gunicorn respawns it, and the sibling worker keeps serving HTTP, so the surface recovers within the window — recovery happens via gunicorn's in-container worker supervision, not an external supervisor and not an in-process reconnect. Reframe #5 as a recovery guard (not a known-broken case) in the docstring and README; the prior "FAILS on master / needs external supervisor" wording was wrong. PER-15065's in-process reconnect would avoid the worker churn but recovery already holds. Co-Authored-By: Claude Opus 4.8 (1M context) --- app-tests/git-leak/README.md | 9 +++++++-- app-tests/git-leak/test_resilience.py | 13 ++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/app-tests/git-leak/README.md b/app-tests/git-leak/README.md index bf4060b33..778971705 100644 --- a/app-tests/git-leak/README.md +++ b/app-tests/git-leak/README.md @@ -27,8 +27,13 @@ Useful flags: `--boot-scopes=N` (any N), `--keep-stack` (skip teardown), env `BOOT_TARGET_SECONDS=120` (tighten the boot gate). ## Expected on master -All five flagship tests FAIL (except the boot test, which only fails when -`BOOT_TARGET_SECONDS` is set low). They become the regression gates for the fixes. +The leak tests (#1, #2) and the offline-repo test (#4) FAIL on master — they +target unfixed bugs and become the regression gates for PR2/PR3. The boot test +(#3) passes but only fails when `BOOT_TARGET_SECONDS` is set low (PR4's gate). +The Postgres-bounce test (#5) PASSES on master: it is a recovery guard — when +the broadcaster drops, the worker is respawned by gunicorn while the sibling +worker keeps serving, so the HTTP surface recovers. It guards that property +against regression rather than reproducing a current failure. ## Requires Docker + docker compose v2, plus host Python with `pytest pytest-timeout requests GitPython`. diff --git a/app-tests/git-leak/test_resilience.py b/app-tests/git-leak/test_resilience.py index 7d9427903..8455451eb 100644 --- a/app-tests/git-leak/test_resilience.py +++ b/app-tests/git-leak/test_resilience.py @@ -37,11 +37,14 @@ def test_offline_repo_does_not_block_healthy_scopes(opal, repo_count): @pytest.mark.timeout(300) def test_server_recovers_after_postgres_bounce(opal): - """A transient Postgres outage must not leave the server permanently down. - - FAILS on master: broadcaster disconnect SIGTERMs the worker with no - in-process reconnect; after a bounce the /internal stats endpoint does - not come back within the window without an external supervisor restart. + """A transient Postgres (broadcaster) outage must not leave the server down. + + Recovery guard, not a known-broken case. On current master this PASSES: + when the broadcast channel drops, the affected worker triggers a graceful + shutdown and gunicorn respawns it, while the sibling worker keeps serving + HTTP — so the surface recovers within the window. It guards against a + regression of that property (PER-15065's in-process reconnect would make + recovery cleaner by avoiding the worker churn, but recovery already holds). """ assert opal.stats() # healthy before bounce_postgres(down_seconds=5) From 92353f6277521da05e1f0e5eb782314e4e424fb0 Mon Sep 17 00:00:00 2001 From: David Shoen Date: Tue, 23 Jun 2026 13:03:05 +0300 Subject: [PATCH 05/63] style(git-leak): apply black/isort/docformatter (pre-commit) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run the repo's pinned pre-commit formatters (black 23.1.0, isort 5.12.0, docformatter 1.7.5) over the PR1 files to satisfy the pre-commit CI check. Formatting only — no behavior change. Co-Authored-By: Claude Opus 4.8 (1M context) --- app-tests/git-leak/conftest.py | 1 - app-tests/git-leak/helpers.py | 5 ++++- app-tests/git-leak/seed/seed_gitea.py | 12 +++++++++--- app-tests/git-leak/test_boot.py | 1 - app-tests/git-leak/test_leak.py | 1 - app-tests/git-leak/test_resilience.py | 19 ++++++++++--------- packages/opal-server/opal_server/server.py | 2 +- .../tests/debug_stats_endpoint_test.py | 1 - 8 files changed, 24 insertions(+), 18 deletions(-) diff --git a/app-tests/git-leak/conftest.py b/app-tests/git-leak/conftest.py index ba7052079..7501f99d0 100644 --- a/app-tests/git-leak/conftest.py +++ b/app-tests/git-leak/conftest.py @@ -1,7 +1,6 @@ import os import pytest - from helpers import GiteaAdmin, OpalServerClient, compose diff --git a/app-tests/git-leak/helpers.py b/app-tests/git-leak/helpers.py index 936a7fc61..1a3bd4fcc 100644 --- a/app-tests/git-leak/helpers.py +++ b/app-tests/git-leak/helpers.py @@ -31,7 +31,10 @@ def wait_healthy(self, timeout: int = 180) -> None: last = None while time.time() < deadline: try: - if requests.get(f"{self.base_url}/healthcheck", timeout=5).status_code == 200: + if ( + requests.get(f"{self.base_url}/healthcheck", timeout=5).status_code + == 200 + ): return except requests.RequestException as exc: last = exc diff --git a/app-tests/git-leak/seed/seed_gitea.py b/app-tests/git-leak/seed/seed_gitea.py index fb41dbc2a..ddebe5667 100644 --- a/app-tests/git-leak/seed/seed_gitea.py +++ b/app-tests/git-leak/seed/seed_gitea.py @@ -53,7 +53,9 @@ def _ensure_token(base_url: str, user: str, password: str) -> str: return resp.json()["sha1"] # token already exists -> delete then recreate (Gitea won't reveal an existing secret) requests.delete( - f"{base_url}/api/v1/users/{user}/tokens/{name}", auth=(user, password), timeout=10 + f"{base_url}/api/v1/users/{user}/tokens/{name}", + auth=(user, password), + timeout=10, ) resp = requests.post( f"{base_url}/api/v1/users/{user}/tokens", @@ -81,7 +83,9 @@ def _ensure_repo(base_url: str, token: str, user: str, name: str) -> None: created.raise_for_status() -def _push_policy(base_url: str, token: str, user: str, name: str, workdir: Path) -> None: +def _push_policy( + base_url: str, token: str, user: str, name: str, workdir: Path +) -> None: repo_dir = workdir / name repo_dir.mkdir(parents=True, exist_ok=True) (repo_dir / "example.rego").write_text(POLICY_REGO) @@ -92,7 +96,9 @@ def _push_policy(base_url: str, token: str, user: str, name: str, workdir: Path) author = Actor("seed", "seed@example.com") repo.index.commit("seed policy", author=author, committer=author) - push_url = base_url.replace("http://", f"http://{user}:{token}@") + f"/{user}/{name}.git" + push_url = ( + base_url.replace("http://", f"http://{user}:{token}@") + f"/{user}/{name}.git" + ) origin = repo.create_remote("origin", push_url) origin.push(refspec="main:main") diff --git a/app-tests/git-leak/test_boot.py b/app-tests/git-leak/test_boot.py index b9a80b5d7..928fd6f4e 100644 --- a/app-tests/git-leak/test_boot.py +++ b/app-tests/git-leak/test_boot.py @@ -2,7 +2,6 @@ import time import pytest - from helpers import compose, gitea_repo_url, list_seeded_repos diff --git a/app-tests/git-leak/test_leak.py b/app-tests/git-leak/test_leak.py index 76db3729c..30b9a81a0 100644 --- a/app-tests/git-leak/test_leak.py +++ b/app-tests/git-leak/test_leak.py @@ -1,7 +1,6 @@ import time import pytest - from helpers import gitea_repo_url, list_seeded_repos diff --git a/app-tests/git-leak/test_resilience.py b/app-tests/git-leak/test_resilience.py index 8455451eb..b8fb73c89 100644 --- a/app-tests/git-leak/test_resilience.py +++ b/app-tests/git-leak/test_resilience.py @@ -1,7 +1,6 @@ import time import pytest - from helpers import ( bounce_postgres, gitea_repo_url, @@ -37,14 +36,16 @@ def test_offline_repo_does_not_block_healthy_scopes(opal, repo_count): @pytest.mark.timeout(300) def test_server_recovers_after_postgres_bounce(opal): - """A transient Postgres (broadcaster) outage must not leave the server down. - - Recovery guard, not a known-broken case. On current master this PASSES: - when the broadcast channel drops, the affected worker triggers a graceful - shutdown and gunicorn respawns it, while the sibling worker keeps serving - HTTP — so the surface recovers within the window. It guards against a - regression of that property (PER-15065's in-process reconnect would make - recovery cleaner by avoiding the worker churn, but recovery already holds). + """A transient Postgres (broadcaster) outage must not leave the server + down. + + Recovery guard, not a known-broken case. On current master this + PASSES: when the broadcast channel drops, the affected worker + triggers a graceful shutdown and gunicorn respawns it, while the + sibling worker keeps serving HTTP — so the surface recovers within + the window. It guards against a regression of that property + (PER-15065's in-process reconnect would make recovery cleaner by + avoiding the worker churn, but recovery already holds). """ assert opal.stats() # healthy before bounce_postgres(down_seconds=5) diff --git a/packages/opal-server/opal_server/server.py b/packages/opal-server/opal_server/server.py index 2b88b3db8..1be796d6d 100644 --- a/packages/opal-server/opal_server/server.py +++ b/packages/opal-server/opal_server/server.py @@ -26,8 +26,8 @@ ) from opal_server.config import opal_server_config from opal_server.data.api import init_data_updates_router -from opal_server.debug_stats import register_internal_stats_route from opal_server.data.data_update_publisher import DataUpdatePublisher +from opal_server.debug_stats import register_internal_stats_route from opal_server.loadlimiting import init_loadlimit_router from opal_server.policy.bundles.api import router as bundles_router from opal_server.policy.watcher.factory import setup_watcher_task diff --git a/packages/opal-server/opal_server/tests/debug_stats_endpoint_test.py b/packages/opal-server/opal_server/tests/debug_stats_endpoint_test.py index 8ba0f678a..00a9c8e1b 100644 --- a/packages/opal-server/opal_server/tests/debug_stats_endpoint_test.py +++ b/packages/opal-server/opal_server/tests/debug_stats_endpoint_test.py @@ -1,6 +1,5 @@ from fastapi import FastAPI from fastapi.testclient import TestClient - from opal_server.debug_stats import register_internal_stats_route From db54ae627824aaea11f72b3ff829dcaddc6fa96d Mon Sep 17 00:00:00 2001 From: David Shoen Date: Tue, 23 Jun 2026 13:13:22 +0300 Subject: [PATCH 06/63] test: scope root pytest collection to packages/ (exclude git-leak bed) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CI `build` job runs `pytest` from the repo root with no path, which recursed into app-tests/git-leak/ and ran the flagship tests — these are designed to FAIL on master (they are the regression gates for PR2-PR5), so they broke the build job. Set `testpaths = packages` so the rootdir run collects only the unit tests under packages/ (matching master's effective behavior, since app-tests/ had no pytest files before). testpaths only applies when pytest is invoked from the rootdir with no args, so `cd app-tests/git-leak && pytest` still collects and runs the test bed. Verified both: root run -> packages only; subdir run -> all 5 flagship tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- pytest.ini | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pytest.ini b/pytest.ini index 16c88ba91..c98ec88b9 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,10 @@ # Handling DeprecationWarning 'asyncio_mode' default value [pytest] asyncio_mode = strict +# Scope the default (rootdir) collection to the unit tests under packages/. +# The heavyweight, docker-compose-driven test bed under app-tests/git-leak/ is +# designed to FAIL on master (it is the regression gate for later PRs) and must +# not be collected by the repo's CI `pytest` run. testpaths only applies when +# pytest is invoked from the rootdir with no args, so running it explicitly +# (e.g. `cd app-tests/git-leak && pytest`) still works. +testpaths = packages From 6f9a0894a1aa937a91a0cbe8f12b9b666306b680 Mon Sep 17 00:00:00 2001 From: David Shoen Date: Tue, 23 Jun 2026 13:24:22 +0300 Subject: [PATCH 07/63] test(git-leak): address Copilot review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - stats(): sample the /internal endpoint several times and merge per key with max(). The caches are per-process on a multi-worker server, so a single read can hit an empty (non-leader) worker — max-merge avoids both false negatives (missing a populated leader) and false positives (an `== 0` drain assertion passing only because it hit an empty worker). - test_leak: assert the initial-load `_wait_until` succeeded before deleting / before taking baseline, so the tests can't pass vacuously when load never completed. - refresh_all(): correct the misleading comment — a 404 is a no-op, there is no client-side fallback. - conftest: skip the suite cleanly if docker is unavailable (defense in depth; it's already excluded from the default pytest run via testpaths). Co-Authored-By: Claude Opus 4.8 (1M context) --- app-tests/git-leak/conftest.py | 7 +++++++ app-tests/git-leak/helpers.py | 35 +++++++++++++++++++++++++-------- app-tests/git-leak/test_leak.py | 6 ++++-- 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/app-tests/git-leak/conftest.py b/app-tests/git-leak/conftest.py index 7501f99d0..9701aff87 100644 --- a/app-tests/git-leak/conftest.py +++ b/app-tests/git-leak/conftest.py @@ -1,4 +1,5 @@ import os +import shutil import pytest from helpers import GiteaAdmin, OpalServerClient, compose @@ -26,6 +27,12 @@ def repo_count(request) -> int: @pytest.fixture(scope="session") def stack(request, repo_count): + # Defense-in-depth: this docker-compose suite is already excluded from the + # repo's default `pytest` run via `testpaths = packages` in pytest.ini, so + # the unit-test CI matrix never collects it. If it is ever collected in an + # environment without docker, skip cleanly instead of erroring. + if shutil.which("docker") is None: + pytest.skip("docker (compose) is required for the git-leak test bed") os.environ["REPO_COUNT"] = str(repo_count) # build + start infra; seed runs to completion then exits compose("up", "-d", "--build") diff --git a/app-tests/git-leak/helpers.py b/app-tests/git-leak/helpers.py index 1a3bd4fcc..8d0acc38a 100644 --- a/app-tests/git-leak/helpers.py +++ b/app-tests/git-leak/helpers.py @@ -41,12 +41,28 @@ def wait_healthy(self, timeout: int = 180) -> None: time.sleep(2) raise RuntimeError(f"opal-server not healthy in {timeout}s (last: {last})") - def stats(self) -> Dict[str, int]: - resp = requests.get( - f"{self.base_url}/internal/git-fetcher-cache-stats", timeout=10 - ) - resp.raise_for_status() - return resp.json() + def stats(self, samples: int = 5, interval: float = 0.1) -> Dict[str, int]: + """Read the git-fetcher cache stats, merged across several reads. + + The server runs multiple workers and the ``GitPolicyFetcher`` caches + are per-process, so a single read may land on a worker with empty + caches (e.g. a non-leader that never fetched). Sample a few times and + take the ``max`` per key, so the harness observes whichever worker + still holds the most entries — this prevents both false negatives + (missing a populated leader) and false positives (an ``== 0`` drain + assertion passing only because it hit an empty worker). + """ + merged: Dict[str, int] = {} + for i in range(max(1, samples)): + resp = requests.get( + f"{self.base_url}/internal/git-fetcher-cache-stats", timeout=10 + ) + resp.raise_for_status() + for key, value in resp.json().items(): + merged[key] = max(merged.get(key, 0), value) + if i < samples - 1: + time.sleep(interval) + return merged def put_scope(self, scope_id: str, repo_url: str, branch: str = "main") -> None: body = { @@ -73,10 +89,13 @@ def delete_scope(self, scope_id: str) -> None: resp.raise_for_status() def refresh_all(self) -> None: - # publishes a refresh on the webhook topic; leader pulls all scopes + # Best-effort: POST /scopes/refresh publishes on the webhook topic so + # every leader re-syncs all scopes. If this OPAL build doesn't expose + # the route (404), treat it as a no-op — there is no client-side + # fallback; callers rely on their own stats polling either way. resp = requests.post(f"{self.base_url}/scopes/refresh", timeout=30) if resp.status_code == 404: - return # endpoint name differs across versions; caller falls back to poll + return resp.raise_for_status() diff --git a/app-tests/git-leak/test_leak.py b/app-tests/git-leak/test_leak.py index 30b9a81a0..d2ef0f21b 100644 --- a/app-tests/git-leak/test_leak.py +++ b/app-tests/git-leak/test_leak.py @@ -23,7 +23,8 @@ def test_churn_releases_caches(opal, repo_count): repos = list_seeded_repos(n) for i, name in enumerate(repos): opal.put_scope(f"churn-{i}", gitea_repo_url(name)) - _wait_until(lambda: opal.stats()["repos"] >= n, timeout=600) + loaded = _wait_until(lambda: opal.stats()["repos"] >= n, timeout=600) + assert loaded, f"initial load never reached {n} repos: {opal.stats()}" for i in range(n): opal.delete_scope(f"churn-{i}") @@ -44,7 +45,8 @@ def test_repeat_sync_does_not_grow(opal, repo_count): repos = list_seeded_repos(n) for i, name in enumerate(repos): opal.put_scope(f"stable-{i}", gitea_repo_url(name)) - _wait_until(lambda: opal.stats()["repos"] >= n, timeout=600) + loaded = _wait_until(lambda: opal.stats()["repos"] >= n, timeout=600) + assert loaded, f"initial sync never reached {n} repos: {opal.stats()}" baseline = opal.stats()["repos"] for _ in range(10): From 1b23ac00223f3b827d51546e0be6c196c27a8edf Mon Sep 17 00:00:00 2001 From: David Shoen Date: Tue, 23 Jun 2026 13:47:25 +0300 Subject: [PATCH 08/63] test(git-leak): isolate scopes per test and fix false repeat-sync gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings on the regression gates: - Cross-test contamination: clone paths are keyed by repo URL (source_id = sha256(url)+branch-shard), not scope_id, so scopes from different tests that point at the same seeded repo share one GitPolicyFetcher cache entry. With a session-scoped stack and no teardown, a leftover boot-*/stable-* scope kept those entries alive and would make test_churn's `repos == 0` drain assertion fail on fixed code. OpalServerClient now tracks created scopes and the opal fixture deletes them on teardown (best-effort drain wait, swallows errors so master — where delete never purges — doesn't fail the passing test). - False gate: test_repeat_sync_does_not_grow re-syncs identical scopes, which a path-keyed cache can't grow even on master, so it could never be the leak gate it claimed. Reframed as an honest idempotency guard (passes on master) that points at test_churn_releases_caches as the real leak gate; README's "Expected on master" reclassifies it alongside the postgres-bounce guard. Co-Authored-By: Claude Opus 4.8 (1M context) --- app-tests/git-leak/README.md | 21 ++++++++++++++------- app-tests/git-leak/conftest.py | 9 ++++++++- app-tests/git-leak/helpers.py | 31 +++++++++++++++++++++++++++++++ app-tests/git-leak/test_leak.py | 11 +++++++++-- 4 files changed, 62 insertions(+), 10 deletions(-) diff --git a/app-tests/git-leak/README.md b/app-tests/git-leak/README.md index 778971705..a8abea9f6 100644 --- a/app-tests/git-leak/README.md +++ b/app-tests/git-leak/README.md @@ -27,13 +27,20 @@ Useful flags: `--boot-scopes=N` (any N), `--keep-stack` (skip teardown), env `BOOT_TARGET_SECONDS=120` (tighten the boot gate). ## Expected on master -The leak tests (#1, #2) and the offline-repo test (#4) FAIL on master — they -target unfixed bugs and become the regression gates for PR2/PR3. The boot test -(#3) passes but only fails when `BOOT_TARGET_SECONDS` is set low (PR4's gate). -The Postgres-bounce test (#5) PASSES on master: it is a recovery guard — when -the broadcaster drops, the worker is respawned by gunicorn while the sibling -worker keeps serving, so the HTTP surface recovers. It guards that property -against regression rather than reproducing a current failure. +The churn leak test (`test_churn_releases_caches`) and the offline-repo test +FAIL on master — they target unfixed bugs and become the regression gates for +PR2/PR3. The boot test passes but only fails when `BOOT_TARGET_SECONDS` is set +low (PR4's gate). + +Two tests are guards that PASS on master rather than reproducing a current +failure: +- `test_repeat_sync_does_not_grow` — clone paths are keyed by the repo URL, so + re-syncing identical scopes reuses cache entries and does not grow them. It + guards against a regression that would make repeat sync allocate per-sync. + (The unbounded-growth-then-no-purge leak is the *churn* test's job.) +- `test_server_recovers_after_postgres_bounce` — when the broadcaster drops, the + affected worker is respawned by gunicorn while the sibling keeps serving, so + the HTTP surface recovers. Guards that property against regression. ## Requires Docker + docker compose v2, plus host Python with `pytest pytest-timeout requests GitPython`. diff --git a/app-tests/git-leak/conftest.py b/app-tests/git-leak/conftest.py index 9701aff87..c3c01bdbb 100644 --- a/app-tests/git-leak/conftest.py +++ b/app-tests/git-leak/conftest.py @@ -47,7 +47,14 @@ def stack(request, repo_count): @pytest.fixture() def opal(stack) -> OpalServerClient: - return stack + # The compose stack is session-scoped (one server for the whole run), but + # scopes must not leak between tests: clone paths are keyed by repo URL, so + # a scope left behind by one test shares a cache entry with any later test + # using the same seeded repo and would pollute its drain assertions. Delete + # whatever this test created on teardown to keep each test isolated. + stack._created_scopes.clear() + yield stack + stack.cleanup_created_scopes() @pytest.fixture(scope="session") diff --git a/app-tests/git-leak/helpers.py b/app-tests/git-leak/helpers.py index 8d0acc38a..4fefaf81d 100644 --- a/app-tests/git-leak/helpers.py +++ b/app-tests/git-leak/helpers.py @@ -25,6 +25,12 @@ class OpalServerClient: def __init__(self, base_url: str = OPAL_URL): self.base_url = base_url.rstrip("/") + # scope_ids created via put_scope, so a per-test fixture can delete them + # on teardown. Clone paths are keyed by repo URL (not scope_id), so a + # scope left behind by one test shares a GitPolicyFetcher cache entry + # with any other test pointing at the same seeded repo — without cleanup + # that leftover keeps the entry alive and pollutes a drain assertion. + self._created_scopes: set = set() def wait_healthy(self, timeout: int = 180) -> None: deadline = time.time() + timeout @@ -82,11 +88,36 @@ def put_scope(self, scope_id: str, repo_url: str, branch: str = "main") -> None: # the scope router mounts at prefix="/scopes" with @router.put("") resp = requests.put(f"{self.base_url}/scopes", json=body, timeout=30) resp.raise_for_status() + self._created_scopes.add(scope_id) def delete_scope(self, scope_id: str) -> None: resp = requests.delete(f"{self.base_url}/scopes/{scope_id}", timeout=30) if resp.status_code not in (200, 204, 404): resp.raise_for_status() + self._created_scopes.discard(scope_id) + + def cleanup_created_scopes(self, drain_timeout: int = 15) -> None: + """Delete every scope created on this client, then best-effort wait for + the caches to drain — so the next test starts from a clean slate. + + Best-effort by design: on master, delete never purges the caches (the + leak this suite gates), so the drain wait will simply time out. A + teardown failure must never fail the test that just ran, hence the broad + excepts and the bounded wait. + """ + for scope_id in list(self._created_scopes): + try: + self.delete_scope(scope_id) + except Exception: + self._created_scopes.discard(scope_id) + deadline = time.time() + drain_timeout + while time.time() < deadline: + try: + if self.stats(samples=1)["repos"] == 0: + return + except Exception: + return + time.sleep(1) def refresh_all(self) -> None: # Best-effort: POST /scopes/refresh publishes on the webhook topic so diff --git a/app-tests/git-leak/test_leak.py b/app-tests/git-leak/test_leak.py index d2ef0f21b..8cfee6363 100644 --- a/app-tests/git-leak/test_leak.py +++ b/app-tests/git-leak/test_leak.py @@ -37,9 +37,16 @@ def test_churn_releases_caches(opal, repo_count): @pytest.mark.timeout(900) def test_repeat_sync_does_not_grow(opal, repo_count): - """Re-syncing the same scopes must not grow the caches unboundedly. + """Re-syncing the *same* scopes must not grow the caches. - FAILS on master: repos cache only ever grows. + Idempotency guard, not a known-broken case. On current master this + PASSES: a clone path is keyed by the repo URL (``source_id`` = + sha256(url)+branch-shard), so re-syncing identical scopes reuses the + existing ``repos`` / ``repos_last_fetched`` entries instead of + allocating new ones. It guards against a regression that would make + repeat sync allocate per-sync. The unbounded *growth + no purge on + delete* leak is covered by ``test_churn_releases_caches`` above, which + uses distinct scopes and DOES fail on master. """ n = min(repo_count, 50) repos = list_seeded_repos(n) From 6046a103f229c07616a9e053fb5de0a0267e93f7 Mon Sep 17 00:00:00 2001 From: David Shoen Date: Wed, 24 Jun 2026 16:03:00 +0300 Subject: [PATCH 09/63] test(git-leak): make the regression gates trustworthy (address PR review) Addresses the CHANGES_REQUESTED review on PR #922. Root cause behind most findings: a fresh scope's first sync takes the _clone() branch, which only fills GitPolicyFetcher.repo_locks; repos/repos_last_fetched are filled on a *second* sync. So the load gates on `repos` hung, and the 2-worker per-process caches made `== 0` drain assertions unsafe. Blocking: - Single worker (UVICORN_NUM_WORKERS=1): deterministic per-process cache reads; removes the false `== 0` drain class. - Load gate (CRITICAL + Zivxx HIGH): _load_scopes gates on repo_locks then refresh_all() to force the second sync, so repos/repos_last_fetched are actually populated before any drain/purge assertion. - compose() surfaces captured stdout/stderr on failure. - Seed completeness asserted in conftest; seed script isolates per-repo failures and exits non-zero with a count. Secondary: - test_repeat_sync asserts an RSS bound (count can't grow for any impl); churn asserts all three caches drain + a loose RSS backstop. - blackhole socat sidecar replaces TEST-NET-1 (deterministic hang); offline test saturates the fetch executor with 40 hung clones and recovers via OpalServerClient.hard_reset() (stop -> redis FLUSHALL -> start). - Per-test clean slate deletes all server scopes (fixes orphan-scope leak). - Postgres-bounce proves broadcast recovery (PUT post-bounce, assert sync) and uses `up -d --wait`. - Remove dead 404 branch in refresh_all; boot clock starts at restart; gitea-admin `|| true` -> "already exists"-only guard; README reworded. opal-server: - /internal stats route now takes the JWTAuthenticator dependency (protected when JWT on, no-op in the test bed); unit test asserts enforcement. Co-Authored-By: Claude Opus 4.8 (1M context) --- app-tests/git-leak/README.md | 62 +++++--- app-tests/git-leak/conftest.py | 27 +++- app-tests/git-leak/docker-compose.yml | 26 +++- app-tests/git-leak/helpers.py | 141 ++++++++++++------ app-tests/git-leak/seed/seed_gitea.py | 36 +++-- app-tests/git-leak/test_boot.py | 12 +- app-tests/git-leak/test_leak.py | 99 ++++++++---- app-tests/git-leak/test_resilience.py | 105 +++++++++---- .../opal-server/opal_server/debug_stats.py | 25 +++- packages/opal-server/opal_server/server.py | 4 +- .../tests/debug_stats_endpoint_test.py | 19 ++- 11 files changed, 407 insertions(+), 149 deletions(-) diff --git a/app-tests/git-leak/README.md b/app-tests/git-leak/README.md index a8abea9f6..aabbb4657 100644 --- a/app-tests/git-leak/README.md +++ b/app-tests/git-leak/README.md @@ -1,21 +1,31 @@ # OPAL git-leak / resilience test bed -Reproduces (as failing tests on `master`) the four issues fixed by PR2–PR5: -memory leak, offline-repo hang, slow serial boot, broadcaster no-reconnect. +Reproduces (as failing tests) the four issues fixed by PR2–PR5: memory leak, +offline-repo hang, slow serial boot, broadcaster no-reconnect. + +Every assertion is driven through `GET /internal/git-fetcher-cache-stats`, which +**this PR (PR1) adds** — it does not exist on `master`. So the suite runs against +*this branch*: the leak/offline tests fail here *until PR2/PR3 land*, then go +green. Run against true `master` they would all error at setup on the missing +endpoint, not "fail for the targeted bug." ## Stack -- `opal_server` (2 workers, scopes on, Postgres broadcaster, built from `docker/Dockerfile`) +- `opal_server` (single worker, scopes on, Postgres broadcaster, built from `docker/Dockerfile`) - `redis`, `postgres`, `gitea` (+ one-shot `gitea-admin` and `seed` sidecars) +- `blackhole` (alpine/socat: accepts TCP then never answers — the offline repo) Only `opal_server` (`:7002`) and `gitea` (`:13000` on the host) are published; -Postgres is internal to the compose network. +Postgres and `blackhole` are internal to the compose network. ## Helpers (`helpers.py`) -- `OpalServerClient` — drive opal over HTTP (`stats`, `put_scope`, `delete_scope`, `refresh_all`). +- `OpalServerClient` — drive opal over HTTP (`stats`, `put_scope`, `delete_scope`, + `refresh_all`, `get_scope_policy`, `list_scope_ids`, `delete_all_scopes`). - `GiteaAdmin` — host-side Gitea admin client (`list_repos`, `repo_exists`, `create_repo`, `delete_repo`); also exposed as the `gitea_admin` pytest fixture. -- `make_repo_unreachable(name)` — git URL on a routable-but-dead host (TEST-NET-1) for the offline-repo test. -- `bounce_postgres(down_seconds)` — stop/start Postgres to simulate a broadcaster outage. +- `make_repo_unreachable(name)` — git URL on the `blackhole` sidecar (completes + the TCP handshake, never answers) so the clone hangs for the offline-repo test. +- `bounce_postgres(down_seconds)` — stop Postgres, then `up -d --wait` it back to + simulate a broadcaster outage and await readiness before the recovery poll. ## Run ```bash @@ -26,21 +36,21 @@ python -m pytest test_leak.py -v --boot-scopes=20 # just the leak gates Useful flags: `--boot-scopes=N` (any N), `--keep-stack` (skip teardown), env `BOOT_TARGET_SECONDS=120` (tighten the boot gate). -## Expected on master +## Expected behavior The churn leak test (`test_churn_releases_caches`) and the offline-repo test -FAIL on master — they target unfixed bugs and become the regression gates for -PR2/PR3. The boot test passes but only fails when `BOOT_TARGET_SECONDS` is set -low (PR4's gate). +FAIL on this branch *without the PR2/PR3 fix* — they target unfixed bugs and +become the regression gates for PR2/PR3, flipping green when those land. The +boot test passes but fails when `BOOT_TARGET_SECONDS` is set low (PR4's gate). -Two tests are guards that PASS on master rather than reproducing a current -failure: +Two tests are guards that PASS rather than reproducing a current failure: - `test_repeat_sync_does_not_grow` — clone paths are keyed by the repo URL, so - re-syncing identical scopes reuses cache entries and does not grow them. It - guards against a regression that would make repeat sync allocate per-sync. - (The unbounded-growth-then-no-purge leak is the *churn* test's job.) + re-syncing identical scopes reuses cache entries and the cache *counts* can't + grow for any implementation; the load-bearing assertion is therefore on RSS, + guarding against a regression that leaks per-sync allocations. - `test_server_recovers_after_postgres_bounce` — when the broadcaster drops, the - affected worker is respawned by gunicorn while the sibling keeps serving, so - the HTTP surface recovers. Guards that property against regression. + worker is respawned by gunicorn and the broadcaster reconnects once Postgres + is back; the test PUTs a fresh scope post-bounce and asserts it syncs, proving + the broadcast path (not just HTTP) recovered. ## Requires Docker + docker compose v2, plus host Python with `pytest pytest-timeout requests GitPython`. @@ -48,8 +58,14 @@ Docker + docker compose v2, plus host Python with `pytest pytest-timeout request ## Notes - Auth is disabled in the stack: `OPAL_AUTH_PUBLIC_KEY` is left unset so the JWT verifier is disabled and the harness can call scope routes without minting JWTs. - Local test bed only; never a production setting. -- The server runs 2 uvicorn workers with a Postgres broadcaster, mirroring a - realistic multi-worker deployment. The `GitPolicyFetcher` caches read by the - `/internal/git-fetcher-cache-stats` endpoint are per-process, so the harness - polls with generous timeouts to let the leader worker converge. + Local test bed only; never a production setting. (The `/internal` endpoint is + registered with the same `JWTAuthenticator` dependency as the other routes, so + it is protected when JWT verification is enabled and open only here.) +- The server runs a **single** uvicorn worker. The `GitPolicyFetcher` caches read + by `/internal/git-fetcher-cache-stats` are per-process, so a multi-worker stack + would make a round-robin read miss the worker that fetched and let a `== 0` + drain assertion pass falsely. One worker makes every cache read deterministic; + the leak/boot/offline bugs all reproduce single-worker. +- First-sync of a fresh scope takes the clone path, which fills only `repo_locks`; + `repos` / `repos_last_fetched` are filled by the discover/fetch path on a second + sync, so the load helpers issue a `refresh_all()` before asserting on `repos`. diff --git a/app-tests/git-leak/conftest.py b/app-tests/git-leak/conftest.py index c3c01bdbb..1660d596b 100644 --- a/app-tests/git-leak/conftest.py +++ b/app-tests/git-leak/conftest.py @@ -2,7 +2,7 @@ import shutil import pytest -from helpers import GiteaAdmin, OpalServerClient, compose +from helpers import GiteaAdmin, OpalServerClient, compose, list_seeded_repos def pytest_addoption(parser): @@ -36,8 +36,20 @@ def stack(request, repo_count): os.environ["REPO_COUNT"] = str(repo_count) # build + start infra; seed runs to completion then exits compose("up", "-d", "--build") - # block until seeding sidecar has finished creating repos + # block until seeding sidecar has finished creating repos. compose() raises + # (with output) if the seed container exited non-zero, so a hard seed + # failure surfaces here rather than as a confusing later test failure. compose("wait", "seed") + # Verify the seed actually produced all N repos before any test runs: a + # partial seed would otherwise look like a server bug when the load gate + # can't reach N. Fail loudly with the gap. + expected = set(list_seeded_repos(repo_count)) + present = set(GiteaAdmin().list_repos()) + missing = expected - present + assert not missing, ( + f"seed incomplete: {len(missing)}/{repo_count} repos missing " + f"(e.g. {sorted(missing)[:5]})" + ) client = OpalServerClient() client.wait_healthy() yield client @@ -50,11 +62,14 @@ def opal(stack) -> OpalServerClient: # The compose stack is session-scoped (one server for the whole run), but # scopes must not leak between tests: clone paths are keyed by repo URL, so # a scope left behind by one test shares a cache entry with any later test - # using the same seeded repo and would pollute its drain assertions. Delete - # whatever this test created on teardown to keep each test isolated. - stack._created_scopes.clear() + # using the same seeded repo and would pollute its drain assertions. + # + # Delete every scope the *server* currently knows (not just this client's + # tracked set) at setup, so a scope orphaned by a prior failed test can't + # contaminate this one; then again on teardown. + stack.delete_all_scopes() yield stack - stack.cleanup_created_scopes() + stack.delete_all_scopes() @pytest.fixture(scope="session") diff --git a/app-tests/git-leak/docker-compose.yml b/app-tests/git-leak/docker-compose.yml index 3704e5c83..bd48c26dc 100644 --- a/app-tests/git-leak/docker-compose.yml +++ b/app-tests/git-leak/docker-compose.yml @@ -51,15 +51,30 @@ services: condition: service_healthy user: git entrypoint: ["/bin/sh", "-c"] + # Tolerate only the idempotent "already exists" case; any other failure + # must abort so `seed` (which depends on this completing) doesn't run + # against a Gitea with no admin user and fail with a confusing 401. command: - > - gitea admin user create --username opaladmin --password opaladmin + out=$$(gitea admin user create --username opaladmin --password opaladmin --email admin@example.com --admin --must-change-password=false - --config /data/gitea/conf/app.ini || true + --config /data/gitea/conf/app.ini 2>&1); rc=$$?; + echo "$$out"; + if [ $$rc -ne 0 ] && ! echo "$$out" | grep -qi "already exist"; then + exit $$rc; + fi volumes: - gitea-data:/data restart: "no" + blackhole: + # Accepts the TCP handshake then never answers — a clone connects and + # blocks reading the git smart-HTTP response, holding the fetch executor. + # Deterministic, unlike a TEST-NET-1 address which many networks reject + # fast with ICMP-unreachable (so the clone would fail fast, not hang). + image: alpine/socat:latest + command: ["TCP-LISTEN:80,fork,reuseaddr", "SYSTEM:sleep 3600"] + seed: build: ./seed depends_on: @@ -82,7 +97,12 @@ services: dockerfile: docker/Dockerfile target: server environment: - UVICORN_NUM_WORKERS: "2" + # Single worker on purpose: the GitPolicyFetcher caches read by + # /internal/git-fetcher-cache-stats are per-process, so with >1 worker a + # round-robin read can miss the worker that fetched and make a `== 0` + # drain assertion pass falsely. One worker makes every cache read + # deterministic. The leak/boot/offline bugs all reproduce single-worker. + UVICORN_NUM_WORKERS: "1" OPAL_SCOPES: "1" OPAL_REDIS_URL: "redis://redis:6379" OPAL_BROADCAST_URI: "postgres://opal:opal@postgres:5432/opal" diff --git a/app-tests/git-leak/helpers.py b/app-tests/git-leak/helpers.py index 4fefaf81d..3cb66a36a 100644 --- a/app-tests/git-leak/helpers.py +++ b/app-tests/git-leak/helpers.py @@ -14,9 +14,12 @@ GITEA_USER = "opaladmin" GITEA_PASSWORD = "opaladmin" -# TEST-NET-1 (RFC 5737): routable but runs no git server, so a clone hangs -# instead of failing fast — used to simulate an offline/unreachable repo. -UNREACHABLE_HOST = "192.0.2.1" +# the `blackhole` compose service (alpine/socat) accepts the TCP handshake then +# never answers, so a clone connects and blocks reading the response — a +# deterministic hang. Reachable from the opal_server container on the compose +# network. (A TEST-NET-1 address was rejected too fast on many networks, so the +# clone failed fast instead of hanging and the offline scenario wasn't exercised.) +UNREACHABLE_HOST = "blackhole" # the compose project lives next to this file; compose() runs from here _COMPOSE_DIR = str(Path(__file__).resolve().parent) @@ -47,16 +50,15 @@ def wait_healthy(self, timeout: int = 180) -> None: time.sleep(2) raise RuntimeError(f"opal-server not healthy in {timeout}s (last: {last})") - def stats(self, samples: int = 5, interval: float = 0.1) -> Dict[str, int]: - """Read the git-fetcher cache stats, merged across several reads. + def stats(self, samples: int = 3, interval: float = 0.1) -> Dict[str, int]: + """Read the git-fetcher cache stats, merged across a few reads. - The server runs multiple workers and the ``GitPolicyFetcher`` caches - are per-process, so a single read may land on a worker with empty - caches (e.g. a non-leader that never fetched). Sample a few times and - take the ``max`` per key, so the harness observes whichever worker - still holds the most entries — this prevents both false negatives - (missing a populated leader) and false positives (an ``== 0`` drain - assertion passing only because it hit an empty worker). + The stack runs a single uvicorn worker (see docker-compose.yml), so the + per-process ``GitPolicyFetcher`` caches are read deterministically — a + read can't miss the worker that fetched. Sampling a few times and taking + the ``max`` per key only smooths over a read that races an in-flight + mutation; it is not relied on to paper over multi-worker nondeterminism + (which the single-worker setup removes outright). """ merged: Dict[str, int] = {} for i in range(max(1, samples)): @@ -96,37 +98,75 @@ def delete_scope(self, scope_id: str) -> None: resp.raise_for_status() self._created_scopes.discard(scope_id) - def cleanup_created_scopes(self, drain_timeout: int = 15) -> None: - """Delete every scope created on this client, then best-effort wait for - the caches to drain — so the next test starts from a clean slate. - - Best-effort by design: on master, delete never purges the caches (the - leak this suite gates), so the drain wait will simply time out. A - teardown failure must never fail the test that just ran, hence the broad - excepts and the bounded wait. + def list_scope_ids(self) -> List[str]: + """All scope ids the server currently knows (GET /scopes).""" + resp = requests.get(f"{self.base_url}/scopes", timeout=30) + resp.raise_for_status() + return [s["scope_id"] for s in resp.json()] + + def hard_reset(self, timeout: int = 600) -> None: + """Recover the server from a saturated fetch executor by wiping state. + + When a test leaves many clones hung (the offline-repo test saturates the + executor on purpose), per-scope DELETEs would queue *behind* those hung + threads, and a plain restart would have ``preload_scopes`` re-clone the + offline scopes and saturate again. Instead: stop the server (killing the + hung threads), flush the Redis scope store so nothing is re-cloned, then + start clean. Used in that test's teardown so the session-scoped stack is + usable by every later test. """ - for scope_id in list(self._created_scopes): - try: - self.delete_scope(scope_id) - except Exception: - self._created_scopes.discard(scope_id) + compose("stop", "opal_server") + compose("exec", "-T", "redis", "redis-cli", "FLUSHALL") + compose("start", "opal_server") + self._created_scopes.clear() + self.wait_healthy(timeout=timeout) + + def delete_all_scopes(self, drain_timeout: int = 20) -> None: + """Delete every scope the *server* knows (not just this client's), then + best-effort wait for the caches to drain — a clean slate independent of + what any prior, possibly-failed, test left behind. + + Best-effort drain by design: on master, delete never purges the caches + (the leak this suite gates), so the wait simply times out. This runs in + fixture setup/teardown, so a failure here must not mask the test, hence + the broad excepts and bounded wait. + """ + try: + for scope_id in self.list_scope_ids(): + try: + self.delete_scope(scope_id) + except Exception: + self._created_scopes.discard(scope_id) + except Exception: + pass + self._created_scopes.clear() deadline = time.time() + drain_timeout while time.time() < deadline: try: - if self.stats(samples=1)["repos"] == 0: + if self.stats()["repo_locks"] == 0: return except Exception: return time.sleep(1) + def get_scope_policy(self, scope_id: str) -> requests.Response: + """Fetch a scope's policy bundle (GET /scopes/{id}/policy). + + A 200 here proves the scope's repo was cloned and is being served — + the signal that a healthy scope still works while another scope's + clone is hanging. + """ + return requests.get( + f"{self.base_url}/scopes/{scope_id}/policy", timeout=30 + ) + def refresh_all(self) -> None: - # Best-effort: POST /scopes/refresh publishes on the webhook topic so - # every leader re-syncs all scopes. If this OPAL build doesn't expose - # the route (404), treat it as a no-op — there is no client-side - # fallback; callers rely on their own stats polling either way. + # POST /scopes/refresh publishes on the webhook topic so the leader + # re-syncs all scopes. The second sync takes the discover/fetch path + # (not the first-sync clone path), which is what populates the `repos` + # and `repos_last_fetched` caches. A missing route is a real error and + # is surfaced via raise_for_status. resp = requests.post(f"{self.base_url}/scopes/refresh", timeout=30) - if resp.status_code == 404: - return resp.raise_for_status() @@ -203,32 +243,47 @@ def gitea_repo_url(name: str) -> str: def make_repo_unreachable(name: str) -> str: - """Return a git URL for ``name`` pointing at a routable-but-dead host. - - Simulates an offline/unreachable policy repo: the address is in - TEST-NET-1 (RFC 5737), which is routable but runs no git server, so a - clone hangs rather than failing fast — exercising the missing fetch - timeout on the scopes path (the bug PR3 fixes). The URL keeps the same - ``/{user}/{name}.git`` shape as a real Gitea repo so the scope looks - ordinary apart from the unreachable host. + """Return a git URL for ``name`` pointing at the ``blackhole`` sidecar. + + Simulates an offline/unreachable policy repo: ``blackhole`` (alpine/socat) + accepts the TCP handshake then never answers, so the clone connects and + blocks reading the git smart-HTTP response — a deterministic hang that + exercises the missing fetch timeout on the scopes path (the bug PR3 fixes). + The URL keeps the same ``/{user}/{name}.git`` shape as a real Gitea repo so + the scope looks ordinary apart from the unreachable host. """ return f"http://{UNREACHABLE_HOST}/{GITEA_USER}/{name}.git" def compose(*args: str) -> subprocess.CompletedProcess: - return subprocess.run( + """Run `docker compose `; on failure, surface the captured output. + + `capture_output=True` keeps compose noise out of passing tests, but a raw + CalledProcessError shows only the exit code — so on failure we re-raise + with the captured stdout/stderr embedded, otherwise a broken build/seed/ + restart is opaque to debug. + """ + proc = subprocess.run( ["docker", "compose", *args], cwd=_COMPOSE_DIR, capture_output=True, text=True, - check=True, ) + if proc.returncode != 0: + raise RuntimeError( + f"`docker compose {' '.join(args)}` failed (exit {proc.returncode})\n" + f"--- stdout ---\n{proc.stdout}\n--- stderr ---\n{proc.stderr}" + ) + return proc def bounce_postgres(down_seconds: int = 5) -> None: compose("stop", "postgres") time.sleep(down_seconds) - compose("start", "postgres") + # `up -d --wait` blocks until Postgres passes its healthcheck again (plain + # `compose start` has no --wait), so a recovery poll that follows isn't + # racing an unready broadcaster. --no-recreate keeps the same container. + compose("up", "-d", "--wait", "--no-recreate", "postgres") def list_seeded_repos(count: int) -> List[str]: diff --git a/app-tests/git-leak/seed/seed_gitea.py b/app-tests/git-leak/seed/seed_gitea.py index ddebe5667..0b8bf2e33 100644 --- a/app-tests/git-leak/seed/seed_gitea.py +++ b/app-tests/git-leak/seed/seed_gitea.py @@ -113,18 +113,36 @@ def main() -> int: token = _ensure_token(base_url, user, password) workdir = Path("/tmp/seed-work") + failures = [] for i in range(count): name = f"policy-repo-{i:04d}" - _ensure_repo(base_url, token, user, name) - # only push if the repo is empty (freshly created) - head = requests.get( - f"{base_url}/api/v1/repos/{user}/{name}/branches/main", - headers={"Authorization": f"token {token}"}, - timeout=10, + # Isolate per-repo failures: one bad push must not abort the loop and + # leave an indeterminate subset seeded. Collect failures and exit + # non-zero with a count so the harness sees a real seed error (and + # `docker compose wait seed` surfaces it) instead of a later, confusing + # load-gate timeout. + try: + _ensure_repo(base_url, token, user, name) + # only push if the repo is empty (freshly created) + head = requests.get( + f"{base_url}/api/v1/repos/{user}/{name}/branches/main", + headers={"Authorization": f"token {token}"}, + timeout=10, + ) + if head.status_code != 200: + _push_policy(base_url, token, user, name, workdir) + print(f"seeded {name}", flush=True) + except Exception as exc: # noqa: BLE001 - report, don't abort the loop + failures.append((name, repr(exc))) + print(f"FAILED {name}: {exc!r}", flush=True) + + if failures: + print( + f"ERROR: seeded {count - len(failures)}/{count} repos; " + f"{len(failures)} failed (e.g. {failures[:3]})", + flush=True, ) - if head.status_code != 200: - _push_policy(base_url, token, user, name, workdir) - print(f"seeded {name}", flush=True) + return 1 # write the token where the test harness can read it Path("/seed-output/token").write_text(token) diff --git a/app-tests/git-leak/test_boot.py b/app-tests/git-leak/test_boot.py index 928fd6f4e..ab9f1eb4b 100644 --- a/app-tests/git-leak/test_boot.py +++ b/app-tests/git-leak/test_boot.py @@ -17,14 +17,18 @@ def test_boot_loads_all_scopes(opal, repo_count): for i, name in enumerate(repos): opal.put_scope(f"boot-{i}", gitea_repo_url(name)) - # restart only the server so it re-runs sync_scopes on boot + # Start the clock at the restart, not after wait_healthy: preload_scopes() + # runs in gunicorn's `when_ready` (before workers accept traffic), so by the + # time /healthcheck answers, boot-sync may already be partly done — starting + # the clock later would undercount it. Measuring from the restart captures + # the whole boot-sync window. + start = time.time() compose("restart", "opal_server") opal.wait_healthy(timeout=600) - start = time.time() deadline = start + 2000 while time.time() < deadline: - if opal.stats()["repos"] >= n: + if opal.stats()["repo_locks"] >= n: break time.sleep(2) elapsed = time.time() - start @@ -32,5 +36,5 @@ def test_boot_loads_all_scopes(opal, repo_count): # PR1 records the baseline (loose). PR4 will set BOOT_TARGET_SECONDS low. BOOT_TARGET_SECONDS = int(os.environ.get("BOOT_TARGET_SECONDS", "2000")) print(f"boot loaded {n} scopes in {elapsed:.1f}s (target {BOOT_TARGET_SECONDS}s)") - assert opal.stats()["repos"] >= n, "not all scopes loaded after boot" + assert opal.stats()["repo_locks"] >= n, "not all scopes loaded after boot" assert elapsed < BOOT_TARGET_SECONDS, f"boot too slow: {elapsed:.1f}s" diff --git a/app-tests/git-leak/test_leak.py b/app-tests/git-leak/test_leak.py index 8cfee6363..11b6da82d 100644 --- a/app-tests/git-leak/test_leak.py +++ b/app-tests/git-leak/test_leak.py @@ -13,52 +13,101 @@ def _wait_until(predicate, timeout=30, interval=0.5): return False +def _load_scopes(opal, prefix, names): + """PUT a scope per repo, then force a second sync so all three caches fill. + + The first sync of a fresh scope takes the clone path, which populates only + ``repo_locks`` — ``repos`` and ``repos_last_fetched`` are filled solely by + the discover/fetch path on a *subsequent* sync. ``refresh_all()`` triggers + that second sync, so after this returns all three caches reflect the N + scopes and the drain assertions test a cache the sync path actually fills. + + Returns the per-key load count reached (max over a wait). + """ + n = len(names) + for i, name in enumerate(names): + opal.put_scope(f"{prefix}-{i}", gitea_repo_url(name)) + # repo_locks is populated on the first sync (the clone path), so it is the + # deterministic signal that every scope was at least picked up. + locked = _wait_until(lambda: opal.stats()["repo_locks"] >= n, timeout=600) + assert locked, f"initial load never locked {n} repos: {opal.stats()}" + # force the discover/fetch path so `repos` / `repos_last_fetched` fill too + opal.refresh_all() + fetched = _wait_until(lambda: opal.stats()["repos"] >= n, timeout=600) + assert fetched, f"refresh never populated {n} repos: {opal.stats()}" + return opal.stats() + + @pytest.mark.timeout(900) def test_churn_releases_caches(opal, repo_count): """Create then delete many scopes; the three caches must return to empty. - FAILS on master: delete_scope never purges GitPolicyFetcher caches. + FAILS on this branch (without PR2): delete_scope never purges the + GitPolicyFetcher caches, so they stay populated after every scope is gone. + Becomes green once PR2 lands. """ n = min(repo_count, 100) repos = list_seeded_repos(n) - for i, name in enumerate(repos): - opal.put_scope(f"churn-{i}", gitea_repo_url(name)) - loaded = _wait_until(lambda: opal.stats()["repos"] >= n, timeout=600) - assert loaded, f"initial load never reached {n} repos: {opal.stats()}" + loaded = _load_scopes(opal, "churn", repos) + assert loaded["repo_locks"] >= n and loaded["repos"] >= n, loaded + rss_loaded = loaded["rss_kb"] for i in range(n): opal.delete_scope(f"churn-{i}") - released = _wait_until(lambda: opal.stats()["repos"] == 0, timeout=30) + # all three caches must drain to empty once every scope is deleted + released = _wait_until( + lambda: opal.stats()["repo_locks"] == 0 + and opal.stats()["repos"] == 0 + and opal.stats()["repos_last_fetched"] == 0, + timeout=60, + ) stats = opal.stats() - assert released, f"repos cache did not drain: {stats}" - assert stats["repos_last_fetched"] == 0, stats + assert released, f"caches did not drain after deleting all scopes: {stats}" + + # The cache drain above is the gate. RSS is only a loose backstop here: + # freeing the caches need not return memory to the OS (glibc/Python keep + # arenas), so this guards against a *gross* leak — RSS ballooning well past + # the loaded peak — without false-failing on allocator slack once PR2 lands. + rss_budget = rss_loaded + max(100_000, rss_loaded // 2) + assert stats["rss_kb"] <= rss_budget, ( + f"RSS ballooned across churn: {rss_loaded} -> {stats['rss_kb']} kb " + f"(budget {rss_budget})" + ) @pytest.mark.timeout(900) def test_repeat_sync_does_not_grow(opal, repo_count): - """Re-syncing the *same* scopes must not grow the caches. - - Idempotency guard, not a known-broken case. On current master this - PASSES: a clone path is keyed by the repo URL (``source_id`` = - sha256(url)+branch-shard), so re-syncing identical scopes reuses the - existing ``repos`` / ``repos_last_fetched`` entries instead of - allocating new ones. It guards against a regression that would make - repeat sync allocate per-sync. The unbounded *growth + no purge on - delete* leak is covered by ``test_churn_releases_caches`` above, which - uses distinct scopes and DOES fail on master. + """Re-syncing the *same* scopes must not grow memory unboundedly. + + Idempotency guard, not a known-broken case — PASSES on this branch. Clone + paths are keyed by the repo URL (``source_id`` = sha256(url)+branch-shard), + so re-syncing identical scopes reuses the existing ``repos`` / + ``repos_last_fetched`` entries rather than allocating new ones; the cache + *counts* therefore can't grow for any implementation, which is why the + load-bearing assertion here is on **RSS**, not on ``len(repos)``. It guards + against a regression where each repeat sync leaks per-sync allocations. The + unbounded growth-then-no-purge-on-delete leak is covered by + ``test_churn_releases_caches`` above, which uses distinct scopes. """ n = min(repo_count, 50) repos = list_seeded_repos(n) - for i, name in enumerate(repos): - opal.put_scope(f"stable-{i}", gitea_repo_url(name)) - loaded = _wait_until(lambda: opal.stats()["repos"] >= n, timeout=600) - assert loaded, f"initial sync never reached {n} repos: {opal.stats()}" + loaded = _load_scopes(opal, "stable", repos) + baseline_repos = loaded["repos"] + baseline_rss = loaded["rss_kb"] - baseline = opal.stats()["repos"] for _ in range(10): opal.refresh_all() time.sleep(2) - grown = opal.stats()["repos"] - assert grown <= baseline, f"repos cache grew on repeat sync: {baseline} -> {grown}" + grown = opal.stats() + assert grown["repos"] <= baseline_repos, ( + f"repos cache count grew on repeat sync: {baseline_repos} -> {grown['repos']}" + ) + # allow generous headroom for allocator slack; fail only on a real per-sync + # leak (10 refreshes of N scopes ballooning RSS). + rss_budget = baseline_rss + max(50_000, baseline_rss // 5) + assert grown["rss_kb"] <= rss_budget, ( + f"RSS grew unboundedly on repeat sync: " + f"{baseline_rss} -> {grown['rss_kb']} kb (budget {rss_budget})" + ) diff --git a/app-tests/git-leak/test_resilience.py b/app-tests/git-leak/test_resilience.py index b8fb73c89..11a25f90e 100644 --- a/app-tests/git-leak/test_resilience.py +++ b/app-tests/git-leak/test_resilience.py @@ -9,48 +9,83 @@ ) -@pytest.mark.timeout(300) +# Enough hanging clones to exhaust opal's default fetch executor +# (run_sync -> run_in_executor(None, ...), a ThreadPoolExecutor of +# min(32, cpu+4) workers). One hang wouldn't starve a multi-thread pool, so we +# saturate it with many; after PR3's fetch timeout these give up and free their +# threads, letting the healthy scope through. +OFFLINE_REPOS = 40 + + +@pytest.mark.timeout(420) def test_offline_repo_does_not_block_healthy_scopes(opal, repo_count): - """An unreachable repo must not stop healthy scopes from serving. + """Unreachable repos must not stop a healthy scope from serving. - FAILS on master: the scopes path has no fetch timeout, so a hung - clone occupies the shared executor and the server stalls. + FAILS on this branch (without PR3): the scopes path has no fetch timeout, + so the hung clones of the offline repos occupy the shared fetch executor + and the healthy scope's bundle never becomes available. """ - # a routable-but-dead address (TEST-NET-1, RFC 5737): the clone hangs - opal.put_scope("offline", make_repo_unreachable("dead-repo"), branch="main") + # the `blackhole` sidecar accepts the TCP handshake then never answers, so + # each of these clones hangs (holding a fetch-executor thread) rather than + # failing fast; enough of them saturate the pool. + for i in range(OFFLINE_REPOS): + opal.put_scope(f"offline-{i}", make_repo_unreachable(f"dead-{i}"), branch="main") healthy = list_seeded_repos(1)[0] opal.put_scope("healthy", gitea_repo_url(healthy)) - # the healthy scope's repo must appear in the cache within a bounded time, - # even though the offline scope is hanging - deadline = time.time() + 60 - served = False - while time.time() < deadline: - if opal.stats()["repos"] >= 1: - served = True - break - time.sleep(2) - assert served, "healthy scope never loaded while an offline repo was hanging" + try: + # The healthy scope must become *servable* within a bounded time even + # while the offline scopes hang. A 200 from its policy bundle proves the + # clone completed and the scope is served — a stronger signal than a + # cache count, and exactly what the offline hang starves on master. + deadline = time.time() + 90 + served = False + last = None + while time.time() < deadline: + try: + resp = opal.get_scope_policy("healthy") + last = resp.status_code + if resp.status_code == 200: + served = True + break + except Exception as exc: # request itself may stall when starved + last = repr(exc) + time.sleep(2) + assert served, ( + f"healthy scope never served while {OFFLINE_REPOS} offline repos " + f"were hanging (last policy response: {last})" + ) + finally: + # The offline clones hang for the blackhole's full duration, occupying + # executor threads. On the session-scoped stack that would starve every + # later test, and per-scope DELETEs would queue behind the hung threads. + # hard_reset stops the server (killing the hung threads), flushes the + # Redis scope store so preload doesn't re-clone them, and restarts clean. + opal.hard_reset() @pytest.mark.timeout(300) -def test_server_recovers_after_postgres_bounce(opal): - """A transient Postgres (broadcaster) outage must not leave the server - down. - - Recovery guard, not a known-broken case. On current master this - PASSES: when the broadcast channel drops, the affected worker - triggers a graceful shutdown and gunicorn respawns it, while the - sibling worker keeps serving HTTP — so the surface recovers within - the window. It guards against a regression of that property - (PER-15065's in-process reconnect would make recovery cleaner by - avoiding the worker churn, but recovery already holds). +def test_server_recovers_after_postgres_bounce(opal, repo_count): + """A transient Postgres (broadcaster) outage must not break propagation. + + Recovery guard, not a known-broken case — PASSES on this branch: when the + broadcast channel drops, the affected worker triggers a graceful shutdown + and gunicorn respawns it, and once Postgres is back the broadcaster + reconnects. We prove recovery of the *broadcast path*, not just HTTP + liveness: after the bounce we PUT a fresh scope and assert it actually + syncs (its repo lands in the cache), which only happens if the leader + received the sync notification over the recovered broadcaster. + (PER-15065's in-process reconnect would make recovery cleaner by avoiding + the worker churn, but recovery already holds.) """ assert opal.stats() # healthy before + baseline_locks = opal.stats()["repo_locks"] + bounce_postgres(down_seconds=5) - deadline = time.time() + 60 + # wait for the HTTP surface to come back first + deadline = time.time() + 90 recovered = False while time.time() < deadline: try: @@ -60,4 +95,16 @@ def test_server_recovers_after_postgres_bounce(opal): break except Exception: time.sleep(2) - assert recovered, "server did not recover within 60s of a postgres bounce" + assert recovered, "server did not recover HTTP within 90s of a postgres bounce" + + # prove the broadcast path itself recovered: a freshly PUT scope must sync + healthy = list_seeded_repos(1)[0] + opal.put_scope("post-bounce", gitea_repo_url(healthy)) + synced = False + deadline = time.time() + 120 + while time.time() < deadline: + if opal.stats()["repo_locks"] > baseline_locks: + synced = True + break + time.sleep(2) + assert synced, "scope PUT after the bounce never synced; broadcaster did not recover" diff --git a/packages/opal-server/opal_server/debug_stats.py b/packages/opal-server/opal_server/debug_stats.py index 20ea6b341..af77ff4ea 100644 --- a/packages/opal-server/opal_server/debug_stats.py +++ b/packages/opal-server/opal_server/debug_stats.py @@ -4,9 +4,9 @@ observe the cache growth that the memory-leak fix (PR2) eliminates. """ from pathlib import Path -from typing import Dict +from typing import Dict, List, Optional -from fastapi import FastAPI +from fastapi import FastAPI, params from opal_server.git_fetcher import GitPolicyFetcher @@ -31,11 +31,26 @@ def git_fetcher_cache_stats() -> Dict[str, int]: } -def register_internal_stats_route(app: FastAPI, enabled: bool) -> None: - """Mount GET /internal/git-fetcher-cache-stats only when enabled.""" +def register_internal_stats_route( + app: FastAPI, + enabled: bool, + dependencies: Optional[List[params.Depends]] = None, +) -> None: + """Mount GET /internal/git-fetcher-cache-stats only when enabled. + + ``dependencies`` are applied to the route (e.g. the server's + ``JWTAuthenticator``) so the endpoint is protected when JWT verification + is enabled. When verification is disabled — as in the test bed, which + leaves ``OPAL_AUTH_PUBLIC_KEY`` unset — the authenticator is a no-op and + the route stays reachable without a token. + """ if not enabled: return - @app.get("/internal/git-fetcher-cache-stats", include_in_schema=False) + @app.get( + "/internal/git-fetcher-cache-stats", + include_in_schema=False, + dependencies=dependencies or [], + ) def _git_fetcher_cache_stats() -> Dict[str, int]: return git_fetcher_cache_stats() diff --git a/packages/opal-server/opal_server/server.py b/packages/opal-server/opal_server/server.py index 1be796d6d..9702e372f 100644 --- a/packages/opal-server/opal_server/server.py +++ b/packages/opal-server/opal_server/server.py @@ -295,7 +295,9 @@ def healthcheck(): return {"status": "ok"} register_internal_stats_route( - app, enabled=opal_server_config.DEBUG_INTERNAL_STATS + app, + enabled=opal_server_config.DEBUG_INTERNAL_STATS, + dependencies=[Depends(authenticator)], ) return app diff --git a/packages/opal-server/opal_server/tests/debug_stats_endpoint_test.py b/packages/opal-server/opal_server/tests/debug_stats_endpoint_test.py index 00a9c8e1b..3e36cd330 100644 --- a/packages/opal-server/opal_server/tests/debug_stats_endpoint_test.py +++ b/packages/opal-server/opal_server/tests/debug_stats_endpoint_test.py @@ -1,4 +1,4 @@ -from fastapi import FastAPI +from fastapi import Depends, FastAPI, HTTPException from fastapi.testclient import TestClient from opal_server.debug_stats import register_internal_stats_route @@ -20,3 +20,20 @@ def test_endpoint_present_when_enabled(): assert resp.status_code == 200 body = resp.json() assert set(body) == {"repo_locks", "repos", "repos_last_fetched", "rss_kb"} + + +def test_endpoint_applies_passed_dependencies(): + """A route dependency (e.g. the server's authenticator) is enforced. + + Mirrors how server.py wires the real JWTAuthenticator: when verification is + enabled the dependency rejects unauthenticated reads; when disabled it is a + no-op (covered by the test above, which passes no dependency). + """ + + def _deny(): + raise HTTPException(status_code=401, detail="unauthorized") + + app = FastAPI() + register_internal_stats_route(app, enabled=True, dependencies=[Depends(_deny)]) + resp = TestClient(app).get("/internal/git-fetcher-cache-stats") + assert resp.status_code == 401 From f810db8faf7f8ab953f179c956cd51f17852ff47 Mon Sep 17 00:00:00 2001 From: David Shoen Date: Wed, 24 Jun 2026 16:15:05 +0300 Subject: [PATCH 10/63] style(git-leak): apply black/isort/docformatter (pre-commit) Co-Authored-By: Claude Opus 4.8 (1M context) --- app-tests/git-leak/helpers.py | 18 ++++++++---------- app-tests/git-leak/test_leak.py | 10 +++++----- app-tests/git-leak/test_resilience.py | 16 ++++++++++------ .../tests/debug_stats_endpoint_test.py | 7 ++++--- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/app-tests/git-leak/helpers.py b/app-tests/git-leak/helpers.py index 3cb66a36a..3dc549421 100644 --- a/app-tests/git-leak/helpers.py +++ b/app-tests/git-leak/helpers.py @@ -152,13 +152,11 @@ def delete_all_scopes(self, drain_timeout: int = 20) -> None: def get_scope_policy(self, scope_id: str) -> requests.Response: """Fetch a scope's policy bundle (GET /scopes/{id}/policy). - A 200 here proves the scope's repo was cloned and is being served — - the signal that a healthy scope still works while another scope's - clone is hanging. + A 200 here proves the scope's repo was cloned and is being + served — the signal that a healthy scope still works while + another scope's clone is hanging. """ - return requests.get( - f"{self.base_url}/scopes/{scope_id}/policy", timeout=30 - ) + return requests.get(f"{self.base_url}/scopes/{scope_id}/policy", timeout=30) def refresh_all(self) -> None: # POST /scopes/refresh publishes on the webhook topic so the leader @@ -258,10 +256,10 @@ def make_repo_unreachable(name: str) -> str: def compose(*args: str) -> subprocess.CompletedProcess: """Run `docker compose `; on failure, surface the captured output. - `capture_output=True` keeps compose noise out of passing tests, but a raw - CalledProcessError shows only the exit code — so on failure we re-raise - with the captured stdout/stderr embedded, otherwise a broken build/seed/ - restart is opaque to debug. + `capture_output=True` keeps compose noise out of passing tests, but + a raw CalledProcessError shows only the exit code — so on failure we + re-raise with the captured stdout/stderr embedded, otherwise a + broken build/seed/ restart is opaque to debug. """ proc = subprocess.run( ["docker", "compose", *args], diff --git a/app-tests/git-leak/test_leak.py b/app-tests/git-leak/test_leak.py index 11b6da82d..7ded9eded 100644 --- a/app-tests/git-leak/test_leak.py +++ b/app-tests/git-leak/test_leak.py @@ -43,8 +43,8 @@ def test_churn_releases_caches(opal, repo_count): """Create then delete many scopes; the three caches must return to empty. FAILS on this branch (without PR2): delete_scope never purges the - GitPolicyFetcher caches, so they stay populated after every scope is gone. - Becomes green once PR2 lands. + GitPolicyFetcher caches, so they stay populated after every scope is + gone. Becomes green once PR2 lands. """ n = min(repo_count, 100) repos = list_seeded_repos(n) @@ -101,9 +101,9 @@ def test_repeat_sync_does_not_grow(opal, repo_count): time.sleep(2) grown = opal.stats() - assert grown["repos"] <= baseline_repos, ( - f"repos cache count grew on repeat sync: {baseline_repos} -> {grown['repos']}" - ) + assert ( + grown["repos"] <= baseline_repos + ), f"repos cache count grew on repeat sync: {baseline_repos} -> {grown['repos']}" # allow generous headroom for allocator slack; fail only on a real per-sync # leak (10 refreshes of N scopes ballooning RSS). rss_budget = baseline_rss + max(50_000, baseline_rss // 5) diff --git a/app-tests/git-leak/test_resilience.py b/app-tests/git-leak/test_resilience.py index 11a25f90e..76515dbe0 100644 --- a/app-tests/git-leak/test_resilience.py +++ b/app-tests/git-leak/test_resilience.py @@ -8,7 +8,6 @@ make_repo_unreachable, ) - # Enough hanging clones to exhaust opal's default fetch executor # (run_sync -> run_in_executor(None, ...), a ThreadPoolExecutor of # min(32, cpu+4) workers). One hang wouldn't starve a multi-thread pool, so we @@ -21,15 +20,18 @@ def test_offline_repo_does_not_block_healthy_scopes(opal, repo_count): """Unreachable repos must not stop a healthy scope from serving. - FAILS on this branch (without PR3): the scopes path has no fetch timeout, - so the hung clones of the offline repos occupy the shared fetch executor - and the healthy scope's bundle never becomes available. + FAILS on this branch (without PR3): the scopes path has no fetch + timeout, so the hung clones of the offline repos occupy the shared + fetch executor and the healthy scope's bundle never becomes + available. """ # the `blackhole` sidecar accepts the TCP handshake then never answers, so # each of these clones hangs (holding a fetch-executor thread) rather than # failing fast; enough of them saturate the pool. for i in range(OFFLINE_REPOS): - opal.put_scope(f"offline-{i}", make_repo_unreachable(f"dead-{i}"), branch="main") + opal.put_scope( + f"offline-{i}", make_repo_unreachable(f"dead-{i}"), branch="main" + ) healthy = list_seeded_repos(1)[0] opal.put_scope("healthy", gitea_repo_url(healthy)) @@ -107,4 +109,6 @@ def test_server_recovers_after_postgres_bounce(opal, repo_count): synced = True break time.sleep(2) - assert synced, "scope PUT after the bounce never synced; broadcaster did not recover" + assert ( + synced + ), "scope PUT after the bounce never synced; broadcaster did not recover" diff --git a/packages/opal-server/opal_server/tests/debug_stats_endpoint_test.py b/packages/opal-server/opal_server/tests/debug_stats_endpoint_test.py index 3e36cd330..4abd1abb1 100644 --- a/packages/opal-server/opal_server/tests/debug_stats_endpoint_test.py +++ b/packages/opal-server/opal_server/tests/debug_stats_endpoint_test.py @@ -25,9 +25,10 @@ def test_endpoint_present_when_enabled(): def test_endpoint_applies_passed_dependencies(): """A route dependency (e.g. the server's authenticator) is enforced. - Mirrors how server.py wires the real JWTAuthenticator: when verification is - enabled the dependency rejects unauthenticated reads; when disabled it is a - no-op (covered by the test above, which passes no dependency). + Mirrors how server.py wires the real JWTAuthenticator: when + verification is enabled the dependency rejects unauthenticated + reads; when disabled it is a no-op (covered by the test above, which + passes no dependency). """ def _deny(): From 82ff33a7300adfe857c115d64ccea5ec135ee2be Mon Sep 17 00:00:00 2001 From: David Shoen Date: Wed, 24 Jun 2026 19:57:23 +0300 Subject: [PATCH 11/63] test(git-leak): tighten stat polling and pin test-bed images (PR review) - snapshot a single /internal stats read per poll in the churn-drain and boot gates (consistent multi-key observation; fewer HTTP round-trips) - document why a 200 from the healthy scope can't be a masked default bundle, and why the stats route is intentionally a sync def - pin alpine/socat and the seed image's pip deps for reproducibility Co-Authored-By: Claude Opus 4.8 (1M context) --- app-tests/git-leak/docker-compose.yml | 2 +- app-tests/git-leak/seed/Dockerfile | 2 +- app-tests/git-leak/test_boot.py | 6 ++++-- app-tests/git-leak/test_leak.py | 15 ++++++++------- app-tests/git-leak/test_resilience.py | 5 +++++ packages/opal-server/opal_server/debug_stats.py | 5 +++++ 6 files changed, 24 insertions(+), 11 deletions(-) diff --git a/app-tests/git-leak/docker-compose.yml b/app-tests/git-leak/docker-compose.yml index bd48c26dc..a863faed4 100644 --- a/app-tests/git-leak/docker-compose.yml +++ b/app-tests/git-leak/docker-compose.yml @@ -72,7 +72,7 @@ services: # blocks reading the git smart-HTTP response, holding the fetch executor. # Deterministic, unlike a TEST-NET-1 address which many networks reject # fast with ICMP-unreachable (so the clone would fail fast, not hang). - image: alpine/socat:latest + image: alpine/socat:1.8.0.3 command: ["TCP-LISTEN:80,fork,reuseaddr", "SYSTEM:sleep 3600"] seed: diff --git a/app-tests/git-leak/seed/Dockerfile b/app-tests/git-leak/seed/Dockerfile index ee9d776d2..540118cd2 100644 --- a/app-tests/git-leak/seed/Dockerfile +++ b/app-tests/git-leak/seed/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.11-slim RUN apt-get update && apt-get install -y --no-install-recommends git \ && rm -rf /var/lib/apt/lists/* -RUN pip install --no-cache-dir requests GitPython +RUN pip install --no-cache-dir requests==2.32.3 GitPython==3.1.50 COPY seed_gitea.py /seed_gitea.py ENTRYPOINT ["python", "/seed_gitea.py"] diff --git a/app-tests/git-leak/test_boot.py b/app-tests/git-leak/test_boot.py index ab9f1eb4b..b28637ec1 100644 --- a/app-tests/git-leak/test_boot.py +++ b/app-tests/git-leak/test_boot.py @@ -27,8 +27,10 @@ def test_boot_loads_all_scopes(opal, repo_count): opal.wait_healthy(timeout=600) deadline = start + 2000 + loaded = 0 while time.time() < deadline: - if opal.stats()["repo_locks"] >= n: + loaded = opal.stats()["repo_locks"] + if loaded >= n: break time.sleep(2) elapsed = time.time() - start @@ -36,5 +38,5 @@ def test_boot_loads_all_scopes(opal, repo_count): # PR1 records the baseline (loose). PR4 will set BOOT_TARGET_SECONDS low. BOOT_TARGET_SECONDS = int(os.environ.get("BOOT_TARGET_SECONDS", "2000")) print(f"boot loaded {n} scopes in {elapsed:.1f}s (target {BOOT_TARGET_SECONDS}s)") - assert opal.stats()["repo_locks"] >= n, "not all scopes loaded after boot" + assert loaded >= n, "not all scopes loaded after boot" assert elapsed < BOOT_TARGET_SECONDS, f"boot too slow: {elapsed:.1f}s" diff --git a/app-tests/git-leak/test_leak.py b/app-tests/git-leak/test_leak.py index 7ded9eded..8ef2258f7 100644 --- a/app-tests/git-leak/test_leak.py +++ b/app-tests/git-leak/test_leak.py @@ -55,13 +55,14 @@ def test_churn_releases_caches(opal, repo_count): for i in range(n): opal.delete_scope(f"churn-{i}") - # all three caches must drain to empty once every scope is deleted - released = _wait_until( - lambda: opal.stats()["repo_locks"] == 0 - and opal.stats()["repos"] == 0 - and opal.stats()["repos_last_fetched"] == 0, - timeout=60, - ) + # all three caches must drain to empty once every scope is deleted. Read a + # single stats snapshot per poll so the three keys reflect the same + # observation (and to avoid 3x the HTTP round-trips per iteration). + def _all_caches_empty() -> bool: + s = opal.stats() + return s["repo_locks"] == 0 and s["repos"] == 0 and s["repos_last_fetched"] == 0 + + released = _wait_until(_all_caches_empty, timeout=60) stats = opal.stats() assert released, f"caches did not drain after deleting all scopes: {stats}" diff --git a/app-tests/git-leak/test_resilience.py b/app-tests/git-leak/test_resilience.py index 76515dbe0..b18829e12 100644 --- a/app-tests/git-leak/test_resilience.py +++ b/app-tests/git-leak/test_resilience.py @@ -41,6 +41,11 @@ def test_offline_repo_does_not_block_healthy_scopes(opal, repo_count): # while the offline scopes hang. A 200 from its policy bundle proves the # clone completed and the scope is served — a stronger signal than a # cache count, and exactly what the offline hang starves on master. + # + # A 200 here can't be a *masked* default bundle: GET /{scope}/policy + # falls back to the "default" scope on a bad/missing repo, but this bed + # never creates a "default" scope, so that fallback raises instead of + # returning 200. The only way to get 200 is the healthy clone completing. deadline = time.time() + 90 served = False last = None diff --git a/packages/opal-server/opal_server/debug_stats.py b/packages/opal-server/opal_server/debug_stats.py index af77ff4ea..5439ce950 100644 --- a/packages/opal-server/opal_server/debug_stats.py +++ b/packages/opal-server/opal_server/debug_stats.py @@ -47,6 +47,11 @@ def register_internal_stats_route( if not enabled: return + # Deliberately a sync def: Starlette runs it in its own threadpool, which is + # independent of the default loop executor opal uses for git fetches + # (run_sync -> run_in_executor(None, ...)). So this endpoint keeps answering + # even when hung clones saturate the fetch executor — which is exactly the + # condition the offline-repo test observes through it. @app.get( "/internal/git-fetcher-cache-stats", include_in_schema=False, From 75ad43a4a90ac23843d0e0e68f2775d3d717584c Mon Sep 17 00:00:00 2001 From: David Shoen Date: Sun, 28 Jun 2026 18:45:58 +0300 Subject: [PATCH 12/63] test(git-leak): isolate offline-hang healthy probe to a never-cloned repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ziv (PR review, round 2) caught that the offline-resilience gate can false-PASS in the full-suite run. The "healthy" scope pointed at policy-repo-0000 (list_seeded_repos(1)[0]), but on-disk clones are keyed by URL-hash and survive compose restart/stop/start (opal_server mounts no volume at /opal; only `down -v` wipes them). test_boot/test_leak run first (alphabetical) and already clone every seeded repo, so the healthy scope hit the existing clone via _discover_repository, skipped _clone(), and served 200 without ever touching the saturated fetch executor — the gate that must FAIL on this branch (no PR3 timeout) passed. Fix: seed a reserved repo (policy-repo-healthy-probe) outside the numeric policy-repo-NNNN range that no boot/leak test enumerates, and point the healthy probe at it. A never-cloned repo forces a genuine fresh clone through the starved pool, so the gate fails correctly. The seed- completeness check in conftest now covers the reserved repo too. Co-Authored-By: Claude Opus 4.8 (1M context) --- app-tests/git-leak/conftest.py | 12 ++++++++++-- app-tests/git-leak/helpers.py | 14 ++++++++++++++ app-tests/git-leak/seed/seed_gitea.py | 19 +++++++++++++++---- app-tests/git-leak/test_resilience.py | 11 +++++++++-- 4 files changed, 48 insertions(+), 8 deletions(-) diff --git a/app-tests/git-leak/conftest.py b/app-tests/git-leak/conftest.py index 1660d596b..446376e3f 100644 --- a/app-tests/git-leak/conftest.py +++ b/app-tests/git-leak/conftest.py @@ -2,7 +2,13 @@ import shutil import pytest -from helpers import GiteaAdmin, OpalServerClient, compose, list_seeded_repos +from helpers import ( + HEALTHY_PROBE_REPO, + GiteaAdmin, + OpalServerClient, + compose, + list_seeded_repos, +) def pytest_addoption(parser): @@ -43,7 +49,9 @@ def stack(request, repo_count): # Verify the seed actually produced all N repos before any test runs: a # partial seed would otherwise look like a server bug when the load gate # can't reach N. Fail loudly with the gap. - expected = set(list_seeded_repos(repo_count)) + # include the reserved probe repo the resilience test relies on, so a + # partial seed of it is caught here too rather than as a later test failure + expected = set(list_seeded_repos(repo_count)) | {HEALTHY_PROBE_REPO} present = set(GiteaAdmin().list_repos()) missing = expected - present assert not missing, ( diff --git a/app-tests/git-leak/helpers.py b/app-tests/git-leak/helpers.py index 3dc549421..f819d0d01 100644 --- a/app-tests/git-leak/helpers.py +++ b/app-tests/git-leak/helpers.py @@ -286,3 +286,17 @@ def bounce_postgres(down_seconds: int = 5) -> None: def list_seeded_repos(count: int) -> List[str]: return [f"policy-repo-{i:04d}" for i in range(count)] + + +# A reserved repo seeded *outside* the numeric ``policy-repo-NNNN`` range that +# ``list_seeded_repos`` enumerates, so no boot/leak test ever clones it. The +# resilience offline-hang test uses it as its "healthy" probe: clones live at +# ``base_dir/`` keyed by URL-hash and survive ``compose +# restart/stop/start`` (opal_server mounts no volume at ``/opal``; only +# ``down -v`` wipes them), so pointing the probe at any shared seeded repo would +# let the healthy scope reuse an on-disk clone and serve 200 *without* touching +# the saturated fetch executor — false-passing a gate that must FAIL on this +# branch. A dedicated never-cloned repo forces a genuine fresh clone through the +# starved executor. Keep this name in sync with ``RESERVED_REPOS`` in +# ``seed/seed_gitea.py``. +HEALTHY_PROBE_REPO = "policy-repo-healthy-probe" diff --git a/app-tests/git-leak/seed/seed_gitea.py b/app-tests/git-leak/seed/seed_gitea.py index 0b8bf2e33..7e1de1a4c 100644 --- a/app-tests/git-leak/seed/seed_gitea.py +++ b/app-tests/git-leak/seed/seed_gitea.py @@ -28,6 +28,14 @@ DATA_JSON = '{"roles": {"admin": ["read", "write"]}}\n' +# Reserved repos seeded in addition to the numeric ``policy-repo-NNNN`` set. +# These sit outside the range the boot/leak tests enumerate (so no test ever +# clones them) and back the resilience offline-hang test's "healthy" probe, +# which must force a fresh clone through the saturated executor rather than +# reuse a surviving on-disk clone. Keep in sync with ``HEALTHY_PROBE_REPO`` in +# ``helpers.py``. +RESERVED_REPOS = ("policy-repo-healthy-probe",) + def _wait_for_gitea(base_url: str, timeout: int = 120) -> None: deadline = time.time() + timeout @@ -114,8 +122,10 @@ def main() -> int: workdir = Path("/tmp/seed-work") failures = [] - for i in range(count): - name = f"policy-repo-{i:04d}" + # the numeric set the boot/leak tests enumerate, plus the reserved repos + # (e.g. the resilience offline-hang test's never-cloned healthy probe) + names = [f"policy-repo-{i:04d}" for i in range(count)] + list(RESERVED_REPOS) + for name in names: # Isolate per-repo failures: one bad push must not abort the loop and # leave an indeterminate subset seeded. Collect failures and exit # non-zero with a count so the harness sees a real seed error (and @@ -136,9 +146,10 @@ def main() -> int: failures.append((name, repr(exc))) print(f"FAILED {name}: {exc!r}", flush=True) + total = len(names) if failures: print( - f"ERROR: seeded {count - len(failures)}/{count} repos; " + f"ERROR: seeded {total - len(failures)}/{total} repos; " f"{len(failures)} failed (e.g. {failures[:3]})", flush=True, ) @@ -146,7 +157,7 @@ def main() -> int: # write the token where the test harness can read it Path("/seed-output/token").write_text(token) - print(f"DONE: ensured {count} repos", flush=True) + print(f"DONE: ensured {total} repos", flush=True) return 0 diff --git a/app-tests/git-leak/test_resilience.py b/app-tests/git-leak/test_resilience.py index b18829e12..60f28338f 100644 --- a/app-tests/git-leak/test_resilience.py +++ b/app-tests/git-leak/test_resilience.py @@ -2,6 +2,7 @@ import pytest from helpers import ( + HEALTHY_PROBE_REPO, bounce_postgres, gitea_repo_url, list_seeded_repos, @@ -33,8 +34,14 @@ def test_offline_repo_does_not_block_healthy_scopes(opal, repo_count): f"offline-{i}", make_repo_unreachable(f"dead-{i}"), branch="main" ) - healthy = list_seeded_repos(1)[0] - opal.put_scope("healthy", gitea_repo_url(healthy)) + # Point the healthy scope at a repo *no other test clones* (HEALTHY_PROBE_REPO + # is seeded outside the numeric range the boot/leak tests enumerate). Clones + # survive compose restart/stop/start, so reusing a shared seeded repo here + # would let this scope hit the existing on-disk clone via _discover_repository, + # skip _clone(), and serve 200 without ever touching the saturated executor — + # false-passing this gate. A never-cloned repo forces a real clone through the + # starved pool, so the gate fails correctly on this branch (no PR3 timeout). + opal.put_scope("healthy", gitea_repo_url(HEALTHY_PROBE_REPO)) try: # The healthy scope must become *servable* within a bounded time even From 15f3cfe2495735936652400edd6ea8fe3fe8bb77 Mon Sep 17 00:00:00 2001 From: David Shoen Date: Sun, 28 Jun 2026 19:05:33 +0300 Subject: [PATCH 13/63] test(git-leak): harden harness teardown and tighten assertions (PR review) Follow-up to PR review of the git-leak/resilience test bed: - hard_reset: always restart opal_server + wait_healthy via a finally, so a failed redis FLUSHALL can't leave the server stopped (which would fail every later session-scoped test and, running in a test finally, mask the result). - delete_all_scopes drain: a transient /internal read error no longer counts as a successful drain (was `except: return`); keep polling to the deadline so a not-yet-drained cache can't leak into the next test once PR2 lands. - use stats(samples=1) for the zero-waiting drain/empty polls (the peak-merge only matters for load assertions; this also drops 3x HTTP per poll). - resilience: narrow broad `except Exception` to requests.RequestException (and RuntimeError for wait_healthy timeout) so harness bugs surface instead of masquerading as "never served"/"never recovered". - resilience: collapse `assert opal.stats()` + redundant re-read into one read. - debug_stats_test: assert rss_kb > 0 on Linux (was `>= 0`, which passed for the wrong reason where /proc is absent and RSS reads fall back to 0). Co-Authored-By: Claude Opus 4.8 (1M context) --- app-tests/git-leak/helpers.py | 22 ++++++++++++++----- app-tests/git-leak/test_leak.py | 2 +- app-tests/git-leak/test_resilience.py | 11 ++++++---- .../opal_server/tests/debug_stats_test.py | 9 +++++++- 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/app-tests/git-leak/helpers.py b/app-tests/git-leak/helpers.py index f819d0d01..452d16b99 100644 --- a/app-tests/git-leak/helpers.py +++ b/app-tests/git-leak/helpers.py @@ -116,10 +116,15 @@ def hard_reset(self, timeout: int = 600) -> None: usable by every later test. """ compose("stop", "opal_server") - compose("exec", "-T", "redis", "redis-cli", "FLUSHALL") - compose("start", "opal_server") - self._created_scopes.clear() - self.wait_healthy(timeout=timeout) + try: + compose("exec", "-T", "redis", "redis-cli", "FLUSHALL") + finally: + # Always bring the server back up, even if the flush failed: leaving + # it stopped would fail every later session-scoped test, and since + # this runs in a test's `finally` it would also mask the real result. + compose("start", "opal_server") + self._created_scopes.clear() + self.wait_healthy(timeout=timeout) def delete_all_scopes(self, drain_timeout: int = 20) -> None: """Delete every scope the *server* knows (not just this client's), then @@ -143,10 +148,15 @@ def delete_all_scopes(self, drain_timeout: int = 20) -> None: deadline = time.time() + drain_timeout while time.time() < deadline: try: - if self.stats()["repo_locks"] == 0: + # Single snapshot: we're waiting for zero, so the peak-merge + # (max over samples) would only delay observing the drain. + if self.stats(samples=1)["repo_locks"] == 0: return except Exception: - return + # A transient stats-read failure is not proof of a drain — keep + # polling until the deadline rather than returning early, which + # would let a not-yet-drained cache leak into the next test. + pass time.sleep(1) def get_scope_policy(self, scope_id: str) -> requests.Response: diff --git a/app-tests/git-leak/test_leak.py b/app-tests/git-leak/test_leak.py index 8ef2258f7..09fe9034d 100644 --- a/app-tests/git-leak/test_leak.py +++ b/app-tests/git-leak/test_leak.py @@ -59,7 +59,7 @@ def test_churn_releases_caches(opal, repo_count): # single stats snapshot per poll so the three keys reflect the same # observation (and to avoid 3x the HTTP round-trips per iteration). def _all_caches_empty() -> bool: - s = opal.stats() + s = opal.stats(samples=1) return s["repo_locks"] == 0 and s["repos"] == 0 and s["repos_last_fetched"] == 0 released = _wait_until(_all_caches_empty, timeout=60) diff --git a/app-tests/git-leak/test_resilience.py b/app-tests/git-leak/test_resilience.py index 60f28338f..04702448b 100644 --- a/app-tests/git-leak/test_resilience.py +++ b/app-tests/git-leak/test_resilience.py @@ -1,6 +1,7 @@ import time import pytest +import requests from helpers import ( HEALTHY_PROBE_REPO, bounce_postgres, @@ -63,7 +64,7 @@ def test_offline_repo_does_not_block_healthy_scopes(opal, repo_count): if resp.status_code == 200: served = True break - except Exception as exc: # request itself may stall when starved + except requests.RequestException as exc: # may stall when starved last = repr(exc) time.sleep(2) assert served, ( @@ -93,8 +94,9 @@ def test_server_recovers_after_postgres_bounce(opal, repo_count): (PER-15065's in-process reconnect would make recovery cleaner by avoiding the worker churn, but recovery already holds.) """ - assert opal.stats() # healthy before - baseline_locks = opal.stats()["repo_locks"] + baseline = opal.stats() # healthy before + assert baseline + baseline_locks = baseline["repo_locks"] bounce_postgres(down_seconds=5) @@ -107,7 +109,8 @@ def test_server_recovers_after_postgres_bounce(opal, repo_count): opal.stats() recovered = True break - except Exception: + except (requests.RequestException, RuntimeError): + # RequestException: HTTP not back yet; RuntimeError: wait_healthy timed out time.sleep(2) assert recovered, "server did not recover HTTP within 90s of a postgres bounce" diff --git a/packages/opal-server/opal_server/tests/debug_stats_test.py b/packages/opal-server/opal_server/tests/debug_stats_test.py index b2d7935db..c67fb3530 100644 --- a/packages/opal-server/opal_server/tests/debug_stats_test.py +++ b/packages/opal-server/opal_server/tests/debug_stats_test.py @@ -1,3 +1,5 @@ +import sys + from opal_server.config import opal_server_config from opal_server.debug_stats import git_fetcher_cache_stats from opal_server.git_fetcher import GitPolicyFetcher @@ -14,7 +16,12 @@ def test_stats_report_dict_sizes(monkeypatch): assert stats["repos"] == 2 assert stats["repos_last_fetched"] == 0 assert isinstance(stats["rss_kb"], int) - assert stats["rss_kb"] >= 0 + # On Linux /proc/self/status exists, so RSS reading must actually work; on + # other platforms _read_rss_kb falls back to 0 and the wiring is untestable. + if sys.platform.startswith("linux"): + assert stats["rss_kb"] > 0 + else: + assert stats["rss_kb"] >= 0 def test_internal_stats_flag_defaults_off(): From ac29c2e9c424ed3e6f8a0d166b90f04a09ceb049 Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 23 Jun 2026 12:50:30 +0300 Subject: [PATCH 14/63] feat(opal-server): add GitPolicyFetcher.forget_repo to release cached repo handles Co-Authored-By: Claude Opus 4.8 (1M context) --- .../opal-server/opal_server/git_fetcher.py | 20 ++++++++++++ .../opal_server/tests/forget_repo_test.py | 31 +++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 packages/opal-server/opal_server/tests/forget_repo_test.py diff --git a/packages/opal-server/opal_server/git_fetcher.py b/packages/opal-server/opal_server/git_fetcher.py index e52083c98..8daee2a8e 100644 --- a/packages/opal-server/opal_server/git_fetcher.py +++ b/packages/opal-server/opal_server/git_fetcher.py @@ -366,6 +366,26 @@ def base_dir(base_dir: Path) -> Path: def repo_clone_path(base_dir: Path, source: GitPolicyScopeSource) -> Path: return GitPolicyFetcher.base_dir(base_dir) / GitPolicyFetcher.source_id(source) + @staticmethod + def forget_repo(path: str) -> None: + """Drop the cached repository for a clone path and release its handles. + + The cached ``pygit2.Repository`` keeps OS file descriptors and mmapped + pack indexes open; without this, a deleted scope's repo pins memory and + inodes for the lifetime of the process even after the clone is removed. + ``Repository.free()`` is called only when available (it is not part of + every pygit2 release); otherwise the dropped reference is reclaimed by GC. + """ + repo = GitPolicyFetcher.repos.pop(path, None) + if repo is None: + return + free = getattr(repo, "free", None) + if callable(free): + try: + free() + except Exception: + logger.debug("pygit2 Repository.free() failed; relying on GC") + class GitCallback(RemoteCallbacks): def __init__(self, source: GitPolicyScopeSource): diff --git a/packages/opal-server/opal_server/tests/forget_repo_test.py b/packages/opal-server/opal_server/tests/forget_repo_test.py new file mode 100644 index 000000000..b6e223cf1 --- /dev/null +++ b/packages/opal-server/opal_server/tests/forget_repo_test.py @@ -0,0 +1,31 @@ +from opal_server.git_fetcher import GitPolicyFetcher + + +class _FakeRepo: + def __init__(self): + self.freed = False + + def free(self): + self.freed = True + + +def test_forget_repo_pops_and_frees(monkeypatch): + fake = _FakeRepo() + monkeypatch.setattr(GitPolicyFetcher, "repos", {"/clones/x": fake}) + + GitPolicyFetcher.forget_repo("/clones/x") + + assert "/clones/x" not in GitPolicyFetcher.repos + assert fake.freed is True + + +def test_forget_repo_unknown_path_is_noop(monkeypatch): + monkeypatch.setattr(GitPolicyFetcher, "repos", {}) + GitPolicyFetcher.forget_repo("/clones/missing") # must not raise + assert GitPolicyFetcher.repos == {} + + +def test_forget_repo_without_free_method(monkeypatch): + monkeypatch.setattr(GitPolicyFetcher, "repos", {"/clones/y": object()}) + GitPolicyFetcher.forget_repo("/clones/y") # object() has no .free(); must not raise + assert "/clones/y" not in GitPolicyFetcher.repos From b792f239a43ee27e7eef049ff49cefaf27fa8658 Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 23 Jun 2026 12:52:04 +0300 Subject: [PATCH 15/63] fix(opal-server): purge git fetcher caches on scope delete Capture the deleted scope's source before scanning siblings (fixing the loop-variable shadowing bug where the clone path was computed from the last-iterated scope). Drop the cached pygit2.Repository handle via forget_repo and pop repos_last_fetched when no sibling shares the source. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../opal-server/opal_server/scopes/service.py | 40 ++++-- .../tests/delete_scope_cache_purge_test.py | 132 ++++++++++++++++++ 2 files changed, 157 insertions(+), 15 deletions(-) create mode 100644 packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py diff --git a/packages/opal-server/opal_server/scopes/service.py b/packages/opal-server/opal_server/scopes/service.py index d3df4972f..64cf3b8a5 100644 --- a/packages/opal-server/opal_server/scopes/service.py +++ b/packages/opal-server/opal_server/scopes/service.py @@ -162,24 +162,34 @@ async def delete_scope(self, scope_id: str): with tracer.trace("scopes_service.delete_scope", resource=scope_id): logger.info(f"Delete scope: {scope_id}") scope = await self._scopes.get(scope_id) - url = scope.policy.url - - scopes = await self._scopes.all() - remove_repo_clone = True - - for scope in scopes: - if scope.scope_id != scope_id and scope.policy.url == url: - logger.info( - f"found another scope with same remote url ({scope.scope_id}), skipping clone deletion" - ) - remove_repo_clone = False - break + deleted_source = cast(GitPolicyScopeSource, scope.policy) + deleted_source_id = GitPolicyFetcher.source_id(deleted_source) + scope_dir = GitPolicyFetcher.repo_clone_path(self._base_dir, deleted_source) + + # Clone dir, the `repos` handle cache, and `repos_last_fetched` are + # all keyed by source_id (= the clone path). A sibling only shares + # storage when it resolves to the same source_id; same url with a + # different branch can shard to a different source_id (and a + # different clone dir) when SCOPES_REPO_CLONES_SHARDS > 1, so gate on + # source_id, not url — otherwise the deleted scope's clone + pygit2 + # handle leak. + other_scopes = [ + s for s in await self._scopes.all() if s.scope_id != scope_id + ] + source_id_shared = any( + isinstance(s.policy, GitPolicyScopeSource) + and GitPolicyFetcher.source_id(s.policy) == deleted_source_id + for s in other_scopes + ) - if remove_repo_clone: - scope_dir = GitPolicyFetcher.repo_clone_path( - self._base_dir, cast(GitPolicyScopeSource, scope.policy) + if source_id_shared: + logger.info( + "Another scope shares the same clone (source id), skipping clone deletion" ) + else: shutil.rmtree(scope_dir, ignore_errors=True) + GitPolicyFetcher.forget_repo(str(scope_dir)) + GitPolicyFetcher.repos_last_fetched.pop(deleted_source_id, None) await self._scopes.delete(scope_id) diff --git a/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py b/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py new file mode 100644 index 000000000..4a4e0a5e7 --- /dev/null +++ b/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py @@ -0,0 +1,132 @@ +import pytest +from opal_common.schemas.policy_source import GitPolicyScopeSource, NoAuthData +from opal_common.schemas.scopes import Scope +from opal_server.git_fetcher import GitPolicyFetcher +from opal_server.scopes.scope_repository import ScopeNotFoundError +from opal_server.scopes.service import ScopesService + + +class FakeScopeRepository: + def __init__(self, scopes): + self._scopes = {s.scope_id: s for s in scopes} + + async def get(self, scope_id): + if scope_id not in self._scopes: + raise ScopeNotFoundError(scope_id) + return self._scopes[scope_id] + + async def all(self): + return list(self._scopes.values()) + + async def delete(self, scope_id): + self._scopes.pop(scope_id, None) + + +def _scope(scope_id, url, branch="main"): + return Scope( + scope_id=scope_id, + policy=GitPolicyScopeSource( + source_type="git", + url=url, + branch=branch, + auth=NoAuthData(auth_type="none"), + ), + data={"entries": []}, + ) + + +@pytest.fixture(autouse=True) +def clear_caches(): + GitPolicyFetcher.repos.clear() + GitPolicyFetcher.repos_last_fetched.clear() + GitPolicyFetcher.repo_locks.clear() + yield + GitPolicyFetcher.repos.clear() + GitPolicyFetcher.repos_last_fetched.clear() + GitPolicyFetcher.repo_locks.clear() + + +@pytest.mark.asyncio +async def test_delete_unique_scope_purges_caches(tmp_path, monkeypatch): + scope = _scope("only", "https://git/repo-a.git") + repo = FakeScopeRepository([scope]) + svc = ScopesService(base_dir=tmp_path, scopes=repo, pubsub_endpoint=None) + + src = scope.policy + sid = GitPolicyFetcher.source_id(src) + clone_path = str(GitPolicyFetcher.repo_clone_path(tmp_path, src)) + GitPolicyFetcher.repos[clone_path] = object() + GitPolicyFetcher.repos_last_fetched[sid] = "ts" + + monkeypatch.setattr( + "opal_server.scopes.service.shutil.rmtree", lambda *a, **k: None + ) + + await svc.delete_scope("only") + + assert clone_path not in GitPolicyFetcher.repos + assert sid not in GitPolicyFetcher.repos_last_fetched + + +@pytest.mark.asyncio +async def test_delete_keeps_caches_when_sibling_shares_source(tmp_path, monkeypatch): + a = _scope("a", "https://git/shared.git") + b = _scope("b", "https://git/shared.git") # same url+branch -> same source_id + repo = FakeScopeRepository([a, b]) + svc = ScopesService(base_dir=tmp_path, scopes=repo, pubsub_endpoint=None) + + sid = GitPolicyFetcher.source_id(a.policy) + clone_path = str(GitPolicyFetcher.repo_clone_path(tmp_path, a.policy)) + GitPolicyFetcher.repos[clone_path] = object() + GitPolicyFetcher.repos_last_fetched[sid] = "ts" + + rmtree_calls = [] + monkeypatch.setattr( + "opal_server.scopes.service.shutil.rmtree", + lambda p, **k: rmtree_calls.append(p), + ) + + await svc.delete_scope("a") + + assert rmtree_calls == [] # sibling shares the source id; clone must survive + assert clone_path in GitPolicyFetcher.repos + assert sid in GitPolicyFetcher.repos_last_fetched + + +@pytest.mark.asyncio +async def test_delete_purges_when_sibling_shares_url_but_not_source( + tmp_path, monkeypatch +): + """Same url, different branch, sharded clones (SCOPES_REPO_CLONES_SHARDS>1) + resolve to different source_ids -> different clone dirs. + + Deleting one must still purge its own clone + caches; the url- + sharing sibling lives elsewhere. + """ + # shards=4: branch "main" -> index 1, "dev" -> index 3 (distinct source_ids). + monkeypatch.setattr( + "opal_server.git_fetcher.opal_server_config.SCOPES_REPO_CLONES_SHARDS", 4 + ) + a = _scope("a", "https://git/shared.git", branch="main") + b = _scope("b", "https://git/shared.git", branch="dev") # same url, diff source_id + assert GitPolicyFetcher.source_id(a.policy) != GitPolicyFetcher.source_id(b.policy) + + repo = FakeScopeRepository([a, b]) + svc = ScopesService(base_dir=tmp_path, scopes=repo, pubsub_endpoint=None) + + sid_a = GitPolicyFetcher.source_id(a.policy) + clone_path_a = str(GitPolicyFetcher.repo_clone_path(tmp_path, a.policy)) + GitPolicyFetcher.repos[clone_path_a] = object() + GitPolicyFetcher.repos_last_fetched[sid_a] = "ts" + + rmtree_calls = [] + monkeypatch.setattr( + "opal_server.scopes.service.shutil.rmtree", + lambda p, **k: rmtree_calls.append(str(p)), + ) + + await svc.delete_scope("a") + + assert rmtree_calls == [clone_path_a] # its own clone dir removed + assert clone_path_a not in GitPolicyFetcher.repos + assert sid_a not in GitPolicyFetcher.repos_last_fetched From 8db1b6bd8be41244a53da98e30ae50718973e686 Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 23 Jun 2026 12:53:04 +0300 Subject: [PATCH 16/63] fix(opal-server): clean all finished webhook tasks (no remove-while-iterating) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../opal_server/policy/watcher/task.py | 9 +++-- .../tests/webhook_task_cleanup_test.py | 35 +++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 packages/opal-server/opal_server/tests/webhook_task_cleanup_test.py diff --git a/packages/opal-server/opal_server/policy/watcher/task.py b/packages/opal-server/opal_server/policy/watcher/task.py index d420d183c..1afba7ef2 100644 --- a/packages/opal-server/opal_server/policy/watcher/task.py +++ b/packages/opal-server/opal_server/policy/watcher/task.py @@ -28,11 +28,10 @@ async def __aexit__(self, exc_type, exc, tb): async def _on_webhook(self, topic: Topic, data: Any): logger.info(f"Webhook listener triggered ({len(self._webhook_tasks)})") - for task in self._webhook_tasks: - if task.done(): - # Clean references to finished tasks - self._webhook_tasks.remove(task) - + # Rebuild rather than remove-while-iterating: list.remove() inside a + # `for t in self._webhook_tasks` loop skips the element after each removal, + # so finished tasks accumulate. + self._webhook_tasks = [t for t in self._webhook_tasks if not t.done()] self._webhook_tasks.append(asyncio.create_task(self.trigger(topic, data))) async def _listen_to_webhook_notifications(self): diff --git a/packages/opal-server/opal_server/tests/webhook_task_cleanup_test.py b/packages/opal-server/opal_server/tests/webhook_task_cleanup_test.py new file mode 100644 index 000000000..a576d924c --- /dev/null +++ b/packages/opal-server/opal_server/tests/webhook_task_cleanup_test.py @@ -0,0 +1,35 @@ +import asyncio + +import pytest +from opal_server.policy.watcher.task import BasePolicyWatcherTask + + +class _Watcher(BasePolicyWatcherTask): + async def trigger(self, topic, data): + return None # fast no-op so the created task finishes quickly + + +@pytest.mark.asyncio +async def test_done_tasks_are_all_removed(): + w = _Watcher(pubsub_endpoint=None) + + async def _done(): + return None + + # three already-finished tasks pre-loaded into the list + finished = [asyncio.create_task(_done()) for _ in range(3)] + await asyncio.gather(*finished) + w._webhook_tasks = list(finished) + + await w._on_webhook("webhook", None) + await asyncio.sleep(0) # let the newly created trigger task finish + + # all 3 done ones removed... + remaining_done = [t for t in w._webhook_tasks if t in finished] + assert remaining_done == [], f"stale done tasks leaked: {remaining_done}" + # ...and exactly the one freshly scheduled trigger task survives. + survivors = [t for t in w._webhook_tasks if t not in finished] + assert len(w._webhook_tasks) == 1 + assert len(survivors) == 1, f"new trigger task not scheduled: {w._webhook_tasks}" + + await asyncio.gather(*survivors) # drain the dangling task From 92c8515abfdf2374eb0a45ab9497f2c2f4c62df8 Mon Sep 17 00:00:00 2001 From: zivxx Date: Sun, 12 Jul 2026 16:07:05 +0300 Subject: [PATCH 17/63] fix(opal-server): actually purge fetcher caches on scope DELETE The PR2 purge in ScopesService.delete_scope was dead code: the HTTP DELETE /scopes/{scope_id} route deleted the scope straight from the ScopeRepository, so the git-leak churn gate stayed red with all cache entries intact. Wire the route to the service (keeping delete-of-missing a 204 no-op), and also drop the source_id-keyed repo_locks entry, which neither forget_repo (path-keyed) nor the service purge released. Verified against the app-tests/git-leak bed: test_churn_releases_caches now passes and test_shared_repo_survives_sibling_scope_delete still guards the shared-clone case. Co-Authored-By: Claude Fable 5 --- .../opal-server/opal_server/scopes/api.py | 11 ++- .../opal-server/opal_server/scopes/service.py | 1 + packages/opal-server/opal_server/server.py | 11 ++- .../tests/delete_scope_cache_purge_test.py | 4 + .../tests/delete_scope_route_test.py | 98 +++++++++++++++++++ 5 files changed, 122 insertions(+), 3 deletions(-) create mode 100644 packages/opal-server/opal_server/tests/delete_scope_route_test.py diff --git a/packages/opal-server/opal_server/scopes/api.py b/packages/opal-server/opal_server/scopes/api.py index d54b1074c..d012b4bf9 100644 --- a/packages/opal-server/opal_server/scopes/api.py +++ b/packages/opal-server/opal_server/scopes/api.py @@ -44,6 +44,7 @@ from opal_server.data.data_update_publisher import DataUpdatePublisher from opal_server.git_fetcher import GitPolicyFetcher from opal_server.scopes.scope_repository import ScopeNotFoundError, ScopeRepository +from opal_server.scopes.service import ScopesService def verify_private_key(private_key: str, key_format: EncryptionKeyFormat) -> bool: @@ -80,6 +81,7 @@ def init_scope_router( scopes: ScopeRepository, authenticator: JWTAuthenticator, pubsub_endpoint: PubSubEndpoint, + scopes_service: ScopesService, ): router = APIRouter() @@ -171,8 +173,13 @@ async def delete_scope( logger.error(f"Unauthorized to delete scope: {repr(ex)}") raise - # TODO: This should also asynchronously clean the repo from the disk (if it's not used by other scopes) - await scopes.delete(scope_id) + try: + # Deletes the scope and also cleans the repo clone from disk and the + # GitPolicyFetcher in-memory caches (unless another scope shares them). + await scopes_service.delete_scope(scope_id) + except ScopeNotFoundError: + # Deleting a missing scope was always a silent no-op (204); keep it. + pass return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/packages/opal-server/opal_server/scopes/service.py b/packages/opal-server/opal_server/scopes/service.py index 2c7dd917d..891a3a23b 100644 --- a/packages/opal-server/opal_server/scopes/service.py +++ b/packages/opal-server/opal_server/scopes/service.py @@ -191,6 +191,7 @@ async def delete_scope(self, scope_id: str): shutil.rmtree(scope_dir, ignore_errors=True) GitPolicyFetcher.forget_repo(str(scope_dir)) GitPolicyFetcher.repos_last_fetched.pop(deleted_source_id, None) + GitPolicyFetcher.repo_locks.pop(deleted_source_id, None) await self._scopes.delete(scope_id) diff --git a/packages/opal-server/opal_server/server.py b/packages/opal-server/opal_server/server.py index 0946d3d2c..7c9e4552f 100644 --- a/packages/opal-server/opal_server/server.py +++ b/packages/opal-server/opal_server/server.py @@ -4,6 +4,7 @@ import sys import traceback from functools import partial +from pathlib import Path from typing import List, Optional from fastapi import Depends, FastAPI, Request @@ -40,6 +41,7 @@ from opal_server.scopes.api import init_scope_router from opal_server.scopes.loader import load_scopes from opal_server.scopes.scope_repository import ScopeRepository +from opal_server.scopes.service import ScopesService from opal_server.security.api import init_security_router from opal_server.security.jwks import JwksStaticEndpoint from opal_server.statistics import OpalStatistics, init_statistics_router @@ -270,8 +272,15 @@ def _configure_api_routes(self, app: FastAPI): ) if opal_server_config.SCOPES: + scopes_service = ScopesService( + base_dir=Path(opal_server_config.BASE_DIR), + scopes=self._scopes, + pubsub_endpoint=self.pubsub.endpoint, + ) app.include_router( - init_scope_router(self._scopes, authenticator, self.pubsub.endpoint), + init_scope_router( + self._scopes, authenticator, self.pubsub.endpoint, scopes_service + ), tags=["Scopes"], prefix="/scopes", ) diff --git a/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py b/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py index 4a4e0a5e7..15cae916e 100644 --- a/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py +++ b/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py @@ -57,6 +57,7 @@ async def test_delete_unique_scope_purges_caches(tmp_path, monkeypatch): clone_path = str(GitPolicyFetcher.repo_clone_path(tmp_path, src)) GitPolicyFetcher.repos[clone_path] = object() GitPolicyFetcher.repos_last_fetched[sid] = "ts" + GitPolicyFetcher.repo_locks[sid] = object() monkeypatch.setattr( "opal_server.scopes.service.shutil.rmtree", lambda *a, **k: None @@ -66,6 +67,7 @@ async def test_delete_unique_scope_purges_caches(tmp_path, monkeypatch): assert clone_path not in GitPolicyFetcher.repos assert sid not in GitPolicyFetcher.repos_last_fetched + assert sid not in GitPolicyFetcher.repo_locks @pytest.mark.asyncio @@ -79,6 +81,7 @@ async def test_delete_keeps_caches_when_sibling_shares_source(tmp_path, monkeypa clone_path = str(GitPolicyFetcher.repo_clone_path(tmp_path, a.policy)) GitPolicyFetcher.repos[clone_path] = object() GitPolicyFetcher.repos_last_fetched[sid] = "ts" + GitPolicyFetcher.repo_locks[sid] = object() rmtree_calls = [] monkeypatch.setattr( @@ -91,6 +94,7 @@ async def test_delete_keeps_caches_when_sibling_shares_source(tmp_path, monkeypa assert rmtree_calls == [] # sibling shares the source id; clone must survive assert clone_path in GitPolicyFetcher.repos assert sid in GitPolicyFetcher.repos_last_fetched + assert sid in GitPolicyFetcher.repo_locks @pytest.mark.asyncio diff --git a/packages/opal-server/opal_server/tests/delete_scope_route_test.py b/packages/opal-server/opal_server/tests/delete_scope_route_test.py new file mode 100644 index 000000000..a980dba3c --- /dev/null +++ b/packages/opal-server/opal_server/tests/delete_scope_route_test.py @@ -0,0 +1,98 @@ +from fastapi import FastAPI +from fastapi.testclient import TestClient +from opal_common.schemas.policy_source import GitPolicyScopeSource, NoAuthData +from opal_common.schemas.scopes import Scope +from opal_server.git_fetcher import GitPolicyFetcher +from opal_server.scopes.api import init_scope_router +from opal_server.scopes.scope_repository import ScopeNotFoundError +from opal_server.scopes.service import ScopesService + +import pytest + + +class FakeScopeRepository: + def __init__(self, scopes): + self._scopes = {s.scope_id: s for s in scopes} + + async def get(self, scope_id): + if scope_id not in self._scopes: + raise ScopeNotFoundError(scope_id) + return self._scopes[scope_id] + + async def all(self): + return list(self._scopes.values()) + + async def delete(self, scope_id): + self._scopes.pop(scope_id, None) + + +class FakeAuthenticator: + """Mimics a JWTAuthenticator whose verifier is disabled (no public key).""" + + enabled = False + + def __call__(self): + return {} + + +def _scope(scope_id, url, branch="main"): + return Scope( + scope_id=scope_id, + policy=GitPolicyScopeSource( + source_type="git", + url=url, + branch=branch, + auth=NoAuthData(auth_type="none"), + ), + data={"entries": []}, + ) + + +def _client(repo, base_dir): + service = ScopesService(base_dir=base_dir, scopes=repo, pubsub_endpoint=None) + app = FastAPI() + app.include_router( + init_scope_router(repo, FakeAuthenticator(), None, service), + prefix="/scopes", + ) + return TestClient(app) + + +@pytest.fixture(autouse=True) +def clear_caches(): + GitPolicyFetcher.repos.clear() + GitPolicyFetcher.repos_last_fetched.clear() + GitPolicyFetcher.repo_locks.clear() + yield + GitPolicyFetcher.repos.clear() + GitPolicyFetcher.repos_last_fetched.clear() + GitPolicyFetcher.repo_locks.clear() + + +def test_delete_route_purges_fetcher_caches(tmp_path, monkeypatch): + """DELETE /scopes/{id} must flow through ScopesService.delete_scope so the + GitPolicyFetcher caches drain (the git-leak churn gate).""" + scope = _scope("only", "https://git/repo-a.git") + repo = FakeScopeRepository([scope]) + + sid = GitPolicyFetcher.source_id(scope.policy) + clone_path = str(GitPolicyFetcher.repo_clone_path(tmp_path, scope.policy)) + GitPolicyFetcher.repos[clone_path] = object() + GitPolicyFetcher.repos_last_fetched[sid] = "ts" + + monkeypatch.setattr( + "opal_server.scopes.service.shutil.rmtree", lambda *a, **k: None + ) + + resp = _client(repo, tmp_path).delete("/scopes/only") + + assert resp.status_code == 204 + assert clone_path not in GitPolicyFetcher.repos + assert sid not in GitPolicyFetcher.repos_last_fetched + + +def test_delete_route_missing_scope_stays_204(tmp_path): + """Deleting a nonexistent scope was a silent no-op (204) before the purge + wiring and must remain one.""" + resp = _client(FakeScopeRepository([]), tmp_path).delete("/scopes/ghost") + assert resp.status_code == 204 From 8d5cfdf44b1ea7808ec9f485ab3b2dc04c494d77 Mon Sep 17 00:00:00 2001 From: zivxx Date: Sun, 12 Jul 2026 16:21:58 +0300 Subject: [PATCH 18/63] style(opal-server): fix import order in delete_scope_route_test (isort) CI fixes: - pre-commit: isort wanted `import pytest` in the top import block of the new test file; it was missed locally because `pre-commit run --all-files` only covers tracked files and the file was untracked when formatters ran. Co-Authored-By: Claude Fable 5 --- .../opal-server/opal_server/tests/delete_scope_route_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/opal-server/opal_server/tests/delete_scope_route_test.py b/packages/opal-server/opal_server/tests/delete_scope_route_test.py index a980dba3c..70ff5b7f5 100644 --- a/packages/opal-server/opal_server/tests/delete_scope_route_test.py +++ b/packages/opal-server/opal_server/tests/delete_scope_route_test.py @@ -1,3 +1,4 @@ +import pytest from fastapi import FastAPI from fastapi.testclient import TestClient from opal_common.schemas.policy_source import GitPolicyScopeSource, NoAuthData @@ -7,8 +8,6 @@ from opal_server.scopes.scope_repository import ScopeNotFoundError from opal_server.scopes.service import ScopesService -import pytest - class FakeScopeRepository: def __init__(self, scopes): From 89e090be440e373b443504a03147cf3b513e0e17 Mon Sep 17 00:00:00 2001 From: zivxx Date: Mon, 13 Jul 2026 13:36:50 +0300 Subject: [PATCH 19/63] fix(opal-server): serialize the scope-delete cache purge under the repo lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit delete_scope mutated GitPolicyFetcher's lock-guarded shared state (rmtree of the clone dir, Repository.free(), cache pops) without holding repo_locks[source_id], racing in-flight fetches that run on executor threads — a native use-after-free. Popping the lock entry also let the next fetcher mint a fresh lock and run unserialized (lock identity hazard). Replace _get_repo_lock with GitPolicyFetcher.lock_source(source_id), an async context manager that re-checks the repo_locks entry after acquiring and retries on the fresh lock if a delete popped it. delete_scope now purges under that lock and pops the entry while holding it, so repo_locks still drains to zero (keeping the PR1 churn gate green) without the identity hazard. Also, per the same review pass: guard delete_scope against non-git policy sources like sync_scope does, restore the sharing scope's id in the skip-deletion log line, and document that the purge is process-local best-effort (fleet-wide purge incl. the leader is tracked for PR3). Addresses review comments: - https://github.com/permitio/opal/pull/923#discussion_r3566536975 (@zeevmoney) - https://github.com/permitio/opal/pull/923#discussion_r3566537005 (@zeevmoney) - https://github.com/permitio/opal/pull/923#discussion_r3566737392 (@zeevmoney) - https://github.com/permitio/opal/pull/923#discussion_r3566737413 (@zeevmoney) Co-Authored-By: Claude Fable 5 --- .../opal-server/opal_server/git_fetcher.py | 29 +++++--- .../opal-server/opal_server/scopes/api.py | 3 + .../opal-server/opal_server/scopes/service.py | 46 ++++++++++--- .../tests/delete_scope_cache_purge_test.py | 67 ++++++++++++++++++- 4 files changed, 123 insertions(+), 22 deletions(-) diff --git a/packages/opal-server/opal_server/git_fetcher.py b/packages/opal-server/opal_server/git_fetcher.py index 305abf4b4..0d67bb49a 100644 --- a/packages/opal-server/opal_server/git_fetcher.py +++ b/packages/opal-server/opal_server/git_fetcher.py @@ -4,6 +4,7 @@ import hashlib import shutil import time +from contextlib import asynccontextmanager from pathlib import Path from typing import Optional, cast @@ -140,13 +141,24 @@ def __init__( f"Initializing git fetcher: scope_id={scope_id}, url={redact_url(source.url)}, branch={self._source.branch}, source_id={self._source_id}" ) - async def _get_repo_lock(self): - # Previous file based implementation worked across multiple processes/threads, but wasn't fair (next acquiree is random) - # This implementation works only within the same process/thread, but is fair (next acquiree is the earliest to enter the lock) - lock = GitPolicyFetcher.repo_locks[ - self._source_id - ] = GitPolicyFetcher.repo_locks.get(self._source_id, asyncio.Lock()) - return lock + @staticmethod + @asynccontextmanager + async def lock_source(source_id: str): + """Serialize all mutation of a source's clone dir and cached handles. + + Locks are minted on demand into ``repo_locks`` (asyncio.Lock: process- + local but fair, unlike the previous file-based lock). A scope delete + pops the dict entry while holding the lock, so after acquiring we must + re-check that ``repo_locks`` still maps ``source_id`` to the lock we + acquired — a waiter woken after a delete would otherwise proceed under + the stale lock, unserialized against holders of the freshly-minted one. + """ + while True: + lock = GitPolicyFetcher.repo_locks.setdefault(source_id, asyncio.Lock()) + async with lock: + if GitPolicyFetcher.repo_locks.get(source_id) is lock: + yield + return async def _was_fetched_after(self, t: datetime.datetime): last_fetched = GitPolicyFetcher.repos_last_fetched.get(self._source_id, None) @@ -168,8 +180,7 @@ async def fetch_and_notify_on_changes( - if the hinted commit hash is provided and is already found in the local clone we use this hint to avoid an necessary fetch. """ - repo_lock = await self._get_repo_lock() - async with repo_lock: + async with GitPolicyFetcher.lock_source(self._source_id): with tracer.trace( "git_policy_fetcher.fetch_and_notify_on_changes", resource=self._scope_id, diff --git a/packages/opal-server/opal_server/scopes/api.py b/packages/opal-server/opal_server/scopes/api.py index d012b4bf9..cf90c76eb 100644 --- a/packages/opal-server/opal_server/scopes/api.py +++ b/packages/opal-server/opal_server/scopes/api.py @@ -176,6 +176,9 @@ async def delete_scope( try: # Deletes the scope and also cleans the repo clone from disk and the # GitPolicyFetcher in-memory caches (unless another scope shares them). + # The cache purge is process-local best-effort: it runs on whichever + # worker serves this DELETE; a fleet-wide purge (leader included) is + # tracked for PR3 of the leak series. await scopes_service.delete_scope(scope_id) except ScopeNotFoundError: # Deleting a missing scope was always a silent no-op (204); keep it. diff --git a/packages/opal-server/opal_server/scopes/service.py b/packages/opal-server/opal_server/scopes/service.py index 891a3a23b..107261c76 100644 --- a/packages/opal-server/opal_server/scopes/service.py +++ b/packages/opal-server/opal_server/scopes/service.py @@ -163,7 +163,18 @@ async def delete_scope(self, scope_id: str): with tracer.trace("scopes_service.delete_scope", resource=scope_id): logger.info(f"Delete scope: {scope_id}") scope = await self._scopes.get(scope_id) - deleted_source = cast(GitPolicyScopeSource, scope.policy) + + if not isinstance(scope.policy, GitPolicyScopeSource): + # Mirrors sync_scope: only git sources have a clone dir and + # fetcher caches to clean up. + logger.warning( + f"Scope {scope_id} has a non-git policy source, " + "deleting the scope record only" + ) + await self._scopes.delete(scope_id) + return + + deleted_source = scope.policy deleted_source_id = GitPolicyFetcher.source_id(deleted_source) scope_dir = GitPolicyFetcher.repo_clone_path(self._base_dir, deleted_source) @@ -177,21 +188,34 @@ async def delete_scope(self, scope_id: str): other_scopes = [ s for s in await self._scopes.all() if s.scope_id != scope_id ] - source_id_shared = any( - isinstance(s.policy, GitPolicyScopeSource) - and GitPolicyFetcher.source_id(s.policy) == deleted_source_id - for s in other_scopes + sharing_scope_id = next( + ( + s.scope_id + for s in other_scopes + if isinstance(s.policy, GitPolicyScopeSource) + and GitPolicyFetcher.source_id(s.policy) == deleted_source_id + ), + None, ) - if source_id_shared: + if sharing_scope_id is not None: logger.info( - "Another scope shares the same clone (source id), skipping clone deletion" + f"Scope {sharing_scope_id} shares the same clone (source id), " + "skipping clone deletion" ) else: - shutil.rmtree(scope_dir, ignore_errors=True) - GitPolicyFetcher.forget_repo(str(scope_dir)) - GitPolicyFetcher.repos_last_fetched.pop(deleted_source_id, None) - GitPolicyFetcher.repo_locks.pop(deleted_source_id, None) + # NOTE: this purge is process-local best-effort — it cleans the + # caches of whichever worker serves the DELETE. Broadcasting the + # purge so the leader (which accumulates most handles via + # continuous sync) also drops its caches is tracked for PR3. + async with GitPolicyFetcher.lock_source(deleted_source_id): + shutil.rmtree(scope_dir, ignore_errors=True) + GitPolicyFetcher.forget_repo(str(scope_dir)) + GitPolicyFetcher.repos_last_fetched.pop(deleted_source_id, None) + # Popped while the lock is held: lock_source waiters re-check + # the dict entry after acquiring and retry on the fresh lock, + # so nobody proceeds under the stale one. + GitPolicyFetcher.repo_locks.pop(deleted_source_id, None) await self._scopes.delete(scope_id) diff --git a/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py b/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py index 15cae916e..1d6f0867d 100644 --- a/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py +++ b/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py @@ -1,3 +1,5 @@ +import asyncio + import pytest from opal_common.schemas.policy_source import GitPolicyScopeSource, NoAuthData from opal_common.schemas.scopes import Scope @@ -57,7 +59,7 @@ async def test_delete_unique_scope_purges_caches(tmp_path, monkeypatch): clone_path = str(GitPolicyFetcher.repo_clone_path(tmp_path, src)) GitPolicyFetcher.repos[clone_path] = object() GitPolicyFetcher.repos_last_fetched[sid] = "ts" - GitPolicyFetcher.repo_locks[sid] = object() + GitPolicyFetcher.repo_locks[sid] = asyncio.Lock() monkeypatch.setattr( "opal_server.scopes.service.shutil.rmtree", lambda *a, **k: None @@ -81,7 +83,7 @@ async def test_delete_keeps_caches_when_sibling_shares_source(tmp_path, monkeypa clone_path = str(GitPolicyFetcher.repo_clone_path(tmp_path, a.policy)) GitPolicyFetcher.repos[clone_path] = object() GitPolicyFetcher.repos_last_fetched[sid] = "ts" - GitPolicyFetcher.repo_locks[sid] = object() + GitPolicyFetcher.repo_locks[sid] = asyncio.Lock() rmtree_calls = [] monkeypatch.setattr( @@ -134,3 +136,64 @@ async def test_delete_purges_when_sibling_shares_url_but_not_source( assert rmtree_calls == [clone_path_a] # its own clone dir removed assert clone_path_a not in GitPolicyFetcher.repos assert sid_a not in GitPolicyFetcher.repos_last_fetched + + +@pytest.mark.asyncio +async def test_delete_serializes_against_inflight_repo_lock(tmp_path, monkeypatch): + """The purge must wait for the repo lock held by an in-flight fetch — + otherwise it rmtree's the clone and free()s the pygit2 handle out from + under the fetch thread.""" + scope = _scope("only", "https://git/repo-a.git") + repo = FakeScopeRepository([scope]) + svc = ScopesService(base_dir=tmp_path, scopes=repo, pubsub_endpoint=None) + + sid = GitPolicyFetcher.source_id(scope.policy) + clone_path = str(GitPolicyFetcher.repo_clone_path(tmp_path, scope.policy)) + GitPolicyFetcher.repos[clone_path] = object() + + rmtree_calls = [] + monkeypatch.setattr( + "opal_server.scopes.service.shutil.rmtree", + lambda p, **k: rmtree_calls.append(str(p)), + ) + + # Simulate an in-flight fetch holding the repo lock. + lock = GitPolicyFetcher.repo_locks.setdefault(sid, asyncio.Lock()) + await lock.acquire() + try: + delete_task = asyncio.create_task(svc.delete_scope("only")) + for _ in range(10): # give the delete every chance to (wrongly) proceed + await asyncio.sleep(0) + assert not delete_task.done(), "delete_scope did not wait for the repo lock" + assert rmtree_calls == [], "purge ran while the fetch held the repo lock" + finally: + lock.release() + + await asyncio.wait_for(delete_task, timeout=5) + assert rmtree_calls == [clone_path] + assert clone_path not in GitPolicyFetcher.repos + assert sid not in GitPolicyFetcher.repo_locks + + +@pytest.mark.asyncio +async def test_lock_source_waiter_retries_after_delete_pops_entry(): + """A waiter queued on the old lock must not proceed under it once a delete + popped the entry — it retries and serializes on the freshly-minted lock.""" + sid = "some-source-id" + events = [] + + async def deleter(): + async with GitPolicyFetcher.lock_source(sid): + events.append("deleter-in") + await asyncio.sleep(0.01) # let the waiter queue on this lock + GitPolicyFetcher.repo_locks.pop(sid, None) + events.append("deleter-out") + + async def waiter(): + async with GitPolicyFetcher.lock_source(sid): + events.append("waiter-in") + # We must hold the *current* dict entry, not the popped one. + assert GitPolicyFetcher.repo_locks.get(sid) is not None + + await asyncio.gather(deleter(), waiter()) + assert events == ["deleter-in", "deleter-out", "waiter-in"] From 1e7df1cb320a03e76880468b483d091d5d130748 Mon Sep 17 00:00:00 2001 From: zivxx Date: Mon, 13 Jul 2026 13:37:30 +0300 Subject: [PATCH 20/63] fix(opal-server): surface rmtree and Repository.free() failures in scope cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rmtree(ignore_errors=True) silently orphaned a deleted tenant's policy clone on any filesystem failure (EBUSY, permissions, stale NFS handle) while the scope record was deleted anyway — no log, no way to ever find the leftovers. Replace with an explicit try/except that tolerates a missing dir and logs any other OSError at WARNING with the scope id and path, still proceeding with the scope deletion. forget_repo likewise swallowed Repository.free() exceptions at DEBUG with no exception detail or path; a systematic free() failure would be invisible in production. Log at WARNING with the clone path and error, and fix the docstring claim about free() availability (the pinned pygit2 always ships it). Add the missing test for the free()-raises branch. Addresses review comments: - https://github.com/permitio/opal/pull/923#discussion_r3566537022 (@zeevmoney) - https://github.com/permitio/opal/pull/923#discussion_r3566537065 (@zeevmoney) - https://github.com/permitio/opal/pull/923#discussion_r3566537101 (@zeevmoney) Co-Authored-By: Claude Fable 5 --- packages/opal-server/opal_server/git_fetcher.py | 12 ++++++++---- packages/opal-server/opal_server/scopes/service.py | 13 ++++++++++++- .../opal_server/tests/forget_repo_test.py | 11 +++++++++++ 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/packages/opal-server/opal_server/git_fetcher.py b/packages/opal-server/opal_server/git_fetcher.py index 0d67bb49a..a5ec9ef2a 100644 --- a/packages/opal-server/opal_server/git_fetcher.py +++ b/packages/opal-server/opal_server/git_fetcher.py @@ -387,8 +387,9 @@ def forget_repo(path: str) -> None: The cached ``pygit2.Repository`` keeps OS file descriptors and mmapped pack indexes open; without this, a deleted scope's repo pins memory and inodes for the lifetime of the process even after the clone is removed. - ``Repository.free()`` is called only when available (it is not part of - every pygit2 release); otherwise the dropped reference is reclaimed by GC. + ``Repository.free()`` is called only when available (the pinned pygit2 + always has it; the guard defends against test doubles and future API + changes); otherwise the dropped reference is reclaimed by GC. """ repo = GitPolicyFetcher.repos.pop(path, None) if repo is None: @@ -397,8 +398,11 @@ def forget_repo(path: str) -> None: if callable(free): try: free() - except Exception: - logger.debug("pygit2 Repository.free() failed; relying on GC") + except Exception as e: + logger.warning( + f"pygit2 Repository.free() failed for {path}: {e!r}; " + "relying on GC to release the handles" + ) class GitCallback(RemoteCallbacks): diff --git a/packages/opal-server/opal_server/scopes/service.py b/packages/opal-server/opal_server/scopes/service.py index 107261c76..01888486a 100644 --- a/packages/opal-server/opal_server/scopes/service.py +++ b/packages/opal-server/opal_server/scopes/service.py @@ -209,7 +209,18 @@ async def delete_scope(self, scope_id: str): # purge so the leader (which accumulates most handles via # continuous sync) also drops its caches is tracked for PR3. async with GitPolicyFetcher.lock_source(deleted_source_id): - shutil.rmtree(scope_dir, ignore_errors=True) + try: + shutil.rmtree(scope_dir) + except FileNotFoundError: + pass # never cloned (or already gone) — nothing to clean + except OSError as e: + # Deliberately not fatal: the scope record must still be + # deleted, but an orphaned clone on disk has to be + # discoverable rather than silently leaked. + logger.warning( + f"Failed to remove clone dir {scope_dir} of deleted " + f"scope {scope_id}: {e!r}" + ) GitPolicyFetcher.forget_repo(str(scope_dir)) GitPolicyFetcher.repos_last_fetched.pop(deleted_source_id, None) # Popped while the lock is held: lock_source waiters re-check diff --git a/packages/opal-server/opal_server/tests/forget_repo_test.py b/packages/opal-server/opal_server/tests/forget_repo_test.py index b6e223cf1..826c2fdc6 100644 --- a/packages/opal-server/opal_server/tests/forget_repo_test.py +++ b/packages/opal-server/opal_server/tests/forget_repo_test.py @@ -29,3 +29,14 @@ def test_forget_repo_without_free_method(monkeypatch): monkeypatch.setattr(GitPolicyFetcher, "repos", {"/clones/y": object()}) GitPolicyFetcher.forget_repo("/clones/y") # object() has no .free(); must not raise assert "/clones/y" not in GitPolicyFetcher.repos + + +class _ExplodingRepo: + def free(self): + raise RuntimeError("free failed") + + +def test_forget_repo_free_raises_entry_still_popped(monkeypatch): + monkeypatch.setattr(GitPolicyFetcher, "repos", {"/clones/z": _ExplodingRepo()}) + GitPolicyFetcher.forget_repo("/clones/z") # must swallow (and log) the error + assert "/clones/z" not in GitPolicyFetcher.repos From deb6a5f8fdaef59514de79b2cdfb448f12f2ed06 Mon Sep 17 00:00:00 2001 From: zivxx Date: Mon, 13 Jul 2026 13:38:30 +0300 Subject: [PATCH 21/63] fix(opal-server): retrieve webhook task exceptions before dropping them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The webhook task sweep dropped finished tasks without inspecting t.exception(), and stop() only gathered self._tasks — so an error raised inside trigger() never reached the app logger, surfacing only as asyncio's generic "Task exception was never retrieved" at GC time. Log failed trigger tasks at ERROR during the sweep and include webhook tasks in the stop() gather. Also de-flake the cleanup test: await the freshly scheduled trigger task directly instead of relying on an asyncio.sleep(0) scheduler tick, and add a test asserting the failure of a trigger task is retrieved and logged. Addresses review comments: - https://github.com/permitio/opal/pull/923#discussion_r3566537082 (@zeevmoney) - https://github.com/permitio/opal/pull/923#discussion_r3566537101 (@zeevmoney) Co-Authored-By: Claude Fable 5 --- .../opal_server/policy/watcher/task.py | 9 ++++- .../tests/webhook_task_cleanup_test.py | 37 ++++++++++++++++++- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/packages/opal-server/opal_server/policy/watcher/task.py b/packages/opal-server/opal_server/policy/watcher/task.py index 1afba7ef2..0e8cd0afd 100644 --- a/packages/opal-server/opal_server/policy/watcher/task.py +++ b/packages/opal-server/opal_server/policy/watcher/task.py @@ -30,7 +30,12 @@ async def _on_webhook(self, topic: Topic, data: Any): logger.info(f"Webhook listener triggered ({len(self._webhook_tasks)})") # Rebuild rather than remove-while-iterating: list.remove() inside a # `for t in self._webhook_tasks` loop skips the element after each removal, - # so finished tasks accumulate. + # so finished tasks accumulate. Retrieve exceptions before dropping the + # references — otherwise a failed trigger() is only reported by asyncio's + # generic "Task exception was never retrieved" at GC time. + for t in self._webhook_tasks: + if t.done() and not t.cancelled() and t.exception() is not None: + logger.error(f"Webhook trigger task failed: {t.exception()!r}") self._webhook_tasks = [t for t in self._webhook_tasks if not t.done()] self._webhook_tasks.append(asyncio.create_task(self.trigger(topic, data))) @@ -72,7 +77,7 @@ async def stop(self): for task in self._tasks + self._webhook_tasks: if not task.done(): task.cancel() - await asyncio.gather(*self._tasks, return_exceptions=True) + await asyncio.gather(*self._tasks, *self._webhook_tasks, return_exceptions=True) async def trigger(self, topic: Topic, data: Any): """Triggers the policy watcher from outside to check for changes (git diff --git a/packages/opal-server/opal_server/tests/webhook_task_cleanup_test.py b/packages/opal-server/opal_server/tests/webhook_task_cleanup_test.py index a576d924c..b3e7d994f 100644 --- a/packages/opal-server/opal_server/tests/webhook_task_cleanup_test.py +++ b/packages/opal-server/opal_server/tests/webhook_task_cleanup_test.py @@ -22,7 +22,6 @@ async def _done(): w._webhook_tasks = list(finished) await w._on_webhook("webhook", None) - await asyncio.sleep(0) # let the newly created trigger task finish # all 3 done ones removed... remaining_done = [t for t in w._webhook_tasks if t in finished] @@ -32,4 +31,38 @@ async def _done(): assert len(w._webhook_tasks) == 1 assert len(survivors) == 1, f"new trigger task not scheduled: {w._webhook_tasks}" - await asyncio.gather(*survivors) # drain the dangling task + # Await the trigger task directly (not sleep(0)) so the test doesn't depend + # on scheduler tick ordering. + await asyncio.gather(*survivors) + + +class _FailingWatcher(BasePolicyWatcherTask): + async def trigger(self, topic, data): + raise RuntimeError("trigger blew up") + + +@pytest.mark.asyncio +async def test_failed_trigger_exception_is_retrieved_and_logged(): + """Sweeping a failed trigger task must retrieve and log its exception, not + silently drop the reference (asyncio's 'exception was never retrieved').""" + from opal_common.logger import logger as opal_logger + + w = _FailingWatcher(pubsub_endpoint=None) + + await w._on_webhook("webhook", None) # schedules a trigger that raises + failed = list(w._webhook_tasks) + await asyncio.gather(*failed, return_exceptions=True) + + records = [] + sink_id = opal_logger.add(lambda m: records.append(str(m)), level="ERROR") + try: + await w._on_webhook("webhook", None) # sweep must log the failure + finally: + opal_logger.remove(sink_id) + + assert failed[0] not in w._webhook_tasks, "failed task not swept" + assert any( + "Webhook trigger task failed" in r and "trigger blew up" in r for r in records + ), f"failure not logged: {records}" + + await asyncio.gather(*w._webhook_tasks, return_exceptions=True) From 36b020f5418d416225bb0517847085b6ac55f35b Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 03:31:57 +0300 Subject: [PATCH 22/63] feat(opal-server): expose pid and cache keys in internal fetcher stats Per-worker attribution for the git-leak bed: the caches are per-process, so the pid identifies which worker answered and the key lists let the invariant checker compare cache contents against live scopes and disk. Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/helpers.py | 7 +++++- .../opal-server/opal_server/debug_stats.py | 13 ++++++++--- .../tests/debug_stats_endpoint_test.py | 22 +++++++++++++++++-- .../opal_server/tests/debug_stats_test.py | 14 ++++++++++++ 4 files changed, 50 insertions(+), 6 deletions(-) diff --git a/app-tests/git-leak/helpers.py b/app-tests/git-leak/helpers.py index 64a32abbe..f763e317f 100644 --- a/app-tests/git-leak/helpers.py +++ b/app-tests/git-leak/helpers.py @@ -67,7 +67,12 @@ def stats(self, samples: int = 3, interval: float = 0.1) -> Dict[str, int]: ) resp.raise_for_status() for key, value in resp.json().items(): - merged[key] = max(merged.get(key, 0), value) + if key != "pid" and isinstance(value, (int, float)): + merged[key] = max(merged.get(key, 0), value) + else: + # pid and the *_keys lists: last-wins (single-worker stack + # makes every read hit the same worker anyway) + merged[key] = value if i < samples - 1: time.sleep(interval) return merged diff --git a/packages/opal-server/opal_server/debug_stats.py b/packages/opal-server/opal_server/debug_stats.py index 5439ce950..28dbff3d8 100644 --- a/packages/opal-server/opal_server/debug_stats.py +++ b/packages/opal-server/opal_server/debug_stats.py @@ -3,6 +3,7 @@ Used only by the off-by-default /internal stats endpoint so tests can observe the cache growth that the memory-leak fix (PR2) eliminates. """ +import os from pathlib import Path from typing import Dict, List, Optional @@ -21,13 +22,19 @@ def _read_rss_kb() -> int: return 0 -def git_fetcher_cache_stats() -> Dict[str, int]: - """Sizes of the three process-global GitPolicyFetcher caches + RSS.""" +def git_fetcher_cache_stats() -> Dict: + """Sizes + keys of the three process-global GitPolicyFetcher caches, RSS, + and the worker pid (per-process caches: the pid identifies WHICH worker + answered, so multi-worker bed tests can assert per-worker drain).""" return { + "pid": os.getpid(), "repo_locks": len(GitPolicyFetcher.repo_locks), "repos": len(GitPolicyFetcher.repos), "repos_last_fetched": len(GitPolicyFetcher.repos_last_fetched), "rss_kb": _read_rss_kb(), + "repo_locks_keys": sorted(GitPolicyFetcher.repo_locks.keys()), + "repos_keys": sorted(GitPolicyFetcher.repos.keys()), + "repos_last_fetched_keys": sorted(GitPolicyFetcher.repos_last_fetched.keys()), } @@ -57,5 +64,5 @@ def register_internal_stats_route( include_in_schema=False, dependencies=dependencies or [], ) - def _git_fetcher_cache_stats() -> Dict[str, int]: + def _git_fetcher_cache_stats() -> Dict: return git_fetcher_cache_stats() diff --git a/packages/opal-server/opal_server/tests/debug_stats_endpoint_test.py b/packages/opal-server/opal_server/tests/debug_stats_endpoint_test.py index 87e392910..00cc3247a 100644 --- a/packages/opal-server/opal_server/tests/debug_stats_endpoint_test.py +++ b/packages/opal-server/opal_server/tests/debug_stats_endpoint_test.py @@ -23,7 +23,16 @@ def test_endpoint_present_when_enabled(): resp = client.get("/internal/git-fetcher-cache-stats") assert resp.status_code == 200 body = resp.json() - assert set(body) == {"repo_locks", "repos", "repos_last_fetched", "rss_kb"} + assert set(body) == { + "pid", + "repo_locks", + "repos", + "repos_last_fetched", + "rss_kb", + "repo_locks_keys", + "repos_keys", + "repos_last_fetched_keys", + } def test_endpoint_applies_passed_dependencies(): @@ -77,7 +86,16 @@ def test_server_wiring_mounts_endpoint_when_flag_enabled(): client = TestClient(_build_server().app) resp = client.get("/internal/git-fetcher-cache-stats") assert resp.status_code == 200 - assert set(resp.json()) == {"repo_locks", "repos", "repos_last_fetched", "rss_kb"} + assert set(resp.json()) == { + "pid", + "repo_locks", + "repos", + "repos_last_fetched", + "rss_kb", + "repo_locks_keys", + "repos_keys", + "repos_last_fetched_keys", + } def test_server_wiring_omits_endpoint_when_flag_disabled(): diff --git a/packages/opal-server/opal_server/tests/debug_stats_test.py b/packages/opal-server/opal_server/tests/debug_stats_test.py index c67fb3530..cb16a1b10 100644 --- a/packages/opal-server/opal_server/tests/debug_stats_test.py +++ b/packages/opal-server/opal_server/tests/debug_stats_test.py @@ -1,3 +1,4 @@ +import os import sys from opal_server.config import opal_server_config @@ -26,3 +27,16 @@ def test_stats_report_dict_sizes(monkeypatch): def test_internal_stats_flag_defaults_off(): assert opal_server_config.DEBUG_INTERNAL_STATS is False + + +def test_stats_include_pid_and_cache_keys(monkeypatch): + monkeypatch.setattr(GitPolicyFetcher, "repos", {"/clones/x": object()}) + monkeypatch.setattr(GitPolicyFetcher, "repos_last_fetched", {"sid-1": "ts"}) + monkeypatch.setattr(GitPolicyFetcher, "repo_locks", {"sid-1": object()}) + + stats = git_fetcher_cache_stats() + + assert stats["pid"] == os.getpid() + assert stats["repos_keys"] == ["/clones/x"] + assert stats["repos_last_fetched_keys"] == ["sid-1"] + assert stats["repo_locks_keys"] == ["sid-1"] From 6b4435159b69b9b2cc90cb6914848532df74428b Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 03:37:27 +0300 Subject: [PATCH 23/63] fix(opal-server): only stamp repos_last_fetched after a successful fetch A failed fetch left the pre-written stamp in place, so _was_fetched_after() suppressed the next webhook-requested forced refresh and the scope served stale policy until an unrelated force. Stamp the fetch START time, written only on success. Co-Authored-By: Claude Fable 5 --- .../opal-server/opal_server/git_fetcher.py | 14 ++++-- .../test_git_fetcher_repos_last_fetched.py | 46 +++++++++++++++++++ 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/packages/opal-server/opal_server/git_fetcher.py b/packages/opal-server/opal_server/git_fetcher.py index a5ec9ef2a..9320dab64 100644 --- a/packages/opal-server/opal_server/git_fetcher.py +++ b/packages/opal-server/opal_server/git_fetcher.py @@ -199,13 +199,21 @@ async def fetch_and_notify_on_changes( logger.debug( f"Fetching remote (force_fetch={force_fetch}): {self._remote} ({redact_url(self._source.url)})" ) - GitPolicyFetcher.repos_last_fetched[ - self._source_id - ] = datetime.datetime.now() + # Record the START time but write it only on + # success: a failed fetch must not look "fresh" + # to _was_fetched_after(), or it suppresses the + # forced refresh a webhook just asked for. The + # start time (not completion) is what req_time + # comparisons need: a fetch that STARTED after + # the request already satisfies it. + fetch_started = datetime.datetime.now() await run_sync( repo.remotes[self._remote].fetch, callbacks=self._auth_callbacks, ) + GitPolicyFetcher.repos_last_fetched[ + self._source_id + ] = fetch_started logger.debug( f"Fetch completed: {redact_url(self._source.url)}" ) diff --git a/packages/opal-server/opal_server/tests/test_git_fetcher_repos_last_fetched.py b/packages/opal-server/opal_server/tests/test_git_fetcher_repos_last_fetched.py index fd12e7f35..dcfafa065 100644 --- a/packages/opal-server/opal_server/tests/test_git_fetcher_repos_last_fetched.py +++ b/packages/opal-server/opal_server/tests/test_git_fetcher_repos_last_fetched.py @@ -1,6 +1,7 @@ import datetime from pathlib import Path +import pygit2 import pytest from opal_common.schemas.policy_source import GitPolicyScopeSource, NoAuthData from opal_server.git_fetcher import GitPolicyFetcher @@ -88,3 +89,48 @@ async def test_force_fetch_not_downgraded_by_sibling_source(monkeypatch, tmp_pat ) is True ) + + +class _FailingRemote: + def fetch(self, *args, **kwargs): + raise pygit2.GitError("network down") + + +class _OkRemote: + def fetch(self, *args, **kwargs): + return None + + +class _FakeRepo: + def __init__(self, remote): + self.remotes = {"origin": remote} + + +@pytest.mark.asyncio +async def test_failed_fetch_does_not_stamp_last_fetched(monkeypatch, tmp_path): + """Bug B: stamping before the fetch means a FAILED fetch still records + "fetched now", which suppresses the next webhook-requested forced refresh + via _was_fetched_after().""" + fetcher = _make_fetcher(tmp_path, "scope_x", "https://example.com/repo-x.git") + monkeypatch.setattr(fetcher, "_discover_repository", lambda path: True) + monkeypatch.setattr(fetcher, "_get_valid_repo", lambda: _FakeRepo(_FailingRemote())) + + with pytest.raises(pygit2.GitError): + await fetcher.fetch_and_notify_on_changes(force_fetch=True) + + assert fetcher._source_id not in GitPolicyFetcher.repos_last_fetched + + +@pytest.mark.asyncio +async def test_successful_fetch_stamps_last_fetched(monkeypatch, tmp_path): + fetcher = _make_fetcher(tmp_path, "scope_x", "https://example.com/repo-x.git") + monkeypatch.setattr(fetcher, "_discover_repository", lambda path: True) + monkeypatch.setattr(fetcher, "_get_valid_repo", lambda: _FakeRepo(_OkRemote())) + + async def _no_notify(repo): + return None + + monkeypatch.setattr(fetcher, "_notify_on_changes", _no_notify) + await fetcher.fetch_and_notify_on_changes(force_fetch=True) + + assert fetcher._source_id in GitPolicyFetcher.repos_last_fetched From b7f1ba0a3dae516a50fa777b2b82daadd64b37c6 Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 03:42:24 +0300 Subject: [PATCH 24/63] test(opal-server): pin fetch-START semantics of the repos_last_fetched stamp Review follow-up: a completion-time regression would have passed the existing tests. Co-Authored-By: Claude Fable 5 --- .../test_git_fetcher_repos_last_fetched.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/packages/opal-server/opal_server/tests/test_git_fetcher_repos_last_fetched.py b/packages/opal-server/opal_server/tests/test_git_fetcher_repos_last_fetched.py index dcfafa065..e27e08ece 100644 --- a/packages/opal-server/opal_server/tests/test_git_fetcher_repos_last_fetched.py +++ b/packages/opal-server/opal_server/tests/test_git_fetcher_repos_last_fetched.py @@ -134,3 +134,36 @@ async def _no_notify(repo): await fetcher.fetch_and_notify_on_changes(force_fetch=True) assert fetcher._source_id in GitPolicyFetcher.repos_last_fetched + + +@pytest.mark.asyncio +async def test_stamp_records_fetch_start_time_not_completion(monkeypatch, tmp_path): + """The stamp must be the fetch START time: a fetch that STARTED after a + refresh's req_time already satisfies it; completion time would wrongly + suppress refreshes requested mid-fetch.""" + import time as _time + + class _SlowRemote: + def __init__(self): + self.entered_at = None + + def fetch(self, *args, **kwargs): + self.entered_at = datetime.datetime.now() + _time.sleep(0.05) + + remote = _SlowRemote() + fetcher = _make_fetcher(tmp_path, "scope_x", "https://example.com/repo-x.git") + monkeypatch.setattr(fetcher, "_discover_repository", lambda path: True) + monkeypatch.setattr(fetcher, "_get_valid_repo", lambda: _FakeRepo(remote)) + + async def _no_notify(repo): + return None + + monkeypatch.setattr(fetcher, "_notify_on_changes", _no_notify) + await fetcher.fetch_and_notify_on_changes(force_fetch=True) + + stamp = GitPolicyFetcher.repos_last_fetched[fetcher._source_id] + assert stamp <= remote.entered_at, ( + "stamp is later than the fetch's entry time — completion time was " + "stored instead of start time" + ) From 138deeb404c125665f7f0e8b99c1df0ea2df2c0f Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 03:46:29 +0300 Subject: [PATCH 25/63] fix(opal-server): make invalid-clone recovery drop the stale handle and clear partial dirs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recovery path rmtree'd a bad clone but kept its cached pygit2.Repository, so the stale handle re-invalidated every fresh clone (infinite re-clone loop, leaked handle per cycle) — recovery only worked on a cold cache. forget_repo the handle first, clear any pre-existing (partial) dir before cloning, and cache the fresh clone's handle. Co-Authored-By: Claude Fable 5 --- .../opal-server/opal_server/git_fetcher.py | 18 ++- .../tests/invalid_repo_recovery_test.py | 116 ++++++++++++++++++ 2 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 packages/opal-server/opal_server/tests/invalid_repo_recovery_test.py diff --git a/packages/opal-server/opal_server/git_fetcher.py b/packages/opal-server/opal_server/git_fetcher.py index 9320dab64..53dc8ff44 100644 --- a/packages/opal-server/opal_server/git_fetcher.py +++ b/packages/opal-server/opal_server/git_fetcher.py @@ -222,11 +222,17 @@ async def fetch_and_notify_on_changes( await self._notify_on_changes(repo) return else: - # repo dir exists but invalid -> we must delete the directory + # repo dir exists but invalid -> drop the cached handle + # FIRST (it is the thing judging the dir invalid; kept, + # it would re-invalidate the fresh clone on every sync + # -> infinite re-clone loop), then delete the directory. logger.warning( "Deleting invalid repo: {path}", path=self._repo_path ) - shutil.rmtree(self._repo_path) + GitPolicyFetcher.forget_repo(str(self._repo_path)) + # ignore_errors: with a stale handle the dir may already + # be partially or fully gone. + shutil.rmtree(self._repo_path, ignore_errors=True) else: logger.info("Repo not found at {path}", path=self._repo_path) @@ -238,6 +244,11 @@ def _discover_repository(self, path: Path) -> bool: return discover_repository(str(path)) and git_path.exists() async def _clone(self): + if self._repo_path.exists(): + # A failed/interrupted clone leaves a partial dir; + # clone_repository refuses a non-empty destination, which would + # wedge every retry for this source. + shutil.rmtree(self._repo_path, ignore_errors=True) logger.info( "Cloning repo at '{url}' to '{path}'", url=redact_url(self._source.url), @@ -254,6 +265,9 @@ async def _clone(self): logger.exception(f"Could not clone repo at {redact_url(self._source.url)}") else: logger.info(f"Clone completed: {redact_url(self._source.url)}") + # Cache the fresh handle so the next sync's _get_repo() reuses it + # instead of reopening (or hitting a stale predecessor). + GitPolicyFetcher.repos[str(self._repo_path)] = repo await self._notify_on_changes(repo) def _get_repo(self) -> Repository: diff --git a/packages/opal-server/opal_server/tests/invalid_repo_recovery_test.py b/packages/opal-server/opal_server/tests/invalid_repo_recovery_test.py new file mode 100644 index 000000000..8781aebb9 --- /dev/null +++ b/packages/opal-server/opal_server/tests/invalid_repo_recovery_test.py @@ -0,0 +1,116 @@ +from pathlib import Path + +import pygit2 +import pytest +from opal_common.schemas.policy_source import GitPolicyScopeSource, NoAuthData +from opal_server.git_fetcher import GitPolicyFetcher + + +def _make_fetcher(base_dir: Path, scope_id: str, url: str) -> GitPolicyFetcher: + source = GitPolicyScopeSource( + source_type="git", + url=url, + branch="main", + auth=NoAuthData(auth_type="none"), + ) + return GitPolicyFetcher(base_dir=base_dir, scope_id=scope_id, source=source) + + +@pytest.fixture(autouse=True) +def _reset_class_state(): + GitPolicyFetcher.repos.clear() + GitPolicyFetcher.repos_last_fetched.clear() + GitPolicyFetcher.repo_locks.clear() + yield + GitPolicyFetcher.repos.clear() + GitPolicyFetcher.repos_last_fetched.clear() + GitPolicyFetcher.repo_locks.clear() + + +class _BrokenRepo: + """Simulates a cached pygit2 handle whose backing clone dir went bad.""" + + freed = False + + @property + def remotes(self): + raise pygit2.GitError("stale handle: backing files gone") + + def free(self): + self.freed = True + + +@pytest.mark.asyncio +async def test_recovery_forgets_stale_cached_handle(monkeypatch, tmp_path): + """Bug A: without forget_repo in the recovery branch, the broken cached + handle survives and re-invalidates every fresh clone -> infinite loop.""" + fetcher = _make_fetcher(tmp_path, "s", "https://example.com/r.git") + path = str(fetcher._repo_path) + broken = _BrokenRepo() + GitPolicyFetcher.repos[path] = broken + monkeypatch.setattr(fetcher, "_discover_repository", lambda p: True) + monkeypatch.setattr("opal_server.git_fetcher.shutil.rmtree", lambda p, **k: None) + clone_calls = [] + + async def fake_clone(): + clone_calls.append(True) + + monkeypatch.setattr(fetcher, "_clone", fake_clone) + + await fetcher.fetch_and_notify_on_changes() + + assert clone_calls == [True] + assert GitPolicyFetcher.repos.get(path) is not broken, ( + "recovery left the stale handle cached — next sync re-invalidates " + "the fresh clone (infinite re-clone loop)" + ) + + +@pytest.mark.asyncio +async def test_clone_caches_the_fresh_handle(monkeypatch, tmp_path): + fetcher = _make_fetcher(tmp_path, "s", "https://example.com/r.git") + fresh = object() + monkeypatch.setattr( + "opal_server.git_fetcher.clone_repository", lambda *a, **k: fresh + ) + + async def _no_notify(repo): + return None + + monkeypatch.setattr(fetcher, "_notify_on_changes", _no_notify) + + await fetcher._clone() + + assert GitPolicyFetcher.repos.get(str(fetcher._repo_path)) is fresh, ( + "fresh clone's handle not cached — _get_repo would reopen (or worse, " + "return a stale entry) on the next sync" + ) + + +@pytest.mark.asyncio +async def test_clone_clears_partial_dir_before_cloning(monkeypatch, tmp_path): + """U9: a failed clone leaves a partial dir; clone_repository refuses a + non-empty destination, wedging every retry.""" + fetcher = _make_fetcher(tmp_path, "s", "https://example.com/r.git") + fetcher._repo_path.mkdir(parents=True) + (fetcher._repo_path / "leftover.pack").write_text("partial clone debris") + seen = {} + + def fake_clone(url, path, callbacks=None): + p = Path(path) + seen["nonempty"] = p.exists() and any(p.iterdir()) + return object() + + monkeypatch.setattr("opal_server.git_fetcher.clone_repository", fake_clone) + + async def _no_notify(repo): + return None + + monkeypatch.setattr(fetcher, "_notify_on_changes", _no_notify) + + await fetcher._clone() + + assert seen["nonempty"] is False, ( + "partial dir not cleared before clone — a real clone_repository " + "raises on a non-empty destination, wedging the scope forever" + ) From d980c9c373d0e3a3a02076c4949575f0b907a841 Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 03:53:50 +0300 Subject: [PATCH 26/63] test(opal-server): delete-then-recreate serializes on the repo lock Co-Authored-By: Claude Fable 5 --- .../tests/delete_scope_cache_purge_test.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py b/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py index 1d6f0867d..de952ebb7 100644 --- a/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py +++ b/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py @@ -197,3 +197,45 @@ async def waiter(): await asyncio.gather(deleter(), waiter()) assert events == ["deleter-in", "deleter-out", "waiter-in"] + + +@pytest.mark.asyncio +async def test_recreate_after_delete_serializes_and_sees_clean_caches( + tmp_path, monkeypatch +): + """A re-create's first sync queued during a delete must run only after the + purge completed, on the freshly-minted lock, and see empty caches.""" + scope = _scope("only", "https://git/repo-a.git") + repo = FakeScopeRepository([scope]) + svc = ScopesService(base_dir=tmp_path, scopes=repo, pubsub_endpoint=None) + sid = GitPolicyFetcher.source_id(scope.policy) + clone_path = str(GitPolicyFetcher.repo_clone_path(tmp_path, scope.policy)) + GitPolicyFetcher.repos[clone_path] = object() + monkeypatch.setattr( + "opal_server.scopes.service.shutil.rmtree", lambda *a, **k: None + ) + + order = [] + gate = GitPolicyFetcher.repo_locks.setdefault(sid, asyncio.Lock()) + await gate.acquire() # simulate the in-flight fetch the delete must wait on + try: + + async def deleter(): + await svc.delete_scope("only") + order.append("delete-done") + + async def recreator(): # stands in for the re-created scope's first sync + async with GitPolicyFetcher.lock_source(sid): + order.append(("recreate-in", clone_path in GitPolicyFetcher.repos)) + + d = asyncio.create_task(deleter()) + for _ in range(5): + await asyncio.sleep(0) # deleter queues on the held lock first + r = asyncio.create_task(recreator()) + for _ in range(5): + await asyncio.sleep(0) # recreator queues behind it + finally: + gate.release() + + await asyncio.wait_for(asyncio.gather(d, r), timeout=5) + assert order == ["delete-done", ("recreate-in", False)] From 6b07f9c5696e05f3cc81d48f2b66b9183c3d49ca Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 03:59:25 +0300 Subject: [PATCH 27/63] test(opal-server): stop() cancels and gathers in-flight webhook triggers Co-Authored-By: Claude Fable 5 --- .../tests/webhook_task_cleanup_test.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/packages/opal-server/opal_server/tests/webhook_task_cleanup_test.py b/packages/opal-server/opal_server/tests/webhook_task_cleanup_test.py index b3e7d994f..1261bfcd0 100644 --- a/packages/opal-server/opal_server/tests/webhook_task_cleanup_test.py +++ b/packages/opal-server/opal_server/tests/webhook_task_cleanup_test.py @@ -66,3 +66,24 @@ async def test_failed_trigger_exception_is_retrieved_and_logged(): ), f"failure not logged: {records}" await asyncio.gather(*w._webhook_tasks, return_exceptions=True) + + +class _HangingWatcher(BasePolicyWatcherTask): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.started = asyncio.Event() + + async def trigger(self, topic, data): + self.started.set() + await asyncio.Event().wait() # hangs until cancelled + + +@pytest.mark.asyncio +async def test_stop_cancels_and_gathers_inflight_trigger(): + w = _HangingWatcher(pubsub_endpoint=None) + await w._on_webhook("webhook", None) + await asyncio.wait_for(w.started.wait(), timeout=5) + + await w.stop() # must cancel the hung trigger AND await it (gather) + + assert all(t.cancelled() for t in w._webhook_tasks) From 436a8a00660e93dfa79dd5b70d7dc666c8d14924 Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 10:26:43 +0300 Subject: [PATCH 28/63] fix(opal-server): read fetcher cache stats from atomic snapshots git_fetcher_cache_stats() runs on a Starlette worker thread while the GitPolicyFetcher repo_locks/repos/repos_last_fetched dicts are mutated on the event-loop/executor threads; iterating a live dict's keys() across that race can raise "dictionary changed size during iteration", and computing len() and sorted(keys()) as two independent reads of a mutating dict can return a self-contradictory count/keys pair (e.g. repos=1 while listing 2 keys). Fix by snapshotting each cache once via dict.copy() (a single, GIL-atomic C-level op) and deriving both the count and the key list from that same snapshot. Co-Authored-By: Claude Fable 5 --- .../opal-server/opal_server/debug_stats.py | 21 ++++++++---- .../opal_server/tests/debug_stats_test.py | 34 +++++++++++++++++++ 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/packages/opal-server/opal_server/debug_stats.py b/packages/opal-server/opal_server/debug_stats.py index 28dbff3d8..249548d55 100644 --- a/packages/opal-server/opal_server/debug_stats.py +++ b/packages/opal-server/opal_server/debug_stats.py @@ -26,15 +26,24 @@ def git_fetcher_cache_stats() -> Dict: """Sizes + keys of the three process-global GitPolicyFetcher caches, RSS, and the worker pid (per-process caches: the pid identifies WHICH worker answered, so multi-worker bed tests can assert per-worker drain).""" + # Snapshot each cache once (dict.copy() is a single C-level operation, + # atomic under the GIL): this handler runs on a Starlette worker thread + # while the caches are mutated on the event-loop/executor threads, so + # iterating the live dicts can raise "dictionary changed size during + # iteration", and reading len() and keys() separately can return a + # self-contradictory count/keys pair. + repo_locks = GitPolicyFetcher.repo_locks.copy() + repos = GitPolicyFetcher.repos.copy() + repos_last_fetched = GitPolicyFetcher.repos_last_fetched.copy() return { "pid": os.getpid(), - "repo_locks": len(GitPolicyFetcher.repo_locks), - "repos": len(GitPolicyFetcher.repos), - "repos_last_fetched": len(GitPolicyFetcher.repos_last_fetched), + "repo_locks": len(repo_locks), + "repos": len(repos), + "repos_last_fetched": len(repos_last_fetched), "rss_kb": _read_rss_kb(), - "repo_locks_keys": sorted(GitPolicyFetcher.repo_locks.keys()), - "repos_keys": sorted(GitPolicyFetcher.repos.keys()), - "repos_last_fetched_keys": sorted(GitPolicyFetcher.repos_last_fetched.keys()), + "repo_locks_keys": sorted(repo_locks.keys()), + "repos_keys": sorted(repos.keys()), + "repos_last_fetched_keys": sorted(repos_last_fetched.keys()), } diff --git a/packages/opal-server/opal_server/tests/debug_stats_test.py b/packages/opal-server/opal_server/tests/debug_stats_test.py index cb16a1b10..b2efc5987 100644 --- a/packages/opal-server/opal_server/tests/debug_stats_test.py +++ b/packages/opal-server/opal_server/tests/debug_stats_test.py @@ -40,3 +40,37 @@ def test_stats_include_pid_and_cache_keys(monkeypatch): assert stats["repos_keys"] == ["/clones/x"] assert stats["repos_last_fetched_keys"] == ["sid-1"] assert stats["repo_locks_keys"] == ["sid-1"] + + +class _ChurningDict(dict): + """Simulates a dict being resized by another thread mid-iteration: any + direct keys() iteration blows up; only an atomic .copy() snapshot is safe.""" + + def keys(self): + raise RuntimeError("dictionary changed size during iteration (simulated)") + + +def test_stats_survive_concurrent_cache_mutation(monkeypatch): + monkeypatch.setattr(GitPolicyFetcher, "repos", _ChurningDict({"/c/x": object()})) + monkeypatch.setattr( + GitPolicyFetcher, "repos_last_fetched", _ChurningDict({"sid-1": "ts"}) + ) + monkeypatch.setattr(GitPolicyFetcher, "repo_locks", _ChurningDict({"sid-1": 1})) + + stats = git_fetcher_cache_stats() # must not iterate the live dicts + + assert stats["repos_keys"] == ["/c/x"] + assert stats["repo_locks_keys"] == ["sid-1"] + assert stats["repos_last_fetched_keys"] == ["sid-1"] + + +def test_stats_counts_derive_from_the_same_snapshot_as_keys(monkeypatch): + monkeypatch.setattr(GitPolicyFetcher, "repos", {"/c/a": object(), "/c/b": object()}) + monkeypatch.setattr(GitPolicyFetcher, "repos_last_fetched", {"s1": "t", "s2": "t"}) + monkeypatch.setattr(GitPolicyFetcher, "repo_locks", {"s1": object()}) + + stats = git_fetcher_cache_stats() + + assert stats["repos"] == len(stats["repos_keys"]) == 2 + assert stats["repos_last_fetched"] == len(stats["repos_last_fetched_keys"]) == 2 + assert stats["repo_locks"] == len(stats["repo_locks_keys"]) == 1 From b0255c6e56b6c8b278a2b8c9c145a8632d9f65ae Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 10:26:48 +0300 Subject: [PATCH 29/63] test(opal-server): assert recovery actually free()s the stale handle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_recovery_forgets_stale_cached_handle only checked that the broken handle was evicted from the cache, not that it was free()d — a regression to a bare .pop() would still pass while reintroducing the fd/mmap leak the recovery path is meant to close. Co-Authored-By: Claude Fable 5 --- .../opal-server/opal_server/tests/invalid_repo_recovery_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/opal-server/opal_server/tests/invalid_repo_recovery_test.py b/packages/opal-server/opal_server/tests/invalid_repo_recovery_test.py index 8781aebb9..6e9ee55d2 100644 --- a/packages/opal-server/opal_server/tests/invalid_repo_recovery_test.py +++ b/packages/opal-server/opal_server/tests/invalid_repo_recovery_test.py @@ -64,6 +64,7 @@ async def fake_clone(): "recovery left the stale handle cached — next sync re-invalidates " "the fresh clone (infinite re-clone loop)" ) + assert broken.freed is True, "recovery evicted the handle without free()ing it" @pytest.mark.asyncio From 788ae68833fe65dc9071725b73499045a7546c37 Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 10:34:37 +0300 Subject: [PATCH 30/63] test(opal-server): invariant checker + pid-stability teardown for the git-leak bed Every bed test now asserts at teardown: no orphan clone dirs (I1), cache keys consistent with disk and live scopes (I2-I4), worker pid-set unchanged (I5 - a crashed worker resets the caches and can fake a clean drain), server liveness (I6). Red gates exempt named invariants via marker. Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/conftest.py | 13 +++- app-tests/git-leak/helpers.py | 20 ++++++ app-tests/git-leak/invariants.py | 89 +++++++++++++++++++++++++++ app-tests/git-leak/pytest.ini | 4 ++ app-tests/git-leak/test_leak.py | 1 + app-tests/git-leak/test_resilience.py | 2 + 6 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 app-tests/git-leak/invariants.py diff --git a/app-tests/git-leak/conftest.py b/app-tests/git-leak/conftest.py index 547c22404..5e5e4ae61 100644 --- a/app-tests/git-leak/conftest.py +++ b/app-tests/git-leak/conftest.py @@ -10,6 +10,7 @@ list_seeded_repos, worker_pids, ) +from invariants import check_invariants def pytest_addoption(parser): @@ -67,7 +68,7 @@ def stack(request, repo_count): @pytest.fixture() -def opal(stack) -> OpalServerClient: +def opal(stack, request) -> OpalServerClient: # The compose stack is session-scoped (one server for the whole run), but # scopes must not leak between tests: clone paths are keyed by repo URL, so # a scope left behind by one test shares a cache entry with any later test @@ -84,8 +85,18 @@ def opal(stack) -> OpalServerClient: assert ( len(worker_pids()) == 1 ), f"expected a single-worker stack, found workers {sorted(worker_pids())}" + pids_before = worker_pids() yield stack stack.delete_all_scopes() + exempt_marker = request.node.get_closest_marker("invariant_exempt") + exempt = set(exempt_marker.args) if exempt_marker else set() + if request.node.get_closest_marker("allow_worker_restart") is None: + assert worker_pids() == pids_before, ( + f"I5: worker pid-set changed during the test " + f"({sorted(pids_before)} -> {sorted(worker_pids())}) — a crash/" + f"respawn resets the caches and can fake a clean drain" + ) + check_invariants(stack, exempt=exempt) @pytest.fixture() diff --git a/app-tests/git-leak/helpers.py b/app-tests/git-leak/helpers.py index f763e317f..7bd32c81b 100644 --- a/app-tests/git-leak/helpers.py +++ b/app-tests/git-leak/helpers.py @@ -401,6 +401,26 @@ def bounce_postgres(down_seconds: int = 5, during=None) -> None: compose("up", "-d", "--wait", "--no-recreate", "postgres") +def wait_until(predicate, timeout: float, interval: float = 2.0) -> bool: + """Poll ``predicate()`` until truthy or ``timeout`` elapses. + + Swallows transient exceptions from the predicate (a stats read + racing a restart is not a verdict) — only the final state decides. + """ + deadline = time.time() + timeout + while time.time() < deadline: + try: + if predicate(): + return True + except Exception: + pass + time.sleep(interval) + try: + return bool(predicate()) + except Exception: + return False + + def list_seeded_repos(count: int) -> List[str]: return [f"policy-repo-{i:04d}" for i in range(count)] diff --git a/app-tests/git-leak/invariants.py b/app-tests/git-leak/invariants.py new file mode 100644 index 000000000..f8163d241 --- /dev/null +++ b/app-tests/git-leak/invariants.py @@ -0,0 +1,89 @@ +"""Quiescence invariants I1-I6 for the git-leak bed. + +Asserted at every test's teardown (see the ``opal`` fixture) after +``delete_all_scopes()``, and callable mid-test. Red-gate tests that +knowingly leave violations exempt specific IDs via +``@pytest.mark.invariant_exempt("I1", ...)``. + +Deviation from the design spec: I1's "none missing" direction is NOT +asserted — scopes clone lazily, so a just-created scope legitimately has +no dir yet. Only the orphan direction (dir with no live scope) signals a +leak. +""" +import hashlib + +import requests +from helpers import compose + +BASE_DIR_IN_CONTAINER = "/opal/git_sources" + + +def source_id(url: str, branch: str = "main", shards: int = 1) -> str: + """Host-side mirror of GitPolicyFetcher.source_id (sha256(url) + shard).""" + base = hashlib.sha256(url.encode("utf-8")).hexdigest() + index = hashlib.sha256(branch.encode("utf-8")).digest()[0] % shards + return f"{base}-{index}" + + +def clone_dirs(service: str = "opal_server") -> set: + out = compose( + "exec", + "-T", + service, + "sh", + "-c", + f"ls -1 {BASE_DIR_IN_CONTAINER} 2>/dev/null || true", + ).stdout + return {line.strip() for line in out.splitlines() if line.strip()} + + +def live_source_ids(opal, shards: int = 1) -> set: + resp = requests.get(f"{opal.base_url}/scopes", timeout=30) + resp.raise_for_status() + return { + source_id(s["policy"]["url"], s["policy"].get("branch", "main"), shards) + for s in resp.json() + } + + +def check_invariants(opal, exempt=frozenset(), shards: int = 1) -> None: + exempt = set(exempt) + failures = [] + live = live_source_ids(opal, shards) + disk = clone_dirs() + stats = opal.stats(samples=1) + + if "I1" not in exempt: + orphans = disk - live + if orphans: + failures.append( + f"I1: {len(orphans)} orphan clone dir(s) on disk, e.g. {sorted(orphans)[:3]}" + ) + if "I2" not in exempt: + cached_dirs = {k.rsplit("/", 1)[-1] for k in stats.get("repos_keys", [])} + stray = cached_dirs - disk + if stray: + failures.append( + f"I2: cached repo handle(s) with no dir on disk: {sorted(stray)[:3]}" + ) + if "I3" not in exempt: + stray = set(stats.get("repos_last_fetched_keys", [])) - live + if stray: + failures.append( + f"I3: repos_last_fetched key(s) with no live scope: {sorted(stray)[:3]}" + ) + if "I4" not in exempt: + stray = set(stats.get("repo_locks_keys", [])) - live + if stray: + failures.append( + f"I4: repo_locks key(s) with no live scope: {sorted(stray)[:3]}" + ) + if "I6" not in exempt: + if requests.get(f"{opal.base_url}/scopes", timeout=10).status_code != 200: + failures.append("I6: scopes API not answering 200") + + assert not failures, ( + "invariant violations:\n " + + "\n ".join(failures) + + f"\nstats={stats}\ndisk(sample)={sorted(disk)[:10]}" + ) diff --git a/app-tests/git-leak/pytest.ini b/app-tests/git-leak/pytest.ini index 0d501534f..df0c03a10 100644 --- a/app-tests/git-leak/pytest.ini +++ b/app-tests/git-leak/pytest.ini @@ -9,3 +9,7 @@ # marker makes that guarantee explicit — it pins the rootdir to this directory # when run in place, independent of the root config — while the root run (from # the repo root) still never descends here. + +markers = + invariant_exempt(ids): exempt named invariants (I1..I6) from the opal fixture's teardown check for this (red-gate) test + allow_worker_restart: this test restarts opal_server on purpose; skip the I5 pid-stability teardown check diff --git a/app-tests/git-leak/test_leak.py b/app-tests/git-leak/test_leak.py index 971823cdd..61137392c 100644 --- a/app-tests/git-leak/test_leak.py +++ b/app-tests/git-leak/test_leak.py @@ -178,6 +178,7 @@ def test_shared_repo_survives_sibling_scope_delete(opal, repo_count): @pytest.mark.timeout(900) +@pytest.mark.invariant_exempt("I1", "I3", "I4") def test_scope_repoint_releases_old_repo_cache(opal, repo_count): """Re-pointing a scope to a new repo URL, then deleting it, must drain ALL cache entries — including the orphaned old URL's. diff --git a/app-tests/git-leak/test_resilience.py b/app-tests/git-leak/test_resilience.py index d2df056f0..401fa4aed 100644 --- a/app-tests/git-leak/test_resilience.py +++ b/app-tests/git-leak/test_resilience.py @@ -41,6 +41,8 @@ def _wait_served(opal, scope_id: str, timeout: int): @pytest.mark.timeout(420) +@pytest.mark.allow_worker_restart +@pytest.mark.invariant_exempt("I1") def test_offline_repo_does_not_block_healthy_scopes(opal, repo_count): """Unreachable repos must not stop a healthy scope from serving. From 04273d31d85f62415189da848b89c86c91032f10 Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 10:39:55 +0300 Subject: [PATCH 31/63] docs(opal-server): document stats() merge semantics and fix stale hints Review follow-up (deferred behind in-flight work): counts merge with max() while pid/key-lists are last-wins, so cross-field consistency is only guaranteed at samples=1; annotate Dict[str, Any] accordingly. Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/helpers.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/app-tests/git-leak/helpers.py b/app-tests/git-leak/helpers.py index 7bd32c81b..875901640 100644 --- a/app-tests/git-leak/helpers.py +++ b/app-tests/git-leak/helpers.py @@ -2,7 +2,7 @@ import subprocess import time from pathlib import Path -from typing import Dict, List +from typing import Any, Dict, List import requests @@ -50,7 +50,7 @@ def wait_healthy(self, timeout: int = 180) -> None: time.sleep(2) raise RuntimeError(f"opal-server not healthy in {timeout}s (last: {last})") - def stats(self, samples: int = 3, interval: float = 0.1) -> Dict[str, int]: + def stats(self, samples: int = 3, interval: float = 0.1) -> Dict[str, Any]: """Read the git-fetcher cache stats, merged across a few reads. The stack runs a single uvicorn worker (see docker-compose.yml), so the @@ -59,8 +59,17 @@ def stats(self, samples: int = 3, interval: float = 0.1) -> Dict[str, int]: the ``max`` per key only smooths over a read that races an in-flight mutation; it is not relied on to paper over multi-worker nondeterminism (which the single-worker setup removes outright). + + Merge semantics: numeric counts take the max across samples (peak + smoothing); ``pid`` and the ``*_keys`` lists are last-wins. The + count fields and their paired ``*_keys`` lists are therefore only + guaranteed mutually consistent at ``samples=1`` — which is what + every consistency-sensitive consumer (the invariant checker, + per-pid sampling) uses. Do not assert + ``len(stats["repos_keys"]) == stats["repos"]`` on a multi-sample + merge. """ - merged: Dict[str, int] = {} + merged: Dict[str, Any] = {} for i in range(max(1, samples)): resp = requests.get( f"{self.base_url}/internal/git-fetcher-cache-stats", timeout=10 From 7358ce4fffe1f6a1d0c44e88b236696b315d3de8 Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 10:42:03 +0300 Subject: [PATCH 32/63] test(opal-server): bed gate for delete/recreate storms on one source Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/test_transitions.py | 35 ++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 app-tests/git-leak/test_transitions.py diff --git a/app-tests/git-leak/test_transitions.py b/app-tests/git-leak/test_transitions.py new file mode 100644 index 000000000..80ac00c32 --- /dev/null +++ b/app-tests/git-leak/test_transitions.py @@ -0,0 +1,35 @@ +"""Transition tests: interleavings the end-state gates never drive. + +See docs/superpowers/specs/2026-07-13-scope-fetcher-lifecycle-tests-design.md. +""" +import time + +import pytest +from helpers import ( + compose, + gitea_repo_url, + list_seeded_repos, + make_repo_unreachable, + wait_until, + worker_pids, +) +from invariants import clone_dirs, live_source_ids, source_id + + +@pytest.mark.timeout(900) +def test_delete_recreate_storm(opal, repo_count): + """Rapid delete/re-create of the same source must serialize on the repo + lock (lock re-mint path) and end with clean caches. + + Guards 89e090be. + """ + url = gitea_repo_url(list_seeded_repos(1)[0]) + for i in range(5): + opal.put_scope("storm", url) + assert wait_until( + lambda: opal.get_scope_policy("storm").status_code == 200, timeout=300 + ), f"round {i}: recreated scope never served" + opal.delete_scope("storm") + assert wait_until( + lambda: opal.stats(samples=1)["repo_locks"] == 0, timeout=60 + ), f"caches did not drain after the storm: {opal.stats()}" From 138dd48d3c0349c4ec130faf3ec2fcfef97c0195 Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 10:48:27 +0300 Subject: [PATCH 33/63] =?UTF-8?q?test(opal-server):=20warm-boot=20gate=20?= =?UTF-8?q?=E2=80=94=20restart=20must=20reuse=20on-disk=20clones?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/test_boot_states.py | 49 ++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 app-tests/git-leak/test_boot_states.py diff --git a/app-tests/git-leak/test_boot_states.py b/app-tests/git-leak/test_boot_states.py new file mode 100644 index 000000000..7dc58862b --- /dev/null +++ b/app-tests/git-leak/test_boot_states.py @@ -0,0 +1,49 @@ +"""Start-state tests: what the server boots INTO (S2-S7 in the design spec). + +The cold-empty start (S1) is covered by test_boot.py. +""" +import time + +import pytest +from helpers import ( + HEALTHY_PROBE_REPO, + compose, + gitea_repo_url, + list_seeded_repos, + make_repo_unreachable, + wait_until, +) +from invariants import clone_dirs + + +def _clone_log_count() -> int: + return compose("logs", "--no-log-prefix", "opal_server").stdout.count( + "Cloning repo" + ) + + +@pytest.mark.timeout(900) +@pytest.mark.allow_worker_restart +def test_warm_boot_reuses_clones(opal, repo_count): + """A restart with intact clones must serve without re-cloning (S2).""" + n = min(repo_count, 10) + for i, repo in enumerate(list_seeded_repos(n)): + opal.put_scope(f"warm-{i}", gitea_repo_url(repo)) + for i in range(n): + assert wait_until( + lambda i=i: opal.get_scope_policy(f"warm-{i}").status_code == 200, + timeout=600, + ), f"warm-{i} never served before the restart" + + clones_before = _clone_log_count() + compose("restart", "opal_server") + opal.wait_healthy() + + for i in range(n): + assert wait_until( + lambda i=i: opal.get_scope_policy(f"warm-{i}").status_code == 200, + timeout=300, + ), f"warm-{i} not served after warm restart" + assert ( + _clone_log_count() == clones_before + ), "warm boot re-cloned instead of reusing the on-disk clones" From 9206590df2fe9886767cd25083b96564e876d010 Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 13:29:58 +0300 Subject: [PATCH 34/63] test(opal-server): seeded randomized churn driver with invariant checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seeded random put/refresh bursts with settled end-of-round deletes (plus a ghost delete to keep the 204 no-op path exercised); repo_locks must not exceed live sources at every settle point. Replay any failure with CHURN_SEED=. Two deliberate constraints, both to be lifted when PR3's fleet purge lands (no silent caps): - repoints are excluded: a put on a live scope reuses its existing repo, since a repoint orphans the old source's caches by design today (the red repoint gate covers it). - deletes run only after every live scope has settled: the driver's development run surfaced (deterministically, seed 309006536) that a DELETE racing an in-flight sync loses its purge — the sync re-populates the dead scope's caches and clone dir. Same PR3 class; deterministic red-gate coverage lands in test_repoint_during_inflight_fetch_drains_old_source. Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/test_transitions.py | 62 ++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/app-tests/git-leak/test_transitions.py b/app-tests/git-leak/test_transitions.py index 80ac00c32..14c9e09e7 100644 --- a/app-tests/git-leak/test_transitions.py +++ b/app-tests/git-leak/test_transitions.py @@ -33,3 +33,65 @@ def test_delete_recreate_storm(opal, repo_count): assert wait_until( lambda: opal.stats(samples=1)["repo_locks"] == 0, timeout=60 ), f"caches did not drain after the storm: {opal.stats()}" + + +@pytest.mark.timeout(1200) +def test_randomized_churn_holds_invariants(opal, repo_count): + """Seeded random put/refresh churn with settled deletes; invariants must + hold at every settle point. Replay a failure with CHURN_SEED=. + + Two deliberate constraints, both lifted when PR3's fleet purge lands + (no silent caps): + - 'repoint' ops are EXCLUDED: a repoint orphans the old source's cache + entries by design today (the red repoint gate covers it). A `put` on a + live scope therefore reuses that scope's existing repo. + - Deletes run only at round END, after every live scope has settled: a + DELETE racing an in-flight sync loses its purge (the sync re-populates + the caches for the dead scope — same PR3 class; proven deterministically + by seed 309006536 during this test's development; deterministic red-gate + coverage lands in test_repoint_during_inflight_fetch_drains_old_source). + A bounded residual window remains (a served scope's re-sync can still be + in flight); in practice the settle polling latency dwarfs a tiny repo's + sync time. + """ + import os + import random + + seed = int(os.environ.get("CHURN_SEED", "0")) or random.randrange(1, 2**31) + print(f"\nCHURN_SEED={seed}") + rng = random.Random(seed) + repos = list_seeded_repos(min(repo_count, 6)) + live = {} + + for round_no in range(4): + # burst: puts + refreshes only (deletes deferred to round end) + for _ in range(10): + op = rng.choice(["put", "refresh"]) + sid_ = f"rand-{rng.randrange(3)}" + if op == "put": + repo = live.get(sid_) or rng.choice(repos) + opal.put_scope(sid_, gitea_repo_url(repo)) + live[sid_] = repo + else: + opal.refresh_all() + # settle every live scope before any delete + for sid_ in list(live): + assert wait_until( + lambda s=sid_: opal.get_scope_policy(s).status_code == 200, + timeout=300, + ), f"round {round_no}: live scope {sid_} never settled (seed {seed})" + # settled deletes: each live scope has a coin-flip chance to go + for sid_ in list(live): + if rng.random() < 0.5: + opal.delete_scope(sid_) + live.pop(sid_) + opal.delete_scope(f"ghost-{round_no}") # delete-missing stays a 204 no-op + drained = wait_until( + lambda: opal.stats(samples=1)["repo_locks"] + <= len({r for r in live.values()}), + timeout=120, + ) + assert drained, ( + f"round {round_no}: locks exceed live sources (seed {seed}): " + f"{opal.stats()}" + ) From 779dad532e462e16d36968a369623bc8f0b6007d Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 13:38:13 +0300 Subject: [PATCH 35/63] test(opal-server): characterize upstream force-push and branch-delete behavior Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/helpers.py | 40 ++++++++++++- app-tests/git-leak/test_remote_transitions.py | 60 +++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 app-tests/git-leak/test_remote_transitions.py diff --git a/app-tests/git-leak/helpers.py b/app-tests/git-leak/helpers.py index 875901640..36146bd28 100644 --- a/app-tests/git-leak/helpers.py +++ b/app-tests/git-leak/helpers.py @@ -248,7 +248,12 @@ def create_repo(self, name: str) -> None: return resp = requests.post( f"{self.base_url}/api/v1/user/repos", - json={"name": name, "private": False, "auto_init": True}, + json={ + "name": name, + "private": False, + "auto_init": True, + "default_branch": "main", + }, auth=self._auth, timeout=10, ) @@ -264,6 +269,39 @@ def delete_repo(self, name: str) -> None: resp.raise_for_status() +class RepoMutator: + """Host-side git mutations against a bed Gitea repo (force-push, branch + ops) — the remote-transition tests' hands. + + Uses GitPython over the published host port with admin basic-auth. + """ + + def __init__(self, name: str, workdir: Path): + import git as gitpython + + self._url = ( + f"http://{GITEA_USER}:{GITEA_PASSWORD}@localhost:13000/" + f"{GITEA_USER}/{name}.git" + ) + self._clone = gitpython.Repo.clone_from(self._url, str(workdir / name)) + with self._clone.config_writer() as cw: + cw.set_value("user", "name", "bed-mutator") + cw.set_value("user", "email", "bed@test.local") + + def force_push_rewrite(self, branch: str = "main") -> None: + self._clone.git.checkout(branch) + self._clone.git.commit("--amend", "--allow-empty", "-m", "rewritten history") + self._clone.git.push("--force", "origin", branch) + + def push_new_branch(self, branch: str) -> None: + self._clone.git.checkout("-b", branch) + self._clone.git.commit("--allow-empty", "-m", f"seed {branch}") + self._clone.git.push("origin", branch) + + def delete_remote_branch(self, branch: str) -> None: + self._clone.git.push("origin", f":{branch}") + + def gitea_repo_url(name: str) -> str: # url reachable from inside the opal_server container return f"{GITEA_INTERNAL_URL}/{GITEA_USER}/{name}.git" diff --git a/app-tests/git-leak/test_remote_transitions.py b/app-tests/git-leak/test_remote_transitions.py new file mode 100644 index 000000000..9c1ae80ed --- /dev/null +++ b/app-tests/git-leak/test_remote_transitions.py @@ -0,0 +1,60 @@ +"""Remote-side transitions (T16): the tenant rewrites or deletes what we track. + +Both are CHARACTERIZATION tests — they pin today's behavior. If one fails, +the behavior changed: decide deliberately, don't just update the assert. +""" +import pytest +from helpers import GiteaAdmin, RepoMutator, gitea_repo_url, wait_until + + +@pytest.mark.timeout(900) +def test_force_push_rewrite_recovers(opal, gitea_admin, tmp_path): + """A force-pushed (rewritten) head must be picked up on refresh: pygit2's + default fetch refspec is forced, and set_target just moves the local + ref.""" + gitea_admin.create_repo("mutation-force-push") + try: + opal.put_scope("fp", gitea_repo_url("mutation-force-push")) + assert wait_until( + lambda: opal.get_scope_policy("fp").status_code == 200, timeout=300 + ) + old_bundle = opal.get_scope_policy("fp").json() + + RepoMutator("mutation-force-push", tmp_path).force_push_rewrite() + opal.refresh_all() + + assert wait_until( + lambda: opal.get_scope_policy("fp").status_code == 200 + and opal.get_scope_policy("fp").json().get("hash") + != old_bundle.get("hash"), + timeout=300, + ), "rewritten head never served after refresh" + finally: + opal.delete_scope("fp") + gitea_admin.delete_repo("mutation-force-push") + + +@pytest.mark.timeout(900) +def test_deleted_branch_keeps_serving_last_head(opal, gitea_admin, tmp_path): + """Deleting the tracked branch upstream must not crash anything; fetch + doesn't prune, so OPAL silently keeps serving the last known head — + documented (not necessarily desirable) behavior.""" + gitea_admin.create_repo("mutation-branch-del") + mut = RepoMutator("mutation-branch-del", tmp_path) + mut.push_new_branch("extra") + try: + opal.put_scope("bd", gitea_repo_url("mutation-branch-del"), branch="extra") + assert wait_until( + lambda: opal.get_scope_policy("bd").status_code == 200, timeout=300 + ) + + mut.delete_remote_branch("extra") + opal.refresh_all() + + # settle, then pin: still healthy, still serving the last head + assert wait_until( + lambda: opal.get_scope_policy("bd").status_code == 200, timeout=120 + ), "scope stopped serving after upstream branch deletion" + finally: + opal.delete_scope("bd") + gitea_admin.delete_repo("mutation-branch-del") From a987bde59445a678e641f0be06619baf04cb2d97 Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 13:46:44 +0300 Subject: [PATCH 36/63] test(opal-server): delete-during-hung-fetch gates (no-crash green, bounded-return red until PR3) Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/test_transitions.py | 46 ++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/app-tests/git-leak/test_transitions.py b/app-tests/git-leak/test_transitions.py index 14c9e09e7..6dbc2cea7 100644 --- a/app-tests/git-leak/test_transitions.py +++ b/app-tests/git-leak/test_transitions.py @@ -95,3 +95,49 @@ def test_randomized_churn_holds_invariants(opal, repo_count): f"round {round_no}: locks exceed live sources (seed {seed}): " f"{opal.stats()}" ) + + +@pytest.mark.timeout(900) +@pytest.mark.allow_worker_restart +@pytest.mark.invariant_exempt("I1", "I3", "I4") +def test_delete_during_hung_fetch_no_crash(opal): + """Deleting a scope whose clone is hung must never crash a worker (the use- + after-free class 89e090be fixed). + + GREEN since that fix. + """ + import requests as _requests + + pids = worker_pids() + opal.put_scope("hung-nc", make_repo_unreachable("hung-nc-repo")) + time.sleep(5) # let the leader's clone start and block on the blackhole + try: + try: + opal.delete_scope("hung-nc") + except _requests.RequestException: + pass # a slow/blocked DELETE is the OTHER test's concern + assert ( + worker_pids() == pids + ), "worker crashed/respawned during delete-vs-hung-fetch" + finally: + opal.hard_reset() + + +@pytest.mark.timeout(900) +@pytest.mark.allow_worker_restart +@pytest.mark.invariant_exempt("I1", "I3", "I4") +def test_delete_during_hung_fetch_returns_bounded(opal): + """RED until PR3 (fetch timeout): the purge waits on the repo lock, and a + hung clone holds that lock indefinitely, so the DELETE hangs with it.""" + import requests as _requests + + opal.put_scope("hung-b", make_repo_unreachable("hung-b-repo")) + time.sleep(5) + try: + start = time.time() + resp = _requests.delete(f"{opal.base_url}/scopes/hung-b", timeout=90) + assert ( + resp.status_code in (200, 204) and time.time() - start < 90 + ), "DELETE of a hung-fetch scope did not return in bounded time" + finally: + opal.hard_reset() From b0bc27be4a1e921c94b487adfc86897a243bd430 Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 13:57:53 +0300 Subject: [PATCH 37/63] test(opal-server): repoint-during-hung-fetch gate (red until PR3 update-path purge) Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/test_transitions.py | 37 ++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/app-tests/git-leak/test_transitions.py b/app-tests/git-leak/test_transitions.py index 6dbc2cea7..09ebe3e94 100644 --- a/app-tests/git-leak/test_transitions.py +++ b/app-tests/git-leak/test_transitions.py @@ -141,3 +141,40 @@ def test_delete_during_hung_fetch_returns_bounded(opal): ), "DELETE of a hung-fetch scope did not return in bounded time" finally: opal.hard_reset() + + +@pytest.mark.timeout(900) +@pytest.mark.allow_worker_restart +@pytest.mark.invariant_exempt("I1", "I3", "I4") +def test_repoint_during_inflight_fetch_drains_old_source(opal, repo_count): + """RED until PR3 (update-path purge). + + Repointing a scope while its old source's clone is hung must still + serve the new source (green half) and eventually drop the old + source's cache entries (red half). + """ + repo_b = list_seeded_repos(2)[1] + old_url = make_repo_unreachable("repoint-hang-repo") + opal.put_scope("rp", old_url) + time.sleep(5) # old source's clone is now in flight, holding its lock + try: + opal.put_scope("rp", gitea_repo_url(repo_b)) # repoint while hung + assert wait_until( + lambda: opal.get_scope_policy("rp").status_code == 200, timeout=300 + ), "repointed scope never served its new source" + + old_sid = source_id(old_url) + + def _old_entries_gone(): + s = opal.stats(samples=1) + return ( + old_sid not in s["repo_locks_keys"] + and old_sid not in s["repos_last_fetched_keys"] + ) + + assert wait_until(_old_entries_gone, timeout=60), ( + f"old source {old_sid[:12]}… cache entries leaked after repoint " + f"(PR3 update-path purge gate): {opal.stats()}" + ) + finally: + opal.hard_reset() From d5a740f95e03928a2c5bb181acf7223bdf71c628 Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 14:03:46 +0300 Subject: [PATCH 38/63] test(opal-server): system gate for corrupt-clone recovery (no re-clone loop) Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/test_boot_states.py | 36 ++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/app-tests/git-leak/test_boot_states.py b/app-tests/git-leak/test_boot_states.py index 7dc58862b..f8dca5b7b 100644 --- a/app-tests/git-leak/test_boot_states.py +++ b/app-tests/git-leak/test_boot_states.py @@ -47,3 +47,39 @@ def test_warm_boot_reuses_clones(opal, repo_count): assert ( _clone_log_count() == clones_before ), "warm boot re-cloned instead of reusing the on-disk clones" + + +@pytest.mark.timeout(900) +def test_corrupt_clone_recovers_without_clone_loop(opal): + """S3/T7: corrupting a clone under a warm handle cache must recover with + exactly one re-clone (pre-Bug-A-fix this looped forever).""" + repo = list_seeded_repos(3)[2] + opal.put_scope("dirty", gitea_repo_url(repo)) + assert wait_until( + lambda: opal.get_scope_policy("dirty").status_code == 200, timeout=300 + ) + + # corrupt the clone's git metadata in place; the server keeps its cached + # pygit2 handle (warm cache) — the exact Bug A precondition + compose( + "exec", + "-T", + "opal_server", + "sh", + "-c", + 'for d in /opal/git_sources/*/; do rm -rf "$d/.git/objects"; done', + ) + opal.refresh_all() + + assert wait_until( + lambda: opal.get_scope_policy("dirty").status_code == 200, timeout=300 + ), "scope never recovered after clone corruption" + + clones_1 = _clone_log_count() + opal.refresh_all() + time.sleep(10) + clones_2 = _clone_log_count() + assert clones_2 == clones_1, ( + f"re-clone loop: clone count kept growing after recovery " + f"({clones_1} -> {clones_2})" + ) From 52a1fe9a452e2520fac2095616d6e07290e4834b Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 14:24:40 +0300 Subject: [PATCH 39/63] test(opal-server): corrupt clone contents in place so recovery exercises the warm-handle branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous corruption (rm -rf .git/objects) deleted the directory NODE, so pygit2.discover_repository returned None and recovery went through the repo-not-found -> _clone() branch, never touching the cached-handle (_get_valid_repo -> forget_repo) branch the Bug A fix lives in. Emptying the objects dir's CONTENTS keeps discovery succeeding and forces recovery through the warm cached handle; a new assert on the "Deleting invalid repo" log pins the branch taken. NOTE: this gate is currently RED — a real finding. With objects/ emptied in place, _get_valid_repo() still judges the repo valid (open + remote-URL verification raise no pygit2.GitError), so the invalid-repo recovery branch never fires; the failure surfaces later in make_bundle (_get_current_branch_head -> "SHA ... missing") and the scope serves 500 forever with no re-clone. The corrupted-but-discoverable case is not covered by the landed Bug A fix. Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/test_boot_states.py | 27 +++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/app-tests/git-leak/test_boot_states.py b/app-tests/git-leak/test_boot_states.py index f8dca5b7b..ae77db537 100644 --- a/app-tests/git-leak/test_boot_states.py +++ b/app-tests/git-leak/test_boot_states.py @@ -51,23 +51,32 @@ def test_warm_boot_reuses_clones(opal, repo_count): @pytest.mark.timeout(900) def test_corrupt_clone_recovers_without_clone_loop(opal): - """S3/T7: corrupting a clone under a warm handle cache must recover with - exactly one re-clone (pre-Bug-A-fix this looped forever).""" + """S3/T7: emptying a clone's object store in place (objects/ kept as an + empty dir, so pygit2.discover_repository still finds the repo) while the + server holds a warm cached handle must recover through the invalid-repo + branch with exactly one re-clone (pre-Bug-A-fix this looped forever).""" repo = list_seeded_repos(3)[2] opal.put_scope("dirty", gitea_repo_url(repo)) assert wait_until( lambda: opal.get_scope_policy("dirty").status_code == 200, timeout=300 ) - # corrupt the clone's git metadata in place; the server keeps its cached - # pygit2 handle (warm cache) — the exact Bug A precondition + invalid_before = compose("logs", "--no-log-prefix", "opal_server").stdout.count( + "Deleting invalid repo" + ) + + # empty the object store's CONTENTS in place, keeping the objects/ dir + # itself so discover_repository still resolves the repo; the server keeps + # its cached pygit2 handle (warm cache) — the exact Bug A precondition + # (deleting the objects/ node instead would route recovery through the + # repo-not-found -> _clone() branch and never touch the cached handle) compose( "exec", "-T", "opal_server", "sh", "-c", - 'for d in /opal/git_sources/*/; do rm -rf "$d/.git/objects"; done', + 'for d in /opal/git_sources/*/; do rm -rf "$d/.git/objects"/*; done', ) opal.refresh_all() @@ -75,6 +84,14 @@ def test_corrupt_clone_recovers_without_clone_loop(opal): lambda: opal.get_scope_policy("dirty").status_code == 200, timeout=300 ), "scope never recovered after clone corruption" + invalid_after = compose("logs", "--no-log-prefix", "opal_server").stdout.count( + "Deleting invalid repo" + ) + assert invalid_after > invalid_before, ( + "recovery did not take the invalid-repo (warm cached handle) branch — " + "the corruption failed to exercise the Bug A path" + ) + clones_1 = _clone_log_count() opal.refresh_all() time.sleep(10) From 8ccd3ace70bc7d4d07ca6e2c047ce0af13aabe32 Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 14:51:02 +0300 Subject: [PATCH 40/63] fix(opal-server): detect gutted object stores during clone validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A clone can be discoverable yet unusable: refs and config intact but the object store gutted (crash mid-gc, disk corruption). _get_valid_repo() judged such a repo valid (open + remote-URL verification raise no GitError), a forced fetch then negotiated "up to date" against the intact refs and downloaded nothing, and every policy read failed in make_bundle with "SHA ... missing" — the scope served 500s forever with no self-heal. Fix: validate that the tracked branch's head object is actually readable FROM DISK before declaring the repo valid. Crucially the check uses a short-lived fresh Repository handle, not the cached warm handle: the warm handle keeps deleted pack files readable through its open mmaps (unlink does not invalidate them) and reports the object as present even after the store is emptied on disk — verified empirically in the test bed, where a cached-handle check never fired. An unreadable head object now routes the repo to the existing invalid-repo recovery branch (forget cached handle -> delete dir -> re-clone, exactly once). Residual: partial corruption deeper in the object graph is not caught (that would need fsck-grade checks); only head-object readability is validated. The system gate added in 52a1fe9a (in-place object-store gutting under a warm handle cache) flips green with this fix, including its "Deleting invalid repo" branch-pin assert: detection warning, one re-clone, stable clone count. Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/test_boot_states.py | 8 +- .../opal-server/opal_server/git_fetcher.py | 32 +++++++- .../tests/invalid_repo_recovery_test.py | 81 +++++++++++++++++++ 3 files changed, 117 insertions(+), 4 deletions(-) diff --git a/app-tests/git-leak/test_boot_states.py b/app-tests/git-leak/test_boot_states.py index ae77db537..69394dfa9 100644 --- a/app-tests/git-leak/test_boot_states.py +++ b/app-tests/git-leak/test_boot_states.py @@ -53,8 +53,10 @@ def test_warm_boot_reuses_clones(opal, repo_count): def test_corrupt_clone_recovers_without_clone_loop(opal): """S3/T7: emptying a clone's object store in place (objects/ kept as an empty dir, so pygit2.discover_repository still finds the repo) while the - server holds a warm cached handle must recover through the invalid-repo - branch with exactly one re-clone (pre-Bug-A-fix this looped forever).""" + server holds a warm cached handle must be detected as invalid (gutted + object store: refs intact, head object unreadable) and recover through + the invalid-repo branch with exactly one re-clone — no serve-500s- + forever wedge and no re-clone loop.""" repo = list_seeded_repos(3)[2] opal.put_scope("dirty", gitea_repo_url(repo)) assert wait_until( @@ -67,7 +69,7 @@ def test_corrupt_clone_recovers_without_clone_loop(opal): # empty the object store's CONTENTS in place, keeping the objects/ dir # itself so discover_repository still resolves the repo; the server keeps - # its cached pygit2 handle (warm cache) — the exact Bug A precondition + # its cached pygit2 handle (warm cache) — the gutted-object-store case # (deleting the objects/ node instead would route recovery through the # repo-not-found -> _clone() branch and never touch the cached handle) compose( diff --git a/packages/opal-server/opal_server/git_fetcher.py b/packages/opal-server/opal_server/git_fetcher.py index 53dc8ff44..188a9d054 100644 --- a/packages/opal-server/opal_server/git_fetcher.py +++ b/packages/opal-server/opal_server/git_fetcher.py @@ -280,7 +280,37 @@ def _get_valid_repo(self) -> Optional[Repository]: try: repo = self._get_repo() RepoInterface.verify_found_repo_matches_remote(repo, self._source.url) - return repo + # A clone can be discoverable yet unusable: refs and config + # intact but the object store gutted (crash mid-gc, disk + # corruption). A fetch then negotiates "up to date" against the + # intact refs and downloads nothing, so without this check the + # scope serves 500s forever with no self-heal. Validate that the + # tracked branch's head object is actually readable FROM DISK: + # the check must use a short-lived fresh handle, because the + # cached warm handle keeps deleted pack files readable through + # its open mmaps (unlink does not invalidate them) and would + # report the object as present. Partial corruption deeper in + # the tree is NOT caught here (that would need fsck-grade + # checks). + probe = Repository(str(self._repo_path)) + try: + try: + ref = probe.lookup_reference( + f"refs/remotes/{self._remote}/{self._source.branch}" + ) + except KeyError: + # Branch not fetched yet — the fetch path handles that. + return repo + if probe.get(ref.target) is None: + logger.warning( + "Repo at {path} has refs but an unreadable object " + "store (missing head object) — treating as invalid", + path=self._repo_path, + ) + return None + return repo + finally: + probe.free() except pygit2.GitError: logger.warning("Invalid repo at: {path}", path=self._repo_path) return None diff --git a/packages/opal-server/opal_server/tests/invalid_repo_recovery_test.py b/packages/opal-server/opal_server/tests/invalid_repo_recovery_test.py index 6e9ee55d2..7c160ca4c 100644 --- a/packages/opal-server/opal_server/tests/invalid_repo_recovery_test.py +++ b/packages/opal-server/opal_server/tests/invalid_repo_recovery_test.py @@ -115,3 +115,84 @@ async def _no_notify(repo): "partial dir not cleared before clone — a real clone_repository " "raises on a non-empty destination, wedging the scope forever" ) + + +class _RemoteStub: + def __init__(self, url): + self.url = url + self.name = "origin" + + +class _Ref: + target = "deadbeef" * 5 + + +class _WarmHandle: + """Cached handle over a gutted clone: its open mmaps keep the deleted + pack files readable (unlink does not invalidate them), so through THIS + handle the repo looks perfectly healthy.""" + + def __init__(self, url): + self.remotes = [_RemoteStub(url)] + self.freed = False + + def lookup_reference(self, name): + return _Ref() + + def get(self, oid): + return object() # mmap still serves the object + + def free(self): + self.freed = True + + +class _GuttedProbe: + """What a fresh on-disk handle sees: refs intact, head object missing.""" + + def __init__(self, path): + self.freed = False + + def lookup_reference(self, name): + return _Ref() + + def get(self, oid): + return None # object store gutted on disk + + def free(self): + self.freed = True + + +@pytest.mark.asyncio +async def test_gutted_object_store_triggers_recovery(monkeypatch, tmp_path): + """Refs intact + objects missing on disk must be treated as invalid even + though the warm cached handle still reads everything via its mmaps: + fetch would negotiate "up to date" and the scope would serve 500s + forever otherwise.""" + url = "https://example.com/r.git" + fetcher = _make_fetcher(tmp_path, "s", url) + path = str(fetcher._repo_path) + warm = _WarmHandle(url) + GitPolicyFetcher.repos[path] = warm + probes = [] + + def fake_repository(p): + probe = _GuttedProbe(p) + probes.append(probe) + return probe + + monkeypatch.setattr("opal_server.git_fetcher.Repository", fake_repository) + monkeypatch.setattr(fetcher, "_discover_repository", lambda p: True) + monkeypatch.setattr("opal_server.git_fetcher.shutil.rmtree", lambda p, **k: None) + clone_calls = [] + + async def fake_clone(): + clone_calls.append(True) + + monkeypatch.setattr(fetcher, "_clone", fake_clone) + + await fetcher.fetch_and_notify_on_changes() + + assert clone_calls == [True], "gutted clone was not routed to recovery" + assert path not in GitPolicyFetcher.repos + assert warm.freed is True, "recovery evicted the warm handle without free()" + assert probes and all(p.freed for p in probes), "disk probe handle leaked" From 3839b6aad961408a99abe968347f6813656fe06f Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 15:00:14 +0300 Subject: [PATCH 41/63] test(opal-server): orphan-dir and redis-wipe boot gates (red until orphan sweep) Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/test_boot_states.py | 54 ++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/app-tests/git-leak/test_boot_states.py b/app-tests/git-leak/test_boot_states.py index 69394dfa9..3c2569400 100644 --- a/app-tests/git-leak/test_boot_states.py +++ b/app-tests/git-leak/test_boot_states.py @@ -102,3 +102,57 @@ def test_corrupt_clone_recovers_without_clone_loop(opal): f"re-clone loop: clone count kept growing after recovery " f"({clones_1} -> {clones_2})" ) + + +@pytest.mark.timeout(900) +@pytest.mark.invariant_exempt("I1") +def test_orphan_clone_dir_is_reclaimed(opal): + """RED until an orphan sweep exists (PR3+, currently unowned): a clone dir + with no live scope must eventually be removed.""" + fake_sid = "f" * 64 + "-0" + compose( + "exec", + "-T", + "opal_server", + "sh", + "-c", + f"mkdir -p /opal/git_sources/{fake_sid} && touch /opal/git_sources/{fake_sid}/junk", + ) + try: + opal.refresh_all() + assert wait_until( + lambda: fake_sid not in clone_dirs(), timeout=60 + ), "orphan clone dir never reclaimed (needs an orphan sweep)" + finally: + # red gate leaves state on purpose; clean it so later tests' I1 holds + compose( + "exec", + "-T", + "opal_server", + "sh", + "-c", + f"rm -rf /opal/git_sources/{fake_sid}", + ) + + +@pytest.mark.timeout(900) +@pytest.mark.allow_worker_restart +@pytest.mark.invariant_exempt("I1") +def test_redis_wiped_boot_reclaims_clones(opal): + """RED until the orphan sweep (same class as the orphan-dir gate): after a + scope-store wipe, on-disk clones reference nothing and must be + reclaimed.""" + opal.put_scope("wipe-0", gitea_repo_url(list_seeded_repos(1)[0])) + assert wait_until( + lambda: opal.get_scope_policy("wipe-0").status_code == 200, timeout=300 + ) + try: + compose("stop", "opal_server") + compose("exec", "-T", "redis", "redis-cli", "FLUSHALL") + compose("start", "opal_server") + opal.wait_healthy() + assert wait_until( + lambda: clone_dirs() == set(), timeout=60 + ), f"clones of the wiped scope store never reclaimed: {sorted(clone_dirs())[:5]}" + finally: + opal.hard_reset() From f9ee0bd2ad6196d8d045b406c2c930ecd5ea15e3 Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 15:14:37 +0300 Subject: [PATCH 42/63] test(opal-server): clean the wiped scope's clone dir so later I1 checks stay meaningful Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/test_boot_states.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/app-tests/git-leak/test_boot_states.py b/app-tests/git-leak/test_boot_states.py index 3c2569400..3099597cc 100644 --- a/app-tests/git-leak/test_boot_states.py +++ b/app-tests/git-leak/test_boot_states.py @@ -145,7 +145,7 @@ def test_redis_wiped_boot_reclaims_clones(opal): opal.put_scope("wipe-0", gitea_repo_url(list_seeded_repos(1)[0])) assert wait_until( lambda: opal.get_scope_policy("wipe-0").status_code == 200, timeout=300 - ) + ), "wipe-0 never served before the wipe" try: compose("stop", "opal_server") compose("exec", "-T", "redis", "redis-cli", "FLUSHALL") @@ -156,3 +156,15 @@ def test_redis_wiped_boot_reclaims_clones(opal): ), f"clones of the wiped scope store never reclaimed: {sorted(clone_dirs())[:5]}" finally: opal.hard_reset() + # hard_reset never touches git_sources/, and post-FLUSHALL no scope + # record exists to route a DELETE's rmtree at the leftover clone — + # remove it explicitly so later tests' I1 stays meaningful. The scope + # store is empty here, so a blanket clean is safe. + compose( + "exec", + "-T", + "opal_server", + "sh", + "-c", + "rm -rf /opal/git_sources/*", + ) From 8d5c61281713b1d9bae12c031c85ec4a22114874 Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 17:42:05 +0300 Subject: [PATCH 43/63] test(opal-server): boot-while-remotes-down gate (red until PR3 fetch timeout) Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/test_boot_states.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/app-tests/git-leak/test_boot_states.py b/app-tests/git-leak/test_boot_states.py index 3099597cc..55e49ce30 100644 --- a/app-tests/git-leak/test_boot_states.py +++ b/app-tests/git-leak/test_boot_states.py @@ -168,3 +168,27 @@ def test_redis_wiped_boot_reclaims_clones(opal): "-c", "rm -rf /opal/git_sources/*", ) + + +@pytest.mark.timeout(1200) +@pytest.mark.allow_worker_restart +@pytest.mark.invariant_exempt("I1", "I3", "I4") +def test_boot_with_unreachable_remotes_still_serves_healthy(opal): + """RED until PR3 (fetch timeout) — WATCH THIS FLIP when PR3 merges. + + Boot-time cousin of the offline gate: unreachable remotes present at boot + hang the preload/first-sync clones and starve the executor, so a healthy + scope can't serve. + """ + for i in range(10): + opal.put_scope(f"down-{i}", make_repo_unreachable(f"down-{i}-repo")) + opal.put_scope("boot-healthy", gitea_repo_url(HEALTHY_PROBE_REPO)) + try: + compose("restart", "opal_server") + opal.wait_healthy(timeout=300) + assert wait_until( + lambda: opal.get_scope_policy("boot-healthy").status_code == 200, + timeout=300, + ), "healthy scope starved at boot by unreachable remotes (PR3 gate)" + finally: + opal.hard_reset() From ac3f9daed73670701df66e2651d4d6d4fb8a36ae Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 17:44:14 +0300 Subject: [PATCH 44/63] test(opal-server): clean stranded blackhole clones after the boot-down gate Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/test_boot_states.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/app-tests/git-leak/test_boot_states.py b/app-tests/git-leak/test_boot_states.py index 55e49ce30..13c167cfd 100644 --- a/app-tests/git-leak/test_boot_states.py +++ b/app-tests/git-leak/test_boot_states.py @@ -192,3 +192,15 @@ def test_boot_with_unreachable_remotes_still_serves_healthy(opal): ), "healthy scope starved at boot by unreachable remotes (PR3 gate)" finally: opal.hard_reset() + # hard_reset flushes Redis but never touches git_sources/ — the + # blackhole scopes' partial clone dirs would poison later tests' I1 + # checks. The scope store is empty post-reset, so a blanket clean is + # safe. (Same rationale as test_redis_wiped_boot_reclaims_clones.) + compose( + "exec", + "-T", + "opal_server", + "sh", + "-c", + "rm -rf /opal/git_sources/*", + ) From a2e73599743fab252c0650550c4d673598d57539 Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 17:51:56 +0300 Subject: [PATCH 45/63] test(opal-server): shard-reconfig boot gate (serves green, orphans red until sweep) Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/docker-compose.yml | 4 +++ app-tests/git-leak/test_boot_states.py | 41 ++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/app-tests/git-leak/docker-compose.yml b/app-tests/git-leak/docker-compose.yml index 66b5d7a59..3dcc2bc42 100644 --- a/app-tests/git-leak/docker-compose.yml +++ b/app-tests/git-leak/docker-compose.yml @@ -114,6 +114,10 @@ services: # (references/debug-pubsub.md §3-4) — a single worker can't tell #915's # in-place broadcaster reconnect from a plain worker respawn. UVICORN_NUM_WORKERS: "${OPAL_TEST_WORKERS:-1}" + # Shard-count knob for the reshard boot test (test_boot_states.py): + # changing SCOPES_REPO_CLONES_SHARDS between boots moves every source_id, + # orphaning all previous clone dirs — a real ops event worth gating. + OPAL_SCOPES_REPO_CLONES_SHARDS: "${OPAL_TEST_SHARDS:-1}" OPAL_SCOPES: "1" OPAL_REDIS_URL: "redis://redis:6379" OPAL_BROADCAST_URI: "postgres://opal:opal@postgres:5432/opal" diff --git a/app-tests/git-leak/test_boot_states.py b/app-tests/git-leak/test_boot_states.py index 13c167cfd..99e8548e3 100644 --- a/app-tests/git-leak/test_boot_states.py +++ b/app-tests/git-leak/test_boot_states.py @@ -204,3 +204,44 @@ def test_boot_with_unreachable_remotes_still_serves_healthy(opal): "-c", "rm -rf /opal/git_sources/*", ) + + +@pytest.mark.timeout(1200) +@pytest.mark.allow_worker_restart +@pytest.mark.invariant_exempt("I1") +def test_shard_reconfig_still_serves_but_orphans_old_clones(opal, tmp_path): + """S5: SCOPES_REPO_CLONES_SHARDS reconfig moves every source_id. + GREEN half: serving must survive the reshard (re-clone under new ids). + RED half (until the orphan sweep): the old-shard dirs are orphaned.""" + import os + + from invariants import live_source_ids + + opal.put_scope("shard-0", gitea_repo_url(list_seeded_repos(1)[0])) + assert wait_until( + lambda: opal.get_scope_policy("shard-0").status_code == 200, timeout=300 + ) + # preserve the shards=1 clones across the recreate (recreate wipes the fs) + compose("cp", "opal_server:/opal/git_sources", str(tmp_path / "saved")) + + os.environ["OPAL_TEST_SHARDS"] = "4" + try: + compose("up", "-d", "--no-deps", "--force-recreate", "opal_server") + opal.wait_healthy() + # restore the old-shard dirs next to whatever the new boot creates + compose("cp", str(tmp_path / "saved") + "/.", "opal_server:/opal/git_sources") + opal.refresh_all() + assert wait_until( + lambda: opal.get_scope_policy("shard-0").status_code == 200, + timeout=300, + ), "scope stopped serving after the shard reconfig (green half broken!)" + assert wait_until( + lambda: clone_dirs() <= live_source_ids(opal, shards=4), timeout=60 + ), ( + "old-shard clone dirs orphaned after reshard (red half — orphan " + f"sweep gate): {sorted(clone_dirs())[:5]}" + ) + finally: + os.environ["OPAL_TEST_SHARDS"] = "1" + compose("up", "-d", "--no-deps", "--force-recreate", "opal_server") + opal.wait_healthy() From 0ff543a4b99e9cb02ca4757397cca426087184fd Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 18:08:40 +0300 Subject: [PATCH 46/63] =?UTF-8?q?test(opal-server):=20multi-worker=20churn?= =?UTF-8?q?=20gate=20=E2=80=94=20every=20worker=20must=20drain=20(red=20un?= =?UTF-8?q?til=20PR3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/helpers.py | 18 ++++++++++++++ app-tests/git-leak/test_transitions.py | 34 ++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/app-tests/git-leak/helpers.py b/app-tests/git-leak/helpers.py index 36146bd28..c2e14516e 100644 --- a/app-tests/git-leak/helpers.py +++ b/app-tests/git-leak/helpers.py @@ -468,6 +468,24 @@ def wait_until(predicate, timeout: float, interval: float = 2.0) -> bool: return False +def stats_by_pid(opal, min_pids: int = 2, attempts: int = 200, interval: float = 0.05): + """Sample the stats endpoint repeatedly; keep the LATEST snapshot per pid. + + Requests land on arbitrary workers, so repeated single-sample reads + eventually observe each worker. Returns {pid: latest_stats}; the + caller decides whether len() >= min_pids is enough. + """ + seen = {} + for _ in range(attempts): + snap = opal.stats(samples=1) + seen[snap["pid"]] = snap + if len(seen) >= min_pids: + # keep going a short while so every seen pid has a FRESH snapshot + pass + time.sleep(interval) + return seen + + def list_seeded_repos(count: int) -> List[str]: return [f"policy-repo-{i:04d}" for i in range(count)] diff --git a/app-tests/git-leak/test_transitions.py b/app-tests/git-leak/test_transitions.py index 09ebe3e94..d1e57f863 100644 --- a/app-tests/git-leak/test_transitions.py +++ b/app-tests/git-leak/test_transitions.py @@ -178,3 +178,37 @@ def _old_entries_gone(): ) finally: opal.hard_reset() + + +@pytest.mark.timeout(1200) +def test_multiworker_churn_drains_every_worker(opal_multiworker, repo_count): + """RED until PR3 (broadcast purge): DELETE purges only the worker that + serves it; the leader (which fetched) keeps its caches. + + The HIGH finding from the PR2 review, as a gate. + """ + from helpers import stats_by_pid + + opal = opal_multiworker + n = min(repo_count, 10) + for i, repo in enumerate(list_seeded_repos(n)): + opal.put_scope(f"mw-{i}", gitea_repo_url(repo)) + assert wait_until( + lambda: any(s["repos"] >= 1 for s in stats_by_pid(opal, attempts=40).values()), + timeout=600, + ), "no worker ever populated its repo cache" + + for i in range(n): + opal.delete_scope(f"mw-{i}") + + def _every_worker_drained(): + snaps = stats_by_pid(opal, min_pids=2, attempts=60) + return len(snaps) >= 2 and all( + s["repo_locks"] == 0 and s["repos"] == 0 and s["repos_last_fetched"] == 0 + for s in snaps.values() + ) + + assert wait_until(_every_worker_drained, timeout=120), ( + "a worker (the leader) kept its caches after full churn (PR3 gate): " + f"{ {p: {k: v for k, v in s.items() if isinstance(v, int)} for p, s in stats_by_pid(opal).items()} }" + ) From 06a0b00ca42683d10c91519e2bbabef6de5b5b7d Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 18:16:48 +0300 Subject: [PATCH 47/63] docs(opal-server): correct the multi-worker leak mechanism in the churn gate docstring Purges are process-local; the LEADER syncs accumulate via its watcher (scopes/task.py), and any bundle-serving worker additionally caches a pygit2 handle. Only the DELETE-serving worker purges, so every accumulation on a different worker outlives the scope. Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/test_transitions.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/app-tests/git-leak/test_transitions.py b/app-tests/git-leak/test_transitions.py index d1e57f863..71ba7be61 100644 --- a/app-tests/git-leak/test_transitions.py +++ b/app-tests/git-leak/test_transitions.py @@ -182,10 +182,18 @@ def _old_entries_gone(): @pytest.mark.timeout(1200) def test_multiworker_churn_drains_every_worker(opal_multiworker, repo_count): - """RED until PR3 (broadcast purge): DELETE purges only the worker that - serves it; the leader (which fetched) keeps its caches. - - The HIGH finding from the PR2 review, as a gate. + """RED until PR3 (broadcast purge): cache purges are process-local, so any + worker whose caches were populated by something other than the DELETE it + serves leaks permanently. + + Who populates what: the LEADER accumulates handles/locks via its watcher's + syncs (scopes/task.py); ANY worker additionally caches a pygit2 handle when + it serves a policy bundle (make_bundle -> _get_current_branch_head -> + _get_repo). The purge runs only on whichever worker happens to serve the + DELETE — every accumulation on a different worker outlives the scope. In + this test only the leader accumulates (nothing GETs bundles), so the + leader's retained entries are the observable leak. The HIGH finding from + the PR2 review, as a gate. """ from helpers import stats_by_pid @@ -209,6 +217,6 @@ def _every_worker_drained(): ) assert wait_until(_every_worker_drained, timeout=120), ( - "a worker (the leader) kept its caches after full churn (PR3 gate): " + "a worker kept caches the churn's DELETEs never reached (PR3 gate): " f"{ {p: {k: v for k, v in s.items() if isinstance(v, int)} for p, s in stats_by_pid(opal).items()} }" ) From 4b7a99c9ddb9a56199910f51fe22b12bfd188f80 Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 18:19:38 +0300 Subject: [PATCH 48/63] docs(opal-server): extend git-leak gate matrix with lifecycle transition tests Adds 14 matrix rows for the new test_transitions.py/test_boot_states.py/test_remote_transitions.py tests and an Invariants section documenting I1-I6 and the invariant_exempt/allow_worker_restart markers. Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/README.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/app-tests/git-leak/README.md b/app-tests/git-leak/README.md index a569975db..a2bab848c 100644 --- a/app-tests/git-leak/README.md +++ b/app-tests/git-leak/README.md @@ -51,6 +51,33 @@ Gate-coverage matrix (what each flagship test actually does): | `test_boot_loads_all_scopes` | **baseline → gate (PR4)** | PASSES with the loose default target; set `BOOT_TARGET_SECONDS` low (plan: 120 @ 50) on PR4 to gate the parallel-boot fix | | `test_repeat_sync_rss_stays_bounded` | **RSS guard** | PASSES; an RSS-budget guard against per-sync allocation leaks (the cache *count* can't grow for any impl, so there is no count assertion — see below) | | `test_server_recovers_after_postgres_bounce` | **guard (PER-15065 + gap publishes)** | PASSES on this branch (which has #915); guards the in-place broadcaster reconnect and that a scope PUT *during* the outage is buffered/replayed, not dropped | +| `test_delete_recreate_storm` | **guard (lock re-mint)** | PASSES — rapid delete/re-create of the same source serializes on the repo lock and ends with clean caches; guards 89e090be | +| `test_randomized_churn_holds_invariants` | **guard (seeded churn)** | PASSES — seeded random put/refresh/settled-delete churn holds invariants at every settle point (replay a failure with `CHURN_SEED=`); repoints and delete-vs-inflight-sync races are deliberately excluded, both lifted when PR3's fleet purge lands | +| `test_delete_during_hung_fetch_no_crash` | **guard (use-after-free)** | PASSES — deleting a scope whose clone is hung never crashes a worker; guards the use-after-free class 89e090be fixed | +| `test_delete_during_hung_fetch_returns_bounded` | **gate (PR3)** | FAILS without the PR3 fetch timeout — the purge waits on the repo lock and a hung clone holds that lock indefinitely, so the DELETE never returns in bounded time | +| `test_repoint_during_inflight_fetch_drains_old_source` | **gate (PR3, update path)** | Green half passes today — repointing while the old source's clone is hung still serves the new source; red half FAILS without PR3's update-path purge — the old source's cache entries never drain | +| `test_multiworker_churn_drains_every_worker` | **gate (PR3, broadcast)** | FAILS without PR3's broadcast purge — cache purges are process-local, so a worker whose caches were populated by something other than the DELETE it served (e.g. the leader's watcher syncs) leaks permanently; the HIGH finding from the PR2 review, as a gate | +| `test_warm_boot_reuses_clones` | **guard (S2)** | PASSES — a restart with intact clones must serve without re-cloning | +| `test_corrupt_clone_recovers_without_clone_loop` | **guard (S3/T7)** | PASSES — emptying a clone's object store in place while the server holds a warm cached handle is detected as invalid and recovers through the invalid-repo branch with exactly one re-clone, no serve-500s wedge and no re-clone loop; verifies the gutted-object-store detection fix | +| `test_orphan_clone_dir_is_reclaimed` | **gate (orphan sweep, unowned)** | FAILS — a clone dir with no live scope is never reclaimed; no orphan sweep exists yet (PR3+, currently unowned) | +| `test_redis_wiped_boot_reclaims_clones` | **gate (orphan sweep, unowned)** | FAILS — after a scope-store wipe, on-disk clones referencing nothing are never reclaimed; same missing-sweep class as the orphan-dir gate | +| `test_boot_with_unreachable_remotes_still_serves_healthy` | **gate (PR3) — watch this flip** | FAILS without the PR3 fetch timeout — unreachable remotes present at boot hang the preload/first-sync clones and starve the executor, so a healthy scope can't serve; boot-time cousin of the offline gate | +| `test_shard_reconfig_still_serves_but_orphans_old_clones` | **half-gate (S5, orphan sweep)** | Green half PASSES — serving survives a `SCOPES_REPO_CLONES_SHARDS` reconfig (re-clone under new ids); red half FAILS — the old-shard dirs are orphaned until the orphan sweep lands | +| `test_force_push_rewrite_recovers` | **characterization** | PASSES — a force-pushed (rewritten) head is picked up on refresh, pinning today's behavior (pygit2's forced default fetch refspec plus `set_target` moving the local ref) | +| `test_deleted_branch_keeps_serving_last_head` | **characterization** | PASSES — deleting the tracked branch upstream doesn't crash anything; fetch doesn't prune, so OPAL silently keeps serving the last known head (documented, not necessarily desirable, behavior) | + +## Invariants +`invariants.py` defines quiescence invariants I1-I6, checked at every `opal`-fixture +teardown (I1-I4 and I6 via `check_invariants`; I5 lives directly in `conftest.py`, +since it needs the worker pids captured at fixture setup): I1 no orphan clone dir on +disk, I2 no cached repo handle with no dir on disk, I3 no `repos_last_fetched` key +with no live scope, I4 no `repo_locks` key with no live scope, I5 the worker pid set +is unchanged across the test (proving no crash/respawn), I6 the scopes API still +answers 200. Two markers let a test opt out deliberately instead of by accident: +`@pytest.mark.invariant_exempt("I1", ...)` for red-gate tests that knowingly leave a +violation behind (named per test, not a blanket skip), and +`@pytest.mark.allow_worker_restart` for tests that restart `opal_server` on purpose, +which would otherwise trip I5. Notes on the guards: - `test_repeat_sync_rss_stays_bounded` — clone paths are keyed by the repo URL, From bf53477ad8479c0b8e5e3d1a7e61bb734a39bfdc Mon Sep 17 00:00:00 2001 From: zivxx Date: Tue, 14 Jul 2026 18:48:16 +0300 Subject: [PATCH 49/63] test(opal-server): make stats_by_pid min_pids an actual early-exit Once min_pids distinct pids are observed, take one grace sample to ensure freshness, then break. Co-Authored-By: Claude Fable 5 --- app-tests/git-leak/helpers.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/app-tests/git-leak/helpers.py b/app-tests/git-leak/helpers.py index c2e14516e..31972b69f 100644 --- a/app-tests/git-leak/helpers.py +++ b/app-tests/git-leak/helpers.py @@ -472,16 +472,19 @@ def stats_by_pid(opal, min_pids: int = 2, attempts: int = 200, interval: float = """Sample the stats endpoint repeatedly; keep the LATEST snapshot per pid. Requests land on arbitrary workers, so repeated single-sample reads - eventually observe each worker. Returns {pid: latest_stats}; the - caller decides whether len() >= min_pids is enough. + eventually observe each worker. Returns {pid: latest_stats} once min_pids + distinct pids are seen (plus one grace sample for freshness), or after + `attempts` samples; the caller decides whether the count is sufficient. """ seen = {} + grace_sweep_done = False for _ in range(attempts): snap = opal.stats(samples=1) seen[snap["pid"]] = snap if len(seen) >= min_pids: - # keep going a short while so every seen pid has a FRESH snapshot - pass + if grace_sweep_done: + break + grace_sweep_done = True time.sleep(interval) return seen From 4826a7d07c0f6cfb66af29dc757da55b855b4b90 Mon Sep 17 00:00:00 2001 From: zivxx Date: Wed, 15 Jul 2026 16:15:41 +0300 Subject: [PATCH 50/63] fix(opal-server): stop make_bundle reading the shared pygit2 handle off-lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _get_current_branch_head() (called from make_bundle, which is invoked via run_sync(fetcher.make_bundle, ...) on executor threads by the policy-bundle route, and directly on the loop by _generate_default_scope_bundle) was calling self._get_repo(), which reads/populates the shared class-level repos cache. Both call sites run outside lock_source, so the cached handle can be concurrently .free()'d by forget_repo() during a scope delete or invalid-repo recovery — a native use-after-free on the pygit2 handle. asyncio locks don't exclude executor threads, so the lock around fetch_and_notify_on_changes does not protect this path. Fixed by opening a short-lived fresh Repository handle per call (freed in a finally), the same fresh-probe pattern _get_valid_repo already uses for its disk-truth check. All remaining shared cached-handle access now happens under lock_source. Co-Authored-By: Claude Fable 5 --- .../opal-server/opal_server/git_fetcher.py | 20 ++++++++++++---- .../tests/invalid_repo_recovery_test.py | 23 +++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/packages/opal-server/opal_server/git_fetcher.py b/packages/opal-server/opal_server/git_fetcher.py index 188a9d054..f9e804404 100644 --- a/packages/opal-server/opal_server/git_fetcher.py +++ b/packages/opal-server/opal_server/git_fetcher.py @@ -385,10 +385,22 @@ async def _notify_on_changes(self, repo: Repository): local_branch.set_target(new_revision) def _get_current_branch_head(self) -> str: - repo = self._get_repo() - head_commit_hash = RepoInterface.get_commit_hash( - repo, self._source.branch, self._remote - ) + # Opened fresh per call instead of using the shared cached handle: + # this runs on executor threads (run_sync(make_bundle) in the policy- + # bundle route) and outside lock_source, where the cached handle can + # be free()'d concurrently by a scope delete or invalid-repo recovery. + # asyncio locks don't exclude executor threads — sharing the handle + # here is a use-after-free. Same fresh-probe pattern as + # _get_valid_repo's disk-truth check. + repo = Repository(str(self._repo_path)) + try: + head_commit_hash = RepoInterface.get_commit_hash( + repo, self._source.branch, self._remote + ) + finally: + free = getattr(repo, "free", None) + if callable(free): + free() if not head_commit_hash: logger.error("Could not find current branch head") raise ValueError("Could not find current branch head") diff --git a/packages/opal-server/opal_server/tests/invalid_repo_recovery_test.py b/packages/opal-server/opal_server/tests/invalid_repo_recovery_test.py index 7c160ca4c..5663435a1 100644 --- a/packages/opal-server/opal_server/tests/invalid_repo_recovery_test.py +++ b/packages/opal-server/opal_server/tests/invalid_repo_recovery_test.py @@ -196,3 +196,26 @@ async def fake_clone(): assert path not in GitPolicyFetcher.repos assert warm.freed is True, "recovery evicted the warm handle without free()" assert probes and all(p.freed for p in probes), "disk probe handle leaked" + + +class _PoisonRepo: + """Any attribute access means the shared cached handle was touched.""" + + def __getattr__(self, name): + raise AssertionError( + "shared cached handle was read outside lock_source (UAF hazard)" + ) + + +def test_branch_head_does_not_touch_shared_handle(monkeypatch, tmp_path): + fetcher = _make_fetcher(tmp_path, "s", "https://example.com/r.git") + path = str(fetcher._repo_path) + GitPolicyFetcher.repos[path] = _PoisonRepo() + fresh = object() + monkeypatch.setattr("opal_server.git_fetcher.Repository", lambda p: fresh) + monkeypatch.setattr( + "opal_server.git_fetcher.RepoInterface.get_commit_hash", + lambda repo, branch, remote: "abc123" if repo is fresh else None, + ) + + assert fetcher._get_current_branch_head() == "abc123" From bd7d007ca61425b87043e701fe4ad7d7ff7bdb6b Mon Sep 17 00:00:00 2001 From: zivxx Date: Wed, 15 Jul 2026 16:17:19 +0300 Subject: [PATCH 51/63] fix(opal-server): serialize sibling-check and purge on the source lock (delete TOCTOU) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit delete_scope's sibling-scope check ran before lock_source and before the record delete. Two concurrent deletes of scopes sharing a source_id could each see the other still present in the scope store and both skip the purge, permanently orphaning the clone dir, pygit2 handle cache, and repos_last_fetched entry. Restructured so the record delete, sibling re-check, and purge all happen inside a single lock_source(deleted_source_id) critical section, with the record delete first — so whichever of the two concurrent deletes finishes last sees no remaining sharer and performs the purge. Also switched the rmtree call to run_sync so it no longer blocks the event loop. Co-Authored-By: Claude Fable 5 --- .../opal-server/opal_server/scopes/service.py | 89 ++++++++++--------- .../tests/delete_scope_cache_purge_test.py | 31 +++++++ 2 files changed, 77 insertions(+), 43 deletions(-) diff --git a/packages/opal-server/opal_server/scopes/service.py b/packages/opal-server/opal_server/scopes/service.py index 01888486a..35a32ecdd 100644 --- a/packages/opal-server/opal_server/scopes/service.py +++ b/packages/opal-server/opal_server/scopes/service.py @@ -7,6 +7,7 @@ import git from ddtrace import tracer from fastapi_websocket_pubsub import PubSubEndpoint +from opal_common.async_utils import run_sync from opal_common.git_utils.commit_viewer import VersionedFile from opal_common.http_utils import redact_url from opal_common.logger import logger @@ -185,50 +186,52 @@ async def delete_scope(self, scope_id: str): # different clone dir) when SCOPES_REPO_CLONES_SHARDS > 1, so gate on # source_id, not url — otherwise the deleted scope's clone + pygit2 # handle leak. - other_scopes = [ - s for s in await self._scopes.all() if s.scope_id != scope_id - ] - sharing_scope_id = next( - ( - s.scope_id - for s in other_scopes - if isinstance(s.policy, GitPolicyScopeSource) - and GitPolicyFetcher.source_id(s.policy) == deleted_source_id - ), - None, - ) - - if sharing_scope_id is not None: - logger.info( - f"Scope {sharing_scope_id} shares the same clone (source id), " - "skipping clone deletion" + # + # Serialize record-delete + sibling-check + purge on the source + # lock: two concurrent sibling deletes could otherwise both read + # the other as still-live (the check and the record delete span + # separate store round-trips) and BOTH skip the purge, orphaning + # the clone and cache entries permanently. Deleting our record + # before re-checking makes the last deleter see no sharer. + async with GitPolicyFetcher.lock_source(deleted_source_id): + await self._scopes.delete(scope_id) + sharing_scope_id = next( + ( + s.scope_id + for s in await self._scopes.all() + if s.scope_id != scope_id + and isinstance(s.policy, GitPolicyScopeSource) + and GitPolicyFetcher.source_id(s.policy) == deleted_source_id + ), + None, ) - else: - # NOTE: this purge is process-local best-effort — it cleans the - # caches of whichever worker serves the DELETE. Broadcasting the - # purge so the leader (which accumulates most handles via - # continuous sync) also drops its caches is tracked for PR3. - async with GitPolicyFetcher.lock_source(deleted_source_id): - try: - shutil.rmtree(scope_dir) - except FileNotFoundError: - pass # never cloned (or already gone) — nothing to clean - except OSError as e: - # Deliberately not fatal: the scope record must still be - # deleted, but an orphaned clone on disk has to be - # discoverable rather than silently leaked. - logger.warning( - f"Failed to remove clone dir {scope_dir} of deleted " - f"scope {scope_id}: {e!r}" - ) - GitPolicyFetcher.forget_repo(str(scope_dir)) - GitPolicyFetcher.repos_last_fetched.pop(deleted_source_id, None) - # Popped while the lock is held: lock_source waiters re-check - # the dict entry after acquiring and retry on the fresh lock, - # so nobody proceeds under the stale one. - GitPolicyFetcher.repo_locks.pop(deleted_source_id, None) - - await self._scopes.delete(scope_id) + if sharing_scope_id is not None: + logger.info( + f"Scope {sharing_scope_id} shares the same clone " + "(source id), skipping clone deletion" + ) + return + # NOTE (PR3): delete must ultimately route through the leader + # like put/refresh — only the leader should mutate the shared + # clone tree. Today this purge is process-local best-effort + # (the leader's caches leak until PR3's broadcast purge), and + # a non-leader DELETE rmtree's the shared tree unserialized + # against the leader's in-flight fetches (cross-process; + # bounded and self-healing, but an invariant break). + try: + await run_sync(shutil.rmtree, str(scope_dir)) + except FileNotFoundError: + pass # never cloned (or already gone) — nothing to clean + except OSError as e: + logger.warning( + f"Failed to remove clone dir {scope_dir} of deleted " + f"scope {scope_id}: {e!r}" + ) + GitPolicyFetcher.forget_repo(str(scope_dir)) + GitPolicyFetcher.repos_last_fetched.pop(deleted_source_id, None) + # Popped while the lock is held: lock_source waiters re-check + # the dict entry after acquiring and retry on the fresh lock. + GitPolicyFetcher.repo_locks.pop(deleted_source_id, None) async def sync_scopes(self, only_poll_updates=False, notify_on_changes=True): with tracer.trace("scopes_service.sync_scopes"): diff --git a/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py b/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py index de952ebb7..5e7e19e8a 100644 --- a/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py +++ b/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py @@ -13,14 +13,17 @@ def __init__(self, scopes): self._scopes = {s.scope_id: s for s in scopes} async def get(self, scope_id): + await asyncio.sleep(0) if scope_id not in self._scopes: raise ScopeNotFoundError(scope_id) return self._scopes[scope_id] async def all(self): + await asyncio.sleep(0) return list(self._scopes.values()) async def delete(self, scope_id): + await asyncio.sleep(0) self._scopes.pop(scope_id, None) @@ -99,6 +102,34 @@ async def test_delete_keeps_caches_when_sibling_shares_source(tmp_path, monkeypa assert sid in GitPolicyFetcher.repo_locks +@pytest.mark.asyncio +async def test_concurrent_sibling_deletes_still_purge(tmp_path, monkeypatch): + """Two concurrent DELETEs of source-sharing scopes must not BOTH skip the + purge (TOCTOU on the sibling check) — whichever finishes last must + purge.""" + a = _scope("a", "https://git/shared.git") + b = _scope("b", "https://git/shared.git") + repo = FakeScopeRepository([a, b]) + svc = ScopesService(base_dir=tmp_path, scopes=repo, pubsub_endpoint=None) + sid = GitPolicyFetcher.source_id(a.policy) + clone_path = str(GitPolicyFetcher.repo_clone_path(tmp_path, a.policy)) + GitPolicyFetcher.repos[clone_path] = object() + GitPolicyFetcher.repos_last_fetched[sid] = "ts" + rmtree_calls = [] + monkeypatch.setattr( + "opal_server.scopes.service.shutil.rmtree", + lambda p, **k: rmtree_calls.append(str(p)), + ) + + await asyncio.gather(svc.delete_scope("a"), svc.delete_scope("b")) + + assert rmtree_calls == [ + clone_path + ], "concurrent sibling deletes both skipped the purge (TOCTOU)" + assert clone_path not in GitPolicyFetcher.repos + assert sid not in GitPolicyFetcher.repos_last_fetched + + @pytest.mark.asyncio async def test_delete_purges_when_sibling_shares_url_but_not_source( tmp_path, monkeypatch From 0093be6a030a7ced84fa22b80b07b824f20adfc0 Mon Sep 17 00:00:00 2001 From: zivxx Date: Wed, 15 Jul 2026 16:18:19 +0300 Subject: [PATCH 52/63] fix(opal-server): log rmtree failures in recovery and clone pre-clean The invalid-repo recovery rmtree and _clone's partial-dir pre-clean both used ignore_errors=True, silently swallowing any OSError beyond a simple missing directory (e.g. a permissions problem or a busy mount) with no trace in the logs. Replaced both with the same try/except pattern the delete-scope purge path already uses: tolerate FileNotFoundError (already gone is the intended end state) but log other OSErrors as a warning so a persistent failure is discoverable instead of silently retried forever. Co-Authored-By: Claude Fable 5 --- .../opal-server/opal_server/git_fetcher.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/opal-server/opal_server/git_fetcher.py b/packages/opal-server/opal_server/git_fetcher.py index f9e804404..c79d0bb54 100644 --- a/packages/opal-server/opal_server/git_fetcher.py +++ b/packages/opal-server/opal_server/git_fetcher.py @@ -230,9 +230,15 @@ async def fetch_and_notify_on_changes( "Deleting invalid repo: {path}", path=self._repo_path ) GitPolicyFetcher.forget_repo(str(self._repo_path)) - # ignore_errors: with a stale handle the dir may already - # be partially or fully gone. - shutil.rmtree(self._repo_path, ignore_errors=True) + try: + shutil.rmtree(self._repo_path) + except FileNotFoundError: + pass # already gone — the intended end state + except OSError as e: + logger.warning( + f"Failed to remove clone dir " + f"{self._repo_path}: {e!r}" + ) else: logger.info("Repo not found at {path}", path=self._repo_path) @@ -248,7 +254,12 @@ async def _clone(self): # A failed/interrupted clone leaves a partial dir; # clone_repository refuses a non-empty destination, which would # wedge every retry for this source. - shutil.rmtree(self._repo_path, ignore_errors=True) + try: + shutil.rmtree(self._repo_path) + except FileNotFoundError: + pass # already gone — the intended end state + except OSError as e: + logger.warning(f"Failed to remove clone dir {self._repo_path}: {e!r}") logger.info( "Cloning repo at '{url}' to '{path}'", url=redact_url(self._source.url), From 2596cd4283c6e075fa7655890e5a2ef1d834324b Mon Sep 17 00:00:00 2001 From: zivxx Date: Wed, 15 Jul 2026 16:19:12 +0300 Subject: [PATCH 53/63] fix(opal-server): run the clone-validation probe off-loop and stamp recloned repos _get_valid_repo() opens a fresh Repository handle from disk and walks refs to detect a gutted object store; called synchronously inline in fetch_and_notify_on_changes, a slow disk stalls the event loop and every other request being served on the worker. Route it through run_sync so it runs on the executor. Also stamp repos_last_fetched on a successful clone: a reclone just downloaded current remote state, so without the stamp _was_fetched_after() sees no record and _should_fetch() forces a redundant fetch on the very next sync cycle. Co-Authored-By: Claude Fable 5 --- packages/opal-server/opal_server/git_fetcher.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/opal-server/opal_server/git_fetcher.py b/packages/opal-server/opal_server/git_fetcher.py index c79d0bb54..4c7dd3705 100644 --- a/packages/opal-server/opal_server/git_fetcher.py +++ b/packages/opal-server/opal_server/git_fetcher.py @@ -187,7 +187,10 @@ async def fetch_and_notify_on_changes( ): if self._discover_repository(self._repo_path): logger.debug("Repo found at {path}", path=self._repo_path) - repo = self._get_valid_repo() + # The probe opens/parses a fresh Repository handle from + # disk — off the event loop so a slow disk can't stall + # every other request being served on this worker. + repo = await run_sync(self._get_valid_repo) if repo is not None: should_fetch = await self._should_fetch( repo, @@ -279,6 +282,11 @@ async def _clone(self): # Cache the fresh handle so the next sync's _get_repo() reuses it # instead of reopening (or hitting a stale predecessor). GitPolicyFetcher.repos[str(self._repo_path)] = repo + # A reclone just downloaded current remote state — record it so + # _was_fetched_after() doesn't force a redundant fetch next cycle. + GitPolicyFetcher.repos_last_fetched[ + self._source_id + ] = datetime.datetime.now() await self._notify_on_changes(repo) def _get_repo(self) -> Repository: From 152eeee9baac71e5137c01474e1b82645b509476 Mon Sep 17 00:00:00 2001 From: zivxx Date: Wed, 15 Jul 2026 16:19:52 +0300 Subject: [PATCH 54/63] fix(opal-server): redact credentialed URLs in webhook task failure logs Webhook trigger failures were logged with repr(exception) verbatim. Git exceptions from a failed fetch/clone can embed a credentialed remote URL (https://user:token@host/repo) directly in the exception message, so a webhook failure would leak the credential into the server logs. Strip any `://user:pass@`-shaped segment before logging. Added a unit test covering a trigger failure whose message embeds a credentialed URL: the logged ERROR must contain the redacted form and must not contain the raw credential. Co-Authored-By: Claude Fable 5 --- .../opal_server/policy/watcher/task.py | 9 +++++- .../tests/webhook_task_cleanup_test.py | 31 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/packages/opal-server/opal_server/policy/watcher/task.py b/packages/opal-server/opal_server/policy/watcher/task.py index 0e8cd0afd..8a6bbfcfa 100644 --- a/packages/opal-server/opal_server/policy/watcher/task.py +++ b/packages/opal-server/opal_server/policy/watcher/task.py @@ -1,5 +1,6 @@ import asyncio import os +import re import signal from typing import Any, Coroutine, List, Optional @@ -9,6 +10,11 @@ from opal_common.sources.base_policy_source import BasePolicySource from opal_server.config import opal_server_config +# git exceptions (e.g. from a failed fetch/clone) can embed a credentialed +# remote URL (https://user:token@host/repo) verbatim in their message — +# strip the credential before this ever reaches the logs. +_CREDENTIALED_URL_RE = re.compile(r"://[^/@\s]+@") + class BasePolicyWatcherTask: """Manages the asyncio tasks of the policy watcher.""" @@ -35,7 +41,8 @@ async def _on_webhook(self, topic: Topic, data: Any): # generic "Task exception was never retrieved" at GC time. for t in self._webhook_tasks: if t.done() and not t.cancelled() and t.exception() is not None: - logger.error(f"Webhook trigger task failed: {t.exception()!r}") + exc_text = _CREDENTIALED_URL_RE.sub("://***@", repr(t.exception())) + logger.error(f"Webhook trigger task failed: {exc_text}") self._webhook_tasks = [t for t in self._webhook_tasks if not t.done()] self._webhook_tasks.append(asyncio.create_task(self.trigger(topic, data))) diff --git a/packages/opal-server/opal_server/tests/webhook_task_cleanup_test.py b/packages/opal-server/opal_server/tests/webhook_task_cleanup_test.py index 1261bfcd0..12d5284db 100644 --- a/packages/opal-server/opal_server/tests/webhook_task_cleanup_test.py +++ b/packages/opal-server/opal_server/tests/webhook_task_cleanup_test.py @@ -68,6 +68,37 @@ async def test_failed_trigger_exception_is_retrieved_and_logged(): await asyncio.gather(*w._webhook_tasks, return_exceptions=True) +class _CredentialLeakingWatcher(BasePolicyWatcherTask): + async def trigger(self, topic, data): + raise RuntimeError("fetch https://user:secret@host/repo failed") + + +@pytest.mark.asyncio +async def test_failed_trigger_log_redacts_credentialed_url(): + """A git exception can embed a credentialed remote URL verbatim; the + sweep's failure log must not leak it.""" + from opal_common.logger import logger as opal_logger + + w = _CredentialLeakingWatcher(pubsub_endpoint=None) + + await w._on_webhook("webhook", None) # schedules a trigger that raises + failed = list(w._webhook_tasks) + await asyncio.gather(*failed, return_exceptions=True) + + records = [] + sink_id = opal_logger.add(lambda m: records.append(str(m)), level="ERROR") + try: + await w._on_webhook("webhook", None) # sweep must log the failure + finally: + opal_logger.remove(sink_id) + + assert any("Webhook trigger task failed" in r for r in records), records + assert any("://***@" in r for r in records), records + assert not any("secret" in r for r in records), records + + await asyncio.gather(*w._webhook_tasks, return_exceptions=True) + + class _HangingWatcher(BasePolicyWatcherTask): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) From aed67f7b842b045483ae3d88e3346dc5472ac1ac Mon Sep 17 00:00:00 2001 From: zivxx Date: Wed, 15 Jul 2026 16:21:04 +0300 Subject: [PATCH 55/63] test(opal-server): cover _get_valid_repo healthy paths and probe lifecycle _get_valid_repo's failure paths (stale cached handle, gutted object store) were already covered, but its two healthy paths were not: (a) the tracked branch hasn't been fetched yet (probe.lookup_reference raises KeyError, tolerated as "fetch path handles that"), and (b) the probe confirms the head object is actually readable from disk. Both must return the cached repo unchanged and free() the short-lived disk probe. Co-Authored-By: Claude Fable 5 --- .../tests/invalid_repo_recovery_test.py | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/packages/opal-server/opal_server/tests/invalid_repo_recovery_test.py b/packages/opal-server/opal_server/tests/invalid_repo_recovery_test.py index 5663435a1..2e4c5b346 100644 --- a/packages/opal-server/opal_server/tests/invalid_repo_recovery_test.py +++ b/packages/opal-server/opal_server/tests/invalid_repo_recovery_test.py @@ -219,3 +219,92 @@ def test_branch_head_does_not_touch_shared_handle(monkeypatch, tmp_path): ) assert fetcher._get_current_branch_head() == "abc123" + + +class _HealthyCachedRepo: + """The warm cached handle _get_valid_repo returns when the disk probe + confirms the repo is healthy.""" + + def __init__(self, url): + self.remotes = [_RemoteStub(url)] + self.freed = False + + def free(self): + self.freed = True + + +class _BranchNotFetchedProbe: + """Fresh on-disk probe when the tracked branch has never been fetched (e.g. + right after a scope's remote branch config changed).""" + + def __init__(self, path): + self.freed = False + + def lookup_reference(self, name): + raise KeyError(name) + + def free(self): + self.freed = True + + +def test_get_valid_repo_tolerates_branch_not_yet_fetched(monkeypatch, tmp_path): + """The fetch path is responsible for missing branches -- the probe must not + treat that as corruption and must still return the cached repo.""" + url = "https://example.com/r.git" + fetcher = _make_fetcher(tmp_path, "s", url) + path = str(fetcher._repo_path) + cached = _HealthyCachedRepo(url) + GitPolicyFetcher.repos[path] = cached + probes = [] + + def fake_repository(p): + probe = _BranchNotFetchedProbe(p) + probes.append(probe) + return probe + + monkeypatch.setattr("opal_server.git_fetcher.Repository", fake_repository) + + result = fetcher._get_valid_repo() + + assert result is cached + assert probes and all(p.freed for p in probes), "disk probe handle leaked" + + +class _HealthyProbe: + """Fresh on-disk probe when the head object is actually readable from disk + (the healthy case).""" + + def __init__(self, path): + self.freed = False + + def lookup_reference(self, name): + return _Ref() + + def get(self, oid): + return object() # readable from disk + + def free(self): + self.freed = True + + +def test_get_valid_repo_returns_cached_when_probe_confirms_healthy( + monkeypatch, tmp_path +): + url = "https://example.com/r.git" + fetcher = _make_fetcher(tmp_path, "s", url) + path = str(fetcher._repo_path) + cached = _HealthyCachedRepo(url) + GitPolicyFetcher.repos[path] = cached + probes = [] + + def fake_repository(p): + probe = _HealthyProbe(p) + probes.append(probe) + return probe + + monkeypatch.setattr("opal_server.git_fetcher.Repository", fake_repository) + + result = fetcher._get_valid_repo() + + assert result is cached + assert probes and all(p.freed for p in probes), "disk probe handle leaked" From b796c3857216bba9bb51aec7bc366aea1a9ce74a Mon Sep 17 00:00:00 2001 From: zivxx Date: Wed, 15 Jul 2026 16:37:14 +0300 Subject: [PATCH 56/63] fix(opal-server): purge defensively when the post-delete sibling check fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit delete_scope's sibling re-check runs after the record delete; its all() does a full store scan + Scope.parse_raw, so a transient store error or one malformed record raises. That exception escaped the source lock with NOTHING purged, and because the record is already deleted the client's retry is a 204 no-op (ScopeNotFoundError) — the purge became permanently unreachable, orphaning the clone dir and both fetcher caches. The re-check failure is now downgraded to a warning and treated as "no sharer" (purge). Over-purging self-heals — a surviving sibling re-clones on its next sync — while under-purging is a permanent leak with no retry path. The record-delete itself still propagates failures normally. Co-Authored-By: Claude Fable 5 --- .../opal-server/opal_server/scopes/service.py | 35 ++++++++--- .../tests/delete_scope_cache_purge_test.py | 62 +++++++++++++++++++ 2 files changed, 87 insertions(+), 10 deletions(-) diff --git a/packages/opal-server/opal_server/scopes/service.py b/packages/opal-server/opal_server/scopes/service.py index 35a32ecdd..79141de50 100644 --- a/packages/opal-server/opal_server/scopes/service.py +++ b/packages/opal-server/opal_server/scopes/service.py @@ -195,16 +195,31 @@ async def delete_scope(self, scope_id: str): # before re-checking makes the last deleter see no sharer. async with GitPolicyFetcher.lock_source(deleted_source_id): await self._scopes.delete(scope_id) - sharing_scope_id = next( - ( - s.scope_id - for s in await self._scopes.all() - if s.scope_id != scope_id - and isinstance(s.policy, GitPolicyScopeSource) - and GitPolicyFetcher.source_id(s.policy) == deleted_source_id - ), - None, - ) + # The re-check must not be able to skip the purge by raising: + # our record is already deleted, so a client retry is a 204 + # no-op (ScopeNotFoundError) and the purge becomes permanently + # unreachable. all() does a full scan + Scope.parse_raw — a + # transient store error or one malformed record throws. Over- + # purging self-heals (a surviving sibling re-clones on its + # next sync); under-purging is a permanent leak. + try: + sharing_scope_id = next( + ( + s.scope_id + for s in await self._scopes.all() + if s.scope_id != scope_id + and isinstance(s.policy, GitPolicyScopeSource) + and GitPolicyFetcher.source_id(s.policy) + == deleted_source_id + ), + None, + ) + except Exception as e: + logger.warning( + f"sibling check failed after deleting scope " + f"{scope_id}; purging defensively: {e!r}" + ) + sharing_scope_id = None if sharing_scope_id is not None: logger.info( f"Scope {sharing_scope_id} shares the same clone " diff --git a/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py b/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py index 5e7e19e8a..c5e8aeb80 100644 --- a/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py +++ b/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py @@ -130,6 +130,68 @@ async def test_concurrent_sibling_deletes_still_purge(tmp_path, monkeypatch): assert sid not in GitPolicyFetcher.repos_last_fetched +class _AllRaisesAfterDeleteRepository(FakeScopeRepository): + """All() blows up only on the post-delete sibling re-check (e.g. a + transient store error or one malformed record failing parse_raw).""" + + def __init__(self, scopes): + super().__init__(scopes) + self.deleted_once = False + + async def delete(self, scope_id): + await super().delete(scope_id) + self.deleted_once = True + + async def all(self): + if self.deleted_once: + raise RuntimeError("store scan failed (malformed record)") + return await super().all() + + +@pytest.mark.asyncio +async def test_sibling_check_failure_still_purges(tmp_path, monkeypatch): + """If the post-delete sibling re-check raises, the purge must still run: + + the record is already deleted, so the client's retry is a 204 no-op + (ScopeNotFoundError) and a skipped purge is a permanent leak. Over- + purging is safe — a surviving sibling re-clones on its next sync. + """ + scope = _scope("only", "https://git/repo-a.git") + repo = _AllRaisesAfterDeleteRepository([scope]) + svc = ScopesService(base_dir=tmp_path, scopes=repo, pubsub_endpoint=None) + + sid = GitPolicyFetcher.source_id(scope.policy) + clone_path = str(GitPolicyFetcher.repo_clone_path(tmp_path, scope.policy)) + GitPolicyFetcher.repos[clone_path] = object() + GitPolicyFetcher.repos_last_fetched[sid] = "ts" + + rmtree_calls = [] + monkeypatch.setattr( + "opal_server.scopes.service.shutil.rmtree", + lambda p, **k: rmtree_calls.append(str(p)), + ) + + from opal_common.logger import logger as opal_logger + + records = [] + sink_id = opal_logger.add(lambda m: records.append(str(m)), level="WARNING") + try: + # Must not raise: the re-check failure is downgraded to a warning. + await svc.delete_scope("only") + finally: + opal_logger.remove(sink_id) + + assert rmtree_calls == [clone_path], ( + "sibling re-check failure skipped the purge — permanent leak " + "(retry is a 204 no-op, the purge is unreachable)" + ) + assert clone_path not in GitPolicyFetcher.repos + assert sid not in GitPolicyFetcher.repos_last_fetched + assert any( + "sibling check failed" in r and "purging defensively" in r for r in records + ), f"defensive purge not logged: {records}" + + @pytest.mark.asyncio async def test_delete_purges_when_sibling_shares_url_but_not_source( tmp_path, monkeypatch From d8d9f6d013d9ec6ba9c61868e7b15b1939c4d1df Mon Sep 17 00:00:00 2001 From: zivxx Date: Wed, 15 Jul 2026 16:38:36 +0300 Subject: [PATCH 57/63] fix(opal-server): offload recovery rmtrees and widen the PR3 delete-routing note MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The invalid-repo recovery rmtree and _clone's partial-dir pre-clean ran synchronously on the event loop while this same PR offloads the delete-path rmtree — inconsistent, and a large clone dir on a slow disk stalls every request on the worker. Both now go through run_sync inside their existing try/excepts. Also widened delete_scope's PR3 NOTE: besides the cross-process leader gap, the same class exists in-process — a sync that loaded the scope before the delete and acquires the fresh lock after the purge re-clones and re-populates the caches for the dead scope. PR3's purge routing must include a scope-liveness check before clone. Co-Authored-By: Claude Fable 5 --- packages/opal-server/opal_server/git_fetcher.py | 4 ++-- packages/opal-server/opal_server/scopes/service.py | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/opal-server/opal_server/git_fetcher.py b/packages/opal-server/opal_server/git_fetcher.py index 4c7dd3705..89100ac98 100644 --- a/packages/opal-server/opal_server/git_fetcher.py +++ b/packages/opal-server/opal_server/git_fetcher.py @@ -234,7 +234,7 @@ async def fetch_and_notify_on_changes( ) GitPolicyFetcher.forget_repo(str(self._repo_path)) try: - shutil.rmtree(self._repo_path) + await run_sync(shutil.rmtree, str(self._repo_path)) except FileNotFoundError: pass # already gone — the intended end state except OSError as e: @@ -258,7 +258,7 @@ async def _clone(self): # clone_repository refuses a non-empty destination, which would # wedge every retry for this source. try: - shutil.rmtree(self._repo_path) + await run_sync(shutil.rmtree, str(self._repo_path)) except FileNotFoundError: pass # already gone — the intended end state except OSError as e: diff --git a/packages/opal-server/opal_server/scopes/service.py b/packages/opal-server/opal_server/scopes/service.py index 79141de50..645e8dce0 100644 --- a/packages/opal-server/opal_server/scopes/service.py +++ b/packages/opal-server/opal_server/scopes/service.py @@ -233,6 +233,12 @@ async def delete_scope(self, scope_id: str): # a non-leader DELETE rmtree's the shared tree unserialized # against the leader's in-flight fetches (cross-process; # bounded and self-healing, but an invariant break). + # The same class exists IN-process: a sync that loaded the + # scope before this delete and acquires the fresh lock after + # the purge re-clones and re-populates the caches for the dead + # scope (found by the bed's randomized churn driver; + # deterministic seed recorded there). PR3's purge routing must + # include a scope-liveness check before clone. try: await run_sync(shutil.rmtree, str(scope_dir)) except FileNotFoundError: From 0450eac3f85856fc147e48fe4aa67cd82d1b2eef Mon Sep 17 00:00:00 2001 From: zivxx Date: Wed, 15 Jul 2026 16:40:04 +0300 Subject: [PATCH 58/63] fix(opal-server): fall back to the default bundle when a clone dir vanishes mid-request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both bundle-route except clauses (make_bundle and _generate_default_scope_bundle) caught InvalidGitRepositoryError, pygit2.GitError, and ValueError but missed GitPython's NoSuchPathError — raised when a concurrent scope delete or invalid-repo recovery fully rmtree'd the clone dir before Repo() opens it. That turned a transient race into a 500 instead of the intended default-scope fallback. Added NoSuchPathError to both tuples. Co-Authored-By: Claude Fable 5 --- packages/opal-server/opal_server/scopes/api.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/opal-server/opal_server/scopes/api.py b/packages/opal-server/opal_server/scopes/api.py index cf90c76eb..1e3f5385c 100644 --- a/packages/opal-server/opal_server/scopes/api.py +++ b/packages/opal-server/opal_server/scopes/api.py @@ -14,7 +14,7 @@ ) from fastapi.responses import RedirectResponse from fastapi_websocket_pubsub import PubSubEndpoint -from git import InvalidGitRepositoryError +from git import InvalidGitRepositoryError, NoSuchPathError from opal_common.async_utils import run_sync from opal_common.authentication.authz import ( require_peer_type, @@ -280,7 +280,14 @@ async def get_scope_policy( try: return await run_sync(fetcher.make_bundle, base_hash) - except (InvalidGitRepositoryError, pygit2.GitError, ValueError): + except ( + InvalidGitRepositoryError, + # A concurrent delete/recovery can rmtree the clone dir before + # Repo() opens it — fall back to the default scope, not a 500. + NoSuchPathError, + pygit2.GitError, + ValueError, + ): logger.warning( "Requested scope {scope_id} has invalid repo, returning default scope", scope_id=scope_id, @@ -305,6 +312,7 @@ async def _generate_default_scope_bundle(scope_id: str) -> PolicyBundle: except ( ScopeNotFoundError, InvalidGitRepositoryError, + NoSuchPathError, pygit2.GitError, ValueError, ): From a295717d487abd8e685411201c8ec1a654a40418 Mon Sep 17 00:00:00 2001 From: zivxx Date: Wed, 15 Jul 2026 16:41:04 +0300 Subject: [PATCH 59/63] fix(opal-server): stamp clone start time after a reclone The reclone stamp added earlier in this PR recorded completion time, contradicting the start-time rule the fetch path documents at the top of the same function: negotiation reflects remote state at operation START, so a clone that started after a webhook's req_time already satisfies it, while a completion stamp can wrongly suppress a forced refresh requested mid-clone. Capture clone_started immediately before run_sync( clone_repository, ...) and stamp that value in the success branch. Co-Authored-By: Claude Fable 5 --- packages/opal-server/opal_server/git_fetcher.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/opal-server/opal_server/git_fetcher.py b/packages/opal-server/opal_server/git_fetcher.py index 89100ac98..ad4a391eb 100644 --- a/packages/opal-server/opal_server/git_fetcher.py +++ b/packages/opal-server/opal_server/git_fetcher.py @@ -268,6 +268,10 @@ async def _clone(self): url=redact_url(self._source.url), path=self._repo_path, ) + # Same start-time rule as the fetch path above: the clone's + # negotiation reflects remote state at clone START, so that is the + # timestamp req_time comparisons need. + clone_started = datetime.datetime.now() try: repo: Repository = await run_sync( clone_repository, @@ -284,9 +288,7 @@ async def _clone(self): GitPolicyFetcher.repos[str(self._repo_path)] = repo # A reclone just downloaded current remote state — record it so # _was_fetched_after() doesn't force a redundant fetch next cycle. - GitPolicyFetcher.repos_last_fetched[ - self._source_id - ] = datetime.datetime.now() + GitPolicyFetcher.repos_last_fetched[self._source_id] = clone_started await self._notify_on_changes(repo) def _get_repo(self) -> Repository: From bd6c86d580843038ed634657bab7668cdb7752fd Mon Sep 17 00:00:00 2001 From: zivxx Date: Thu, 16 Jul 2026 16:03:37 +0300 Subject: [PATCH 60/63] refactor(opal-server): reuse redact_url_in_text for webhook failure logs The hand-rolled _CREDENTIALED_URL_RE duplicated (a weaker form of) the shared scrubber: it missed ?token=/?access_token= query-string credentials that redact_url_in_text's SENSITIVE_QUERY_PARAMS path covers. Use the vetted helper instead; kept local (rather than relying solely on the global log patcher) because the patcher only exists once configure_logs() has installed it. Addresses review comments: - https://github.com/permitio/opal/pull/923#discussion_r3589521239 (@zeevmoney) Co-Authored-By: Claude Fable 5 --- .../opal-server/opal_server/policy/watcher/task.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/opal-server/opal_server/policy/watcher/task.py b/packages/opal-server/opal_server/policy/watcher/task.py index 8a6bbfcfa..a2b3177f6 100644 --- a/packages/opal-server/opal_server/policy/watcher/task.py +++ b/packages/opal-server/opal_server/policy/watcher/task.py @@ -1,20 +1,15 @@ import asyncio import os -import re import signal from typing import Any, Coroutine, List, Optional from fastapi_websocket_pubsub import Topic from fastapi_websocket_pubsub.pub_sub_server import PubSubEndpoint +from opal_common.http_utils import redact_url_in_text from opal_common.logger import logger from opal_common.sources.base_policy_source import BasePolicySource from opal_server.config import opal_server_config -# git exceptions (e.g. from a failed fetch/clone) can embed a credentialed -# remote URL (https://user:token@host/repo) verbatim in their message — -# strip the credential before this ever reaches the logs. -_CREDENTIALED_URL_RE = re.compile(r"://[^/@\s]+@") - class BasePolicyWatcherTask: """Manages the asyncio tasks of the policy watcher.""" @@ -41,7 +36,10 @@ async def _on_webhook(self, topic: Topic, data: Any): # generic "Task exception was never retrieved" at GC time. for t in self._webhook_tasks: if t.done() and not t.cancelled() and t.exception() is not None: - exc_text = _CREDENTIALED_URL_RE.sub("://***@", repr(t.exception())) + # git exceptions can embed a credentialed remote URL verbatim; + # scrub here as well as in the global log patcher, which only + # runs once configure_logs() has installed it. + exc_text = redact_url_in_text(repr(t.exception())) logger.error(f"Webhook trigger task failed: {exc_text}") self._webhook_tasks = [t for t in self._webhook_tasks if not t.done()] self._webhook_tasks.append(asyncio.create_task(self.trigger(topic, data))) From bb171e6bd268f1f94272b8bbf8530ac69b2cb128 Mon Sep 17 00:00:00 2001 From: zivxx Date: Thu, 16 Jul 2026 16:05:01 +0300 Subject: [PATCH 61/63] fix(opal-server): keep the delete purge reachable and extract the purge helper An ambiguous store outcome (record delete committed server-side, error surfaced to the client) aborted delete_scope before the purge: the retry then hits ScopeNotFoundError -> 204 no-op, so the clone + cache entries leaked permanently. Run the purge in a finally so a record- delete failure can't gate cache cleanup; the error still propagates. Extracting the sibling-check + purge into _purge_source_cache_if_unshared / _find_scope_sharing_source also brings delete_scope from complexity 10 back under the <=8 project limit, with the critical section unchanged. Addresses review comments: - https://github.com/permitio/opal/pull/923#discussion_r3589521583 (@zeevmoney) - https://github.com/permitio/opal/pull/923#discussion_r3589659978 (@zeevmoney) Co-Authored-By: Claude Fable 5 --- .../opal-server/opal_server/scopes/service.py | 148 +++++++++++------- .../tests/delete_scope_cache_purge_test.py | 45 ++++++ 2 files changed, 136 insertions(+), 57 deletions(-) diff --git a/packages/opal-server/opal_server/scopes/service.py b/packages/opal-server/opal_server/scopes/service.py index 645e8dce0..5b07d1329 100644 --- a/packages/opal-server/opal_server/scopes/service.py +++ b/packages/opal-server/opal_server/scopes/service.py @@ -194,65 +194,99 @@ async def delete_scope(self, scope_id: str): # the clone and cache entries permanently. Deleting our record # before re-checking makes the last deleter see no sharer. async with GitPolicyFetcher.lock_source(deleted_source_id): - await self._scopes.delete(scope_id) - # The re-check must not be able to skip the purge by raising: - # our record is already deleted, so a client retry is a 204 - # no-op (ScopeNotFoundError) and the purge becomes permanently - # unreachable. all() does a full scan + Scope.parse_raw — a - # transient store error or one malformed record throws. Over- - # purging self-heals (a surviving sibling re-clones on its - # next sync); under-purging is a permanent leak. - try: - sharing_scope_id = next( - ( - s.scope_id - for s in await self._scopes.all() - if s.scope_id != scope_id - and isinstance(s.policy, GitPolicyScopeSource) - and GitPolicyFetcher.source_id(s.policy) - == deleted_source_id - ), - None, - ) - except Exception as e: - logger.warning( - f"sibling check failed after deleting scope " - f"{scope_id}; purging defensively: {e!r}" - ) - sharing_scope_id = None - if sharing_scope_id is not None: - logger.info( - f"Scope {sharing_scope_id} shares the same clone " - "(source id), skipping clone deletion" - ) - return - # NOTE (PR3): delete must ultimately route through the leader - # like put/refresh — only the leader should mutate the shared - # clone tree. Today this purge is process-local best-effort - # (the leader's caches leak until PR3's broadcast purge), and - # a non-leader DELETE rmtree's the shared tree unserialized - # against the leader's in-flight fetches (cross-process; - # bounded and self-healing, but an invariant break). - # The same class exists IN-process: a sync that loaded the - # scope before this delete and acquires the fresh lock after - # the purge re-clones and re-populates the caches for the dead - # scope (found by the bed's randomized churn driver; - # deterministic seed recorded there). PR3's purge routing must - # include a scope-liveness check before clone. try: - await run_sync(shutil.rmtree, str(scope_dir)) - except FileNotFoundError: - pass # never cloned (or already gone) — nothing to clean - except OSError as e: - logger.warning( - f"Failed to remove clone dir {scope_dir} of deleted " - f"scope {scope_id}: {e!r}" + await self._scopes.delete(scope_id) + finally: + # The purge must stay reachable even when the record + # delete raises an ambiguous outcome (committed server- + # side, error surfaced to the client): the client retry + # then hits ScopeNotFoundError -> 204 no-op, and a purge + # gated on a clean delete would be permanently orphaned. + # If the delete genuinely failed the record survives, + # over-purging self-heals (the scope re-clones on its + # next sync), and the error still propagates as a 500. + await self._purge_source_cache_if_unshared( + deleted_source_id, scope_dir, scope_id ) - GitPolicyFetcher.forget_repo(str(scope_dir)) - GitPolicyFetcher.repos_last_fetched.pop(deleted_source_id, None) - # Popped while the lock is held: lock_source waiters re-check - # the dict entry after acquiring and retry on the fresh lock. - GitPolicyFetcher.repo_locks.pop(deleted_source_id, None) + + async def _purge_source_cache_if_unshared( + self, deleted_source_id: str, scope_dir: Path, scope_id: str + ): + """Remove the clone dir and the GitPolicyFetcher cache entries keyed by + ``deleted_source_id``, unless a surviving scope still shares them. + + Must run under ``lock_source(deleted_source_id)``, after scope_id's + record was deleted (so the last of two concurrent sibling deleters + sees no sharer and purges). + """ + sharing_scope_id = await self._find_scope_sharing_source( + deleted_source_id, scope_id + ) + if sharing_scope_id is not None: + logger.info( + f"Scope {sharing_scope_id} shares the same clone " + "(source id), skipping clone deletion" + ) + return + # NOTE (PR3): delete must ultimately route through the leader + # like put/refresh — only the leader should mutate the shared + # clone tree. Today this purge is process-local best-effort + # (the leader's caches leak until PR3's broadcast purge), and + # a non-leader DELETE rmtree's the shared tree unserialized + # against the leader's in-flight fetches (cross-process; + # bounded and self-healing, but an invariant break). + # The same class exists IN-process: a sync that loaded the + # scope before this delete and acquires the fresh lock after + # the purge re-clones and re-populates the caches for the dead + # scope (found by the bed's randomized churn driver; + # deterministic seed recorded there). PR3's purge routing must + # include a scope-liveness check before clone. + try: + await run_sync(shutil.rmtree, str(scope_dir)) + except FileNotFoundError: + pass # never cloned (or already gone) — nothing to clean + except OSError as e: + logger.warning( + f"Failed to remove clone dir {scope_dir} of deleted " + f"scope {scope_id}: {e!r}" + ) + GitPolicyFetcher.forget_repo(str(scope_dir)) + GitPolicyFetcher.repos_last_fetched.pop(deleted_source_id, None) + # Popped while the lock is held: lock_source waiters re-check + # the dict entry after acquiring and retry on the fresh lock. + GitPolicyFetcher.repo_locks.pop(deleted_source_id, None) + + async def _find_scope_sharing_source( + self, deleted_source_id: str, scope_id: str + ) -> Optional[str]: + """Return the id of a surviving scope that shares deleted_source_id, or + None (= safe to purge). + + Must not be able to skip the purge by raising: scope_id's record + is already deleted, so a client retry is a 204 no-op + (ScopeNotFoundError) and the purge becomes permanently + unreachable. all() does a full scan + Scope.parse_raw — a + transient store error or one malformed record throws. Over- + purging self-heals (a surviving sibling re-clones on its next + sync); under-purging is a permanent leak. + """ + try: + return next( + ( + s.scope_id + for s in await self._scopes.all() + if s.scope_id != scope_id + and isinstance(s.policy, GitPolicyScopeSource) + and GitPolicyFetcher.source_id(s.policy) == deleted_source_id + ), + None, + ) + except Exception as e: + logger.warning( + f"sibling check failed after deleting scope " + f"{scope_id}; purging defensively: {e!r}" + ) + return None async def sync_scopes(self, only_poll_updates=False, notify_on_changes=True): with tracer.trace("scopes_service.sync_scopes"): diff --git a/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py b/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py index c5e8aeb80..8941f5fae 100644 --- a/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py +++ b/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py @@ -292,6 +292,51 @@ async def waiter(): assert events == ["deleter-in", "deleter-out", "waiter-in"] +class _AmbiguousDeleteRepository(FakeScopeRepository): + """Delete commits server-side but the client sees an error (classic + dropped-connection/timeout ambiguous store outcome).""" + + async def delete(self, scope_id): + await super().delete(scope_id) + raise ConnectionError("connection dropped after the delete committed") + + +@pytest.mark.asyncio +async def test_purge_still_runs_when_record_delete_raises_ambiguously( + tmp_path, monkeypatch +): + """If the record delete commits but raises to the caller, the purge must + still run: the record is gone, so a client retry is a 204 no-op + (ScopeNotFoundError) and a purge gated on a clean delete is permanently + orphaned. The error must still propagate (the client sees a 500 and can + retry).""" + scope = _scope("only", "https://git/repo-a.git") + repo = _AmbiguousDeleteRepository([scope]) + svc = ScopesService(base_dir=tmp_path, scopes=repo, pubsub_endpoint=None) + + sid = GitPolicyFetcher.source_id(scope.policy) + clone_path = str(GitPolicyFetcher.repo_clone_path(tmp_path, scope.policy)) + GitPolicyFetcher.repos[clone_path] = object() + GitPolicyFetcher.repos_last_fetched[sid] = "ts" + + rmtree_calls = [] + monkeypatch.setattr( + "opal_server.scopes.service.shutil.rmtree", + lambda p, **k: rmtree_calls.append(str(p)), + ) + + with pytest.raises(ConnectionError): + await svc.delete_scope("only") + + assert rmtree_calls == [clone_path], ( + "record-delete failure skipped the purge — the retry is a 204 no-op, " + "so the clone and cache entries leak permanently" + ) + assert clone_path not in GitPolicyFetcher.repos + assert sid not in GitPolicyFetcher.repos_last_fetched + assert sid not in GitPolicyFetcher.repo_locks + + @pytest.mark.asyncio async def test_recreate_after_delete_serializes_and_sees_clean_caches( tmp_path, monkeypatch From caaf713666d33c3588c057288a92894a2892b018 Mon Sep 17 00:00:00 2001 From: zivxx Date: Thu, 16 Jul 2026 16:05:12 +0300 Subject: [PATCH 62/63] fix(opal-server): re-check scope liveness right before each queued sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sync_scopes passed the stale all() snapshot object into sync_scope, so a DELETE that landed while the scope was still queued re-cloned the dead scope's repo and re-populated repos/repos_last_fetched/repo_locks (leaked until restart) — reintroducing a bounded form of the leak this series closes. Pass scope_id instead so sync_scope re-gets fresh state right before use, shrinking the window from the rest of the sweep to the sync itself; the delete race now surfaces as ScopeNotFoundError and is logged at info. The remaining window (delete landing mid-sync) is a known limitation deferred to PR3's leader-routed purge + scope-liveness check. Addresses review comments: - https://github.com/permitio/opal/pull/923#discussion_r3589521405 (@zeevmoney) Co-Authored-By: Claude Fable 5 --- .../opal-server/opal_server/scopes/service.py | 53 ++++++++++++------- .../tests/delete_scope_cache_purge_test.py | 34 ++++++++++++ 2 files changed, 69 insertions(+), 18 deletions(-) diff --git a/packages/opal-server/opal_server/scopes/service.py b/packages/opal-server/opal_server/scopes/service.py index 5b07d1329..7455a3fe2 100644 --- a/packages/opal-server/opal_server/scopes/service.py +++ b/packages/opal-server/opal_server/scopes/service.py @@ -19,7 +19,11 @@ create_policy_update, create_update_all_directories_in_repo, ) -from opal_server.scopes.scope_repository import Scope, ScopeRepository +from opal_server.scopes.scope_repository import ( + Scope, + ScopeNotFoundError, + ScopeRepository, +) def is_rego_source_file( @@ -309,24 +313,37 @@ async def sync_scopes(self, only_poll_updates=False, notify_on_changes=True): skipped_scopes.append(scope) continue - try: - await self.sync_scope( - scope=scope, - force_fetch=True, - notify_on_changes=notify_on_changes, - ) - except Exception as e: - logger.exception(f"sync_scope failed for {scope.scope_id}") - + await self._sync_snapshotted_scope( + scope, force_fetch=True, notify_on_changes=notify_on_changes + ) fetched_source_ids.add(src_id) for scope in skipped_scopes: # No need to refetch the same repo, just check for changes - try: - await self.sync_scope( - scope=scope, - force_fetch=False, - notify_on_changes=notify_on_changes, - ) - except Exception as e: - logger.exception(f"sync_scope failed for {scope.scope_id}") + await self._sync_snapshotted_scope( + scope, force_fetch=False, notify_on_changes=notify_on_changes + ) + + async def _sync_snapshotted_scope( + self, scope: Scope, force_fetch: bool, notify_on_changes: bool + ): + """Sync one scope taken from a (possibly stale) all() snapshot, + swallowing per-scope errors so the sweep continues. + + Passes scope_id (not the snapshot object) so sync_scope re-gets + fresh state right before use — a delete that landed after the + snapshot surfaces as ScopeNotFoundError instead of re-cloning a + dead scope's repo and re-populating the fetcher caches for it. + """ + try: + await self.sync_scope( + scope_id=scope.scope_id, + force_fetch=force_fetch, + notify_on_changes=notify_on_changes, + ) + except ScopeNotFoundError: + logger.info( + f"scope {scope.scope_id} was deleted while sync was queued, skipping" + ) + except Exception: + logger.exception(f"sync_scope failed for {scope.scope_id}") diff --git a/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py b/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py index 8941f5fae..d8164e53b 100644 --- a/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py +++ b/packages/opal-server/opal_server/tests/delete_scope_cache_purge_test.py @@ -337,6 +337,40 @@ async def test_purge_still_runs_when_record_delete_raises_ambiguously( assert sid not in GitPolicyFetcher.repo_locks +class _DeletedAfterSnapshotRepository(FakeScopeRepository): + """All() still returns the scope (a stale sync_scopes snapshot); get() + reports it deleted — models a DELETE landing between the two.""" + + async def get(self, scope_id): + await asyncio.sleep(0) + raise ScopeNotFoundError(scope_id) + + +@pytest.mark.asyncio +async def test_sync_scopes_skips_scope_deleted_after_snapshot(tmp_path, monkeypatch): + """A scope deleted between sync_scopes' all() snapshot and its queued + sync_scope call must not be fetched — re-cloning it would re-populate + repos/repos_last_fetched/repo_locks for a dead scope (leaked until + restart).""" + scope = _scope("dead", "https://git/repo-a.git") + repo = _DeletedAfterSnapshotRepository([scope]) + svc = ScopesService(base_dir=tmp_path, scopes=repo, pubsub_endpoint=None) + + fetch_calls = [] + + async def fake_fetch(self, *args, **kwargs): + fetch_calls.append(self._scope_id) + + monkeypatch.setattr(GitPolicyFetcher, "fetch_and_notify_on_changes", fake_fetch) + + await svc.sync_scopes() # must not raise: the delete race is expected + + assert fetch_calls == [], "sync fetched a scope whose delete already landed" + assert not GitPolicyFetcher.repos + assert not GitPolicyFetcher.repos_last_fetched + assert not GitPolicyFetcher.repo_locks + + @pytest.mark.asyncio async def test_recreate_after_delete_serializes_and_sees_clean_caches( tmp_path, monkeypatch From bf72ca5f7bccdbfe73adef855f7ac96a7fc66fcd Mon Sep 17 00:00:00 2001 From: zivxx Date: Thu, 16 Jul 2026 16:05:29 +0300 Subject: [PATCH 63/63] test(opal-server): cover the clone-vanish default-bundle fallback The NoSuchPathError -> default-bundle branch of GET /scopes/{id}/policy (a live scope whose clone dir vanished under a concurrent delete/recovery) had no test; lock it in alongside the pre-existing scope-not-found fallback. Returning a retryable error (503) for the live-scope case instead of silently substituting the default bundle is deferred to PR3 with the delete-routing work. Addresses review comments: - https://github.com/permitio/opal/pull/923#discussion_r3589521493 (@zeevmoney) Co-Authored-By: Claude Fable 5 --- .../tests/scope_policy_fallback_test.py | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 packages/opal-server/opal_server/tests/scope_policy_fallback_test.py diff --git a/packages/opal-server/opal_server/tests/scope_policy_fallback_test.py b/packages/opal-server/opal_server/tests/scope_policy_fallback_test.py new file mode 100644 index 000000000..5034bea65 --- /dev/null +++ b/packages/opal-server/opal_server/tests/scope_policy_fallback_test.py @@ -0,0 +1,116 @@ +"""GET /scopes/{scope_id}/policy fallback when the clone dir vanishes. + +A concurrent delete (or invalid-repo recovery) can rmtree a live scope's +clone between the scope-record read and make_bundle opening the repo. +The route must fall back to the default scope's bundle, not 500. (Known +limitation, tracked for PR3: a live scope is briefly served the default +bundle instead of a retryable error.) +""" + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from git import NoSuchPathError +from opal_common.schemas.policy import PolicyBundle +from opal_common.schemas.policy_source import GitPolicyScopeSource, NoAuthData +from opal_common.schemas.scopes import Scope +from opal_server.git_fetcher import GitPolicyFetcher +from opal_server.scopes.api import init_scope_router +from opal_server.scopes.scope_repository import ScopeNotFoundError +from opal_server.scopes.service import ScopesService + + +class FakeScopeRepository: + def __init__(self, scopes): + self._scopes = {s.scope_id: s for s in scopes} + + async def get(self, scope_id): + if scope_id not in self._scopes: + raise ScopeNotFoundError(scope_id) + return self._scopes[scope_id] + + async def all(self): + return list(self._scopes.values()) + + async def delete(self, scope_id): + self._scopes.pop(scope_id, None) + + +class FakeAuthenticator: + """Mimics a JWTAuthenticator whose verifier is disabled (no public key).""" + + enabled = False + + def __call__(self): + return {} + + +def _scope(scope_id, url, branch="main"): + return Scope( + scope_id=scope_id, + policy=GitPolicyScopeSource( + source_type="git", + url=url, + branch=branch, + auth=NoAuthData(auth_type="none"), + ), + data={"entries": []}, + ) + + +def _client(repo, base_dir): + service = ScopesService(base_dir=base_dir, scopes=repo, pubsub_endpoint=None) + app = FastAPI() + app.include_router( + init_scope_router(repo, FakeAuthenticator(), None, service), + prefix="/scopes", + ) + return TestClient(app) + + +def _default_bundle(): + return PolicyBundle( + manifest=[], hash="default-head", data_modules=[], policy_modules=[] + ) + + +def test_live_scope_clone_vanish_falls_back_to_default_bundle(tmp_path, monkeypatch): + """A live scope whose clone dir vanished mid-request (NoSuchPathError from + make_bundle) must be served the default scope's bundle, not a 500.""" + live = _scope("live", "https://git/live.git") + default = _scope("default", "https://git/default.git") + repo = FakeScopeRepository([live, default]) + + def fake_make_bundle(self, base_hash): + if self._scope_id == "live": + raise NoSuchPathError(str(tmp_path / "gone")) + return _default_bundle() + + monkeypatch.setattr(GitPolicyFetcher, "make_bundle", fake_make_bundle) + monkeypatch.setattr( + "opal_server.scopes.api.opal_server_config.BASE_DIR", str(tmp_path) + ) + + resp = _client(repo, tmp_path).get("/scopes/live/policy") + + assert resp.status_code == 200 + assert resp.json()["hash"] == "default-head" + + +def test_missing_scope_still_falls_back_to_default_bundle(tmp_path, monkeypatch): + """The pre-existing scope-not-found fallback must keep working alongside + the clone-vanish branch.""" + default = _scope("default", "https://git/default.git") + repo = FakeScopeRepository([default]) + + monkeypatch.setattr( + GitPolicyFetcher, "make_bundle", lambda self, base_hash: _default_bundle() + ) + monkeypatch.setattr( + "opal_server.scopes.api.opal_server_config.BASE_DIR", str(tmp_path) + ) + + resp = _client(repo, tmp_path).get("/scopes/ghost/policy") + + assert resp.status_code == 200 + assert resp.json()["hash"] == "default-head"