diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51d9d3f..27284b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,9 +67,9 @@ jobs: e2e: # Pure-Python slow e2e tests (no Node/Docker toolchain): they spawn a - # real FastMCP server / DBOS SQLite app and run emitted code end to end, - # so the empirical proof is automated on every PR — not just `-m slow` - # locally. The TS/Docker e2e suites stay opt-in (they need Node/Docker). + # real FastMCP server / DBOS SQLite app and run emitted code end to + # end, so the empirical proof is automated on every PR — not just + # `-m slow` locally. name: Python e2e (slow) runs-on: ubuntu-latest steps: @@ -82,5 +82,93 @@ jobs: - name: Install package with dev extras run: python -m pip install -e ".[dev]" - - name: MCP backend e2e (live mock MCP server + DBOS SQLite) - run: pytest tests/test_mcp_e2e.py -m slow -v + # Every slow suite that needs no external toolchain. Previously + # only test_mcp_e2e ran here, which left the DBOS Python runtime, + # park-on-auth, the OAuth flow and the MCP-trigger server proven + # only on a maintainer's laptop. + - name: Python e2e (MCP backend, DBOS, park-on-auth, OAuth, serve) + run: | + pytest -m slow -v \ + tests/test_mcp_e2e.py \ + tests/test_mcp_park_e2e.py \ + tests/test_mcp_oauth_e2e.py \ + tests/test_dbos_e2e.py \ + tests/test_dbos_serve_e2e.py \ + tests/test_eval_empirical_dbos.py \ + tests/test_serve_server.py + + ts-e2e: + # The TypeScript runtimes' live suites. These were opt-in on the + # assumption that a runner lacks the toolchain — ubuntu-latest ships + # Node and npm, so the only real cost is time. Worth it: the emitted + # TS is three of the six supported runtimes, and a fan_out regression + # that only these suites catch would otherwise reach main unnoticed. + name: TypeScript e2e (slow) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Install package with dev extras + run: python -m pip install -e ".[dev]" + + # Each suite npm-installs into its own emitted directory, so there + # is no lockfile for setup-node to key a cache on; cache the npm + # download cache itself instead. + - name: Cache npm downloads + uses: actions/cache@v4 + with: + path: ~/.npm + key: npm-e2e-${{ runner.os }}-${{ hashFiles('src/rote/adapters/_ts_common.py') }} + restore-keys: npm-e2e-${{ runner.os }}- + + - name: Cloudflare / Inngest / agent-loop / MCP e2e + run: | + pytest -m slow -v \ + tests/test_cloudflare_e2e.py \ + tests/test_inngest_e2e.py \ + tests/test_ts_agent_loop_e2e.py \ + tests/test_mcp_ts_e2e.py \ + tests/test_mcp_park_cf_e2e.py \ + tests/test_mcp_park_inngest_e2e.py + + dbos-ts-e2e: + # Split from ts-e2e because these two also need Docker: the DBOS + # TypeScript SDK is Postgres-only (no SQLite parity with DBOS + # Python), and the suite starts a throwaway container when no + # reachable Postgres is configured. + name: DBOS-TS e2e (slow, Docker) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - uses: actions/setup-node@v4 + with: + node-version: "22" + + - name: Install package with dev extras + run: python -m pip install -e ".[dev]" + + - name: Cache npm downloads + uses: actions/cache@v4 + with: + path: ~/.npm + key: npm-dbosts-${{ runner.os }}-${{ hashFiles('src/rote/adapters/_ts_common.py') }} + restore-keys: npm-dbosts-${{ runner.os }}- + + - name: DBOS TypeScript e2e (live runtime against Docker Postgres) + run: | + pytest -m slow -v \ + tests/test_dbos_ts_e2e.py \ + tests/test_mcp_park_ts_e2e.py diff --git a/tests/test_ci_workflow.py b/tests/test_ci_workflow.py new file mode 100644 index 0000000..d592db1 --- /dev/null +++ b/tests/test_ci_workflow.py @@ -0,0 +1,84 @@ +"""Every slow test suite must actually run somewhere in CI. + +`@pytest.mark.slow` excludes a suite from the default run, which is the +point — they need Node, Docker, or minutes. The failure mode is that +excluding them locally also excluded them from CI: for a long stretch +only ``test_mcp_e2e.py`` ran there, so 24 of 27 slow tests existed +solely on a maintainer's laptop. Three of the six supported runtimes +had no automated proof at all, and a fan_out regression that only the +Cloudflare e2e catches would have reached main unnoticed. + +Marking a suite slow is a statement about the *toolchain it needs*, not +about whether it should run. This test makes adding a slow suite without +wiring it into a CI job a failure rather than an omission. +""" + +from __future__ import annotations + +import re +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +CI_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "ci.yml" + + +def _slow_test_files() -> set[str]: + """Collect the slow suite by asking pytest, not by globbing. + + A file is only slow if pytest actually collects a slow test from it, + so this can't drift from the markers. + """ + proc = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "tests/", + "-m", + "slow", + "-q", + "--collect-only", + "--no-header", + ], + cwd=REPO_ROOT, + capture_output=True, + text=True, + timeout=300, + ) + assert proc.returncode == 0, f"collection failed:\n{proc.stdout}\n{proc.stderr}" + return { + line.split("::")[0].rsplit("/", 1)[-1] + for line in proc.stdout.splitlines() + if line.startswith("tests/") and "::" in line + } + + +def test_every_slow_suite_is_wired_into_a_ci_job() -> None: + collected = _slow_test_files() + # Sanity: if collection returns nothing the assertion below is vacuous. + assert len(collected) >= 10, f"expected a substantial slow suite, got {sorted(collected)}" + + referenced = set(re.findall(r"tests/(test_\w+\.py)", CI_WORKFLOW.read_text(encoding="utf-8"))) + missing = sorted(collected - referenced) + + assert not missing, ( + f"These slow suites run nowhere in CI: {missing}. Add them to a job in " + f"{CI_WORKFLOW.relative_to(REPO_ROOT)} — 'Python e2e' for pure-Python " + f"suites, 'TypeScript e2e' for Node ones, 'DBOS-TS e2e' if they also " + f"need Docker. Marking a suite slow says what toolchain it needs, not " + f"that it should go unrun." + ) + + +def test_ci_does_not_reference_deleted_test_files() -> None: + """A renamed suite must not leave CI silently running nothing. + + `pytest path/that/does/not/exist.py` exits 4, so this would surface + as a red job — but only on the next push that happens to touch CI. + Failing here names the stale path directly. + """ + referenced = set(re.findall(r"tests/(test_\w+\.py)", CI_WORKFLOW.read_text(encoding="utf-8"))) + absent = sorted(name for name in referenced if not (REPO_ROOT / "tests" / name).is_file()) + assert not absent, f"ci.yml references test files that no longer exist: {absent}"