v2.0.0: cleanup release — architecture, telemetry, tenancy, gates

.check_router_core_floor

@@ -0,0 +1 @@
+117

.env.example

@@ -47,6 +47,13 @@
 # backend runs on a different host than the frontend.
 # NEXT_PUBLIC_API_URL=http://127.0.0.1:8000
 
+# ── Observability ──────────────────────────────────────────────────────────────
+# OpenTelemetry exporter. Default 'none' — no spans/metrics leave the process.
+# Set 'console' to dump spans and 60s metric snapshots to stdout (loud; useful
+# locally when chasing a perf regression). Don't set 'console' in prod — it
+# pollutes log aggregation with ~1 MB/min of JSON.
+# OTEL_EXPORTER=console
+
 # ── Docker only ────────────────────────────────────────────────────────────────
 # Set automatically by docker-compose; not needed for local dev.
 # API_PROXY_URL=http://backend:8000

.github/workflows/ci.yml

@@ -32,8 +32,15 @@ jobs:
       - name: Format check (ruff)
         run: uv run ruff format --check .
 
-      - name: Type check (mypy)
-        run: uv run mypy backend/
+      - name: Type check (mypy, filtered through mypy-baseline)
+        # Pre-existing errors accepted via mypy-baseline.txt; the filter
+        # exits non-zero only on NET-NEW errors. Refresh the baseline after
+        # a burndown PR with
+        #   uv run mypy backend/ 2>&1 | uv run mypy-baseline sync
+        # and commit mypy-baseline.txt. Burndown plan +
+        # bucket scoping live in
+        # pending-docs/session_2026-06-10_otel_dump_and_log_extents.md.
+        run: uv run mypy backend/ 2>&1 | uv run mypy-baseline filter
 
       - name: Install falco
         run: |
@@ -85,19 +92,50 @@ jobs:
         env:
           FALCO_REQUIRED: "1"
           TERRAFORM_VALIDATE: "1"
-        # Gate ratcheted as milestones land:
+        # Gate ratcheted as milestones land (convention: current actual − 2pp):
         #   end Milestone A: 44% (baseline 46%, -2pp buffer)
         #   end Milestone E: 47% (current 49% — keeps the 2pp buffer)
         #   post-Milestone E coverage backfill: 55% (current 59% — 4pp buffer)
         #   confidence-batch (insights+admin+services+dashboard+origin+
         #   hypothesis+regression+E2E smoke): 78% (current 83% — 5pp buffer)
+        #   post live-query-monitor (2026-06-11): 80% (current 82%)
+        #   post backend coverage waves (reconciliation/compaction/session_scoring
+        #   /data_migrations/tunnel-state/dashboard-router/views/sqlite_profiler):
+        #     82% (current 83% — 1pp buffer; tight while v2.0 target 85% lands).
+        #   v2.0 final wave (2026-06-12): per-module tests for the post-split
+        #   rollups/ + admin/ packages (rollups/sessions 85, rollups/time_series
+        #   84, rollups/day_bundles 76, rollups/recompute 96, admin/compaction
+        #   100, admin/health 100): 85% (current 85% — the v2.0 target hit).
         #
         # `-n auto` parallelizes via pytest-xdist (TESTING_PLAN_3 item 21).
         # Verified safe: per-service SQLite (`{id}.metadata.db`) + per-test
         # tmp_path give file isolation; autouse `_reset_module_caches` resets
         # the 8 module-level caches between tests; moto fixtures are per-test.
         # Local run: 2268 passed in 58s under `-n auto` vs ~3min serial.
-        run: uv run pytest -n auto --cov=backend --cov-report=term --cov-fail-under=78
+        run: uv run pytest -n auto --cov=backend --cov-report=term --cov-fail-under=85
+
+      - name: Security-regression count gate
+        # v2.0 cleanup Phase 0.8: asserts the
+        # @pytest.mark.security_regression count never drops below the
+        # baseline floor (24 — derived from audit-findings/ verified
+        # fixes). A refactor cannot silently delete coverage of a
+        # verified fix without surfacing the change.
+        run: bash scripts/check_security_regression_count.sh
+
+      - name: Emit perf samples (CI-scale synthetic load)
+        # Produces tests/perf/latest.json from a 100K-row in-memory
+        # DuckDB dataset (~2 s wall). The gate below compares to
+        # tests/perf/baseline.json and fails on >regression_pct_threshold%
+        # over baseline (50 % default; tuned for GH Actions runner
+        # variance at CI scale).
+        run: uv run python scripts/emit_perf_latest.py
+
+      - name: Perf gate (load-harness baseline)
+        # Compares the just-emitted latest.json against baseline.json.
+        # Production targets (≤2800 / ≤1900 ms) are documented in
+        # baseline.json's production_targets_comment for traceability
+        # but enforced by the manual loadtest probe, not this CI gate.
+        run: bash scripts/perf_gate.sh
 
   frontend:
     name: Frontend (Node)
@@ -140,7 +178,17 @@ jobs:
         run: npx tsc --noEmit
 
       - name: Tests (vitest with coverage)
-        # Gate ratcheted as milestones land:
+        # Gate ratcheted as milestones land (convention: current actual − 2pp):
         #   end Milestone A: 40% (baseline 42.7%, -2pp buffer)
-        #   end Milestone E: 44% (current 46.55% — keeps the 2pp buffer)
-        run: npx vitest run --coverage --coverage.thresholds.lines=44
+        #   end Milestone E: 44% (current 46.55%)
+        #   post live-query-monitor (2026-06-11): 53% (current 55.19%)
+        #   post lib/toast + lib/api/custom-fields + lib/workers/parseJson tests
+        #     (2026-06-12): 55% (current 57.12%)
+        #   post ProvisionWizard/wizard-config-helpers tests
+        #     (2026-06-12): 56% (current 58.42%)
+        #   post ProvisionWizard/wizard-api tests
+        #     (2026-06-12): 57% (current 59.8%)
+        #   post ProvisionWizard/wizard-deploy tests
+        #     (2026-06-12): 58% (current 61.66%) — final v2.0 target hit
+        #     per cleanup_plan §10.14.
+        run: npx vitest run --coverage --coverage.thresholds.lines=58

.github/workflows/cidr-refresh.yml

@@ -0,0 +1,53 @@
+name: Refresh Fastly CIDRs
+
+# Weekly refresh of the Fastly edge CIDR list in the repo-root Caddyfile.
+# The @from_fastly_v4 matcher gates X-Forwarded-For rewriting on Fastly's
+# published v4 ranges; a stale list silently classifies traffic from new
+# POPs as direct (untrusted) until somebody refreshes it and reloads
+# Caddy. The script is well-tested (scripts/refresh_fastly_cidrs.py);
+# this workflow just runs it on a cadence and opens a PR if the file
+# changed. Off-minute schedule on purpose so the runner pool isn't
+# hammered at :00 alongside everybody else's hourly jobs.
+
+on:
+  schedule:
+    - cron: '13 9 * * 1'  # Mondays at 09:13 UTC
+  workflow_dispatch: {}
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  refresh:
+    name: Fetch + open PR on diff
+    runs-on: forge-amd64-medium
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+          python-version: "3.13"
+
+      - name: Refresh Caddyfile
+        # No-op if the published list already matches what's in the
+        # Caddyfile (script prints "No changes …" and exits 0). Writes
+        # the updated matcher block otherwise; peter-evans/create-pull-
+        # request below only opens a PR when the working tree is dirty.
+        run: uv run python scripts/refresh_fastly_cidrs.py
+
+      - name: Open PR if Caddyfile changed
+        uses: peter-evans/create-pull-request@v7
+        with:
+          commit-message: 'chore: refresh Fastly edge CIDR list in Caddyfile'
+          branch: chore/refresh-fastly-cidrs
+          delete-branch: true
+          title: 'chore: refresh Fastly edge CIDR list'
+          body: |
+            Automated update from `scripts/refresh_fastly_cidrs.py`, triggered by the weekly `cidr-refresh.yml` workflow.
+
+            The `@from_fastly_v4` matcher in [Caddyfile](../blob/main/Caddyfile) gates the `X-Forwarded-For` rewrite on Fastly-published edge ranges. A stale list silently classifies traffic from new POPs as direct (untrusted) until Caddy reloads.
+
+            After merge: run `~/restart.sh caddy` (or equivalent) on the VM to pick up the new ranges.

.gitignore

@@ -61,6 +61,12 @@ tests/fixtures/scoring/
 # scripts/scoring/train.py against a fresh trace extract.
 compute/scorer/matrix.json
 
+# Per-tenant matrices pulled from FOS on every backend startup
+# (see backend/main.py:_ensure_scoring_matrix). matrix.default.json
+# stays tracked — that's the in-repo fallback the scoring endpoint
+# uses when neither the shared matrix.json nor a tenant matrix exists.
+compute/scorer/matrix_*.json
+
 # Rust build artifacts.
 compute/scorer/target/
 compute/scorer/bin/
@@ -76,6 +82,11 @@ compute/scorer/pkg/
 # split_per_page.py) live here for now; treat the whole tree as throwaway.
 /scratch/
 
+# Performance-audit campaign artifacts: HAR captures, per-sample telemetry,
+# aggregated p50/p95/p99 summaries, per-page reports + improvement plans.
+# Throwaway — regenerable by re-running scratch/perf_audit.mjs.
+/performance-report/
+
 # Local-only VS Code config (file-watcher / Pylance excludes for the
 # regenerating .next + cache trees). Personal to each contributor's editor
 # setup — not promoted to the repo by default.

.pre-commit-config.yaml

@@ -1,26 +1,44 @@
 repos:
+  # Pinned ruff version must stay reasonably close to the version in
+  # pyproject.toml (currently ruff>=0.11) — drift triggers pre-existing
+  # rule changes (UP038, E731 strictness) that the project's actual ruff
+  # has already retired. Bump together when bumping either side.
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.0
+    rev: v0.15.15
     hooks:
       - id: ruff
         args: [--fix]
       - id: ruff-format
 
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.15.0
+  # mypy runs via the project's own uv env (matches what CI runs) and is
+  # piped through mypy-baseline so pre-existing errors stay accepted and
+  # only NET-NEW errors fail the commit. The baseline lives in
+  # mypy-baseline.txt at the repo root; refresh it after a burndown PR with
+  #   uv run mypy backend/ 2>&1 | uv run mypy-baseline sync
+  # and commit the updated file. Burndown plan in
+  # pending-docs/session_2026-06-10_otel_dump_and_log_extents.md.
+  - repo: local
     hooks:
       - id: mypy
-        additional_dependencies:
-          - types-boto3
-          - types-pytz
-          - fastapi
-          - pydantic
+        name: mypy (full backend/, filtered through mypy-baseline)
+        language: system
+        # Always check the whole backend/ tree, not just changed files —
+        # per-file mypy only visits a partial import graph, which makes
+        # mypy-baseline report unrelated baseline entries as "fixed" and
+        # exit non-zero. Cost: ~10s per commit; benefit: matches CI exactly.
+        entry: bash -c 'uv run mypy backend/ 2>&1 | uv run mypy-baseline filter'
+        files: '^backend/.*\.py$'
+        pass_filenames: false
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
+        # openapi-typescript emits openapi.json without a trailing newline;
+        # end-of-file-fixer adds one, then the next regen-openapi run
+        # strips it. Excluding the generated artifact breaks the cycle.
+        exclude: '^frontend/openapi\.json$'
       - id: check-yaml
       - id: check-json
       - id: check-merge-conflict
@@ -60,3 +78,16 @@ repos:
         language: system
         pass_filenames: false
         entry: bash -c 'cd frontend && npx tsc --noEmit'
+
+      # v2.0 cleanup (Phase 0.12): pre-push gate that the
+      # @pytest.mark.security_regression count hasn't dropped below
+      # the Phase 0 floor (24). Catches a refactor that silently
+      # removes coverage of a verified security fix before push,
+      # not in CI. `stages: [pre-push]` keeps it off the per-commit
+      # hot path (the gate takes ~2s to collect 3k+ tests).
+      - id: security-regression-count
+        name: Assert security_regression test count >= floor
+        stages: [pre-push]
+        language: system
+        pass_filenames: false
+        entry: bash scripts/check_security_regression_count.sh