diff --git a/.env.example b/.env.example index 09e6b88..4ceba6a 100644 --- a/.env.example +++ b/.env.example @@ -22,6 +22,7 @@ GITHUB_TOKEN= # repo-scoped PAT for opening PRs (or use `gh auth login`) # VPCOPILOT_PROTECTED_LBS=nimbus-www # comma-separated LBs that refuse mutation without an override # VPCOPILOT_REFINE_ATTEMPTS=3 # max self-heal attempts when a policy doesn't block the exploit # VPCOPILOT_REQUIRE_PROBE=1 # fail-closed: don't validate a finding that has no derived probe +# VPCOPILOT_ACTOR=jane.doe # audit-log attribution — default: the OS user; set it in CI # --- Validation auth (auth-protected targets: let the probe log in so it can demonstrate the exploit) --- # VPCOPILOT_PROBE_USER= # username the validation probe logs in with (cookie or token app) # VPCOPILOT_PROBE_PASS= # its password diff --git a/BACKLOG.md b/BACKLOG.md index 91bbef3..2b19c41 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -2,11 +2,16 @@ Ideas captured for later; not yet scheduled. -- **HTML results dashboard.** Have an agent generate a self-contained HTML dashboard of a - run — findings, triage (band-aids + residual risk), generated XC configs, and code-fix - PRs, plus the benchmark scorecard. A shareable artifact for stakeholders. NOTE: the - interactive console dashboard now exists — this is the *standalone, static, shareable* - export (a single self-contained .html file). _(Requested 2026-07-01.)_ +- ~~**HTML results dashboard.**~~ ✅ **Done** — `report.py` writes a single self-contained + `/report.html` (inline CSS, no external assets, native `
`) at the end of every + `run_pipeline`; `vpcopilot report` rebuilds it from any existing out dir. It carries the hero, + at-a-glance severity/coverage bars, model independence, pipeline metrics, findings + band-aid + coverage, the generated XC policies, the `found → mitigated → remediated → retired` ledger and + the band-aid impact table. Reachable from ② Review and Setup — **Open HTML report ↗** (rebuilt + from the current out dir on every request, so it is never stale) and **Download** for a stamped + `vpcopilot-report--.html`. Not included: the benchmark scorecard — that still lives + only in the console's ⑥ Benchmark step. Original ask: a standalone, static, shareable export of + a run for stakeholders. _(Requested 2026-07-01.)_ - ~~**Ops console admin panel (localhost).**~~ ✅ **Done** in the console MVP — the Admin tab reads/writes the local `.env` (XC creds + model API keys), redacting secrets. - **Benchmark: bonus-vuln handling.** Add a `bonus:` section to `bench/answer_key.yaml` so @@ -16,3 +21,20 @@ Ideas captured for later; not yet scheduled. rate) and discovery duplicates, not just discovery + triage. - **Finding correlation as a first-class step.** The model already remarks "band-aid for A covers B" — make it explicit so overlapping band-aids are deduped/linked in the output. +- **Sign the evidence bundle.** `manifest.json` SHA-256s every member, so tampering with a + *member* is detectable — but nothing binds the manifest to a signer. A detached signature + (sigstore or a plain GPG/minisign detached sig next to the manifest) would make a bundle + attributable after it leaves the machine, not just internally consistent. _(Requested 2026-07-26.)_ +- **`vpcopilot export --verify`.** The other half of the above: re-read a bundle, recompute each + member's digest against the manifest, and print pass/fail. Cheap, and it means the reviewer + doesn't have to trust the sender's word. _(Requested 2026-07-26.)_ +- **Stream audit events to a SIEM.** `audit.record` is the single choke point every mutating path + goes through — an optional sink (syslog / HTTP webhook / stdout JSON) would put the trail + somewhere other than the box that made the change. Fail-soft: a dead collector must never fail + an apply. _(Requested 2026-07-26.)_ +- **Backfill attribution onto historic audit logs.** Entries written before identity was stamped + in `audit.record` have no `run_id` / `actor`, and older `apply_*` entries no `finding_id`; the + export leaves those cells blank rather than guessing. A one-shot `vpcopilot audit backfill` + could fill only what is provably derivable (policy name → finding, via `policies.json`), mark + the rest `unknown`, and record that it did so — a backfill that silently invents an actor is + worse than a blank. _(Requested 2026-07-26.)_ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8e02166..2831cad 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,6 +29,12 @@ the nightly job. roll back on failure, and honor the protected-LB / protected-policy guardrails. - New band-aid controls are added in one place: the `controls.py` registry (attach/detach, LB-wide, validation kind, refine strategy). Wire the handler through the engine, not a new bespoke function. +- Every mutating path must leave an audit record. Call `audit.record(out, "", ...)` with at + least `finding_id` and `namespace` — an entry that can't say *why* an LB changed, or *in which + tenant*, isn't an audit record. Identity (`run_id` / `actor` / `host` / `tool_version`) is stamped + inside `audit.record`, never at the call site, and those keys are stripped from `**detail` so a + caller can't override them. Register the new action in `export.CATEGORY` / `export.CONTROL` so the + evidence bundle names it instead of showing `other`. Dry runs stay unrecorded, by design. - Match the surrounding style; `ruff` enforces the important bits. ## Scope + safety diff --git a/DESIGN.md b/DESIGN.md index 022d1c4..ac4db89 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -90,6 +90,11 @@ abstraction and handle differences with config. - **Self-validation + auto-rollback:** after applying, fire the exploit + a legit request; if the exploit isn't blocked or legit traffic breaks → auto-revert and flag. Validation target is the **live LB** (more demo-dramatic; the snapshot/rollback makes it safe). +- **Audit trail:** every mutating action in the finding lifecycle appends one JSON line to `/audit.log`, stamped + centrally with `run_id` / `actor` / `host` / `tool_version` and carrying `finding_id`, + `namespace`, the LB, the XC object it touched, `rolled_back` **and** `kept` (rolled-back alone + can't say whether the change is still live), plus the before/after exploit proof. Dry runs are + not recorded — nothing changed, so there is nothing to answer for. See *Audit + provenance*. - **Secrets:** scoped XC token + provider keys in env / secret store, never in git. ## XC integration (`xc/`, next increment) @@ -107,18 +112,96 @@ API. Each PR notes it permanently remediates an issue currently held closed by a XC virtual patch (retire the policy on merge). A **ledger** tracks finding state so band-aids don't silently become permanent. +## Audit + provenance (`audit.py`, `runmeta.py`, `export.py`) +A band-aid is a change to production infrastructure. The question a change board asks is not "did +it work" but **"why was this load balancer changed, by whom, in which tenant, and is it still +live?"** These three modules exist to answer exactly that — and nothing more: they are evidence for +a human reviewer, not a compliance certification. + +**What one entry carries.** Every mutating action appends a JSON line to `/audit.log`: + +| stamped centrally (`audit.record`) | supplied by the action | +|---|---| +| `ts` · `run_id` · `actor` · `host` · `tool_version` | `finding_id` · `namespace` · `lb` · the XC object (`policy` / `app_firewall` / `apidef` / …) · `passed` \| `config_enabled` \| `enabled` · `rolled_back` · `kept` · `attempts` · `before_after` | + +15 actions emit records: the 7 `apply_*`, the 3 `create_*`, `refine_apply`, `apply_timing`, +`open_pr`, `retire`, `rollback_failed`. + +### The decisions +1. **Identity is stamped inside `audit.record`, never at the call sites.** Nineteen `record()` call + sites write the trail — `apply.py`, `refiner.py`, `engine.py`, `pr.py`, `retire.py`, the console. + Asking each to remember `run_id`/`actor`/`host`/`tool_version` guarantees one eventually forgets + — and an entry that can't say who made the change, from where, as part of which run, is not an + audit record. `record()` also **strips** those keys out of `**detail`, so a caller cannot + override them. Only the per-action facts (`finding_id`, `namespace`, the object, the outcome) + stay at the call site, because only the call site knows them. `engine.ApplyContext` carries + `finding_id` for the same reason: a rollback failure raised deep in the spine must still be + attributable to the vulnerability that justified the change. +2. **Run identity lives in `/run.json`, keyed by `run_id`.** `runmeta.run_id(out)` mints an id + on first use and persists it (atomic replace), so a scan and a `vpcopilot apply` an hour later + **in a separate process** against the same out dir stamp the same run — the join key is the + directory, not process memory. `write_manifest` never clobbers an existing `run_id`, so a + re-scan keeps the identity the entries already on disk join to. Provenance sits beside it: + scanned repo + commit/branch/dirty, config path, per-agent models, caps, counts, + started/finished, actor/host/tool_version. Because it lives *in the run dir*, an exported bundle + is self-describing — it doesn't need this machine, this checkout, or this console to be read. +3. **Write heterogeneous → normalize at export.** The log stays append-only and per-action: each + record keeps exactly the fields that mattered for that action (a WAF's `config_enabled` is not a + service policy's `passed`). That is right for a log and wrong for a reviewer, who needs one + sortable shape. So `export.build_audit_events` does the flattening at **read** time — coalescing + `passed ?? config_enabled ?? enabled` into one `outcome`, resolving `finding` → `finding_id`, + recovering a finding from the `policies.json` index when an older entry names none, and joining + the ledger + `findings.json` for title/class/severity/state. Normalizing at write time would + have cost per-action fidelity permanently and frozen the schema; normalizing at export keeps + both, and lets old logs improve as the normalizer learns their shapes. It keeps **every** entry: + the report's impact table filters to entries with `before_after` or `behavioral`, which would + silently drop `retire`, `open_pr`, every `create_*` and every config-only apply — an export that + quietly loses rows is worse than no export. +4. **The export is read-only and stdlib-only.** `zipfile` / `csv` / `hashlib`, no new dependencies; + it touches neither XC, GitHub, nor the run's artifacts. Evidence gathering must never be able to + change the thing it is evidence of. The bundle carries the normalized events (`audit.csv` + + `audit-events.json`, with `detail` keeping the raw JSON so flattening loses nothing), the raw + `audit.log` **verbatim**, `run.json`, the scan artifacts, the exact XC configs pushed + (`policies/*`), the pre-change LB snapshots (`snapshots/*`), and a `manifest.json` that SHA-256s + every member — plus an explicit `caveats` list stating what the trail does *not* cover. +5. **Provenance writing is fail-soft.** `git_provenance` on a non-git target contributes nothing; + a `write_manifest` failure logs a warning and returns. Provenance is evidence, not a gate — it + must never fail a scan that already succeeded. Same instinct as the ledger: a half-written + manifest reads as `{}` rather than raising. + +**Surfaces** (repo convention: console and CLI call the same module function) — +`GET /api/audit-events` (the Retire step *shows* the trail before anyone exports it, so you can +check what leaves the machine), `GET /api/audit-export?scope=run|all`, `GET /api/runs`; and +`vpcopilot export [--out DIR] [--output PATH] [--all] [--root DIR]`. + +**Honest limits.** Dry runs are unrecorded by design. `apply_timing` exists only for +console-driven live applies. Entries written by older builds lack `finding_id`/`namespace`/`actor` +and export as blank cells rather than inferred ones. `VPCOPILOT_ACTOR` overrides the OS user — set +it in CI or on a shared jump host so changes name the engineer, not the service account. + ## Repo layout ``` src/vpcopilot/ schemas.py typed agent I/O (the cross-model contract) - config.py per-agent model registry + config.py per-agent model registry (+ AGENT_NAMES) harness.py LiteLLM + instructor (model independence) repo_scan.py collect candidate source files - agents/ discover, verify, triage, generate, remediate - pipeline.py deterministic orchestration (read-only today) - cli.py `vpcopilot scan` -config/agents.yaml model-per-agent -tests/ schema/config smoke tests (no API needed) + agents/ discover, verify, triage, generate, remediate, probe, refine + pipeline.py deterministic orchestration (scan → artifacts + run.json) + engine.py SafeApply spine: snapshot → self-test → attach → validate → keep/rollback + controls.py registry of the 7 XC controls (attach/detach inverse, validation kind) + apply.py per-control apply entry points (CLI + console both call these) + refiner.py validate → refine → retry until the band-aid blocks (or gives up honestly) + xc.py / probe.py XC API client · executable exploit + legit request per finding + ledger.py found → mitigated → remediated → retired + audit.py append-only audit log (stamps run_id/actor/host/tool_version) + runmeta.py run identity + provenance (/run.json, VPCOPILOT_ACTOR) + export.py evidence bundle (.zip): normalized events + raw artifacts + manifest + report.py standalone HTML report · retire.py detach · pr.py the cure + console/ FastAPI ops console (app.py + static/index.html) + cli.py scan · apply · export · console · bench-model · … +config/agents*.yaml model-per-agent, one config per model +tests/ offline tests against fakes (no API needed) ``` ## Roadmap @@ -142,6 +225,13 @@ tests/ schema/config smoke tests (no API needed) and an Admin panel that reads/writes the local `.env`. TODO: remediation ledger, richer before/after panel. _(superseded original bullet below)_ ~~review → approve → apply → undo, with a live before/after panel and the remediation ledger.~~ +6. **Audit + evidence export (done):** ✅ every LB/object-mutating record now carries `finding_id` + + `namespace` (and `kept`), with `run_id`/`actor`/`host`/`tool_version` stamped centrally in + `audit.record`; `/run.json` (`runmeta.py`) gives a run its identity and provenance so a + scan and a later apply join up; `export.py` normalizes the log and zips the run's evidence + (`vpcopilot export [--all]`, `GET /api/audit-export?scope=run|all`); the console's Retire step + shows the trail — when · action · justified by · control · LB · outcome · by — before you export + it. `VPCOPILOT_ACTOR` names who a change is attributed to. ## Open decisions - Remediation output starts as **GitHub PRs** (confirmed). diff --git a/README.md b/README.md index 53e8c10..44ee181 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,17 @@ F5 Distributed Cloud (XC) control, and drafts the real code fix** — so the exp between "AI found a vuln" and "the code fix ships" collapses from weeks to minutes, with a human in the loop and everything reversible. -The band-aid buys time; the PR is the cure. Every mitigated finding also gets a code-fix PR, and -the copilot **validates its own band-aid against the finding's real exploit** — refining until the -exploit is actually blocked — so it never claims a fix that doesn't work. +Three properties make it something you could actually point at production: + +- **The band-aid buys time; the PR is the cure.** Every mitigated finding also gets a code-fix PR, + and a ledger tracks each one `found → mitigated → remediated → retired` so a temporary control + never quietly becomes permanent. +- **It proves its own work.** Each band-aid is validated against the finding's *real exploit*. If + the policy doesn't block, the refiner diagnoses and retries until it does — or gives up honestly + ("code fix required"). It never claims a fix that doesn't work. +- **Every change to a load balancer is on the record.** What changed, in which XC namespace, *which + vulnerability justified it*, whether it's still live, and who ran it — exportable as a + SHA-256-manifested evidence bundle for a change board. See **[docs/AUDIT.md](docs/AUDIT.md)**. It is **model-independent**: every agent's model is chosen in `config/agents.yaml`, so you run it on Claude, OpenAI, Gemini, or local Ollama — per agent or globally — with no code change. @@ -24,6 +32,9 @@ on Claude, OpenAI, Gemini, or local Ollama — per agent or globally — with no repo ─▶ discover ─▶ verify ─▶ triage ─▶ generate ─┬▶ apply (XC band-aid: snapshot → self-test → (find) (refute) (route) (XC config) │ attach → validate → refine → keep/rollback) remediate └▶ open PR (the real code fix — the cure) + │ + ledger: found → mitigated → remediated → retire (detach the band-aid) + audit: every mutating step, appended and exportable as evidence ``` - **discover → verify** find candidates and adversarially refute the weak ones (calibrated, @@ -33,15 +44,13 @@ repo ─▶ discover ─▶ verify ─▶ triage ─▶ generate ─┬▶ apply code-only when no band-aid fits. - **apply** creates/attaches the control to a live LB behind a human gate, then **validates it against the finding's own exploit** (a probe agent derives setup/exploit/legit requests). If the - policy doesn't block, the **refiner** diagnoses and retries until it does — or gives up honestly - ("code fix required"). A deterministic linter catches self-defeating policies before any live - round-trip. -- **remediate** drafts the code cure as a GitHub PR. A **ledger** tracks every finding - `found → mitigated → remediated → retired`, and **retire** detaches the band-aid once the cure + policy doesn't block, the **refiner** diagnoses and retries until it does. A deterministic linter + catches self-defeating policies before any live round-trip. +- **remediate** drafts the code cure as a GitHub PR; **retire** detaches the band-aid once the cure merges. Guardrails throughout: protected LBs/policies refuse mutation unless opted in; every apply -snapshots first and rolls back on failure. +snapshots first and rolls back on failure; dry-run is the default in the console. ## Try it in 2 minutes (no cloud, no keys) @@ -67,34 +76,43 @@ cp .env.example .env # model key(s) + XC creds + GITHUB_TOK vpcopilot console # scan, apply, PR, retire — all from the UI # or headless: vpcopilot scan /path/to/app-repo --out out +vpcopilot export --out out # evidence bundle (.zip) of every change made to an LB ``` `scan` writes `out/` (`findings.json`, `triage.json`, `policies/*.json`, code-fix PR drafts, -`report.html`) and performs **no** XC or GitHub writes — safe to run anywhere. Live changes happen -only in `apply` / `pr` / `retire`, behind the gate. Full command reference: **[docs/USAGE.md](docs/USAGE.md)**. +`report.html`, and a `run.json` provenance manifest) and performs **no** XC or GitHub writes — safe +to run anywhere. Live changes happen only in `apply` / `pr` / `retire`, behind the gate. Full command +reference: **[docs/USAGE.md](docs/USAGE.md)**. ## The console A guided flow that follows the lifecycle — a persistent hero band (N exploitable → mitigated live -in seconds vs. change-control days) sits on top of five steps: +in seconds vs. change-control days) sits on top of six steps: -1. **Scan** — point at a repo; read-only, safe. +1. **Scan** — point at a repo; read-only, safe. The log holds the whole transcript in a scrollable + box, so a long run can be read end-to-end while it's still going. 2. **Review** — findings + the recommended XC control; click a row to inspect exploit / code / policy. 3. **Mitigate** — apply each band-aid live; the refiner streams `before 200 → after 403 BLOCKED` with a *self-healed in N attempts* / *unfixable → ship the code fix* badge. 4. **Cure** — open the code-fix PR for each finding. -5. **Retire** — the four-state ledger track; detach a band-aid once its cure merges. +5. **Retire** — the four-state ledger track, and the **audit trail** of every change made to a load + balancer — exportable as an evidence bundle. +6. **Benchmark** — build a model-tagged report from this run, then compare models side by side. -Credentials, XC status, the per-agent model wiring, and the shareable HTML report live under -**Setup**. +The shareable HTML report opens (or downloads) from **Review** and from **Setup** — it is rebuilt +from the current run dir on every open, so it's always the latest run. Credentials, XC status, and +the per-agent model wiring live under **Setup**. **③ Mitigate** — apply each band-aid and watch it validate: ![Mitigate step](docs/images/3-mitigate.png) -**⑤ Retire** — the four-state ledger (here `crapi-sqli-001` walked all the way to *retired*): +**⑤ Retire** — the four-state ledger (here `crapi-sqli-001` walked all the way to *retired*), and +under it the audit trail: each change tied to the vulnerability that justified it, the LB and XC +namespace it touched, whether it's still live, and who ran it. Dry runs are absent by design — +nothing changed, so there is nothing to answer for. -![Retire step](docs/images/5-retire.png) +![Retire step — ledger and audit trail](docs/images/5-retire.png) Every scan also drops a standalone, shareable **`report.html`** — the same hero plus at-a-glance bars, the self-heal (`200 → 403`, *self-healed ×2*), the rate-limit behavioral proof, and the @@ -109,6 +127,7 @@ ledger: | [docs/TRY_IT.md](docs/TRY_IT.md) | try it on safe repos (VAmPI / crAPI) before your own | | [docs/DEMO.md](docs/DEMO.md) | 5-minute runbook (offline + live) | | [docs/USAGE.md](docs/USAGE.md) | full CLI + console reference | +| [docs/AUDIT.md](docs/AUDIT.md) | the audit trail and the evidence export — what is recorded, and how to verify a bundle | | [DESIGN.md](DESIGN.md) | architecture | | [MODELS.md](MODELS.md) | cross-provider model notes | | [docs/QUALITY_PLAN.md](docs/QUALITY_PLAN.md) | quality burn-down | @@ -124,7 +143,10 @@ cloud needed). See [CONTRIBUTING.md](CONTRIBUTING.md). This is a **dual-use security tool**. `scan` is read-only; `apply` / `pr` / `retire` change live systems and validation fires real exploits. Use it only against systems you own or are explicitly -authorized to test. Reporting and guardrails: [SECURITY.md](SECURITY.md). +authorized to test. The audit trail is a record of what this tool did — it is not tamper-evident, +and it is evidence for a human reviewer rather than a compliance certification +([docs/AUDIT.md](docs/AUDIT.md) is explicit about the limits). Reporting and guardrails: +[SECURITY.md](SECURITY.md). ## License diff --git a/demo/build_demo_out.py b/demo/build_demo_out.py index cf2c77b..b0490fc 100644 --- a/demo/build_demo_out.py +++ b/demo/build_demo_out.py @@ -16,7 +16,8 @@ OUT = ROOT / "out" sys.path.insert(0, str(ROOT.parent / "src")) -from vpcopilot import audit, ledger # noqa: E402 +from vpcopilot import audit, ledger, runmeta # noqa: E402 +from vpcopilot.config import AGENT_NAMES # noqa: E402 # --- the curated crAPI-flavoured findings ----------------------------------- FINDINGS = [ @@ -53,6 +54,8 @@ COV = {"service_policy": "full", "api_schema": "full", "waf": "partial", "rate_limit": "full", "waf_data_guard": "partial"} LB = "crapi-lab" +NS = "crapi-demo" # the XC namespace the curated run "mutated" +ACTOR, HOST = "security-oncall", "vpcopilot-demo" # see _curate_identity() TRIAGE = [] for f in FINDINGS: @@ -90,6 +93,39 @@ def _blocked(status): # helper for before/after cells return {"exploit_status": status, "exploit_blocked": status in (403,), "legit_ok": True} +def _curate_identity(): + """Rewrite `actor` / `host` / `out_dir` in the generated fixture to curated, portable values. + + `audit.record` stamps the real OS user and hostname, and the writers record an absolute out dir + — all correct for a real run, all wrong for a dataset committed to a public repo and + screenshotted into the README. Normalizing here keeps the fiction where it belongs (this curated + builder) rather than adding a spoofing knob to `runmeta`, which the audit trail depends on being + honest.""" + ident = {"actor": ACTOR, "host": HOST} + log = OUT / "audit.log" + if log.exists(): + # Space the timestamps out. The builder writes the whole trail in one burst, which would + # show 13 identical clock times — the shape of a fixture, not of a mitigation session. + from datetime import datetime, timedelta, timezone + t, out = datetime.now(timezone.utc) - timedelta(minutes=9), [] + for ln in log.read_text().splitlines(): + if not ln.strip(): + continue + e = json.loads(ln) + out.append(json.dumps({**e, **ident, "ts": t.isoformat()})) + t += timedelta(seconds=int(e.get("elapsed_s") or 0) + (45 if e["action"] == "retire" else 4)) + log.write_text("\n".join(out) + "\n") + for name in ("run.json", "summary.json"): + p = OUT / name + if not p.exists(): + continue + d = json.loads(p.read_text()) + d.update({k: v for k, v in ident.items() if k in d}) + if "out_dir" in d: + d["out_dir"] = "demo/out" # portable — no home directory in a shared fixture + p.write_text(json.dumps(d, indent=2)) + + def main(): # Start from a clean slate so the dataset is deterministic — audit.record and the ledger APPEND, # so regenerating over an existing out/ would otherwise inflate the action log on every run. @@ -120,26 +156,46 @@ def main(): ledger.mark_remediated(str(OUT), "crapi-tokenleak-006", pr_url="https://github.com/acme/crapi/pull/313", pr_number=313) ledger.mark_retired(str(OUT), "crapi-sqli-001") # cure shipped -> band-aid removed - # audit trail: the SQLi service policy self-heals (attempt 1 fails, attempt 2 blocks) - audit.record(str(OUT), "refine_apply", control="service_policy", policy="deny-login-sqli", lb=LB, - passed=True, attempts=2, before_after={"before": _blocked(200), "after": _blocked(403)}) + # F3) run manifest — so the curated dataset also demonstrates provenance + the evidence export + runmeta.write_manifest(str(OUT), repo="/src/crapi", config_path="config/agents.yaml", + models={a: "anthropic/claude-opus-4-8" for a in AGENT_NAMES}, + caps={"min_confidence": 0.5, "max_files": 200, "max_bytes": 60000, + "draft_code_fixes": True}, + counts={"candidates": 9, "verified": 6, "policies": 5, "code_fix_prs": 6}) + + # audit trail: the SQLi service policy self-heals (attempt 1 fails, attempt 2 blocks). Every + # LB-mutating record carries the finding that justified it and the namespace it happened in — + # the same shape a real run writes, so the Retire step's audit trail is fully populated here. ba = {"before": _blocked(200), "after": _blocked(403)} + audit.record(str(OUT), "refine_apply", finding_id="crapi-sqli-001", namespace=NS, + control="service_policy", policy="deny-login-sqli", lb=LB, + passed=True, attempts=2, before_after=ba) audit.record(str(OUT), "apply_timing", control="service_policy", finding_id="crapi-sqli-001", passed=True, elapsed_s=48.0, attempts=2, before_after=ba) - audit.record(str(OUT), "apply_api_schema", apidef="crapi-lab-apidef", lb=LB, passed=True, before_after=ba) + audit.record(str(OUT), "create_api_definition", finding_id="crapi-bola-002", namespace=NS, + name="crapi-lab-apidef", swagger="crapi-lab-swagger") + audit.record(str(OUT), "apply_api_schema", finding_id="crapi-bola-002", namespace=NS, + apidef="crapi-lab-apidef", lb=LB, passed=True, kept=True, before_after=ba) audit.record(str(OUT), "apply_timing", control="api_schema", finding_id="crapi-bola-002", passed=True, elapsed_s=33.0, attempts=1, before_after=ba) - audit.record(str(OUT), "apply_waf", app_firewall="crapi-lab-waf", lb=LB, passed=True, before_after=ba) + audit.record(str(OUT), "apply_waf", finding_id="crapi-mass-003", namespace=NS, + app_firewall="crapi-lab-waf", lb=LB, config_enabled=True, kept=True, before_after=ba) audit.record(str(OUT), "apply_timing", control="waf", finding_id="crapi-mass-003", passed=True, elapsed_s=21.0, attempts=1, before_after=ba) - audit.record(str(OUT), "apply_rate_limit", rate="5/MINUTE", lb=LB, passed=True, + audit.record(str(OUT), "apply_rate_limit", finding_id="crapi-bruteforce-004", namespace=NS, + rate="5/MINUTE", lb=LB, passed=True, kept=True, behavioral={"sent": 30, "limited": 25, "passed": 5, "codes": {"200": 5, "429": 25}}) audit.record(str(OUT), "apply_timing", control="rate_limit", finding_id="crapi-bruteforce-004", passed=True, elapsed_s=27.0, attempts=1) - audit.record(str(OUT), "apply_data_guard", lb=LB, config_enabled=True) + audit.record(str(OUT), "apply_data_guard", finding_id="crapi-tokenleak-006", namespace=NS, + app_firewall="crapi-lab-waf", lb=LB, enabled=True, kept=True) audit.record(str(OUT), "apply_timing", control="waf_data_guard", finding_id="crapi-tokenleak-006", passed=True, elapsed_s=19.0, attempts=1) - audit.record(str(OUT), "retire", finding_id="crapi-sqli-001", control="service_policy", lb=LB, forced=False) + audit.record(str(OUT), "open_pr", finding_id="crapi-sqli-001", finding="crapi-sqli-001", + repo="acme/crapi", url="https://github.com/acme/crapi/pull/311", number=311) + audit.record(str(OUT), "retire", finding_id="crapi-sqli-001", namespace=NS, + control="service_policy", lb=LB, forced=False) + _curate_identity() from vpcopilot.report import write_report path = write_report(str(OUT)) diff --git a/demo/out/audit.log b/demo/out/audit.log index 964e5f4..1d08f94 100644 --- a/demo/out/audit.log +++ b/demo/out/audit.log @@ -1,11 +1,13 @@ -{"ts": "2026-07-13T18:17:32.618025+00:00", "action": "refine_apply", "control": "service_policy", "policy": "deny-login-sqli", "lb": "crapi-lab", "passed": true, "attempts": 2, "before_after": {"before": {"exploit_status": 200, "exploit_blocked": false, "legit_ok": true}, "after": {"exploit_status": 403, "exploit_blocked": true, "legit_ok": true}}} -{"ts": "2026-07-13T18:17:32.618538+00:00", "action": "apply_timing", "control": "service_policy", "finding_id": "crapi-sqli-001", "passed": true, "elapsed_s": 48.0, "attempts": 2, "before_after": {"before": {"exploit_status": 200, "exploit_blocked": false, "legit_ok": true}, "after": {"exploit_status": 403, "exploit_blocked": true, "legit_ok": true}}} -{"ts": "2026-07-13T18:17:32.618762+00:00", "action": "apply_api_schema", "apidef": "crapi-lab-apidef", "lb": "crapi-lab", "passed": true, "before_after": {"before": {"exploit_status": 200, "exploit_blocked": false, "legit_ok": true}, "after": {"exploit_status": 403, "exploit_blocked": true, "legit_ok": true}}} -{"ts": "2026-07-13T18:17:32.618979+00:00", "action": "apply_timing", "control": "api_schema", "finding_id": "crapi-bola-002", "passed": true, "elapsed_s": 33.0, "attempts": 1, "before_after": {"before": {"exploit_status": 200, "exploit_blocked": false, "legit_ok": true}, "after": {"exploit_status": 403, "exploit_blocked": true, "legit_ok": true}}} -{"ts": "2026-07-13T18:17:32.619159+00:00", "action": "apply_waf", "app_firewall": "crapi-lab-waf", "lb": "crapi-lab", "passed": true, "before_after": {"before": {"exploit_status": 200, "exploit_blocked": false, "legit_ok": true}, "after": {"exploit_status": 403, "exploit_blocked": true, "legit_ok": true}}} -{"ts": "2026-07-13T18:17:32.619333+00:00", "action": "apply_timing", "control": "waf", "finding_id": "crapi-mass-003", "passed": true, "elapsed_s": 21.0, "attempts": 1, "before_after": {"before": {"exploit_status": 200, "exploit_blocked": false, "legit_ok": true}, "after": {"exploit_status": 403, "exploit_blocked": true, "legit_ok": true}}} -{"ts": "2026-07-13T18:17:32.619586+00:00", "action": "apply_rate_limit", "rate": "5/MINUTE", "lb": "crapi-lab", "passed": true, "behavioral": {"sent": 30, "limited": 25, "passed": 5, "codes": {"200": 5, "429": 25}}} -{"ts": "2026-07-13T18:17:32.620505+00:00", "action": "apply_timing", "control": "rate_limit", "finding_id": "crapi-bruteforce-004", "passed": true, "elapsed_s": 27.0, "attempts": 1} -{"ts": "2026-07-13T18:17:32.620750+00:00", "action": "apply_data_guard", "lb": "crapi-lab", "config_enabled": true} -{"ts": "2026-07-13T18:17:32.621467+00:00", "action": "apply_timing", "control": "waf_data_guard", "finding_id": "crapi-tokenleak-006", "passed": true, "elapsed_s": 19.0, "attempts": 1} -{"ts": "2026-07-13T18:17:32.621743+00:00", "action": "retire", "finding_id": "crapi-sqli-001", "control": "service_policy", "lb": "crapi-lab", "forced": false} +{"ts": "2026-07-27T01:40:25.195575+00:00", "action": "refine_apply", "run_id": "14a3e38c7569", "actor": "security-oncall", "host": "vpcopilot-demo", "tool_version": "0.1.0", "finding_id": "crapi-sqli-001", "namespace": "crapi-demo", "control": "service_policy", "policy": "deny-login-sqli", "lb": "crapi-lab", "passed": true, "attempts": 2, "before_after": {"before": {"exploit_status": 200, "exploit_blocked": false, "legit_ok": true}, "after": {"exploit_status": 403, "exploit_blocked": true, "legit_ok": true}}} +{"ts": "2026-07-27T01:40:29.195575+00:00", "action": "apply_timing", "run_id": "14a3e38c7569", "actor": "security-oncall", "host": "vpcopilot-demo", "tool_version": "0.1.0", "control": "service_policy", "finding_id": "crapi-sqli-001", "passed": true, "elapsed_s": 48.0, "attempts": 2, "before_after": {"before": {"exploit_status": 200, "exploit_blocked": false, "legit_ok": true}, "after": {"exploit_status": 403, "exploit_blocked": true, "legit_ok": true}}} +{"ts": "2026-07-27T01:41:21.195575+00:00", "action": "create_api_definition", "run_id": "14a3e38c7569", "actor": "security-oncall", "host": "vpcopilot-demo", "tool_version": "0.1.0", "finding_id": "crapi-bola-002", "namespace": "crapi-demo", "name": "crapi-lab-apidef", "swagger": "crapi-lab-swagger"} +{"ts": "2026-07-27T01:41:25.195575+00:00", "action": "apply_api_schema", "run_id": "14a3e38c7569", "actor": "security-oncall", "host": "vpcopilot-demo", "tool_version": "0.1.0", "finding_id": "crapi-bola-002", "namespace": "crapi-demo", "apidef": "crapi-lab-apidef", "lb": "crapi-lab", "passed": true, "kept": true, "before_after": {"before": {"exploit_status": 200, "exploit_blocked": false, "legit_ok": true}, "after": {"exploit_status": 403, "exploit_blocked": true, "legit_ok": true}}} +{"ts": "2026-07-27T01:41:29.195575+00:00", "action": "apply_timing", "run_id": "14a3e38c7569", "actor": "security-oncall", "host": "vpcopilot-demo", "tool_version": "0.1.0", "control": "api_schema", "finding_id": "crapi-bola-002", "passed": true, "elapsed_s": 33.0, "attempts": 1, "before_after": {"before": {"exploit_status": 200, "exploit_blocked": false, "legit_ok": true}, "after": {"exploit_status": 403, "exploit_blocked": true, "legit_ok": true}}} +{"ts": "2026-07-27T01:42:06.195575+00:00", "action": "apply_waf", "run_id": "14a3e38c7569", "actor": "security-oncall", "host": "vpcopilot-demo", "tool_version": "0.1.0", "finding_id": "crapi-mass-003", "namespace": "crapi-demo", "app_firewall": "crapi-lab-waf", "lb": "crapi-lab", "config_enabled": true, "kept": true, "before_after": {"before": {"exploit_status": 200, "exploit_blocked": false, "legit_ok": true}, "after": {"exploit_status": 403, "exploit_blocked": true, "legit_ok": true}}} +{"ts": "2026-07-27T01:42:10.195575+00:00", "action": "apply_timing", "run_id": "14a3e38c7569", "actor": "security-oncall", "host": "vpcopilot-demo", "tool_version": "0.1.0", "control": "waf", "finding_id": "crapi-mass-003", "passed": true, "elapsed_s": 21.0, "attempts": 1, "before_after": {"before": {"exploit_status": 200, "exploit_blocked": false, "legit_ok": true}, "after": {"exploit_status": 403, "exploit_blocked": true, "legit_ok": true}}} +{"ts": "2026-07-27T01:42:35.195575+00:00", "action": "apply_rate_limit", "run_id": "14a3e38c7569", "actor": "security-oncall", "host": "vpcopilot-demo", "tool_version": "0.1.0", "finding_id": "crapi-bruteforce-004", "namespace": "crapi-demo", "rate": "5/MINUTE", "lb": "crapi-lab", "passed": true, "kept": true, "behavioral": {"sent": 30, "limited": 25, "passed": 5, "codes": {"200": 5, "429": 25}}} +{"ts": "2026-07-27T01:42:39.195575+00:00", "action": "apply_timing", "run_id": "14a3e38c7569", "actor": "security-oncall", "host": "vpcopilot-demo", "tool_version": "0.1.0", "control": "rate_limit", "finding_id": "crapi-bruteforce-004", "passed": true, "elapsed_s": 27.0, "attempts": 1} +{"ts": "2026-07-27T01:43:10.195575+00:00", "action": "apply_data_guard", "run_id": "14a3e38c7569", "actor": "security-oncall", "host": "vpcopilot-demo", "tool_version": "0.1.0", "finding_id": "crapi-tokenleak-006", "namespace": "crapi-demo", "app_firewall": "crapi-lab-waf", "lb": "crapi-lab", "enabled": true, "kept": true} +{"ts": "2026-07-27T01:43:14.195575+00:00", "action": "apply_timing", "run_id": "14a3e38c7569", "actor": "security-oncall", "host": "vpcopilot-demo", "tool_version": "0.1.0", "control": "waf_data_guard", "finding_id": "crapi-tokenleak-006", "passed": true, "elapsed_s": 19.0, "attempts": 1} +{"ts": "2026-07-27T01:43:37.195575+00:00", "action": "open_pr", "run_id": "14a3e38c7569", "actor": "security-oncall", "host": "vpcopilot-demo", "tool_version": "0.1.0", "finding_id": "crapi-sqli-001", "finding": "crapi-sqli-001", "repo": "acme/crapi", "url": "https://github.com/acme/crapi/pull/311", "number": 311} +{"ts": "2026-07-27T01:43:41.195575+00:00", "action": "retire", "run_id": "14a3e38c7569", "actor": "security-oncall", "host": "vpcopilot-demo", "tool_version": "0.1.0", "finding_id": "crapi-sqli-001", "namespace": "crapi-demo", "control": "service_policy", "lb": "crapi-lab", "forced": false} diff --git a/demo/out/report.html b/demo/out/report.html index b36e0a5..68f7305 100644 --- a/demo/out/report.html +++ b/demo/out/report.html @@ -57,7 +57,7 @@ .model .a{color:var(--grey)}.model .m{font-family:ui-monospace,Menlo,monospace;color:var(--f5)}

virtual-patch·copilot — scan report

-
target: crapi-lab · generated 2026-07-13 18:17 UTC
+
target: crapi-lab · generated 2026-07-27 01:49 UTC
6exploitable vulns
→
5mitigated live by XC
29.6stime to mitigate · 72,973× faster
vs
25 daysnormal change control
6code-fix PRs (the cure)

Run summary

9candidates
6verified
5band-aided
1code-cure only
5XC policies
6code-fix PRs
@@ -66,7 +66,7 @@

Model independence each agent's model is set per-agent in

Pipeline metrics

19.2stotal time
6.4sdiscover
4.1sverify
8.7ssynthesize
67%verify confirm-rate
0.86avg confidence
1dupe band-aids collapsed

verify: 9 candidates → 6 verified, 2 refuted, 1 dropped < 0.5 confidence

Findings & band-aid coverage

criticalSQL injection in logincrapi-sqli-001sqliservices/identity/login.js:42
service_policy · full✓ code fix drafted
Details

Description: The email field is concatenated straight into the auth query.

Exploit: email=" OR 1=1 -- lets any password through and dumps the users table.

Code cure: Fix sql injection in login

highBOLA on vehicle locationcrapi-bola-002broken_object_authzservices/identity/vehicle.js:88
api_schema · full✓ code fix drafted
Details

Description: Any authenticated user can read another user's vehicle GPS by id.

Exploit: Swap {id} to another user's vehicle uuid; server returns their live location.

Code cure: Fix bola on vehicle location

highMass assignment on profilecrapi-mass-003mass_assignmentservices/identity/dashboard.js:61
waf · partial✓ code fix drafted
Details

Description: The update handler binds the whole body, so `role` and `credit` are writable.

Exploit: POST {"role":"admin","available_credit":9999} — privilege + balance escalation.

Code cure: Fix mass assignment on profile

highJWT + card data in responsecrapi-tokenleak-006sensitive_dataservices/workshop/mechanic.js:120
waf_data_guard · partial✓ code fix drafted
Details

Description: The receipt payload echoes the full PAN and a signed service token.

Exploit: GET a receipt; response body contains the 16-digit card number in cleartext.

Code cure: Fix jwt + card data in response

mediumNo rate limit on OTP verifycrapi-bruteforce-004rate_abuseservices/identity/otp.js:30
rate_limit · full✓ code fix drafted
Details

Description: The 4-digit OTP endpoint has no throttle — brute-forceable in minutes.

Exploit: Fire all 10k OTPs; no lockout, no delay.

Code cure: Fix no rate limit on otp verify

mediumUsername enumeration on signupcrapi-userenum-005broken_authservices/identity/signup.js:25
no band-aid — code cure only✓ code fix drafted
Residual risk: no positive-security band-aid fits; ships as code-only.
Details

Description: Distinct errors for taken vs free emails leak which accounts exist.

Exploit: Diff the 'already registered' vs 'ok' responses to enumerate users.

Code cure: Fix username enumeration on signup

Generated XC band-aid policies

api_schema crapi-lab-apidef
rate_limit otp-throttle
service_policy deny-login-sqli
waf crapi-lab-waf
waf_data_guard mask-pan
-

Band-aid impact exploit before → after (live validation)

controlpolicyexploit beforeexploit afterlegitresultwhen
service_policy self-healed ×2deny-login-sqli200 allowed403 blockedokPASS2026-07-13T18:17:32
apply_timing self-healed ×2200 allowed403 blockedokPASS2026-07-13T18:17:32
api_schemacrapi-lab-apidef200 allowed403 blockedokPASS2026-07-13T18:17:32
apply_timing200 allowed403 blockedokPASS2026-07-13T18:17:32
wafcrapi-lab-waf200 allowed403 blockedokPASS2026-07-13T18:17:32
apply_timing200 allowed403 blockedokPASS2026-07-13T18:17:32
rate_limit5/MINUTEburst 30 allowed25/30 rate-limited (429)—PASS2026-07-13T18:17:32
+

Band-aid impact exploit before → after (live validation)

controlpolicyexploit beforeexploit afterlegitresultwhen
service_policy self-healed ×2deny-login-sqli200 allowed403 blockedokPASS2026-07-27T01:40:25
apply_timing self-healed ×2200 allowed403 blockedokPASS2026-07-27T01:40:29
api_schemacrapi-lab-apidef200 allowed403 blockedokPASS2026-07-27T01:41:25
apply_timing200 allowed403 blockedokPASS2026-07-27T01:41:29
wafcrapi-lab-waf200 allowed403 blockedokfail2026-07-27T01:42:06
apply_timing200 allowed403 blockedokPASS2026-07-27T01:42:10
rate_limit5/MINUTEburst 30 allowed25/30 rate-limited (429)—PASS2026-07-27T01:42:35

Remediation ledger found → mitigated → remediated → retired

findingstateband-aidcode cure
crapi-sqli-001retired{'control': 'service_policy', 'policy_name': 'deny-login-sqli', 'lb': 'crapi-lab'}{'pr_url': 'https://github.com/acme/crapi/pull/311', 'pr_number': 311}
crapi-bola-002remediated{'control': 'api_schema', 'policy_name': 'crapi-lab-apidef', 'lb': 'crapi-lab'}{'pr_url': 'https://github.com/acme/crapi/pull/312', 'pr_number': 312}
crapi-mass-003mitigated{'control': 'waf', 'policy_name': 'crapi-lab-waf', 'lb': 'crapi-lab'}—
crapi-bruteforce-004mitigated{'control': 'rate_limit', 'policy_name': 'otp-throttle', 'lb': 'crapi-lab'}—
crapi-tokenleak-006remediated{'control': 'waf_data_guard', 'policy_name': 'mask-pan', 'lb': 'crapi-lab'}{'pr_url': 'https://github.com/acme/crapi/pull/313', 'pr_number': 313}
crapi-userenum-005found——

virtual-patch-copilot 0.1.0 · band-aids are temporary — every finding also gets a code-fix PR
diff --git a/demo/out/run.json b/demo/out/run.json new file mode 100644 index 0000000..0936b2b --- /dev/null +++ b/demo/out/run.json @@ -0,0 +1,31 @@ +{ + "run_id": "14a3e38c7569", + "created": "2026-07-27T01:49:25.186311+00:00", + "repo": "/src/crapi", + "config_path": "config/agents.yaml", + "models": { + "discover": "anthropic/claude-opus-4-8", + "verify": "anthropic/claude-opus-4-8", + "triage": "anthropic/claude-opus-4-8", + "generate": "anthropic/claude-opus-4-8", + "remediate": "anthropic/claude-opus-4-8", + "probe": "anthropic/claude-opus-4-8", + "refine": "anthropic/claude-opus-4-8" + }, + "caps": { + "min_confidence": 0.5, + "max_files": 200, + "max_bytes": 60000, + "draft_code_fixes": true + }, + "counts": { + "candidates": 9, + "verified": 6, + "policies": 5, + "code_fix_prs": 6 + }, + "actor": "security-oncall", + "host": "vpcopilot-demo", + "tool_version": "0.1.0", + "out_dir": "demo/out" +} \ No newline at end of file diff --git a/demo/out/summary.json b/demo/out/summary.json index 04a782d..8dc1ef4 100644 --- a/demo/out/summary.json +++ b/demo/out/summary.json @@ -20,5 +20,5 @@ "crapi-userenum-005" ], "correlations": [], - "out_dir": "/Users/d.henley/demos/virtual-patch-copilot/demo/out" + "out_dir": "demo/out" } \ No newline at end of file diff --git a/docs/AUDIT.md b/docs/AUDIT.md new file mode 100644 index 0000000..ff6b5b4 --- /dev/null +++ b/docs/AUDIT.md @@ -0,0 +1,509 @@ +# Audit trail & evidence export + +Every change the copilot makes to live F5 XC config is written to an append-only log, stamped with +who made it, from where, and which run it belongs to — and can be exported as a self-describing +`.zip` for a change board or an auditor. + +The question this exists to answer: **"why was this load balancer changed, by whom, in which +tenant, and is the change still live?"** + +Source of truth: `src/vpcopilot/audit.py` (the sink), `runmeta.py` (identity + provenance), +`export.py` (normalization + bundle). Read those if this doc and the code ever disagree. + +--- + +## 1. What is recorded — and what is not + +**Recorded:** every mutating action in the **finding lifecycle** — creating an XC object, +attaching/enabling a band-aid on an LB, a refine-and-retry loop, opening a code-fix PR, retiring a +band-aid, and a rollback that failed. One JSON object per line in `/audit.log`, never rewritten. + +That scope is the whole band-aid path, but it is not literally every XC write the tool can make: the +lab/teardown utilities `vpcopilot lab-create` (`lab.py`) and `vpcopilot xc-rm` (`cli.py`) mutate XC +outside the finding lifecycle and write **no** audit record. They are setup/cleanup helpers, not +remediation — but if you use them against a real tenant, the trail will not show it. + +**Not recorded:** + +| Not in the trail | Why | +|---|---| +| Dry runs (`--dry-run`) | Nothing changed, so there is nothing to answer for. A dry run does a GET, computes a diff, and prints it — the LB is byte-identical afterwards. Logging previews would pad the trail with non-events and make "N changes to this LB" a lie. | +| Scans | Read-only. The scan's own outputs (`findings.json`, `triage.json`, …) are the record, and `run.json` describes the run. | +| Failed *attempts* inside a refine loop | `refine_apply` records one terminal entry per invocation with `attempts=N`. The per-attempt narration is in the console job log, not the audit log. | +| Anything XC did on its own side | This is the record of what the **copilot** sent. XC's own tenant audit log is the authority for what XC received and enforced. | +| `apply_timing` on CLI runs | Written only by the console's Mitigate step (see §2). | + +The bundle manifest states these caveats **in-band** (`manifest.json → caveats`), so an exported +zip can never imply more coverage than it has: + +```json +"caveats": [ + "Dry runs are not recorded: nothing was changed, so nothing is logged. This is the record of changes MADE, not of every attempt.", + "apply_timing entries are written only by the console's Mitigate step on a live (non-dry) apply; a CLI-driven run has none.", + "Entries written by older builds may lack finding_id / namespace / actor — those cells are blank rather than inferred." +] +``` + +### Honest limits + +- The log is written by the same process that makes the change, to a local file. It is **not + tamper-evident** — anyone who can write the out dir can edit `audit.log`. The manifest's SHA-256s + prove a bundle was not altered *after export*; they say nothing about the authenticity of the log + before it. If you need tamper-evidence, ship the out dir to append-only storage. +- `actor` is self-asserted (`VPCOPILOT_ACTOR`, else the OS user) — it is attribution, not + authentication. +- Nothing in `export.py` calls XC or GitHub. The trail says what the tool *did*; the LB itself is + the authority for what is live *now*. +- The bundle is evidence for a human reviewer. It is not a compliance certification. + +--- + +## 2. The audit entry + +`audit.record(out_dir, action, **detail)` writes one line to `/audit.log`. + +### Always present — stamped centrally + +Identity is stamped inside `record()`, not at the call sites, so no mutating path can forget it and +no caller can spoof it. The reserved keys are stripped from `**detail` before the entry is built: + +```python +_STAMPED = ("ts", "run_id", "actor", "host", "tool_version") +detail = {k: v for k, v in detail.items() if k not in _STAMPED} +``` + +| Field | Meaning | +|---|---| +| `ts` | UTC ISO-8601 timestamp (`runmeta.utc_now()`) | +| `action` | one of the 15 below | +| `run_id` | 12-hex id of the run dir — joins the entry to `/run.json`. Minted on first use and persisted, so a scan and a later `vpcopilot apply` against the same dir share it | +| `actor` | `VPCOPILOT_ACTOR` if set, else the OS user, else `unknown` | +| `host` | `socket.gethostname()`, else `unknown` | +| `tool_version` | `vpcopilot.__version__` | + +Everything else is per-action detail. + +### The 15 actions + +| action | category | what it means | key detail fields | +|---|---|---|---| +| `apply_service_policy` | mitigate | A service policy was attached to an LB and validated by firing the real exploit + a legit request | `finding_id` `lb` `namespace` `policy` `passed` `rolled_back` `kept` `before_after` | +| `create_service_policy` | create | A service policy object was created in the namespace (from-scan path, before attach) | `finding_id` `policy` `namespace` | +| `apply_malicious_user` | mitigate | Malicious-User detection enabled on the LB; validated by config readback (behavioral — not single-request testable) | `finding_id` `lb` `namespace` `enabled` `rolled_back` `kept` | +| `apply_rate_limit` | mitigate | Rate limiting set on the LB; optionally validated by driving a real burst | `finding_id` `lb` `namespace` `enabled` `passed` `rolled_back` `kept` `rate` `behavioral` | +| `apply_bot_defense` | mitigate | Bot Defense enabled on the LB; config readback | `finding_id` `lb` `namespace` `enabled` `rolled_back` `kept` | +| `create_app_firewall` | create | A **Blocking** `app_firewall` object was created (cloned from a template) because the named one did not exist | `finding_id` `name` `namespace` `mode` | +| `apply_waf` | mitigate | An `app_firewall` was attached to the LB; config readback plus an exploit probe (a single-request block is payload-dependent, so it is not scored pass/fail) | `finding_id` `lb` `namespace` `app_firewall` `config_enabled` `rolled_back` `kept` `before_after` | +| `apply_data_guard` | mitigate | Data Guard rules attached (reusing the LB's existing WAF); config readback | `finding_id` `lb` `namespace` `app_firewall` `enabled` `rolled_back` `kept` | +| `create_api_definition` | create | An OpenAPI spec was uploaded and an `api_definition` created/replaced | `finding_id` `name` `namespace` `swagger` | +| `apply_api_schema` | mitigate | The validation-block `api_specification` was attached to the LB and validated live | `finding_id` `lb` `namespace` `apidef` `passed` `rolled_back` `kept` `before_after` | +| `refine_apply` | refine | One full self-healing service-policy loop: apply → validate → diagnose → refine → retry, up to `max_refine`. Exactly one terminal entry per invocation | `finding_id` `namespace` `control` `policy` `lb` `passed` `attempts` + one of `rolled_back`/`before_after`, `unfixable`+`recommend`, or `reason` | +| `retire` | retire | A live band-aid was detached from the LB and the ledger marked `retired` | `finding_id` `control` `lb` `namespace` `forced` | +| `open_pr` | cure | The code-fix PR (the cure) was opened on GitHub | `finding_id` `finding` `repo` `url` `number` | +| `rollback_failed` | rollback | The LB could **not** be confirmed restored to its pre-apply snapshot after N retries. The one entry that must never be anonymous — the LB may be left in a changed state | `finding_id` `lb` `namespace` `reason` | +| `apply_timing` | timing | Wall-clock + outcome for one console Mitigate click. Feeds MTTM on the hero panel and policy quality in the model benchmark | `control` `finding_id` `passed` `elapsed_s` `attempts` `before_after` `unfixable` `reason` `kept` | + +Notes read from the source: + +- `open_pr` and `apply_timing` carry **no** `namespace`, and `apply_timing` carries no `lb` either + — it is a wrapper around whichever action just ran, and that action logged the LB and namespace + itself on the adjacent line. +- `apply_timing.passed` is **optimistic**: it is the wrapped action's `passed`, else + `config_enabled is not False` (`console/app.py::_run_action`). An action reporting neither key + records `passed: true` having proved nothing, and `impact.py` counts that toward MTTM. For actual + proof, read `exploit_before → exploit_after` on the adjacent entry, not this one. +- `apply_rate_limit`'s live proof is `behavioral` — `{sent, limited, passed, codes}` — not + `before_after`. +- `before_after` is `{"before": {…}, "after": {…}}` where each side is the normalized probe result + `{exploit_status, exploit_blocked, legit_ok}`. +- `apply_timing` is written by the console only, and only when `dry_run` is false + (`console/app.py::_run_action`). A CLI-driven run has none — MTTM and `elapsed_s` are simply + absent, not zero. + +### Reading it raw + +```sh +vpcopilot audit --out out # rich table of the log +curl -s 127.0.0.1:8787/api/audit # the raw entries, unnormalized +``` + +--- + +## 3. `run.json` — the run manifest + +`/run.json` is the per-run provenance record every audit entry points back to via `run_id`. +Written atomically (temp file + `os.replace`), merged never clobbered — a re-scan of the same dir +keeps its `run_id`, so audit entries already on disk stay joinable. + +| Field | Source | +|---|---| +| `run_id` | `uuid4().hex[:12]`, minted on first use, never overwritten | +| `created` | UTC when the dir first got an identity | +| `repo` | absolute path of the scanned repo (`root.resolve()`) | +| `repo_commit` / `repo_branch` / `repo_dirty` | `runmeta.git_provenance(repo)` — `git rev-parse HEAD`, `rev-parse --abbrev-ref HEAD`, `git status --porcelain`. **Fail-soft**: a target that is not a git checkout contributes none of these keys at all | +| `config_path` | the `config/agents*.yaml` the scan ran with — **absent** when the scan used the default config (`runmeta.write_manifest` drops `None` fields) | +| `models` | `{agent: model}` for each of `config.AGENT_NAMES` = `discover, verify, triage, generate, remediate, probe, refine` | +| `caps` | `{min_confidence, max_files, max_bytes, draft_code_fixes}` — the limits the scan ran under | +| `counts` | `{candidates, verified, policies, code_fix_prs}` | +| `started` / `finished` | UTC scan bounds | +| `actor` / `host` / `tool_version` / `out_dir` | re-stamped on every manifest write | + +Two things to know: + +1. **A dir where only `vpcopilot apply` ran has a minimal `run.json`** — just `run_id` and + `created`, minted lazily by the first `audit.record()`. No repo, no models. That is expected, not + a bug. +2. **`actor`/`host` in `run.json` describe whoever last wrote the manifest** (the last scan), not + the actor of each change. Each audit entry stamps its own — trust the entry, not the manifest, + for per-change attribution. + +Manifest writing is fail-soft by design: `pipeline.run_pipeline` wraps it and logs +`⚠ could not write the run manifest (run.json): … — an audit export will lack provenance` rather +than failing a completed scan. Provenance is evidence, not a gate. + +--- + +## 4. Attribution — `VPCOPILOT_ACTOR` + +```python +def actor() -> str: + explicit = (os.environ.get("VPCOPILOT_ACTOR") or "").strip() + if explicit: + return explicit + try: + return getpass.getuser() + except Exception: + return "unknown" +``` + +Resolution order: **`VPCOPILOT_ACTOR` → OS user → `"unknown"`**. It never returns blank, and it +never raises — a slim container with no passwd entry must not break an apply mid-flight. + +Set it in CI or on a shared jump host so the trail names the engineer who asked for the change +rather than the service account it happens to run as: + +```sh +VPCOPILOT_ACTOR="dhenley@utexas.edu" vpcopilot apply --from-scan out/policies/service_policy.deny-login-sqli.json \ + --lb crapi-lab --url https://lab.example.com --keep +``` + +It is stamped in `audit.record()`, so a caller passing `actor="someone-else"` is silently +overridden (`tests/test_audit_provenance.py::test_identity_cannot_be_overridden_by_a_caller`). +That stops accidental spoofing from a call site. It does **not** stop someone who controls the +environment — this is attribution, not authentication. + +--- + +## 5. The normalized event model + +`audit.log` is deliberately heterogeneous: each action records what mattered for that action. That +is right for a log and wrong for a reviewer. `export.build_audit_events(out_dir)` produces **one row +per entry**, newest first, joined against `ledger.json`, `findings.json` and the `policies.json` +index. + +Every entry survives. `report.py`'s impact table filters to entries with `before_after` **or** +`behavioral`; an export that did the same would silently drop `retire`, `open_pr`, every `create_*` +and every config-only apply — so the exporter deliberately keeps them all. + +### `export.COLUMNS` + +| Column | Meaning | +|---|---| +| `ts` | UTC timestamp of the entry | +| `run_id` | run this change belongs to → `run.json` | +| `actor` | who | +| `host` | from where | +| `category` | `mitigate` · `create` · `refine` · `timing` · `cure` · `retire` · `rollback` (else `other`) — from `export.CATEGORY` | +| `action` | the raw action name | +| `finding_id` | the vulnerability that justified the change (see resolution below) | +| `title` | finding title, from `findings.json` else the ledger | +| `vuln_class` | e.g. `sqli`, `bola` — same fallback | +| `severity` | `critical` / `high` / … — same fallback | +| `ledger_state` | the finding's state **at export time**: `found → mitigated → remediated → retired`. Not the state when the change was made | +| `control` | the entry's `control`, else inferred from the action via `export.CONTROL` | +| `lb` | the load balancer touched | +| `namespace` | the XC namespace it lives in | +| `object` | the XC object named by the entry — first non-empty of `policy`, `app_firewall`, `apidef`, `name`, `rate`, `swagger` | +| `outcome` | one word for "did it stick, and did it work?" (below) | +| `attempts` | self-heal count — how many refine cycles it took | +| `exploit_before` | `"200 allowed"` — the exploit's status before the change | +| `exploit_after` | `"403 blocked"` — after | +| `legit_ok` | did legitimate traffic still pass after the change (over-block check) | +| `pr_url` | the cure PR — the entry's `url`, else the ledger's `cure.pr_url` | +| `tool_version` | version that wrote the entry | +| `detail` | the whole raw entry minus the stamped keys and `action` (each already has its own column). In CSV it is JSON, so the flattening loses nothing | + +`finding_id` resolution: `entry.finding_id` → `entry.finding` (the legacy key `open_pr` used) → +`policies.json[policy_name == entry.policy].finding_id`. Blank if none of the three resolve. + +### How `outcome` is derived + +`export._outcome()`, in order: + +1. **Fixed by action** — these have no pass/fail to coalesce: + `rollback_failed → rollback_failed`, `retire → retired`, `open_pr → pr_opened`, + `create_service_policy` / `create_app_firewall` / `create_api_definition` → `created`. +2. `unfixable` truthy → **`unfixable`** (the refiner gave up honestly: code fix required). +3. `rolled_back` truthy → **`rolled_back`**. +4. Otherwise coalesce the three success keys the apply paths disagree on: + `ok = passed ?? config_enabled ?? enabled`. + - all three absent → **`recorded`** + - `ok` and `kept` → **`kept`** (the change is still live on the LB) + - `ok` → **`passed`** (it worked, but was not kept) + - else → **`failed`** + +Two consequences worth knowing before you read a table: + +- Step 3 runs **before** the coalesce, so a validated-then-rolled-back apply — the default without + `--keep` — reads `rolled_back`, not `passed`. The proof it worked is still in + `exploit_before → exploit_after`. +- A record that omits `kept` falls through to `passed`/`failed` rather than `kept`. Absence of + `kept` is not evidence of a rollback — cross-check `ledger_state` and, for ground truth, the LB. + +`kept` exists precisely because "was it rolled back?" alone cannot answer "is this change still +live?" — a `rolled_back: false` on a failed apply means something very different from a +`kept: true`. + +--- + +## 6. The evidence bundle + +A `.zip` for one run. Every file is added verbatim; nothing is re-rendered except the two derived +views. + +``` +manifest.json what this bundle is + a SHA-256 per member + the caveats +audit.csv the normalized events, flat, one row per event (export.COLUMNS) +audit-events.json the same events with `detail` as real JSON +run.json the run manifest (§3) +audit.log the raw append-only log, byte-for-byte +ledger.json found → mitigated → remediated → retired, per finding +findings.json triage.json policies.json remediations.json +summary.json metrics.json probes.json correlations.json +lb_snapshot.json the most recent pre-change LB state +report.html the standalone HTML report +policies/* the exact XC configs that were pushed +snapshots/* per-LB timestamped pre-change state (`-.json`) +``` + +Missing members are skipped, not faked — an unscanned dir has no `findings.json`, and a run with no +live apply has no `snapshots/`. (`apply_timing` is an *entry inside* `audit.log`, not a member — a +CLI-driven run simply has none of those lines.) + +`--all` produces one archive with each run under its own folder (`out-claude/`, `demo-out/`, …) +plus a top-level `index.json` listing `{out_dir, folder, events, run_id}` per run. Run dirs are +discovered by `export.find_runs`: everything matching `out*/` plus `demo/out`, keeping only dirs +that have an `audit.log` or a `findings.json`. + +### The manifest + +```json +{ + "kind": "vpcopilot-audit-bundle", "schema": 1, "tool_version": "0.1.0", + "generated": "2026-07-26T19:04:11+00:00", + "generated_by": "dhenley", "generated_on": "jump-01", + "out_dir": "out-claude", + "run": { "run_id": "9f2c1ab30e77", "repo": "…", "repo_commit": "…", "models": {…}, "caps": {…} }, + "events": 14, + "actions": { "apply_service_policy": 3, "retire": 1, "open_pr": 2, … }, + "findings_touched": ["crapi-sqli-001", "crapi-bruteforce-004"], + "caveats": [ … ], + "members": { "audit.csv": {"bytes": 4211, "sha256": "…"}, "audit.log": {…}, … } +} +``` + +`manifest.json` itself is not in `members` (it is written after the hashes are computed). In a +multi-run bundle, member names are relative to that run's folder. + +### Verifying a bundle after it leaves the machine + +Recompute every hash from the zip alone — no vpcopilot install needed: + +```sh +python3 - vpcopilot-audit-out-claude-20260726T190411Z.zip <<'PY' +import hashlib, json, sys, zipfile + +z = zipfile.ZipFile(sys.argv[1]) +bad = z.testzip() +if bad: + sys.exit(f"corrupt member (CRC): {bad}") + +names, fails = set(z.namelist()), 0 +for mpath in sorted(n for n in names if n.endswith("manifest.json")): + prefix = mpath[: -len("manifest.json")] # "" for a single run, "/" for --all + m = json.loads(z.read(mpath)) + run = (m.get("run") or {}).get("run_id", "?") + print(f"\n{mpath} run={run} events={m['events']} members={len(m['members'])}") + for name, meta in sorted(m["members"].items()): + full = prefix + name + if full not in names: + print(f" MISSING {name}"); fails += 1; continue + got = hashlib.sha256(z.read(full)).hexdigest() + ok = got == meta["sha256"] and len(z.read(full)) == meta["bytes"] + fails += not ok + print(f" {'ok ' if ok else 'BAD '} {name} {got[:16]}") + for c in m["caveats"]: + print(f" caveat: {c}") +print(f"\n{fails} mismatch(es)") +sys.exit(1 if fails else 0) +PY +``` + +What this proves: the bundle is internally consistent and was not edited after export. What it does +not prove: that `audit.log` was authentic when it was bundled (see §1, honest limits). + +To re-derive `audit.csv` yourself and confirm the normalization added nothing, run +`vpcopilot export --out ` against the unpacked run and diff. + +--- + +## 7. How to get one + +### Console — ⑤ Retire + +```sh +vpcopilot console # http://127.0.0.1:8787 +``` + +The Retire step has a second card, **"Audit trail — every change made to a load balancer"**. It +*shows* the trail before anything is exported — you can check what leaves the machine first. One row +per change: + +`when (UTC)` · `action` (+ category) · `justified by` (finding title, id, severity) · `control` +(+ the XC object) · `load balancer` (+ namespace) · `outcome` (with a `×N` self-heal badge and the +`200 allowed → 403 blocked` proof) · `by` (actor) + +A filter box matches against the whole row, and `▸` expands any row to its raw JSON detail. Two +buttons: **Export evidence bundle (.zip)** (current run) and **All runs**. + +Downloads are named by `_stamped_name()` so several can share a folder or a ticket without +ambiguity: + +``` +vpcopilot-audit-out-claude-20260726T190411Z.zip # scope=run +vpcopilot-audit-all-out-claude-20260726T190411Z.zip # scope=all +``` + +Backing endpoints, all read-only: + +| Endpoint | Returns | +|---|---| +| `GET /api/audit-events` | `{out, events}` — the normalized events for the current out dir | +| `GET /api/audit-export?scope=run` | the `.zip` for the current run (`400` on an unknown scope) | +| `GET /api/audit-export?scope=all` | the `.zip` for every run dir on disk | +| `GET /api/runs` | `{current, runs}` — run dirs with something to export | +| `GET /api/audit` | the raw log, unnormalized | + +### CLI + +Console and CLI call the same module function — the bundles are identical. + +```sh +vpcopilot export --out out # → out/audit-bundle.zip +vpcopilot export --out out-claude --output ~/tickets/SEC-412/evidence.zip +vpcopilot export --all --root . --output all-runs.zip # every run dir, each in its own folder +``` + +`export` prints the path and the event count, and warns (without failing) when the run has no audit +entries yet — `no audit entries in out — nothing has changed a load balancer yet`. + +--- + +## 8. Reading the trail for a specific question + +All of the below run against `audit-events.json` from a bundle. Live, use +`curl -s 127.0.0.1:8787/api/audit-events | jq .events` instead. + +### "Which vulnerability justified this LB change?" + +```sh +jq -r '.[] | select(.lb=="crapi-lab") + | [.ts, .action, .control, .object, .finding_id, .severity, .title] | @tsv' audit-events.json +``` + +That join is the whole point of the normalizer — `finding_id` → `title`/`severity`/`vuln_class` from +`findings.json`, falling back to the ledger. If `finding_id` is blank, the change predates the +attribution work or names a policy that is no longer in `policies.json` (§9). + +### "What is still live on the LB right now?" + +```sh +# changes that were kept (not rolled back) +jq -r '.[] | select(.outcome=="kept") | [.ts,.lb,.namespace,.control,.object,.finding_id] | @tsv' audit-events.json +# …minus anything later retired +jq -r '.[] | select(.action=="retire") | [.ts,.lb,.control,.finding_id] | @tsv' audit-events.json +``` + +Cross-check `ledger_state`: `mitigated` means a band-aid is live, `retired` means it was detached. +Caveat: this is the tool's record — **the LB is the authority**. For ground truth, `GET` the LB and compare against +`snapshots/-.json` in the bundle. + +### "Did the band-aid actually block the exploit?" + +```sh +jq -r '.[] | select(.exploit_after != "") + | [.finding_id, .control, .exploit_before, .exploit_after, .legit_ok, .attempts] | @tsv' audit-events.json +``` + +`200 allowed → 403 blocked` with `legit_ok=true` is the real proof: the exploit stopped working and +legitimate traffic still passed. `attempts > 1` means the refiner self-healed the policy that many +times before it worked. + +Which controls can produce that proof: + +| Control | Evidence | +|---|---| +| `service_policy`, `api_schema`, `refine_apply` | live exploit + legit probe → `exploit_before`/`exploit_after`/`legit_ok` | +| `waf` | probe recorded, but a single-request signature block is payload-dependent, so it is **not** scored pass/fail — `config_enabled` is the assertion | +| `rate_limit` | `detail.behavioral` = `{sent, limited, passed, codes}` — a real burst was driven and the excess 429'd | +| `malicious_user`, `bot_defense`, `waf_data_guard` | config readback only (`enabled`) — behavioral controls are not single-request testable | + +`outcome=unfixable` is the honest negative: the refiner tried, could not make a band-aid work, and +said so. The finding stays `found` and needs the code fix. + +### "Who made this change?" + +```sh +jq -r '.[] | [.ts,.actor,.host,.run_id,.action,.lb,.finding_id] | @tsv' audit-events.json +jq '.run' manifest.json # → repo, repo_commit, repo_branch, repo_dirty, models, caps +``` + +`run_id` ties every entry to the run manifest: which repo at which commit produced the finding, with +which models, under which caps. Blank `actor` means an entry from an older build (§9). Remember +`actor` is self-asserted. + +--- + +## 9. Backward compatibility + +The normalizer reads old logs without rewriting them. `audit.log` is append-only — nothing +back-fills it. + +| Older shape | What the exporter does | +|---|---| +| No `run_id` / `actor` / `host` / `tool_version` | Those cells export **blank**. They are not inferred from the current environment — a guess in an audit trail is worse than a gap. | +| `open_pr` wrote `finding`, not `finding_id` | Resolved via the `finding` key. Current builds write **both** (`pr.py`), so old and new logs read the same way. | +| `apply_*` recorded no finding at all | Resolved through the `policies.json` index: `entry.policy` → `policy_name` → `finding_id`. Works only for entries that name a `policy`, and only while that scan's `policies.json` is still in the dir. Otherwise blank. | +| No `namespace` | Blank. The `lb` is still there; the tenant/namespace is not recoverable after the fact. | +| No `kept` | Older entries (and any action that omits it) fall through to `passed`/`failed` rather than `kept`. Absence of `kept` is not evidence of rollback. | +| Mixed success keys (`passed` / `config_enabled` / `enabled`) | Coalesced into one `outcome` (§5) — including the seeded demo dataset, which uses its own mix. | +| No `apply_timing` | Expected on any CLI-driven run: those entries come only from the console's Mitigate step on a live apply. MTTM and `elapsed_s` are absent, not zero. | + +The seeded `demo/out/audit.log` is a good worked example of the legacy shape — no `run_id`, no +`actor`, `apply_rate_limit` with `behavioral` but no `namespace`. Export it and see exactly which +cells come out blank: + +```sh +vpcopilot export --out demo/out --output /tmp/demo-evidence.zip +``` + +--- + +## See also + +- `docs/USAGE.md` — the full apply / PR / retire workflow +- `DESIGN.md` — where the audit sink sits in the architecture +- `src/vpcopilot/export.py` — `COLUMNS`, `CATEGORY`, `CONTROL`, `BUNDLE_FILES` +- Tests: `tests/test_audit_provenance.py` (what every entry must carry), + `tests/test_export.py` (normalization, CSV, bundle, multi-run), + `tests/test_console_audit_export.py` (the endpoints) diff --git a/docs/DEMO.md b/docs/DEMO.md index 1511818..26ccf6c 100644 --- a/docs/DEMO.md +++ b/docs/DEMO.md @@ -21,22 +21,38 @@ python3 demo/build_demo_out.py # writes a curated demo/out (crAPI-flavo VPCOPILOT_OUT=demo/out vpcopilot console # http://127.0.0.1:8787 ``` -Walk the tabs top to bottom — the whole arc is already in the data: +Walk the steps top to bottom — the whole arc is already in the data: -1. **Dashboard → hero band.** "6 exploitable vulns → mitigated live in ~30s, vs a 25-day change +1. **② Review → hero band.** "6 exploitable vulns → mitigated live in ~30s, vs a 25-day change window." Five XC control families are in play (service_policy, api_schema, waf, rate_limit, waf_data_guard). One finding ships code-only (no band-aid fits) — honesty, not theatre. -2. **Dashboard → findings.** Click any row to inspect the exploit, the vulnerable code, the +2. **② Review → findings.** Click any row to inspect the exploit, the vulnerable code, the generated band-aid, and the code cure. Note the SQLi row. -3. **Impact tab.** Control-family coverage + the full action log: the SQLi service policy shows +3. **② Review → Open HTML report ↗** (one click, right there in the step — no hunting in Setup). + The same story as a shareable, self-contained `report.html`: hero, severity/coverage bars, + model-independence, and the band-aid impact table where the SQLi service policy shows **self-healed ×2** — the refiner's first policy didn't block, it diagnosed and retried until the - exploit actually returned 403. Rate-limit shows the behavioral proof (25/30 requests 429'd). -4. **Ledger tab.** The four-state track: `found → mitigated → remediated → retired`. `crapi-sqli-001` - is walked all the way to **retired** — its cure PR merged, so the band-aid was detached. -5. **Open HTML report ↗** (Dashboard action bar). The same hero + self-heal + model-independence - chips + severity/coverage bars in one shareable, self-contained `report.html`. + exploit actually returned 403 — and rate-limit shows the behavioral proof (25/30 requests 429'd). + **Download** grabs a stamped copy. It is rebuilt from the current out dir every time you open + it, so it is never a stale file. +4. **⑤ Retire → ledger.** The four-state track: `found → mitigated → remediated → retired`. + `crapi-sqli-001` is walked all the way to **retired** — its cure PR merged, so the band-aid was + detached. +5. **⑤ Retire → audit trail.** *Every change made to a load balancer*, one row each: when (UTC) · + action · justified by (the finding + severity) · control (+ the XC object) · load balancer + (+ namespace) · outcome (with the `200 allowed → 403 blocked` proof and a self-heal ×N badge) · + by. Filter it, expand `▸` for the raw JSON, then **Export evidence bundle (.zip)** — the + normalized `audit.csv`, the raw `audit.log`, the exact XC configs pushed, the pre-change LB + snapshots, and a manifest that SHA-256s every member. **All runs** does the same for every run + dir on disk. The curated log is hand-built, so a couple of rows carry no finding or actor; a + live apply (Path B) stamps both on every record. Detail: [AUDIT.md](AUDIT.md). + +The report also lives at `demo/out/report.html` — open it directly with no server. The same bundle +is available from the CLI: -The report also lives at `demo/out/report.html` — open it directly with no server. +```bash +vpcopilot export --out demo/out # -> demo/out/audit-bundle.zip +``` --- @@ -50,17 +66,27 @@ Prereqs in `.env`: `XC_API_URL`, `XC_API_TOKEN`, `XC_NAMESPACE`, a model key (e. vpcopilot console # http://127.0.0.1:8787 ``` -1. **Run scan** (tab) against a vulnerable app repo (VAmPI / crAPI / Nimbus). Watch discover → - verify → triage → generate → remediate stream live. -2. **Dashboard.** Turn OFF `dry-run`, turn ON `keep live`, and click **Apply service_policy** on a - finding. The refiner streams in the row: attach → validate → (refine → retry)* → **before 200 - through → after 403 BLOCKED · legit ok**, with a *self-healed in N attempts* badge if it took - more than one try. It never claims success unless the live exploit is actually blocked. -3. **XC security dashboard ↗** (hero band) — jump to the native WAF/API-Security telemetry to show +1. **① Scan** a vulnerable app repo (VAmPI / crAPI / Nimbus). Watch discover → verify → triage → + generate → remediate stream live. The log box is scrollable and holds the **whole** transcript — + scroll up mid-scan to re-read the discover output and it stays put; a **↓ follow** chip and a + line counter appear until you scroll back to the bottom. Long scans no longer push the page down. +2. **② Review** the findings, and hit **Open HTML report ↗** if someone wants the artifact now. +3. **③ Mitigate.** With `dry-run` OFF and `keep live` ON (**Run settings** — the collapsible bar at the top of the Mitigate step), click + **Mitigate service_policy ▶** on a finding. The refiner streams in the row: attach → validate → + (refine → retry)* → **before 200 through → after 403 BLOCKED · legit ok**, with a *self-healed in + N attempts* badge if it took more than one try. It never claims success unless the live exploit + is actually blocked. +4. **XC security dashboard ↗** (hero band) — jump to the native WAF/API-Security telemetry to show the block landing in XC. -4. **Open PR** on the same finding to draft the real code fix against your repo. -5. **Ledger → Retire** once the cure merges — the band-aid is detached and the finding goes - `retired`. The loop is closed. +5. **④ Cure → Open PR** on the same finding to draft the real code fix against your repo. +6. **⑤ Retire** once the cure merges — the band-aid is detached and the finding goes `retired`. The + loop is closed. Below the ledger, the **audit trail** now has a row per live change: which + finding justified it, which LB and namespace it touched, whether it stuck, and who ran it + (`VPCOPILOT_ACTOR`, else the OS user). **Export evidence bundle (.zip)** hands that to whoever + asks why the load balancer changed — see [AUDIT.md](AUDIT.md). + +Dry runs are deliberately *not* in the trail: nothing changed, so there is nothing to answer for. +The bundle is evidence for a human reviewer, not a compliance certification. Guardrails hold throughout: protected LBs (`VPCOPILOT_PROTECTED_LBS`, default `nimbus-www`) and `nimbus-*` policies refuse mutation unless you explicitly opt in; every apply snapshots first and @@ -78,6 +104,10 @@ rolls back on failure. Model independence panels show it) — Claude, OpenAI, Gemini, or local Ollama, no code change. - **Reversible + gated.** Snapshot → self-test → attach → validate → keep or rollback. A human approves every live change in the console. +- **Auditable.** Every live change is recorded with the finding that justified it, the LB + + namespace it touched, whether it stuck, and who ran it — exportable as a .zip with a SHA-256 + manifest ([AUDIT.md](AUDIT.md)). It is evidence for a human reviewer, not a compliance + certification. ## Screenshots @@ -86,10 +116,17 @@ story on their own: | Shot | File | |---|---| -| Review — hero band + findings | [`2-review.png`](images/2-review.png) | +| Scan — the target form and its scrollable run log | [`1-scan.png`](images/1-scan.png) | +| Review — hero band + findings + the HTML-report buttons | [`2-review.png`](images/2-review.png) | | Mitigate — per-finding live apply | [`3-mitigate.png`](images/3-mitigate.png) | -| Retire — four-state ledger (`crapi-sqli-001` at *retired*) | [`5-retire.png`](images/5-retire.png) | +| Retire — four-state ledger (`crapi-sqli-001` at *retired*) + the audit trail | [`5-retire.png`](images/5-retire.png) | | The shareable HTML report (self-heal ×2 + rate-limit proof) | [`report.png`](images/report.png) | -To regenerate them: `VPCOPILOT_OUT=demo/out vpcopilot console`, then screenshot the `#review`, -`#mitigate`, `#retire` steps and `demo/out/report.html`. +To regenerate them: rebuild the dataset with `python3 demo/build_demo_out.py`, run +`VPCOPILOT_OUT=demo/out vpcopilot console`, then capture the `#scan`, `#review`, `#mitigate` and +`#retire` steps plus `demo/out/report.html` at 1200px wide / 2× device pixel ratio. + +Point the console at a **credential-free** `.env` when you do (`VPCOPILOT_ENV=…`): with XC creds +loaded, the hero band renders a deep link carrying your tenant hostname and namespace, and that +would ship in the image. `build_demo_out.py` curates `actor`/`host`/`out_dir` in the fixture for the +same reason — no real machine identity in a shared dataset. diff --git a/docs/PROJECT_STATE.md b/docs/PROJECT_STATE.md index ffef3e2..3bfb14b 100644 --- a/docs/PROJECT_STATE.md +++ b/docs/PROJECT_STATE.md @@ -3,7 +3,7 @@ A living snapshot so a fresh session (or a new machine) picks up where we left off. Public repo — kept free of tenant/credential specifics; real values live in `.env` (gitignored). -_Last updated: 2026-07-15._ +_Last updated: 2026-07-26._ ## What this is An agentic AppSec copilot: scan an app → find + verify vulns → triage each to an F5 Distributed @@ -14,14 +14,35 @@ real exploit (self-heal until it blocks, or honestly "unfixable") → open the c ## Status - **Quality plan fully burned down** — `docs/QUALITY_PLAN.md` Phases 0–4 all ✅ (P0 fixes, agent correctness, demoability, the SafeApply engine + control registry, durability/tests/CI). -- **118 tests** (offline against fakes), ruff clean, CI (`.github/workflows/ci.yml`, py3.10–3.12, - coverage floor, `live`/`bench` markers), **v0.1.0** released, repo public (Apache-2.0). +- **185 tests** (offline against fakes), 73% coverage, ruff clean, CI + (`.github/workflows/ci.yml`, py3.10–3.12, coverage floor 65%, `live`/`bench` markers), + **v0.1.0** released, repo public (Apache-2.0). - **Cross-model benchmark harness** built and in use (see below). - **Local open-source model wired** — `config/agents.dgx.yaml` runs every agent on a local Ollama/vLLM server (structured output validated live). The third leg of the benchmark; see below. - **The console is now a full benchmark-driving surface** — model dropdown, data-backed pickers (load balancer, scan target, output dir, PR repo), **Mitigate ALL**, and a **⑥ Benchmark** build+compare step. The whole three-way runs without leaving the UI. +- **Audit capture + evidence export (F3) landed** — the answer to "why was this LB changed, by + whom, in which tenant?". Every LB/object-mutating record now carries `finding_id` + `namespace` + (7 `apply_*`, 3 `create_*`, 5 `refine_apply` sites, `rollback_failed`); `apply_waf` / + `apply_data_guard` also record `kept`. `audit.record` stamps `run_id`/`actor`/`host`/ + `tool_version` on **every** entry. New `runmeta.py` (`/run.json` — run id + repo commit + + models + caps + counts) and `export.py` (normalized events → `audit.csv`/`audit-events.json` + + the raw artifacts + a SHA-256'd `manifest.json`, stdlib-only). Surfaces: `vpcopilot export + [--all]`, `GET /api/audit-events` · `/api/audit-export?scope=run|all` · `/api/runs`, and an + **Audit trail** card on ⑤ Retire that shows the trail before you export it. New env var + **`VPCOPILOT_ACTOR`** (defaults to the OS user). +- **Console log windows are complete and scrollable** — `/api/scan` and `/api/action` now return + the FULL transcript plus `log_total` and take `?since=` for the new tail only (they used to + serve the last 40 / 60 lines, so the rest never existed client-side). `.logbox` caps the box at + 46vh instead of letting it push the page down; the poll appends a **text node** (never + `innerHTML` — pipeline output is untrusted) so scroll position and selection survive, sticking to + the bottom only if you were already there (`↓ follow` chip + line counter otherwise). Bounded + sink `_append` / `LOG_MAX = 20_000` says in-band when it caps. +- **The HTML report is reachable from ② Review** — **Open HTML report ↗** + **Download** + (`/api/report?download=1` → `vpcopilot-report--.html`). It was always rebuilt from + the current out dir on every request; only the affordance was missing (Setup-only). ## Architecture worth knowing (before changing anything) - `engine.py` + `controls.py` — one **SafeApply spine** (snapshot → self-test → attach → validate → @@ -37,6 +58,23 @@ real exploit (self-heal until it blocks, or honestly "unfixable") → open the c a global `OPENAI_API_BASE` hijacking the real OpenAI config), and `temperature`/`timeout`/ `max_retries`. A **live model switcher** in the console header swaps the active config with no relaunch; the Output-dir field is authoritative and the console reads whatever dir you scan into. +- **Run identity joins processes, not memory** — `runmeta.run_id(out_dir)` mints an id on first use + and persists it in `/run.json`; every audit entry carries it. So a console scan and a + `vpcopilot apply` an hour later in a *separate process* against the same out dir belong to the + same run, and an exported bundle explains its own provenance (repo + commit/branch/dirty, config + path, per-agent models, caps, counts, actor/host/tool_version) without needing this machine. + `write_manifest` never clobbers an existing `run_id` — a re-scan keeps the identity the entries + on disk already join to. Provenance is **fail-soft**: it warns and moves on, never failing a + completed scan. +- **Write heterogeneous, normalize at export** — `audit.log` stays append-only and per-action (a + WAF's `config_enabled` is not a service policy's `passed`); `export.build_audit_events` does the + flattening at read time into one row per event (`export.COLUMNS`), coalescing the outcome keys, + mapping legacy `finding` → `finding_id`, recovering a finding from the `policies.json` index, and + joining ledger + findings for title/class/severity/state. Add a field to a record freely — the + normalizer, not the writer, owns the reviewer's shape. It keeps **every** entry on purpose + (report.py's impact table filters to entries with `before_after` or `behavioral` and would drop + `retire`, `open_pr`, `create_*` and config-only applies). Identity is stamped inside `audit.record`, so + add a new mutating path and it is attributable for free — and a caller cannot override it. - **Benchmark harness** — `bench_model.py` + `vpcopilot bench-model` / `bench-compare`, **also in the console's ⑥ Benchmark step** (`POST /api/bench-model` build + `GET /api/benchmarks` compare table — same `benchmarks/*.json` as the CLI). Reads a run's findings/policies + the audit log's @@ -114,6 +152,19 @@ Now fully UI-drivable (headless CLI still works — add `--no-code-fixes` to `vp picker returns case-correct absolute paths, so pick from it rather than typing. - **PR-repo picker needs `gh`** authenticated (`gh auth login`); without it that field just stays free-text (nothing breaks). +- **Dry runs are not recorded** — by design (nothing changed, so there is nothing to answer for). + The Retire step's audit table and the evidence bundle stay **empty until the first LIVE Mitigate**; + a dry-run demo will look like nothing happened. +- **`apply_timing` only exists for console-driven live applies** — the console writes it after a + non-dry action job. A CLI-driven run has none, which is also why `bench-model` needs the console's + Mitigate step for live policy quality. +- **Audit logs written by older builds lack `finding_id` / `namespace` / `actor`** — the export + leaves those cells **blank rather than inferred** (the one exception: a finding recovered from the + `policies.json` index by policy name). There is no backfill; historic runs stay as they were + written. +- **The console reads ONE run dir at a time** (`OUT`, set by the Output-dir field on scan) — so + "Export evidence bundle (.zip)" covers *that* dir only. Use **All runs** (`scope=all` / + `vpcopilot export --all`) for everything on disk; `GET /api/runs` lists what it would find. ## Open threads / next - ✅ **Matched VAmPI three-way — DONE** (dgx/claude/openai at min-conf 0.5, all findings mitigated). @@ -125,6 +176,12 @@ Now fully UI-drivable (headless CLI still works — add `--no-code-fixes` to `vp worth confirming the F5 tenant's API-protection quota if you want api_schema to validate reliably. - Optional: re-run OpenAI on crAPI at min-confidence 0.5 for strict parity with the Claude baseline. - Nice-to-have: `/api/repos` shells to `gh` at console start — could lazy-load it. +- **Audit/export, still open:** no backfill of historic `audit.log` files (pre-F3 entries export + with blank `finding_id`/`namespace`/`actor` — a one-shot rewrite would be a judgement call about + inventing attribution, so it hasn't been done). The export is scoped to **run dirs on the local + filesystem** (`find_runs` = `out*/` + `demo/out` under the given root) — no object store, no + cross-machine collation, and no signing of the bundle beyond the per-member SHA-256s. It is + evidence for a human reviewer, not a compliance certification. ## Running on a fresh machine `README` quickstart + `docs/TRY_IT.md`. In short: clone repo → `pip install -e ".[deploy,console,dev]"` diff --git a/docs/QUALITY_PLAN.md b/docs/QUALITY_PLAN.md index c167b9b..40c6606 100644 --- a/docs/QUALITY_PLAN.md +++ b/docs/QUALITY_PLAN.md @@ -72,3 +72,15 @@ Execute in phase order; check items off as they land. Effort: S/M/L. Source audi refiner `unfixable`/`over_block`; retire/pr/lab mutation paths; schema golden-replay. (M, Test) - [x] **D4** CI (`.github/workflows`, ruff + `pytest -m "not live and not bench"`, py3.10–3.12, coverage floor) + markers + nightly live-smoke + bench-gate (fails below `BASELINE.md`). (M, Test) + +## Phase 5 — F. Auditability & operator experience (from operator feedback, 2026-07-26) +- [x] **F1** Console log windows serve the FULL transcript and scroll. `/api/scan` and `/api/action` + were truncating to the last 40 / 60 lines, so a long run's output did not exist client-side; + bounded `.logbox` + append-with-stick-to-bottom so a running scan can be read end-to-end. (S, UI) +- [x] **F2** The shareable HTML report is reachable from **② Review**, not only Setup, plus a + stamped `?download=1` attachment. It is rebuilt per request, so it is always the latest run. (S, UI) +- [x] **F3** Auditable LB changes end-to-end: identity (`run_id`/`actor`/`host`/`tool_version`) + stamped centrally in `audit.record`; `finding_id` + XC `namespace` on every mutating record; + `/run.json` run manifest with repo commit + per-agent models; `export.py` normalizes the + append-only log into reviewable events and packages a SHA-256-manifested evidence bundle; + Retire step shows the trail and exports it (`vpcopilot export` is the CLI twin). (L, Orch+UI) diff --git a/docs/USAGE.md b/docs/USAGE.md index 795a188..1acd59d 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -21,6 +21,11 @@ cp .env.example .env | `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` / `GEMINI_API_KEY` / `OLLAMA_API_BASE` | the model(s) you run | | `XC_API_URL`, `XC_API_TOKEN`, `XC_NAMESPACE` | deploying band-aids to F5 XC | | `GITHUB_TOKEN` *(or `gh auth login`)* | opening code-fix PRs | +| `VPCOPILOT_ACTOR` *(optional)* | who changes are attributed to in the audit log — defaults to the OS user | + +**`VPCOPILOT_ACTOR`** is what the audit trail records as the person who made a change. On your own +machine the OS user is right. In CI, or on a shared jump host, set it so the record names the +engineer who asked for the change rather than the service account it happens to run as. **Model-independence:** every agent's model is chosen per-agent in `config/agents.yaml` (LiteLLM naming). Swap Claude / OpenAI / Gemini / Ollama — globally or per agent — with no @@ -68,24 +73,70 @@ Uses the full corrected file from `remediate` (no fragile diff apply). Token fro ```sh vpcopilot ledger # found -> mitigated -> remediated -> retired (per finding) vpcopilot audit # append-only log of every applied / rolled-back change +vpcopilot export [--out DIR] [--output PATH] # evidence bundle (.zip) for one run +vpcopilot export --all [--root DIR] # every run dir on disk, each in its own folder vpcopilot report --open # standalone shareable HTML dashboard of the results vpcopilot retire --finding # C2: when the cure PR merges, detach the band-aid + mark retired vpcopilot retire --all # retire every mitigated finding whose cure PR merged (--force to skip the check) ``` -Every scan also drops a self-contained `out/report.html` (no server, no external assets); -the console's Dashboard has an **Open HTML report** button too. +`export` writes `/audit-bundle.zip` (`--all` → `/audit-bundle-all.zip` with a top-level +`index.json`). Inside: `manifest.json` (bundle identity, the run manifest, a SHA-256 per member, and +an explicit `caveats` list), `audit.csv` + `audit-events.json` (one normalized row per change, joined +to the finding that justified it), the raw `audit.log` verbatim, `run.json`, the ledger and scan +artifacts, `policies/*` (the exact XC configs pushed), `snapshots/*` (pre-change LB state), and +`report.html`. It is the same bundle the console's ⑤ Retire step downloads, and it is read-only — +nothing here touches XC or GitHub. + +Dry runs are not in it: nothing changed, so nothing is logged. The bundle is evidence for a human +reviewer, not a compliance certification. Full reference: **[AUDIT.md](AUDIT.md)**. + +Every scan also drops a self-contained `out/report.html` (no server, no external assets). In the +console it's on **② Review** and **⚙ Setup** — **Open HTML report ↗** for a new tab, **Download** +for a timestamped copy. Both rebuild it from the current run dir on every open, so you always get +the latest run. ## 7. Ops console (localhost) ```sh vpcopilot console # http://127.0.0.1:8787 ``` -Tabs: **Dashboard** (findings + inline Apply/PR with an action-settings bar), **Workflow** -(the agent pipeline + each agent's model), **Ledger**, **Run scan**, **Admin** (reads/writes -`.env`), **XC status**. The action bar's **dry-run** is on by default. +A six-step stepper that follows the lifecycle, plus a **⚙ Setup** page. A persistent hero band +(exploitable vulns → mitigated live in seconds, vs. change-control days) sits above every step, the +header carries a live model switcher, and each step is deep-linkable (`#mitigate`, `#retire`, …). + +| Step | What | +|---|---| +| **① Scan** | point at a repo and run the pipeline — read-only, no XC/GitHub writes. Auto-advances to Review when it finishes | +| **② Review** | verified findings + the recommended band-aid; click a row for exploit / code / generated policy. **Open HTML report ↗** + **Download** | +| **③ Mitigate** | apply each band-aid (or **Mitigate ALL**, one at a time, continuing past failures) and watch `before → after` stream, with a *self-healed in N attempts* badge | +| **④ Cure** | open the code-fix PR per finding, or all of them | +| **⑤ Retire** | the four-state ledger track, plus the **Audit trail** table and **Export evidence bundle (.zip)** / **All runs** | +| **⑥ Benchmark** | build a model-tagged report from this run, then compare models side by side per target app | +| **⚙ Setup** | credentials (writes `.env`), XC status, the per-agent model wiring, and the report buttons | + +**Run settings** — the collapsible bar shown on the action steps (**Mitigate / Cure / Retire**): +LB · validate URL · PR repo · base · path prefix, plus **dry-run** (on by default), **refine** + +attempts, **keep live**, and **allow protected LB**. Its summary line spells out the mode you're +about to run in — `dry-run · rollback · LB=… · refine×3`. + +**Log windows.** ① Scan and ③ Mitigate's per-finding job log hold the *whole* transcript in a +scrollable box, not the last N lines. The endpoints serve the full log and the page appends only the +new tail, so scroll position and text selection survive each poll — you can read back through a long +run while it's still going. Both stick to the bottom only while you're already at the bottom; on +① Scan, scrolling up also reveals a **↓ follow** chip and a line count (the Mitigate job log has +neither — it's a small box inside a table row). + +**Audit trail (⑤ Retire).** One row per change made to a load balancer — when (UTC) · action · +justified by (the finding, its id and severity) · control (+ the XC object) · load balancer +(+ namespace) · outcome (with a self-heal ×N badge and the `200 allowed → 403 blocked` proof) · by +(actor). Filter it, expand `▸` for the raw JSON, then **Export evidence bundle (.zip)** for this run +or **All runs** — the same bundle `vpcopilot export` writes (§6). The trail is shown *before* it can +be exported, so you can check what leaves the machine. Dry runs are absent by design. -The action-bar defaults (LB / validate URL / PR repo / base / path-prefix) are **env-overridable** -so the console isn't pinned to one app — set them in `.env` (or the environment) to match what -you're testing: +The LB / validate URL / PR repo fields are **pickers**, not pre-filled defaults — load balancers come +from your XC namespace (with their domains), scan targets from sibling directories, PR repos from +`gh` — so the console is never pinned to one app. `/api/defaults` still reads the +`VPCOPILOT_DEFAULT_*` env vars, and `VPCOPILOT_DEFAULT_LB` is what the hero's +**XC security dashboard ↗** link points at: ```sh VPCOPILOT_DEFAULT_LB=vampi-lab VPCOPILOT_DEFAULT_URL=https://vampi.banknimbus.com @@ -100,7 +151,10 @@ VPCOPILOT_DEFAULT_PREFIX= # usually empty protected LBs (`VPCOPILOT_PROTECTED_LBS`, default `nimbus-www`) can't be mutated without `--allow-protected-lb`. - **Reversible:** every apply snapshots the LB and rolls back on validation failure (or by - default). Every change is written to the audit log. + default). Every change is written to the append-only audit log — the finding that justified it, + the control and the XC object, the load balancer and its namespace, whether it was kept or rolled + back, and who ran it (`VPCOPILOT_ACTOR`, else the OS user) on which host, under which run id. + Dry runs are not recorded: nothing changed, so there is nothing to answer for. - **Band-aids are temporary:** every finding also gets a code-fix PR; the ledger tracks each finding to `retired` (band-aid removed once the cure merges). diff --git a/docs/images/1-scan.png b/docs/images/1-scan.png index 4ffac97..8896f00 100644 Binary files a/docs/images/1-scan.png and b/docs/images/1-scan.png differ diff --git a/docs/images/2-review.png b/docs/images/2-review.png index e50bff9..e17ddd3 100644 Binary files a/docs/images/2-review.png and b/docs/images/2-review.png differ diff --git a/docs/images/3-mitigate.png b/docs/images/3-mitigate.png index 94d6108..33b0447 100644 Binary files a/docs/images/3-mitigate.png and b/docs/images/3-mitigate.png differ diff --git a/docs/images/5-retire.png b/docs/images/5-retire.png index 24f0c38..5d4abb9 100644 Binary files a/docs/images/5-retire.png and b/docs/images/5-retire.png differ diff --git a/docs/images/report.png b/docs/images/report.png index 9dd8499..704fe03 100644 Binary files a/docs/images/report.png and b/docs/images/report.png differ diff --git a/src/vpcopilot/apply.py b/src/vpcopilot/apply.py index c6fd33d..eb9a354 100644 --- a/src/vpcopilot/apply.py +++ b/src/vpcopilot/apply.py @@ -251,17 +251,18 @@ def _log_baseline(before: dict, log: Callable) -> None: def apply_service_policy(lb: str, policy_name: str, target_url: str, *, dry_run: bool = False, keep: bool = False, allow_protected: bool = False, probe: bool = False, retries: int = 8, wait_seconds: int = 8, - out_dir: str = "out", log: Callable = print) -> dict: + finding_id: str | None = None, out_dir: str = "out", log: Callable = print) -> dict: xc = XC() guard_lb(lb, allow_protected=allow_protected, dry_run=dry_run) - ctx = ApplyContext(xc=xc, lb=lb, out_dir=out_dir, log=log).load() + ctx = ApplyContext(xc=xc, lb=lb, out_dir=out_dir, log=log, finding_id=finding_id).load() spec = ctx.spec snap_sp = _sp_block(spec) had = "active_service_policies" in spec log(f"snapshot saved · current LB service-policy = {list(snap_sp) or ['(none set)']}") from . import ledger as _ledger - fid = _ledger.find_finding_for_policy(out_dir, policy_name) + fid = finding_id or _ledger.find_finding_for_policy(out_dir, policy_name) + ctx.finding_id = fid # resolved via the policy index — pin it so a rollback_failed is attributable too exists = xc.service_policy_exists(policy_name) if not exists and not dry_run: # a from-scan policy is created on the live apply, not in dry-run raise RuntimeError(f"service policy '{policy_name}' not found in namespace {xc.ns}") @@ -322,8 +323,9 @@ def rollback(): # B3: verified, retried rollback (LB restored to snapshot or Ro rolled = True before_after = {"before": before, "after": res} from . import audit - audit.record(out_dir, "apply_service_policy", lb=lb, policy=policy_name, passed=passed, - rolled_back=rolled, kept=(passed and keep), before_after=before_after) + audit.record(out_dir, "apply_service_policy", finding_id=fid, lb=lb, namespace=xc.ns, + policy=policy_name, passed=passed, rolled_back=rolled, kept=(passed and keep), + before_after=before_after) return {"mode": "apply", "diff": diff, "validation": res, "before_after": before_after, "passed": passed, "rolled_back": rolled, "kept": passed and keep} @@ -331,7 +333,8 @@ def rollback(): # B3: verified, retried rollback (LB restored to snapshot or Ro def apply_from_scan(artifact_path: str, lb: str, target_url: str, *, name: str | None = None, create_only: bool = False, dry_run: bool = False, keep: bool = False, allow_protected: bool = False, probe: bool = False, retries: int = 8, - wait_seconds: int = 8, out_dir: str = "out", log: Callable = print) -> dict: + wait_seconds: int = 8, finding_id: str | None = None, + out_dir: str = "out", log: Callable = print) -> dict: """End-to-end from a generated artifact: create the policy in XC (if missing), then attach -> validate -> rollback via apply_service_policy. Guarded against clobbering a protected policy.""" @@ -364,17 +367,18 @@ def apply_from_scan(artifact_path: str, lb: str, target_url: str, *, name: str | xc.create_service_policy(body) log(f"created service policy '{policy_name}'") from . import audit - audit.record(out_dir, "create_service_policy", policy=policy_name, namespace=xc.ns) + audit.record(out_dir, "create_service_policy", finding_id=finding_id, policy=policy_name, + namespace=xc.ns) if create_only: return {"mode": "create_only", "policy": policy_name, "created": not dry_run} res = apply_service_policy(lb, policy_name, target_url, dry_run=dry_run, keep=keep, allow_protected=allow_protected, probe=probe, retries=retries, - wait_seconds=wait_seconds, out_dir=out_dir, log=log) + wait_seconds=wait_seconds, finding_id=finding_id, out_dir=out_dir, log=log) if res.get("kept"): from . import ledger - fid = ledger.find_finding_for_policy(out_dir, policy_name) + fid = finding_id or ledger.find_finding_for_policy(out_dir, policy_name) if fid: ledger.mark_mitigated(out_dir, fid, control="service_policy", policy_name=policy_name, lb=lb) @@ -384,6 +388,7 @@ def apply_from_scan(artifact_path: str, lb: str, target_url: str, *, name: str | def apply_malicious_user(lb: str, *, dry_run: bool = False, keep: bool = False, allow_protected: bool = False, finding_id: str | None = None, + user_id_header: str | None = None, user_identification_name: str | None = None, out_dir: str = "out", log: Callable = print) -> dict: """Enable XC Malicious-User Detection on the LB. This is a per-user BEHAVIORAL control set on the LB itself (a oneof: enable/disable), not a separate policy object. Validation @@ -392,7 +397,7 @@ def apply_malicious_user(lb: str, *, dry_run: bool = False, keep: bool = False, self-test + rollback, same safety spine as the service-policy path.""" xc = XC() guard_lb(lb, allow_protected=allow_protected, dry_run=dry_run) - ctx = ApplyContext(xc=xc, lb=lb, out_dir=out_dir, log=log).load() + ctx = ApplyContext(xc=xc, lb=lb, out_dir=out_dir, log=log, finding_id=finding_id).load() spec = ctx.spec already = "enable_malicious_user_detection" in spec has_user_id = ("user_id_client_ip" in spec) or ("user_identification" in spec) @@ -410,7 +415,16 @@ def apply_malicious_user(lb: str, *, dry_run: bool = False, keep: bool = False, new_spec = copy.deepcopy(spec) new_spec.pop("disable_malicious_user_detection", None) new_spec["enable_malicious_user_detection"] = {} - new_spec.setdefault("user_id_client_ip", {}) # per-user tracking needs a user identifier + # user_id_client_ip vs user_identification is a oneof. Keying on a request header + # (e.g. X-Agent-Id) attributes abuse per agent, not per shared egress IP behind an edge. + if user_id_header: + uid_name = user_identification_name or f"{lb}-user-id" + _ensure_user_identification(xc, uid_name, user_id_header, log) + new_spec.pop("user_id_client_ip", None) + new_spec["user_identification"] = _user_id_ref(ctx.lb_obj, uid_name, xc.ns) + log(f"keying malicious-user detection on header {user_id_header} (user_identification '{uid_name}')") + else: + new_spec.setdefault("user_id_client_ip", {}) # per-user tracking needs a user identifier ctx.put(new_spec) log("enabled malicious-user detection on the LB") @@ -435,15 +449,17 @@ def rollback(): rollback() rolled = True from . import audit - audit.record(out_dir, "apply_malicious_user", lb=lb, enabled=enabled, rolled_back=rolled, - kept=(enabled and keep)) + audit.record(out_dir, "apply_malicious_user", finding_id=finding_id, lb=lb, namespace=xc.ns, + enabled=enabled, rolled_back=rolled, kept=(enabled and keep)) return {"mode": "apply_malicious_user", "diff": diff, "config_enabled": enabled, "validation": "config-level (readback)", "rolled_back": rolled, "kept": enabled and keep} def apply_rate_limit(lb: str, *, requests: int = 100, unit: str = "MINUTE", burst: int = 1, behavioral: bool = False, target_url: str = "https://lab.banknimbus.com", - behavioral_path: str = "/login", wait_seconds: int = 8, max_refine: int = 2, + behavioral_path: str = "/login", burst_headers: dict | None = None, + user_id_header: str | None = None, user_identification_name: str | None = None, + wait_seconds: int = 8, max_refine: int = 2, dry_run: bool = False, keep: bool = False, allow_protected: bool = False, finding_id: str | None = None, out_dir: str = "out", log: Callable = print) -> dict: """Enable XC rate limiting on the LB (oneof: disable_rate_limit -> rate_limit). Config-level @@ -452,7 +468,7 @@ def apply_rate_limit(lb: str, *, requests: int = 100, unit: str = "MINUTE", burs the control mitigates real traffic rather than just being configured.""" xc = XC() guard_lb(lb, allow_protected=allow_protected, dry_run=dry_run) - ctx = ApplyContext(xc=xc, lb=lb, out_dir=out_dir, log=log).load() + ctx = ApplyContext(xc=xc, lb=lb, out_dir=out_dir, log=log, finding_id=finding_id).load() spec = ctx.spec already = "rate_limit" in spec diff = {"from": "enabled" if already else "disabled", "to": f"{requests}/{unit} (burst x{burst})"} @@ -469,6 +485,16 @@ def apply_rate_limit(lb: str, *, requests: int = 100, unit: str = "MINUTE", burs "rate_limiter": {"total_number": requests, "unit": unit, "burst_multiplier": burst}, "no_policies": {}, "no_ip_allowed_list": {}, } + # Per-agent limiting: with a user_identification set on the LB, the rate_limiter counts + # per identity (the header value) instead of LB-wide. If the two-identity behavioral burst + # shows a clean agent is ALSO throttled, this keying isn't taking effect and a keyed + # rate_limiter_policy is needed instead (see the Mode-2 plan, A4). + if user_id_header: + uid_name = user_identification_name or f"{lb}-user-id" + _ensure_user_identification(xc, uid_name, user_id_header, log) + new_spec.pop("user_id_client_ip", None) + new_spec["user_identification"] = _user_id_ref(ctx.lb_obj, uid_name, xc.ns) + log(f"keying rate limit per-user on header {user_id_header} (user_identification '{uid_name}')") ctx.put(new_spec) log(f"enabled rate limiting ({requests}/{unit}, burst x{burst})") @@ -490,7 +516,8 @@ def rollback(): for attempt in range(1, (max_refine or 1) + 1): ctx.sleep(wait_seconds) # let the limit propagate to the edge burst = max(cur * 3, 30) - behavioral_res = probe_rate_limit(target_url, count=burst, path=behavioral_path, log=log) + behavioral_res = probe_rate_limit(target_url, count=burst, path=behavioral_path, + headers=burst_headers, log=log) behaved = behavioral_res["limited"] > 0 log(f"behavioral validation (attempt {attempt}, {cur}/{unit}) -> {'PASS' if behaved else 'FAIL'} " f"({behavioral_res['limited']}/{burst} requests rate-limited)") @@ -515,8 +542,9 @@ def rollback(): rollback() rolled = True from . import audit - audit.record(out_dir, "apply_rate_limit", lb=lb, enabled=enabled, passed=passed, rolled_back=rolled, - kept=(passed and keep), rate=f"{requests}/{unit}", behavioral=behavioral_res) + audit.record(out_dir, "apply_rate_limit", finding_id=finding_id, lb=lb, namespace=xc.ns, + enabled=enabled, passed=passed, rolled_back=rolled, kept=(passed and keep), + rate=f"{requests}/{unit}", behavioral=behavioral_res) return {"mode": "apply_rate_limit", "diff": diff, "config_enabled": enabled, "behavioral": behavioral_res, "passed": passed, "rolled_back": rolled, "kept": passed and keep, "unfixable": unfixable, @@ -537,7 +565,7 @@ def apply_bot_defense(lb: str, *, policy: dict | None = None, regional_endpoint: log(f"bot_defense currently {'ENABLED' if already else 'disabled'}") return {"mode": "dry_run", "already_enabled": already, "note": "will enable Bot Defense with a flag-only policy (all paths) unless a policy is given"} - ctx = ApplyContext(xc=xc, lb=lb, out_dir=out_dir, log=log).load() + ctx = ApplyContext(xc=xc, lb=lb, out_dir=out_dir, log=log, finding_id=finding_id).load() spec = ctx.spec already = bool(spec.get("bot_defense")) # disabled state may carry a null bot_defense key log(f"bot_defense currently {'ENABLED' if already else 'disabled'}") @@ -565,13 +593,14 @@ def rollback(): rollback() rolled = True from . import audit - audit.record(out_dir, "apply_bot_defense", lb=lb, enabled=enabled, rolled_back=rolled, - kept=(enabled and keep)) + audit.record(out_dir, "apply_bot_defense", finding_id=finding_id, lb=lb, namespace=xc.ns, + enabled=enabled, rolled_back=rolled, kept=(enabled and keep)) return {"mode": "apply_bot_defense", "config_enabled": enabled, "rolled_back": rolled, "kept": enabled and keep} -def _ensure_blocking_waf(xc, app_firewall: str, template: str, out_dir: str, log: Callable) -> None: +def _ensure_blocking_waf(xc, app_firewall: str, template: str, out_dir: str, log: Callable, + finding_id: str | None = None) -> None: """Create a Blocking app_firewall (cloned from `template`) if `app_firewall` is missing.""" if xc.app_firewall_exists(app_firewall): return @@ -581,7 +610,8 @@ def _ensure_blocking_waf(xc, app_firewall: str, template: str, out_dir: str, log xc.create_app_firewall({"metadata": {"name": app_firewall, "namespace": xc.ns}, "spec": tspec}) log(f"created Blocking app_firewall '{app_firewall}'") from . import audit - audit.record(out_dir, "create_app_firewall", name=app_firewall, mode="blocking") + audit.record(out_dir, "create_app_firewall", finding_id=finding_id, name=app_firewall, + namespace=xc.ns, mode="blocking") def _waf_ref(xc, lb_obj: dict, app_firewall: str) -> dict: @@ -593,6 +623,30 @@ def _waf_ref(xc, lb_obj: dict, app_firewall: str) -> dict: return ref +def _user_id_ref(lb_obj: dict, name: str, ns: str) -> dict: + """Fully-qualified user_identification ref (name+namespace+tenant).""" + ref = {"namespace": ns, "name": name} + tenant = lb_obj.get("system_metadata", {}).get("tenant") + if tenant: + ref["tenant"] = tenant + return ref + + +def _ensure_user_identification(xc, name: str, header: str, log: Callable = print) -> None: + """Create a user_identification object keyed on an HTTP header, if it does not exist. + Lets per-user controls (malicious_user, rate_limit) count per agent identity (e.g. + X-Agent-Id) instead of per client IP — which matters when many agents egress a shared IP + behind an edge. XC's UserIdentifier rule is a oneof; `http_header_name` selects a header.""" + if xc.user_identification_exists(name): + return + body = { + "metadata": {"name": name, "namespace": xc.ns}, + "spec": {"rules": [{"http_header_name": header}]}, + } + xc.create_user_identification(body) + log(f"created user_identification '{name}' keyed on header {header}") + + def apply_waf(lb: str, *, app_firewall: str = "vpcopilot-lab-waf", template: str = "nimbus-waf", target_url: str = "https://lab.banknimbus.com", dry_run: bool = False, keep: bool = False, allow_protected: bool = False, finding_id: str | None = None, @@ -608,9 +662,9 @@ def apply_waf(lb: str, *, app_firewall: str = "vpcopilot-lab-waf", template: str if dry_run: log(f"[dry-run] would create Blocking app_firewall '{app_firewall}' from '{template}'") else: - _ensure_blocking_waf(xc, app_firewall, template, out_dir, log) + _ensure_blocking_waf(xc, app_firewall, template, out_dir, log, finding_id) - ctx = ApplyContext(xc=xc, lb=lb, out_dir=out_dir, log=log).load() + ctx = ApplyContext(xc=xc, lb=lb, out_dir=out_dir, log=log, finding_id=finding_id).load() spec = ctx.spec already = bool(spec.get("app_firewall")) diff = {"from": "on" if already else "off", "to": f"app_firewall:{app_firewall}"} @@ -655,8 +709,9 @@ def rollback(): rolled = True before_after = {"before": before, "after": res} from . import audit - audit.record(out_dir, "apply_waf", lb=lb, app_firewall=app_firewall, config_enabled=enabled, - rolled_back=rolled, before_after=before_after) + audit.record(out_dir, "apply_waf", finding_id=finding_id, lb=lb, namespace=xc.ns, + app_firewall=app_firewall, config_enabled=enabled, rolled_back=rolled, + kept=(enabled and keep), before_after=before_after) return {"mode": "apply_waf", "diff": diff, "config_enabled": enabled, "before_after": before_after, "rolled_back": rolled, "kept": enabled and keep} @@ -673,9 +728,17 @@ def apply_data_guard(lb: str, *, app_firewall: str = "vpcopilot-lab-waf", templa already = bool(xc.get_lb(lb).get("spec", {}).get("data_guard_rules")) return {"mode": "dry_run", "already_on": already, "to": "WAF (blocking) + data_guard_rules: mask all paths"} + # SAFETY (default): never detach a WAF the LB is already carrying. Data Guard only needs *a* + # blocking WAF attached — if one is already there under a different name (e.g. a live + # banknimbus-dev-waf), reuse it instead of creating/attaching `app_firewall` and clobbering it. + attached_name = (xc.get_lb(lb).get("spec", {}).get("app_firewall") or {}).get("name") + if attached_name and attached_name != app_firewall: + log(f"⚠ WARNING: LB '{lb}' already has app_firewall '{attached_name}' attached — reusing it " + f"instead of attaching '{app_firewall}' (refusing to detach the live WAF)") + app_firewall = attached_name if not xc.app_firewall_exists(app_firewall): - _ensure_blocking_waf(xc, app_firewall, template, out_dir, log) - ctx = ApplyContext(xc=xc, lb=lb, out_dir=out_dir, log=log).load() + _ensure_blocking_waf(xc, app_firewall, template, out_dir, log, finding_id) + ctx = ApplyContext(xc=xc, lb=lb, out_dir=out_dir, log=log, finding_id=finding_id).load() spec = ctx.spec had_dg, had_waf = bool(spec.get("data_guard_rules")), bool(spec.get("app_firewall")) ctx.self_test() @@ -706,7 +769,9 @@ def rollback(): rollback() rolled = True from . import audit - audit.record(out_dir, "apply_data_guard", lb=lb, enabled=enabled, rolled_back=rolled) + audit.record(out_dir, "apply_data_guard", finding_id=finding_id, lb=lb, namespace=xc.ns, + app_firewall=app_firewall, enabled=enabled, rolled_back=rolled, + kept=(enabled and keep)) return {"mode": "apply_data_guard", "config_enabled": enabled, "rolled_back": rolled, "kept": enabled and keep} @@ -730,8 +795,13 @@ def rollback(): } +_DEFAULT_VALIDATION_PROPERTIES = ["PROPERTY_HTTP_HEADERS", "PROPERTY_QUERY_PARAMETERS", + "PROPERTY_HTTP_BODY"] + + def apply_api_schema(lb: str, *, openapi: dict | None = None, swagger_name: str | None = None, apidef_name: str | None = None, target_url: str = "https://lab.banknimbus.com", + request_validation_properties: list[str] | None = None, dry_run: bool = False, keep: bool = False, allow_protected: bool = False, finding_id: str | None = None, retries: int = 10, wait_seconds: int = 8, out_dir: str = "out", log: Callable = print) -> dict: @@ -739,7 +809,11 @@ def apply_api_schema(lb: str, *, openapi: dict | None = None, swagger_name: str object store -> create an api_definition -> attach api_specification with validation_all_spec_endpoints(enforcement_block). Validates the FINDING's own exploit (via its derived probe) is blocked as a schema violation while its legit request passes — falling back to - the built-in demo negative-pay probe only when no finding-probe exists; roll back on failure.""" + the built-in demo negative-pay probe only when no finding-probe exists; roll back on failure. + + request_validation_properties selects WHICH parts of the request XC validates against the spec. + It defaults to headers + query params + body: body-only validation enforces nothing for a + credential/token carried in a header on a bodyless GET, which is a common API-auth shape.""" from .probe import probe_negative_pay xc = XC() guard_lb(lb, allow_protected=allow_protected, dry_run=dry_run) @@ -749,6 +823,7 @@ def apply_api_schema(lb: str, *, openapi: dict | None = None, swagger_name: str openapi = _DEFAULT_OPENAPI swagger_name = swagger_name or f"{lb}-swagger" # per-LB objects so apps don't collide apidef_name = apidef_name or f"{lb}-apidef" + validate_props = list(request_validation_properties or _DEFAULT_VALIDATION_PROPERTIES) if dry_run: already = bool(xc.get_lb(lb).get("spec", {}).get("api_specification")) return {"mode": "dry_run", "already_on": already, @@ -766,10 +841,11 @@ def apply_api_schema(lb: str, *, openapi: dict | None = None, swagger_name: str "included_operations": [], "excluded_operations": []}]}}) log(f"created api_definition '{apidef_name}'") from . import audit - audit.record(out_dir, "create_api_definition", name=apidef_name, swagger=swagger_name) + audit.record(out_dir, "create_api_definition", finding_id=finding_id, name=apidef_name, + namespace=xc.ns, swagger=swagger_name) # 3. snapshot + attach the validation-block api_specification - ctx = ApplyContext(xc=xc, lb=lb, out_dir=out_dir, log=log).load() + ctx = ApplyContext(xc=xc, lb=lb, out_dir=out_dir, log=log, finding_id=finding_id).load() spec = ctx.spec had = "api_specification" in spec ctx.self_test() @@ -787,7 +863,7 @@ def apply_api_schema(lb: str, *, openapi: dict | None = None, swagger_name: str "api_definition": ref, "validation_all_spec_endpoints": { "validation_mode": {"validation_mode_active": { - "request_validation_properties": ["PROPERTY_HTTP_BODY"], "enforcement_block": {}}}, + "request_validation_properties": validate_props, "enforcement_block": {}}}, "fall_through_mode": {"fall_through_mode_allow": {}}}, } try: @@ -805,7 +881,8 @@ def apply_api_schema(lb: str, *, openapi: dict | None = None, swagger_name: str "reason": "XC OAS-validation quota/entitlement unavailable (429) — can't attach on this tenant", "before_after": {"before": before, "after": before}} raise - log("attached api_specification (OpenAPI validation, block mode)") + log("attached api_specification (OpenAPI validation, block mode) — validating " + + ", ".join(p.replace("PROPERTY_", "").lower() for p in validate_props)) def rollback(): safe_rollback(ctx, verify=lambda b: ("api_specification" in b) == had) @@ -834,8 +911,9 @@ def rollback(): rollback() rolled = True before_after = {"before": before, "after": res} - audit.record(out_dir, "apply_api_schema", lb=lb, apidef=apidef_name, passed=passed, - rolled_back=rolled, before_after=before_after) + audit.record(out_dir, "apply_api_schema", finding_id=finding_id, lb=lb, namespace=xc.ns, + apidef=apidef_name, passed=passed, rolled_back=rolled, kept=(passed and keep), + before_after=before_after) return {"mode": "apply_api_schema", "before_after": before_after, "passed": passed, "rolled_back": rolled, "kept": passed and keep} diff --git a/src/vpcopilot/audit.py b/src/vpcopilot/audit.py index 86eefd9..5b44766 100644 --- a/src/vpcopilot/audit.py +++ b/src/vpcopilot/audit.py @@ -1,17 +1,26 @@ -"""Append-only audit log of every mutating action (create/attach/enable/rollback/PR). +"""Append-only audit log of every mutating action (create/attach/enable/rollback/PR/retire). -One JSON object per line in `/audit.log` with a UTC timestamp, the action, and -details — so there's a durable record of exactly what the copilot changed, when, and the -result. Read-only actions (scan, dry-run) are not recorded.""" +One JSON object per line in `/audit.log` with a UTC timestamp, the action, and details — so +there's a durable record of exactly what the copilot changed, when, and the result. Read-only +actions (scan, dry-run) are not recorded: nothing changed, so there is nothing to answer for. + +Identity (`run_id` / `actor` / `host` / `tool_version`) is stamped **here** rather than at each call +site, so no mutating path can forget it and no caller can override it — an entry that cannot say who +made the change, from where, and as part of which run is not an audit record.""" from __future__ import annotations import json -from datetime import datetime, timezone from pathlib import Path +from . import __version__, runmeta + +_STAMPED = ("ts", "run_id", "actor", "host", "tool_version") + def record(out_dir, action: str, **detail) -> None: - entry = {"ts": datetime.now(timezone.utc).isoformat(), "action": action, **detail} + detail = {k: v for k, v in detail.items() if k not in _STAMPED} + entry = {"ts": runmeta.utc_now(), "action": action, "run_id": runmeta.run_id(out_dir), + "actor": runmeta.actor(), "host": runmeta.host(), "tool_version": __version__, **detail} p = Path(out_dir) p.mkdir(parents=True, exist_ok=True) with open(p / "audit.log", "a") as f: diff --git a/src/vpcopilot/cli.py b/src/vpcopilot/cli.py index 3f8cb05..4f42b28 100644 --- a/src/vpcopilot/cli.py +++ b/src/vpcopilot/cli.py @@ -131,6 +131,7 @@ def apply( probe_pass: str = typer.Option(None, "--probe-pass", help="validation login password (or VPCOPILOT_PROBE_PASS)"), probe_login_path: str = typer.Option(None, "--probe-login-path", help="login endpoint path, default /api/login (or VPCOPILOT_PROBE_LOGIN_PATH)"), probe_token: str = typer.Option(None, "--probe-token", help="bearer token for validation instead of user/pass (or VPCOPILOT_PROBE_TOKEN)"), + finding: str = typer.Option(None, "--finding", help="finding id whose probe (out/probes.json) validates this policy; overrides the ledger lookup"), out: str = typer.Option("out", help="output directory"), ): """Gated apply: (create from scan) -> snapshot -> self-test -> attach -> validate -> refine/rollback.""" @@ -142,12 +143,13 @@ def apply( if flag: os.environ[key] = flag logf = lambda m: rprint(f"[dim]{m}[/dim]") # noqa: E731 - kw = dict(dry_run=dry_run, keep=keep, allow_protected=allow_protected_lb, probe=probe, out_dir=out, log=logf) + kw = dict(dry_run=dry_run, keep=keep, allow_protected=allow_protected_lb, probe=probe, + finding_id=finding, out_dir=out, log=logf) if from_scan and refine and not dry_run and not create_only: from .refiner import refine_apply_service_policy res = refine_apply_service_policy(from_scan, lb, url, name=name, keep=keep, allow_protected=allow_protected_lb, max_refine=refine_attempts, - out_dir=out, log=logf) + finding_id=finding, out_dir=out, log=logf) elif from_scan: from .apply import apply_from_scan res = apply_from_scan(from_scan, lb, url, name=name, create_only=create_only, **kw) @@ -164,13 +166,17 @@ def apply_maluser( dry_run: bool = typer.Option(False, "--dry-run", help="no mutation; show current + would-be change"), keep: bool = typer.Option(False, "--keep", help="leave detection enabled (default: rollback)"), allow_protected_lb: bool = typer.Option(False, "--allow-protected-lb", help="permit mutating a protected LB"), + user_id_header: str = typer.Option(None, "--user-id-header", help="key detection on this request header (e.g. X-Agent-Id) instead of client IP"), + user_id_name: str = typer.Option(None, "--user-id-name", help="user_identification object name (default -user-id)"), out: str = typer.Option("out", help="output directory"), ): """Enable XC Malicious-User Detection on an LB (behavioral control; config-level validation).""" from .apply import apply_malicious_user res = apply_malicious_user(lb, dry_run=dry_run, keep=keep, allow_protected=allow_protected_lb, - finding_id=finding, out_dir=out, log=lambda m: rprint(f"[dim]{m}[/dim]")) + finding_id=finding, user_id_header=user_id_header, + user_identification_name=user_id_name, + out_dir=out, log=lambda m: rprint(f"[dim]{m}[/dim]")) rprint(Panel.fit("\n".join(f"[bold]{k}[/bold]: {v}" for k, v in res.items()), title="apply-maluser")) @@ -181,7 +187,11 @@ def apply_ratelimit( unit: str = typer.Option("MINUTE", help="SECOND | MINUTE | HOUR"), burst: int = typer.Option(1, help="burst multiplier (>0)"), behavioral: bool = typer.Option(False, "--behavioral", help="B3: drive a burst + confirm 429s (not just config)"), + behavioral_path: str = typer.Option("/login", "--behavioral-path", help="path to burst for --behavioral (use the rate-limited endpoint)"), url: str = typer.Option("https://lab.banknimbus.com", help="live host for the behavioral burst"), + user_id_header: str = typer.Option(None, "--user-id-header", help="key the limit per this request header (e.g. X-Agent-Id) instead of LB-wide"), + user_id_name: str = typer.Option(None, "--user-id-name", help="user_identification object name (default -user-id)"), + burst_header: list[str] = typer.Option(None, "--burst-header", help="header sent on every behavioral burst request, name=value (repeatable)"), finding: str = typer.Option(None, "--finding", help="link to a finding id for the ledger"), dry_run: bool = typer.Option(False, "--dry-run"), keep: bool = typer.Option(False, "--keep", help="leave enabled on success (default: rollback)"), @@ -191,7 +201,15 @@ def apply_ratelimit( """Enable XC rate limiting on an LB (config validation + rollback; --behavioral drives traffic).""" from .apply import apply_rate_limit + burst_headers = {} + for h in (burst_header or []): + if "=" in h: + k, v = h.split("=", 1) + burst_headers[k.strip()] = v.strip() + res = apply_rate_limit(lb, requests=requests, unit=unit, burst=burst, behavioral=behavioral, + behavioral_path=behavioral_path, burst_headers=burst_headers or None, + user_id_header=user_id_header, user_identification_name=user_id_name, target_url=url, finding_id=finding, dry_run=dry_run, keep=keep, allow_protected=allow_protected_lb, out_dir=out, log=lambda m: rprint(f"[dim]{m}[/dim]")) @@ -237,6 +255,8 @@ def apply_waf_cmd( @app.command(name="apply-dataguard") def apply_dataguard_cmd( lb: str = typer.Option("vpcopilot-lab", help="HTTP LB name"), + app_firewall: str = typer.Option("vpcopilot-lab-waf", help="app_firewall to ensure (created Blocking if missing); an already-attached, differently-named WAF is reused, never clobbered"), + template: str = typer.Option("nimbus-waf", help="app_firewall to clone for the Blocking WAF"), finding: str = typer.Option(None, "--finding", help="link to a finding id for the ledger"), dry_run: bool = typer.Option(False, "--dry-run"), keep: bool = typer.Option(False, "--keep", help="leave Data Guard on (default: rollback)"), @@ -246,7 +266,8 @@ def apply_dataguard_cmd( """Enable WAF Data Guard on an LB (mask sensitive data in responses; config validation).""" from .apply import apply_data_guard - res = apply_data_guard(lb, finding_id=finding, dry_run=dry_run, keep=keep, + res = apply_data_guard(lb, app_firewall=app_firewall, template=template, finding_id=finding, + dry_run=dry_run, keep=keep, allow_protected=allow_protected_lb, out_dir=out, log=lambda m: rprint(f"[dim]{m}[/dim]")) rprint(Panel.fit("\n".join(f"[bold]{k}[/bold]: {v}" for k, v in res.items()), title="apply-dataguard")) @@ -257,6 +278,10 @@ def apply_apischema_cmd( lb: str = typer.Option("vpcopilot-lab", help="HTTP LB name"), url: str = typer.Option("https://lab.banknimbus.com", help="live host to validate against"), openapi_file: str = typer.Option(None, "--openapi-file", help="OpenAPI/Swagger JSON to enforce (default: built-in Nimbus spec)"), + validate_properties: str = typer.Option( + "PROPERTY_HTTP_HEADERS,PROPERTY_QUERY_PARAMETERS,PROPERTY_HTTP_BODY", "--validate-properties", + help="comma-separated request parts XC validates against the spec (default: headers, query " + "params and body — body-only enforces nothing on a bodyless GET)"), finding: str = typer.Option(None, "--finding", help="link to a finding id for the ledger"), dry_run: bool = typer.Option(False, "--dry-run"), keep: bool = typer.Option(False, "--keep", help="leave validation enabled on success (default: rollback)"), @@ -271,7 +296,9 @@ def apply_apischema_cmd( from .apply import apply_api_schema openapi = _json.loads(_Path(openapi_file).read_text()) if openapi_file else None + props = [p.strip() for p in (validate_properties or "").split(",") if p.strip()] or None res = apply_api_schema(lb, openapi=openapi, target_url=url, finding_id=finding, dry_run=dry_run, + request_validation_properties=props, keep=keep, allow_protected=allow_protected_lb, out_dir=out, log=lambda m: rprint(f"[dim]{m}[/dim]")) rprint(Panel.fit("\n".join(f"[bold]{k}[/bold]: {v}" for k, v in res.items()), title="apply-apischema")) @@ -407,6 +434,29 @@ def audit(out: str = typer.Option("out", help="output directory")): rprint(t) +@app.command() +def export( + out: str = typer.Option("out", help="run directory to export"), + output: str = typer.Option(None, "--output", help="bundle path (default: /audit-bundle.zip)"), + all_runs: bool = typer.Option(False, "--all", help="bundle every run dir on disk, each in its own folder"), + root: str = typer.Option(".", help="where to look for run dirs when --all is used"), +): + """Export the audit evidence bundle for a run: every change made to a load balancer, the finding + that justified it, the exact XC config pushed, and the pre-change snapshot — with a manifest that + SHA-256s every member. Same bundle the console's Retire step downloads.""" + from .export import build_audit_events, write_bundle, write_bundle_all + + if all_runs: + path = write_bundle_all(root, output) + rprint(f"wrote [bold]{path}[/bold] (all runs under {root})") + return + events = build_audit_events(out) + if not events: + rprint(f"[yellow]no audit entries in {out} — nothing has changed a load balancer yet[/yellow]") + path = write_bundle(out, output) + rprint(f"wrote [bold]{path}[/bold] · {len(events)} audit event(s)") + + @app.command() def ledger(out: str = typer.Option("out", help="output directory")): """Show the remediation ledger: found -> mitigated -> remediated -> retired.""" diff --git a/src/vpcopilot/config.py b/src/vpcopilot/config.py index 67d8bc8..1921fcc 100644 --- a/src/vpcopilot/config.py +++ b/src/vpcopilot/config.py @@ -11,6 +11,10 @@ DEFAULT_MODEL = "anthropic/claude-opus-4-8" +# Every agent in the pipeline, in lifecycle order. Recorded per run so an audit export can say which +# model produced each finding, band-aid and cure. +AGENT_NAMES = ("discover", "verify", "triage", "generate", "remediate", "probe", "refine") + @dataclass class AgentConfig: diff --git a/src/vpcopilot/console/app.py b/src/vpcopilot/console/app.py index a0815e7..00d9e6b 100644 --- a/src/vpcopilot/console/app.py +++ b/src/vpcopilot/console/app.py @@ -7,11 +7,13 @@ import json import os +import re import threading +from datetime import datetime, timezone from pathlib import Path from dotenv import load_dotenv -from fastapi import FastAPI, HTTPException +from fastapi import FastAPI, HTTPException, Response from fastapi.responses import FileResponse from pydantic import BaseModel @@ -71,6 +73,18 @@ def _active_tag() -> str: load_dotenv(ENV_PATH) _scan = {"state": "idle", "log": [], "summary": None, "error": None} +LOG_MAX = 20_000 # the log endpoints serve the FULL transcript now — keep a ceiling on what one run pins in memory + + +def _append(buf: list, msg: str) -> None: + """Bounded log sink. A scan/apply used to be served as a truncated tail, so an unbounded buffer + never mattered; now the console streams the whole transcript, so cap it — and say so in-band + rather than silently dropping the rest.""" + if len(buf) < LOG_MAX: + buf.append(msg) + elif len(buf) == LOG_MAX: + buf.append(f"… log capped at {LOG_MAX} lines — further output suppressed (artifacts are still written to the out dir)") + # ---------------- helpers ---------------- def _read_env() -> dict: @@ -115,6 +129,13 @@ def _rj(name: str, default): return json.loads(p.read_text()) if p.exists() else default +def _stamped_name(kind: str, ext: str) -> str: + """`vpcopilot---.` — a downloaded artifact has to say which run it came + from and when, so several of them can sit in one folder (or one ticket) without ambiguity.""" + run = re.sub(r"[^A-Za-z0-9]+", "-", str(OUT)).strip("-") or "run" + return f"vpcopilot-{kind}-{run}-{datetime.now(timezone.utc):%Y%m%dT%H%M%SZ}.{ext}" + + # ---------------- read endpoints ---------------- @app.get("/api/results") def results(): @@ -142,6 +163,38 @@ def audit(): return load(str(OUT)) +@app.get("/api/audit-events") +def audit_events(): + """F3: the audit log normalized into one shape per event and joined to the ledger + findings, so + the Retire step can SHOW the trail — every change made to a load balancer, and the vulnerability + that justified it — before anyone exports it. Seeing it first is what makes the export + trustworthy: you can check what leaves the machine.""" + from ..export import build_audit_events + return {"out": str(OUT), "events": build_audit_events(str(OUT))} + + +@app.get("/api/runs") +def runs(): + """Run dirs on disk that have something to export — backs the Retire step's 'all runs' bundle.""" + from ..export import find_runs + return {"current": str(OUT), "runs": find_runs(".")} + + +@app.get("/api/audit-export") +def audit_export(scope: str = "run"): + """F3: download the evidence bundle — `scope=run` for the run the console is reading, `scope=all` + for every run dir on disk (each in its own folder, under a top-level index). A .zip because the + evidence is plural: the normalized events, the raw append-only log verbatim, the exact XC configs + that were pushed, the pre-change LB snapshots, and a manifest that SHA-256s every member.""" + from ..export import bundle_all_bytes, bundle_bytes + if scope not in ("run", "all"): + raise HTTPException(400, f"unknown scope '{scope}' (use 'run' or 'all')") + data = bundle_bytes(str(OUT)) if scope == "run" else bundle_all_bytes(".") + name = _stamped_name("audit", "zip") if scope == "run" else _stamped_name("audit-all", "zip") + return Response(content=data, media_type="application/zip", + headers={"content-disposition": f'attachment; filename="{name}"'}) + + AGENT_ROLES = { "discover": "read source → candidate findings", "verify": "adversarially confirm or refute each finding", @@ -281,11 +334,16 @@ def build_benchmark(body: BenchReq): @app.get("/api/report") -def report_html(): - """Build + serve the standalone HTML report (E3) for the current out/ dir.""" +def report_html(download: bool = False): + """Build + serve the standalone HTML report (E3) for the current out/ dir. It is REBUILT on every + request, so whoever opens it always gets the latest run — which is why Review links straight to + it rather than to a stale file on disk (F2). `download=1` names the file and sends it as an + attachment (the report is meant to be shared); the default stays inline for the new-tab view.""" from ..report import write_report path = write_report(str(OUT), str(OUT / "report.html")) - return FileResponse(path, media_type="text/html") + if not download: + return FileResponse(path, media_type="text/html") + return FileResponse(path, media_type="text/html", filename=_stamped_name("report", "html")) @app.get("/api/defaults") @@ -405,7 +463,7 @@ def _run_scan(repo: str, out: str, min_confidence: float = 0.5, from ..pipeline import run_pipeline summary = run_pipeline(repo, out_dir=out, config_path=_active_config, min_confidence=min_confidence, max_files=max_files, max_bytes=max_bytes, draft_code_fixes=draft_code_fixes, - log=lambda m: _scan["log"].append(m)) + log=lambda m: _append(_scan["log"], m)) _scan.update(state="done", summary=summary) except Exception as e: # noqa: BLE001 _scan.update(state="error", error=str(e)) @@ -429,8 +487,12 @@ def start_scan(body: ScanReq): @app.get("/api/scan") -def scan_status(): - return {**_scan, "log": _scan["log"][-40:]} +def scan_status(since: int = 0): + """F1: scan state + log. The log is the FULL transcript so a long run can be scrolled end-to-end; + the console passes `since=` and appends only the new tail (`log_total` + is the running length — a smaller value than `since` means a newer scan reset the buffer).""" + log = _scan["log"] + return {**_scan, "log": log[max(0, since):], "log_total": len(log)} # ---------------- impact + ledger loop (C1/C3/C4) ---------------- @@ -528,7 +590,7 @@ def _run_action(job_id: str, body: ActionReq): job = _jobs[job_id] t0 = time.perf_counter() try: - res = _dispatch_action(body, lambda m: job["log"].append(m)) + res = _dispatch_action(body, lambda m: _append(job["log"], m)) job.update(state="done", result=res) if not body.dry_run: # feed MTTM for the hero + a self-contained record for the model benchmark from ..audit import record @@ -555,11 +617,14 @@ def start_action(body: ActionReq): @app.get("/api/action") -def action_status(job: str): +def action_status(job: str, since: int = 0): + """F1: same full-transcript + `since` tail contract as /api/scan — the refiner's attach → validate → + refine → retry narration is the evidence a band-aid works, so none of it should scroll off.""" j = _jobs.get(job) if not j: raise HTTPException(404, "no such job") - return {**j, "log": j["log"][-60:], "job": job} + log = j["log"] + return {**j, "log": log[max(0, since):], "log_total": len(log), "job": job} # ---------------- action endpoints (gated) ---------------- diff --git a/src/vpcopilot/console/static/index.html b/src/vpcopilot/console/static/index.html index 8c68f63..e2ec60b 100644 --- a/src/vpcopilot/console/static/index.html +++ b/src/vpcopilot/console/static/index.html @@ -62,7 +62,12 @@ /* live apply */ .jobbox { display:none; margin-top:6px; border-left:3px solid var(--line); padding:6px 10px; background:var(--bg); border-radius:6px; } .jobbox.ok { border-left-color:var(--ok); } .jobbox.err { border-left-color:#a1001b; } - .jobstat { font-size:12px; font-weight:600; } .joblog { max-height:150px; margin:6px 0 0; font-size:11px; } + .jobstat { font-size:12px; font-weight:600; } .joblog { max-height:220px; margin:6px 0 0; font-size:11px; } + /* log windows — a bounded height is what makes the global `pre { overflow:auto }` actually scroll */ + .logbox { max-height:46vh; min-height:140px; overflow-y:auto; overscroll-behavior:contain; } + .logbar { display:flex; align-items:center; gap:10px; margin-top:4px; font-size:11px; color:var(--grey); } + .logbar button.mini { margin:0; padding:2px 9px; font-size:11px; } + .rowa { display:flex; align-items:center; gap:10px; flex-wrap:wrap; margin:2px 0 12px; font-size:12px; } .badge { display:inline-block; background:#e7f5ec; color:var(--ok); border:1px solid #bfe3cc; border-radius:20px; padding:1px 9px; font-size:11px; font-weight:700; } .ba { margin-top:5px; font-size:12px; } .ba-b{color:var(--amber)} .ba-a{color:var(--ok);font-weight:700} /* inspect + track */ @@ -150,7 +155,9 @@
-
+
+ @@ -158,6 +165,11 @@

② Review findings

What the copilot found and verified, and the XC control it recommends for each. Click a row to inspect the exploit, code, and generated policy. Ready? Go to ③ Mitigate.

+
+ + + +
Loading…
@@ -187,6 +199,16 @@

Once a cure ships, detach its temporary XC control. The ledger tracks every finding found → mitigated → remediated → retired.

Loading…
+

Audit trail — every change made to a load balancer

+

Each row is one change the copilot made to live XC config, the finding that justified it, and how it ended. Dry runs are not listed: nothing changed, so there is nothing to answer for. Export the bundle for a change board — it carries this table plus the raw append-only log, the exact XC configs pushed, the pre-change LB snapshots, and a manifest that SHA-256s every file.

+
+ + + + +
+
Loading…
+
@@ -215,7 +237,9 @@ - + + +

Agents & models — model-independent, per-agent in config/agents.yaml

@@ -241,17 +265,36 @@ try { history.replaceState(null, "", "#"+id); } catch(e){ location.hash=id; } // deep-linkable steps document.getElementById("settings").style.display = ACTION_STEPS.has(id) ? "block" : "none"; if(id==="scan"){ loadHero(); } - if(id==="review"){ loadResults().then(renderReview); loadHero(); } + if(id==="review"){ loadResults().then(renderReview); loadHero(); reportNote(); } if(id==="mitigate"){ loadResults().then(renderMitigate); loadHero(); } if(id==="cure"){ loadResults().then(renderCure); loadHero(); } - if(id==="retire"){ loadLedger(); loadHero(); } + if(id==="retire"){ loadLedger(); loadAudit(); loadHero(); } if(id==="benchmark"){ loadBenchmarks(); } - if(id==="setup"){ loadConfig(); loadAgents(); } + if(id==="setup"){ loadConfig(); loadAgents(); reportNote(); } +} +// The report is rebuilt server-side on every open, so it always reflects the LATEST run — name the +// run dir it comes from, since a scan can repoint it (out-claude-vampi, demo/out, …). +async function reportNote(){ + let out; try { out=(await jget("/api/models")).out; } catch(e){ return; } + document.querySelectorAll(".repnote").forEach(el=>el.textContent="latest run · rebuilt from "+out+" every time you open it"); } async function jget(u){ const r=await fetch(u); if(!r.ok) throw new Error((await r.json()).detail||r.statusText); return r.json(); } async function jpost(u,b){ const r=await fetch(u,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(b)}); const d=await r.json().catch(()=>({})); if(!r.ok) throw new Error(d.detail||r.statusText); return d; } const esc = s => (s==null?"":String(s)).replace(/[&<>]/g,c=>({"&":"&","<":"<",">":">"}[c])); +// ---- log windows ---- +// The scan/apply endpoints serve the FULL transcript and take ?since=, so we +// APPEND the new tail instead of rewriting the node. That keeps scroll position and text selection +// alive across polls — you can scroll back through a long run while it is still going. +function atBottom(el){ return el.scrollHeight - el.scrollTop - el.clientHeight < 24; } +function followLog(el){ el.scrollTop = el.scrollHeight; el.dispatchEvent(new Event("scroll")); } +function pushLog(el, lines){ + if(!lines || !lines.length) return; + const stick = atBottom(el); // only chase the tail if we were already at it + el.appendChild(document.createTextNode(lines.join("\n")+"\n")); // text node, never innerHTML — this is untrusted output + if(stick) el.scrollTop = el.scrollHeight; + el.dispatchEvent(new Event("scroll")); +} const S = ()=>({ lb:setLb.value.trim(), url:setUrl.value.trim(), repo:setRepo.value.trim(), base:setBase.value.trim(), prefix:setPrefix.value.trim(), dry:setDry.checked, allow:setAllow.checked, refine:setRefine.checked, refineAttempts:parseInt(setRefineAttempts.value)||3, keep:setKeep.checked }); @@ -375,9 +418,10 @@ jpost("/api/action",{control,finding_id:fid,policy_name:policyName,lb:st.lb,url:st.url, dry_run:st.dry,keep:st.keep,refine:st.refine,refine_attempts:st.refineAttempts,allow_protected_lb:st.allow}) .then(job=>{ + let n=0; // lines already appended for this job — same full-transcript + ?since= tail as the scan log const poll=setInterval(async()=>{ - let s; try{ s=await jget("/api/action?job="+job.job); }catch(e){ return; } - logEl.textContent=(s.log||[]).join("\n"); logEl.scrollTop=logEl.scrollHeight; + let s; try{ s=await jget("/api/action?job="+job.job+"&since="+n); }catch(e){ return; } + pushLog(logEl, s.log); n = s.log_total ?? (n + (s.log||[]).length); if(s.state==="running"){ statEl.textContent=`applying ${esc(control)}…`; return; } clearInterval(poll); if(s.state==="error"){ box.className="jobbox err"; statEl.textContent="error: "+s.error; loadHero(); resolve({ok:false}); return; } @@ -491,12 +535,49 @@ }).join(""); ledgerBox.innerHTML = rows ? `${rows}
findingsevprogressmitigationcure
` : '
empty — run a scan.
'; } +// ---- audit trail + evidence export (F3) ---- +// The trail is SHOWN before it can be exported: an operator should be able to check what leaves the +// machine, and a change board is going to ask "which vuln justified this LB change?" — so the join +// back to the finding happens server-side (/api/audit-events) and lands in the table. +let AUDIT = []; +const OUTCOME_CLS = {kept:"st-remediated", passed:"st-remediated", created:"st-mitigated", + pr_opened:"st-remediated", retired:"st-retired", rolled_back:"st-found", + failed:"st-mitigated", unfixable:"st-mitigated", rollback_failed:"st-mitigated"}; +async function loadAudit(){ + let d; try { d=await jget("/api/audit-events"); } catch(err){ auditBox.textContent="error: "+err.message; return; } + AUDIT=d.events||[]; auditNote.textContent=AUDIT.length ? AUDIT.length+" change(s) recorded in "+d.out : ""; + renderAudit(); +} +function renderAudit(){ + const q=(auditFilter.value||"").trim().toLowerCase(); + const rows=AUDIT.filter(e=>!q||JSON.stringify(e).toLowerCase().includes(q)).map((e,i)=>{ + const proof = e.exploit_before||e.exploit_after + ? `
${esc(e.exploit_before||"—")} → ${esc(e.exploit_after||"—")}
` : ""; + const why = e.finding_id + ? `${esc(e.title||e.finding_id)}
${esc(e.finding_id)}${e.severity?" · "+esc(e.severity):""}
` + : '—'; + return ` + ▸ ${esc(String(e.ts).slice(0,19).replace("T"," "))} + ${esc(e.action)}
${esc(e.category)}
+ ${why} + ${esc(e.control||"—")}
${esc(e.object||"")}
+ ${esc(e.lb||"—")}
${esc(e.namespace||"")}
+ ${esc(e.outcome)}${e.attempts>1?` ×${esc(e.attempts)}`:""}${proof} + ${esc(e.actor||"—")} + 
${esc(JSON.stringify(e.detail,null,2))}
`; + }).join(""); + auditBox.innerHTML = rows + ? `${rows}
when (UTC)actionjustified bycontrolload balanceroutcomeby
` + : `
${AUDIT.length?"nothing matches that filter.":"no changes recorded yet — dry runs are not logged, so this fills in on the first live Mitigate."}
`; +} +function exportAudit(scope){ location.href="/api/audit-export?scope="+encodeURIComponent(scope); } + async function doRetire(fid){ const st=S(); const out=document.getElementById("ret-"+fid); if(!confirm(`Retire the band-aid for ${fid}? (detaches the live XC control${st.allow?"":"; needs allow-protected for a protected LB"})`)) return; out.className="act"; out.textContent=" retiring…"; try { const r=await jpost("/api/retire",{finding_id:fid,force:true,allow_protected_lb:st.allow}); - out.className="act ok"; out.textContent=" "+(r.status||"retired"); loadLedger(); loadHero(); + out.className="act ok"; out.textContent=" "+(r.status||"retired"); loadLedger(); loadAudit(); loadHero(); } catch(e){ out.className="act err"; out.textContent=" "+e.message; } } @@ -586,13 +667,22 @@

Full matrix

${head} try { out.textContent=JSON.stringify(await jget("/api/xc-status?lb="+encodeURIComponent(xcLb.value.trim())),null,2); } catch(e){ out.textContent="error: "+e.message; } } // ---- scan (auto-advances to Review on done) ---- +let SCAN_N = 0; // lines already appended — the server streams the transcript from here +scanLog.addEventListener("scroll",()=>{ scanFollow.style.display = atBottom(scanLog) ? "none" : "inline-block"; }); async function runScan(){ const repo=scanRepo.value.trim(); if(!repo){alert("enter a repo path");return;} scanState.textContent="starting…"; try { await jpost("/api/scan",{repo,out:scanOut.value.trim()||"out",min_confidence:parseFloat(scanMinConf.value)||0.5,max_files:parseInt(scanMaxFiles.value)||200,max_bytes:parseInt(scanMaxBytes.value)||60000,draft_code_fixes:scanRemediate.checked}); } catch(e){ scanState.textContent="error: "+e.message; return; } - scanLog.style.display="block"; - const poll=setInterval(async()=>{ const s=await jget("/api/scan"); scanState.textContent="state: "+s.state; - scanLog.textContent=(s.log||[]).join("\n")+(s.error?("\nERROR: "+s.error):""); + SCAN_N=0; scanLog.textContent=""; scanLogCount.textContent=""; scanLogWrap.style.display="block"; + const poll=setInterval(async()=>{ + let s; try { s=await jget("/api/scan?since="+SCAN_N); } catch(e){ return; } // transient fetch blip — keep polling + scanState.textContent="state: "+s.state; + // a newer scan reset the buffer (log_total went backwards) — start the transcript over + if(s.log_total < SCAN_N){ scanLog.textContent=""; SCAN_N=0; try { s=await jget("/api/scan"); } catch(e){ return; } } + pushLog(scanLog, s.log); + SCAN_N = s.log_total ?? (SCAN_N + (s.log||[]).length); + scanLogCount.textContent = SCAN_N ? SCAN_N+" line"+(SCAN_N===1?"":"s") : ""; if(s.state==="done"||s.state==="error"){ clearInterval(poll); + if(s.error) pushLog(scanLog,["ERROR: "+s.error]); if(s.state==="done"){ scanState.textContent+=" — review below"; await loadResults(); show("review"); } } },1500); } diff --git a/src/vpcopilot/engine.py b/src/vpcopilot/engine.py index 564e687..dcde405 100644 --- a/src/vpcopilot/engine.py +++ b/src/vpcopilot/engine.py @@ -43,6 +43,7 @@ class ApplyContext: lb: str out_dir: str = "out" log: Callable = print + finding_id: str | None = None # the vuln this change is justified by — carried for the audit trail sleep: Callable = None # DI: tests pass a no-op so polls don't wait lb_obj: dict = field(default_factory=dict) spec: dict = field(default_factory=dict) @@ -121,6 +122,8 @@ def safe_rollback(ctx: ApplyContext, *, retries: int = 3, verify: Callable[[dict if i < retries: ctx.sleep(2) from . import audit - audit.record(ctx.out_dir, "rollback_failed", lb=ctx.lb, reason=last) + # The one entry that MUST be attributable: the LB may be left in a changed state. + audit.record(ctx.out_dir, "rollback_failed", finding_id=ctx.finding_id, lb=ctx.lb, + namespace=getattr(ctx.xc, "ns", None), reason=last) ctx.log(f"!! ROLLBACK FAILED after {retries} tries: {last} — the LB may be in a changed state") raise RollbackError(f"could not restore {ctx.lb} to snapshot after {retries} tries: {last}") diff --git a/src/vpcopilot/export.py b/src/vpcopilot/export.py new file mode 100644 index 0000000..f15c6eb --- /dev/null +++ b/src/vpcopilot/export.py @@ -0,0 +1,264 @@ +"""F3 — audit export: turn a run's artifacts into evidence someone else can review. + +`audit.log` is append-only and deliberately heterogeneous — each action records what mattered for +that action. That is right for a log and wrong for a reviewer, who needs one shape they can sort, +filter and hand to a change board. `build_audit_events` normalizes it: one row per auditable event, +joined against the ledger and the scan artifacts so every LB change carries the vulnerability that +justified it, its severity, and the outcome. + +`write_bundle` packages the whole run — the normalized events, the raw log verbatim, the exact XC +configs that were pushed, and the pre-change LB snapshot — with a manifest that SHA-256s every +member, so a bundle can be checked for completeness after it leaves this machine. + +Read-only: nothing here touches XC, GitHub, or the run's artifacts.""" +from __future__ import annotations + +import csv +import hashlib +import io +import json +import zipfile +from pathlib import Path + +from . import __version__, audit, ledger, runmeta + +# What each action DID, for a reviewer scanning a column rather than reading action names. +CATEGORY = { + "apply_service_policy": "mitigate", "apply_malicious_user": "mitigate", + "apply_rate_limit": "mitigate", "apply_bot_defense": "mitigate", "apply_waf": "mitigate", + "apply_data_guard": "mitigate", "apply_api_schema": "mitigate", + "create_service_policy": "create", "create_app_firewall": "create", + "create_api_definition": "create", + "refine_apply": "refine", "apply_timing": "timing", + "open_pr": "cure", "retire": "retire", "rollback_failed": "rollback", +} +# The XC control each action acted on, where the action name alone implies it. +CONTROL = { + "apply_service_policy": "service_policy", "create_service_policy": "service_policy", + "refine_apply": "service_policy", "apply_malicious_user": "malicious_user", + "apply_rate_limit": "rate_limit", "apply_bot_defense": "bot_defense", + "apply_waf": "waf", "create_app_firewall": "waf", "apply_data_guard": "waf_data_guard", + "apply_api_schema": "api_schema", "create_api_definition": "api_schema", +} +# Flat CSV columns, in reading order: when · who · what · why · where · outcome. +COLUMNS = ["ts", "run_id", "actor", "host", "category", "action", "finding_id", "title", + "vuln_class", "severity", "ledger_state", "control", "lb", "namespace", "object", + "outcome", "attempts", "exploit_before", "exploit_after", "legit_ok", "pr_url", + "tool_version", "detail"] +# Every artifact worth carrying as evidence. Missing ones are simply skipped — an unscanned dir has +# no findings.json, a run with no live apply has no lb_snapshot.json. +BUNDLE_FILES = ["run.json", "audit.log", "ledger.json", "findings.json", "triage.json", + "policies.json", "remediations.json", "summary.json", "metrics.json", + "probes.json", "correlations.json", "lb_snapshot.json", "report.html"] + + +def _rj(out: Path, name: str, default): + p = out / name + if not p.exists(): + return default + try: + return json.loads(p.read_text()) + except json.JSONDecodeError: + return default + + +def _outcome(e: dict) -> str: + """One word for 'did the change stick, and did it work?'. The apply paths disagree on the key + they use for success (`passed` vs `config_enabled` vs `enabled`, and the curated demo dataset + uses yet another mix), so coalesce rather than trust one name.""" + fixed = {"rollback_failed": "rollback_failed", "retire": "retired", "open_pr": "pr_opened", + "create_service_policy": "created", "create_app_firewall": "created", + "create_api_definition": "created"}.get(e.get("action", "")) + if fixed: + return fixed + if e.get("unfixable"): + return "unfixable" + if e.get("rolled_back"): + return "rolled_back" + ok = e.get("passed") + if ok is None: + ok = e.get("config_enabled") + if ok is None: + ok = e.get("enabled") + if ok is None: + return "recorded" + return "kept" if (ok and e.get("kept")) else ("passed" if ok else "failed") + + +def _ba(e: dict, side: str) -> dict: + return ((e.get("before_after") or {}).get(side) or {}) if isinstance(e.get("before_after"), dict) else {} + + +def _exploit(cell: dict) -> str: + """`403 blocked` / `200 allowed` — the actual proof a band-aid worked, flattened for a cell.""" + blocked = cell.get("exploit_blocked") + if blocked is None: + return "" + return f"{cell.get('exploit_status', '')} {'blocked' if blocked else 'allowed'}".strip() + + +def _object(e: dict) -> str: + """The XC object the action touched, whichever field the action happens to name it in.""" + for k in ("policy", "app_firewall", "apidef", "name", "rate", "swagger"): + if e.get(k): + return str(e[k]) + return "" + + +def build_audit_events(out_dir: str = "out") -> list[dict]: + """Normalized, newest-first audit events for a run. Every entry in audit.log survives — an + export that silently drops `retire`, `open_pr` or a failed rollback is worse than no export.""" + out = Path(out_dir) + entries = audit.load(out_dir) + led = ledger.load(out_dir) + findings = {f.get("id"): f for f in _rj(out, "findings.json", [])} + # a policy name is often the only link back to a finding on older entries + by_policy = {a.get("policy_name"): a.get("finding_id") for a in _rj(out, "policies.json", [])} + + events = [] + for e in entries: + # `open_pr` wrote `finding` before every other action settled on `finding_id`; older logs + # attributed nothing at all, so fall back to the policy index. + fid = e.get("finding_id") or e.get("finding") or by_policy.get(e.get("policy")) + f, le = findings.get(fid, {}), (led.get(fid) or {}) + before, after = _ba(e, "before"), _ba(e, "after") + detail = {k: v for k, v in e.items() + if k not in ("ts", "action", "run_id", "actor", "host", "tool_version")} + events.append({ + "ts": e.get("ts", ""), "run_id": e.get("run_id", ""), "actor": e.get("actor", ""), + "host": e.get("host", ""), + "category": CATEGORY.get(e.get("action", ""), "other"), "action": e.get("action", ""), + "finding_id": fid or "", "title": f.get("title") or le.get("title") or "", + "vuln_class": f.get("vuln_class") or le.get("vuln_class") or "", + "severity": f.get("severity") or le.get("severity") or "", + "ledger_state": le.get("state", ""), + "control": e.get("control") or CONTROL.get(e.get("action", ""), ""), + "lb": e.get("lb", ""), "namespace": e.get("namespace", ""), "object": _object(e), + "outcome": _outcome(e), "attempts": e.get("attempts", ""), + "exploit_before": _exploit(before), "exploit_after": _exploit(after), + "legit_ok": after.get("legit_ok", ""), + "pr_url": e.get("url") or (le.get("cure") or {}).get("pr_url") or "", + "tool_version": e.get("tool_version", ""), "detail": detail, + }) + events.sort(key=lambda x: x["ts"], reverse=True) + return events + + +def to_csv(events: list[dict]) -> str: + """Flat CSV for a spreadsheet or a change board. `detail` keeps the raw JSON so nothing that was + recorded is lost in the flattening.""" + buf = io.StringIO() + w = csv.DictWriter(buf, fieldnames=COLUMNS, extrasaction="ignore", lineterminator="\n") + w.writeheader() + for e in events: + w.writerow({**e, "detail": json.dumps(e.get("detail", {}), sort_keys=True)}) + return buf.getvalue() + + +def build_manifest(out_dir: str = "out", *, members: dict | None = None) -> dict: + """What this bundle is, which run it covers, and a SHA-256 per member so it can be checked for + completeness later. `caveats` states out loud what the trail does NOT contain — an export that + implies more coverage than it has is the one way this feature could mislead someone.""" + events = build_audit_events(out_dir) + return { + "kind": "vpcopilot-audit-bundle", "schema": 1, "tool_version": __version__, + "generated": runmeta.utc_now(), "generated_by": runmeta.actor(), "generated_on": runmeta.host(), + "out_dir": str(out_dir), "run": runmeta.load(out_dir), + "events": len(events), + "actions": {a: sum(1 for e in events if e["action"] == a) for a in sorted({e["action"] for e in events})}, + "findings_touched": sorted({e["finding_id"] for e in events if e["finding_id"]}), + "caveats": [ + "Dry runs are not recorded: nothing was changed, so nothing is logged. This is the record " + "of changes MADE, not of every attempt.", + "apply_timing entries are written only by the console's Mitigate step on a live (non-dry) " + "apply; a CLI-driven run has none.", + "Entries written by older builds may lack finding_id / namespace / actor — those cells are " + "blank rather than inferred.", + ], + "members": members or {}, + } + + +def write_bundle(out_dir: str = "out", output: str | None = None) -> str: + """Zip the run's evidence to `output` (default `/audit-bundle.zip`) and return the path.""" + path = Path(output) if output else Path(out_dir) / "audit-bundle.zip" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(bundle_bytes(out_dir)) + return str(path) + + +def bundle_bytes(out_dir: str = "out", *, prefix: str = "") -> bytes: + """The evidence bundle as bytes, so the console can stream it without touching disk.""" + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z: + _add_run(z, out_dir, prefix) + return buf.getvalue() + + +def _add_run(z: zipfile.ZipFile, out_dir: str, prefix: str = "") -> dict: + """Write one run into an open archive under `prefix`, returning its manifest (so a multi-run + bundle can index what it contains).""" + out = Path(out_dir) + events = build_audit_events(out_dir) + members: dict[str, dict] = {} + + def add(name: str, data: bytes) -> None: + z.writestr(f"{prefix}{name}", data) + members[name] = {"bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()} + + add("audit.csv", to_csv(events).encode()) + add("audit-events.json", json.dumps(events, indent=2).encode()) + for name in BUNDLE_FILES: + p = out / name + if p.exists() and p.is_file(): + add(name, p.read_bytes()) # raw and verbatim — audit.log especially + # the exact XC configs that were pushed, plus every per-LB snapshot taken before a change + for sub in ("policies", "snapshots"): + for p in sorted((out / sub).glob("*")): + if p.is_file(): + add(f"{sub}/{p.name}", p.read_bytes()) + + manifest = build_manifest(out_dir, members=members) + z.writestr(f"{prefix}manifest.json", json.dumps(manifest, indent=2)) + return manifest + + +def find_runs(root: str = ".") -> list[str]: + """Run dirs on disk worth exporting — anything with an audit log or scan results. Matches how + the console's output-dir picker discovers runs (out*/ plus the seeded demo/out).""" + base = Path(root) + cands = [p for p in sorted(base.glob("out*")) if p.is_dir()] + demo = base / "demo" / "out" + if demo.is_dir(): + cands.append(demo) + return [str(p) for p in cands if (p / "audit.log").exists() or (p / "findings.json").exists()] + + +def bundle_all_bytes(root: str = ".") -> bytes: + """Every run on disk in one archive, each under its own folder, with a top-level index. For the + case an auditor asks for "everything", not one engagement.""" + runs = find_runs(root) + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z: + index = [] + for r in runs: + try: # name folders by the run dir RELATIVE to root, so an absolute root doesn't + rel = Path(r).relative_to(root) # bury each run under its full filesystem path + except ValueError: + rel = Path(r) + folder = str(rel).replace("/", "-").replace("\\", "-").strip("-") or "run" + m = _add_run(z, r, prefix=f"{folder}/") + index.append({"out_dir": str(r), "folder": folder, "events": m["events"], + "run_id": (m.get("run") or {}).get("run_id", "")}) + z.writestr("index.json", json.dumps({ + "kind": "vpcopilot-audit-bundle-multi", "schema": 1, "tool_version": __version__, + "generated": runmeta.utc_now(), "generated_by": runmeta.actor(), + "generated_on": runmeta.host(), "root": str(root), "runs": index}, indent=2)) + return buf.getvalue() + + +def write_bundle_all(root: str = ".", output: str | None = None) -> str: + path = Path(output) if output else Path(root) / "audit-bundle-all.zip" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(bundle_all_bytes(root)) + return str(path) diff --git a/src/vpcopilot/pipeline.py b/src/vpcopilot/pipeline.py index 2ec369e..d69a8d4 100644 --- a/src/vpcopilot/pipeline.py +++ b/src/vpcopilot/pipeline.py @@ -13,8 +13,9 @@ from pathlib import Path from typing import Callable -from . import correlate +from . import correlate, runmeta from .agents import discover, generate, remediate, triage, verify +from .config import AGENT_NAMES from .routes import collect_route_context from .harness import Harness from .repo_scan import collect_files, read_numbered @@ -65,7 +66,7 @@ def run_pipeline( n = sum(1 for _, r in skipped if r == reason) if n: log(f" ⚠ {n} file(s) skipped ({reason}) — raise --max-files/--max-bytes to include them") - t0 = time.perf_counter() + t0, started = time.perf_counter(), runmeta.utc_now() # Ground endpoints in the app's DECLARED routes (OpenAPI spec / framework registrations) so a # weaker model looks a finding's path up instead of hallucinating it — and warn loudly if none. @@ -271,6 +272,21 @@ def _remediate(f): } summary = _write_out(out_dir, findings, verified, decisions, artifacts, remediations, correlations, skipped, metrics, probes) + # F3) run manifest — what was scanned, at which commit, by whom, with which models and caps. + # Every audit entry carries this run_id, so an exported bundle explains its own provenance. + # Fail-soft: provenance is evidence, not a gate — never fail a completed scan over it. + try: + cfg = getattr(h, "cfg", None) + runmeta.write_manifest( + out_dir, repo=str(root.resolve()), config_path=config_path, started=started, + models={a: cfg.for_agent(a).model for a in AGENT_NAMES} if cfg else None, + caps={"min_confidence": min_confidence, "max_files": max_files, "max_bytes": max_bytes, + "draft_code_fixes": draft_code_fixes}, + counts={"candidates": len(findings), "verified": len(verified), "policies": len(artifacts), + "code_fix_prs": len(remediations)}, + finished=runmeta.utc_now(), **runmeta.git_provenance(root)) + except Exception as e: # noqa: BLE001 + log(f" ⚠ could not write the run manifest (run.json): {e} — an audit export will lack provenance") from . import report # E3: drop a standalone shareable HTML dashboard of the results log(f"wrote {report.write_report(out_dir)}") return summary diff --git a/src/vpcopilot/pr.py b/src/vpcopilot/pr.py index 9b5bc9f..3974b70 100644 --- a/src/vpcopilot/pr.py +++ b/src/vpcopilot/pr.py @@ -58,5 +58,8 @@ def open_pr(remediation: dict, repo_slug: str, *, base: str = "main", path_prefi log(f"opened PR #{pr.number}: {pr.html_url}") from . import audit, ledger ledger.mark_remediated(out_dir, fid, pr_url=pr.html_url, pr_number=pr.number) - audit.record(out_dir, "open_pr", finding=fid, repo=repo_slug, url=pr.html_url, number=pr.number) + # `finding_id` is the key every other action uses; `finding` is kept so logs written by older + # builds and the ones written now read the same way. + audit.record(out_dir, "open_pr", finding_id=fid, finding=fid, repo=repo_slug, url=pr.html_url, + number=pr.number) return {"mode": "opened", "number": pr.number, "url": pr.html_url, **plan} diff --git a/src/vpcopilot/probe.py b/src/vpcopilot/probe.py index c8df010..ae2bad3 100644 --- a/src/vpcopilot/probe.py +++ b/src/vpcopilot/probe.py @@ -216,14 +216,16 @@ def probe_from_spec(target_url: str, probe: dict, log: Callable = print, def probe_rate_limit(target_url: str, count: int = 40, path: str = "/login", - log: Callable = print) -> dict: - """Behavioral check: fire `count` rapid GETs and report how many were rate-limited (429).""" + headers: dict | None = None, log: Callable = print) -> dict: + """Behavioral check: fire `count` rapid GETs and report how many were rate-limited (429). + `headers` (e.g. {'X-Agent-Id': ...}) rides on every request so a per-identity limit can + be driven from one identity and confirmed against another.""" limited = passed = 0 codes: dict[int, int] = {} with httpx.Client(base_url=target_url, timeout=15, follow_redirects=False) as c: for _ in range(count): try: - s = c.get(path).status_code + s = c.get(path, headers=headers or {}).status_code except Exception: # noqa: BLE001 s = 0 codes[s] = codes.get(s, 0) + 1 diff --git a/src/vpcopilot/refiner.py b/src/vpcopilot/refiner.py index 437037d..80da199 100644 --- a/src/vpcopilot/refiner.py +++ b/src/vpcopilot/refiner.py @@ -99,7 +99,7 @@ def detach(): log(f" refined (lint): {refined.rationale}" + (" [UNFIXABLE]" if refined.unfixable else "")) if refined.unfixable: detach() - audit.record(out_dir, "refine_apply", control="service_policy", policy=policy_name, + audit.record(out_dir, "refine_apply", finding_id=finding_id, namespace=xc.ns, control="service_policy", policy=policy_name, lb=lb, passed=False, attempts=attempt, unfixable=True, recommend=refined.recommend) return {"mode": "refine_apply", "control": "service_policy", "policy": policy_name, "passed": False, "attempts": attempt, "unfixable": True, @@ -125,7 +125,7 @@ def detach(): "exploit_blocked": False, "legit_ok": True}, "xc_rejected") log(f" refined (xc-rejected): {refined.rationale}" + (" [UNFIXABLE]" if refined.unfixable else "")) if refined.unfixable: - audit.record(out_dir, "refine_apply", control="service_policy", policy=policy_name, lb=lb, + audit.record(out_dir, "refine_apply", finding_id=finding_id, namespace=xc.ns, control="service_policy", policy=policy_name, lb=lb, passed=False, attempts=attempt, unfixable=True, recommend=refined.recommend) return {"mode": "refine_apply", "control": "service_policy", "policy": policy_name, "passed": False, "attempts": attempt, "unfixable": True, @@ -151,7 +151,7 @@ def detach(): if not keep: detach() rolled = True - audit.record(out_dir, "refine_apply", control="service_policy", policy=policy_name, lb=lb, + audit.record(out_dir, "refine_apply", finding_id=finding_id, namespace=xc.ns, control="service_policy", policy=policy_name, lb=lb, passed=True, attempts=attempt, rolled_back=rolled, before_after={"before": before, "after": result}) return {"mode": "refine_apply", "control": "service_policy", "policy": policy_name, @@ -167,7 +167,7 @@ def detach(): refined = refine_agent.run(h, finding, "service_policy", spec, probe, result, diagnosis) log(f" refined: {refined.rationale}" + (" [UNFIXABLE]" if refined.unfixable else "")) if refined.unfixable: - audit.record(out_dir, "refine_apply", control="service_policy", policy=policy_name, lb=lb, + audit.record(out_dir, "refine_apply", finding_id=finding_id, namespace=xc.ns, control="service_policy", policy=policy_name, lb=lb, passed=False, attempts=attempt, unfixable=True, recommend=refined.recommend) return {"mode": "refine_apply", "control": "service_policy", "policy": policy_name, "passed": False, "attempts": attempt, "unfixable": True, "recommend": refined.recommend, @@ -177,7 +177,7 @@ def detach(): detach() reason = f"no working policy after {max_refine} attempt(s) ({diagnosis}) — code fix required" - audit.record(out_dir, "refine_apply", control="service_policy", policy=policy_name, lb=lb, + audit.record(out_dir, "refine_apply", finding_id=finding_id, namespace=xc.ns, control="service_policy", policy=policy_name, lb=lb, passed=False, attempts=max_refine, reason=reason) return {"mode": "refine_apply", "control": "service_policy", "policy": policy_name, "passed": False, "attempts": max_refine, "reason": reason, diff --git a/src/vpcopilot/retire.py b/src/vpcopilot/retire.py index ea8e32f..32d9ffd 100644 --- a/src/vpcopilot/retire.py +++ b/src/vpcopilot/retire.py @@ -64,7 +64,8 @@ def retire_finding(out_dir: str, finding_id: str, *, force: bool = False, dry_ru log(f"detached {control} band-aid from {lb}") ledger.mark_retired(out_dir, finding_id) from . import audit - audit.record(out_dir, "retire", finding_id=finding_id, control=control, lb=lb, forced=force) + audit.record(out_dir, "retire", finding_id=finding_id, control=control, lb=lb, + namespace=xc.ns, forced=force) return {"finding_id": finding_id, "status": "retired", "control": control, "lb": lb, "cure_pr": cure.get("pr_url")} diff --git a/src/vpcopilot/runmeta.py b/src/vpcopilot/runmeta.py new file mode 100644 index 0000000..8239c3b --- /dev/null +++ b/src/vpcopilot/runmeta.py @@ -0,0 +1,105 @@ +"""F3 — run identity + provenance: the spine of a defensible audit trail. + +An audit entry has to stand on its own: *what* changed, *when*, *who* changed it, and *which run* +it belongs to. This module supplies the last two. `/run.json` is the per-run manifest — the +scanned repo and its commit, the models that produced the findings, the caps the scan ran under — +and `run_id` is the key every audit entry carries back to it, including entries written later by a +separate `vpcopilot apply` process against the same out dir. + +Both live inside the run's out dir, so an exported evidence bundle is self-describing.""" +from __future__ import annotations + +import getpass +import json +import os +import socket +import subprocess +import uuid +from datetime import datetime, timezone +from pathlib import Path + +from . import __version__ + + +def _path(out_dir) -> Path: + return Path(out_dir) / "run.json" + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def actor() -> str: + """Who a change is attributed to. VPCOPILOT_ACTOR wins, so CI or a shared jump host can name the + engineer who asked for the change rather than the service account it happens to run as.""" + explicit = (os.environ.get("VPCOPILOT_ACTOR") or "").strip() + if explicit: + return explicit + try: + return getpass.getuser() + except Exception: # noqa: BLE001 — no passwd entry (slim containers) must never break an apply + return "unknown" + + +def host() -> str: + try: + return socket.gethostname() + except Exception: # noqa: BLE001 + return "unknown" + + +def load(out_dir) -> dict: + p = _path(out_dir) + if not p.exists(): + return {} + try: + return json.loads(p.read_text()) + except json.JSONDecodeError: # a half-written manifest must not break a read (cf. ledger.load) + return {} + + +def _save(out_dir, meta: dict) -> None: + p = _path(out_dir) + p.parent.mkdir(parents=True, exist_ok=True) + tmp = p.with_suffix(f".json.tmp.{os.getpid()}") + tmp.write_text(json.dumps(meta, indent=2)) + os.replace(tmp, p) # atomic — every export joins on this file + + +def run_id(out_dir) -> str: + """The stable id for this run dir. Minted on first use and persisted, so a scan and a later + apply/retire against the same dir all stamp their audit entries with the same run.""" + meta = load(out_dir) + if meta.get("run_id"): + return meta["run_id"] + rid = uuid.uuid4().hex[:12] + _save(out_dir, {**meta, "run_id": rid, "created": utc_now()}) + return rid + + +def git_provenance(repo) -> dict: + """Commit + branch of the scanned repo, so a finding ties back to the exact source it came from. + Fail-soft: a target that is not a git checkout simply contributes nothing.""" + def _git(*args) -> str: + try: + return subprocess.check_output(["git", "-C", str(repo), *args], text=True, + stderr=subprocess.DEVNULL, timeout=10).strip() + except Exception: # noqa: BLE001 — no git, not a repo, slow FS: all non-fatal + return "" + sha = _git("rev-parse", "HEAD") + if not sha: + return {} + return {"repo_commit": sha, "repo_branch": _git("rev-parse", "--abbrev-ref", "HEAD"), + "repo_dirty": bool(_git("status", "--porcelain"))} + + +def write_manifest(out_dir, **fields) -> dict: + """Merge run facts into run.json, never clobbering an existing run_id (a re-scan of the same dir + keeps its identity, so the audit entries already on disk stay joinable).""" + meta = load(out_dir) + meta.setdefault("run_id", uuid.uuid4().hex[:12]) + meta.setdefault("created", utc_now()) + meta.update({k: v for k, v in fields.items() if v is not None}) + meta.update({"actor": actor(), "host": host(), "tool_version": __version__, "out_dir": str(out_dir)}) + _save(out_dir, meta) + return meta diff --git a/src/vpcopilot/xc.py b/src/vpcopilot/xc.py index 4f03377..b965b11 100644 --- a/src/vpcopilot/xc.py +++ b/src/vpcopilot/xc.py @@ -68,6 +68,20 @@ def put_service_policy(self, name: str, spec: dict) -> dict: def delete_service_policy(self, name: str) -> dict: return self._req("DELETE", f"/config/namespaces/{self.ns}/service_policys/{name}") + # --- user identifications (key per-user controls on a header, not client IP) --- + def get_user_identification(self, name: str) -> dict: + return self._req("GET", f"/config/namespaces/{self.ns}/user_identifications/{name}") + + def create_user_identification(self, obj: dict) -> dict: + return self._req("POST", f"/config/namespaces/{self.ns}/user_identifications", json=obj) + + def user_identification_exists(self, name: str) -> bool: + try: + self.get_user_identification(name) + return True + except XCError: + return False + # --- http load balancer --- def list_http_loadbalancers(self) -> dict: # `?report_fields=` makes the collection GET embed each LB's full spec under `get_spec` diff --git a/tests/conftest.py b/tests/conftest.py index a0cbd2b..96846ee 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,6 +23,7 @@ def __init__(self, lb_spec=None, namespace="test-ns", tenant="test-tenant"): self.app_firewalls: dict[str, dict] = {"nimbus-waf": {"spec": {"blocking": {}}}} self.api_definitions: dict[str, dict] = {} self.swaggers: dict[str, dict] = {} + self.user_identifications: dict[str, dict] = {} self.put_lb_calls: list[dict] = [] # every spec PUT, in order self.fail_put_lb = False # flip on to simulate an XC write failure @@ -78,6 +79,17 @@ def put_swagger(self, name, openapi, version="v1"): self.swaggers[name] = copy.deepcopy(openapi) return f"string:///{name}" + # ---- user identification ---- + def user_identification_exists(self, name): + return name in self.user_identifications + + def get_user_identification(self, name): + return copy.deepcopy(self.user_identifications[name]) + + def create_user_identification(self, obj): + self.user_identifications[obj["metadata"]["name"]] = copy.deepcopy(obj) + return obj + class FakeHarness: """A Harness whose agents return canned typed objects, keyed by role. No LLM calls.""" diff --git a/tests/test_apply_engine.py b/tests/test_apply_engine.py index 5cd6a9a..7ccbdec 100644 --- a/tests/test_apply_engine.py +++ b/tests/test_apply_engine.py @@ -37,6 +37,39 @@ def test_malicious_user_default_rolls_back(monkeypatch, fake_xc, tmp_path): assert "enable_malicious_user_detection" not in fake_xc.lb["spec"] # restored to snapshot +def test_malicious_user_default_keys_on_client_ip(monkeypatch, fake_xc, tmp_path): + # Without --user-id-header the historical IP-based identity is preserved. + _use(monkeypatch, fake_xc) + apply.apply_malicious_user("lab", keep=True, out_dir=str(tmp_path), log=lambda m: None) + assert "user_id_client_ip" in fake_xc.lb["spec"] + assert "user_identification" not in fake_xc.lb["spec"] + assert fake_xc.user_identifications == {} # no object created + + +def test_malicious_user_keys_on_header(monkeypatch, fake_xc, tmp_path): + # A2/A3: with --user-id-header, key per identity via a created user_identification, not client IP. + _use(monkeypatch, fake_xc) + apply.apply_malicious_user("lab", keep=True, user_id_header="X-Agent-Id", + out_dir=str(tmp_path), log=lambda m: None) + spec = fake_xc.lb["spec"] + assert "user_id_client_ip" not in spec + assert spec["user_identification"]["name"] == "lab-user-id" + assert spec["user_identification"]["tenant"] == "test-tenant" # fully qualified + uid = fake_xc.user_identifications["lab-user-id"] + assert uid["spec"]["rules"][0]["http_header_name"] == "X-Agent-Id" + + +def test_rate_limit_keys_on_header(monkeypatch, fake_xc, tmp_path): + # A4: per-agent rate limit sets the same user_identification and drops the IP key. + _use(monkeypatch, fake_xc) + apply.apply_rate_limit("lab", requests=60, unit="MINUTE", keep=True, + user_id_header="X-Agent-Id", out_dir=str(tmp_path), log=lambda m: None) + spec = fake_xc.lb["spec"] + assert "rate_limit" in spec and "user_id_client_ip" not in spec + assert spec["user_identification"]["name"] == "lab-user-id" + assert fake_xc.user_identifications["lab-user-id"]["spec"]["rules"][0]["http_header_name"] == "X-Agent-Id" + + # ---- config-validated control (waf): attaching a blocking WAF is 'applied' (defense-in-depth) ---- def test_waf_config_validated_keeps_and_marks_ledger(monkeypatch, fake_xc, tmp_path): @@ -62,6 +95,64 @@ def test_waf_rolls_back_when_not_kept(monkeypatch, fake_xc, tmp_path): assert fake_xc.lb["spec"] == {"no_service_policies": {}} +# ---- data_guard MUST NOT clobber an already-attached, differently-named WAF (A5 safety gate) ---- + +def test_data_guard_reuses_attached_waf_and_does_not_clobber(monkeypatch, fake_xc, tmp_path): + _use(monkeypatch, fake_xc) + # LB already carries the LIVE banknimbus-dev-waf; apply_data_guard defaults to vpcopilot-lab-waf + fake_xc.lb["spec"] = {"app_firewall": {"namespace": "test-ns", "name": "banknimbus-dev-waf", + "tenant": "test-tenant"}} + fake_xc.app_firewalls["banknimbus-dev-waf"] = {"spec": {"blocking": {}}} + res = apply.apply_data_guard("lab", keep=True, out_dir=str(tmp_path), log=lambda m: None) + assert res["config_enabled"] is True and res["kept"] is True + # (a) the attached WAF is STILL the live one — not replaced by vpcopilot-lab-waf + assert fake_xc.lb["spec"]["app_firewall"]["name"] == "banknimbus-dev-waf" + assert "vpcopilot-lab-waf" not in fake_xc.app_firewalls # the default WAF was never created + # (b) Data Guard rules are present and non-empty + assert fake_xc.lb["spec"].get("data_guard_rules") + + +def test_data_guard_explicit_app_firewall_passthrough(monkeypatch, fake_xc, tmp_path): + _use(monkeypatch, fake_xc) + # no WAF attached, and names match the explicit request -> the requested WAF is attached as-is + res = apply.apply_data_guard("lab", app_firewall="banknimbus-dev-waf", template="nimbus-waf", + keep=True, out_dir=str(tmp_path), log=lambda m: None) + assert res["config_enabled"] is True + assert fake_xc.lb["spec"]["app_firewall"]["name"] == "banknimbus-dev-waf" + assert fake_xc.lb["spec"].get("data_guard_rules") + + +# ---- api_schema must validate HEADERS + QUERY, not just the body (A1) ---- + +def _emitted_validation_props(fake_xc): + return (fake_xc.lb["spec"]["api_specification"]["validation_all_spec_endpoints"] + ["validation_mode"]["validation_mode_active"]["request_validation_properties"]) + + +def test_api_schema_validates_headers_and_query_by_default(monkeypatch, fake_xc, tmp_path): + _use(monkeypatch, fake_xc) + monkeypatch.setattr(apply, "_run_validation", + lambda *a, **k: {"exploit_status": 403, "exploit_blocked": True, "legit_ok": True}) + res = apply.apply_api_schema("lab", target_url="http://x", keep=True, + out_dir=str(tmp_path), log=lambda m: None) + assert res["passed"] is True + props = _emitted_validation_props(fake_xc) + # a credential on a bodyless GET (e.g. an auth/payment header) is only enforced if headers + + # query params are validated — body-only validation would enforce nothing + assert "PROPERTY_HTTP_HEADERS" in props and "PROPERTY_QUERY_PARAMETERS" in props + assert props == ["PROPERTY_HTTP_HEADERS", "PROPERTY_QUERY_PARAMETERS", "PROPERTY_HTTP_BODY"] + + +def test_api_schema_validation_properties_override(monkeypatch, fake_xc, tmp_path): + _use(monkeypatch, fake_xc) + monkeypatch.setattr(apply, "_run_validation", + lambda *a, **k: {"exploit_status": 403, "exploit_blocked": True, "legit_ok": True}) + apply.apply_api_schema("lab", target_url="http://x", keep=True, + request_validation_properties=["PROPERTY_HTTP_BODY"], + out_dir=str(tmp_path), log=lambda m: None) + assert _emitted_validation_props(fake_xc) == ["PROPERTY_HTTP_BODY"] # caller wins + + def test_apply_control_routes_via_registry(monkeypatch, fake_xc, tmp_path): _use(monkeypatch, fake_xc) # service_policy is not LB-wide -> apply_control refuses it (points to the from-scan path) diff --git a/tests/test_audit_provenance.py b/tests/test_audit_provenance.py new file mode 100644 index 0000000..0ea05f1 --- /dev/null +++ b/tests/test_audit_provenance.py @@ -0,0 +1,166 @@ +"""F3 — every audit entry must answer: what changed, WHERE (LB + XC namespace), WHY (the finding), +WHO ran it, and as part of which run. An LB change that can't name its justification is not an +audit record.""" +import json + +import pytest + +from vpcopilot import apply, audit, runmeta + + +@pytest.fixture(autouse=True) +def _fast(monkeypatch, noop_sleep): + import vpcopilot.engine as engine + real_init = engine.ApplyContext.__post_init__ + + def patched(self): + real_init(self) + self.sleep = noop_sleep + monkeypatch.setattr(engine.ApplyContext, "__post_init__", patched) + monkeypatch.setenv("VPCOPILOT_PROTECTED_LBS", "nimbus-www") + + +def _use(monkeypatch, fake_xc): + monkeypatch.setattr(apply, "XC", lambda *a, **k: fake_xc) + + +def _entries(out, action): + return [e for e in audit.load(str(out)) if e["action"] == action] + + +# ---- identity stamped centrally ---- +def test_every_entry_carries_run_actor_host_and_version(tmp_path, monkeypatch): + monkeypatch.setenv("VPCOPILOT_ACTOR", "sec-oncall") + audit.record(str(tmp_path), "apply_waf", lb="lab") + e = audit.load(str(tmp_path))[0] + assert e["actor"] == "sec-oncall" and e["run_id"] and e["host"] and e["tool_version"] + + +def test_identity_cannot_be_overridden_by_a_caller(tmp_path, monkeypatch): + """Identity is stamped in audit.record, not at the call sites — so nothing can spoof it, and no + mutating path can forget it.""" + monkeypatch.setenv("VPCOPILOT_ACTOR", "sec-oncall") + audit.record(str(tmp_path), "apply_waf", lb="lab", actor="someone-else", run_id="forged") + e = audit.load(str(tmp_path))[0] + assert e["actor"] == "sec-oncall" and e["run_id"] != "forged" + + +def test_run_id_is_stable_across_processes_for_one_out_dir(tmp_path): + """A scan and a later `vpcopilot apply` against the same dir belong to the same run.""" + audit.record(str(tmp_path), "apply_waf", lb="lab") + audit.record(str(tmp_path), "retire", lb="lab") + a, b = audit.load(str(tmp_path)) + assert a["run_id"] == b["run_id"] == runmeta.run_id(str(tmp_path)) + + +def test_actor_falls_back_to_the_os_user(tmp_path, monkeypatch): + monkeypatch.delenv("VPCOPILOT_ACTOR", raising=False) + audit.record(str(tmp_path), "apply_waf", lb="lab") + assert audit.load(str(tmp_path))[0]["actor"] # never blank + + +# ---- the finding + namespace on every live control ---- +def test_malicious_user_records_finding_and_namespace(monkeypatch, fake_xc, tmp_path): + _use(monkeypatch, fake_xc) + apply.apply_malicious_user("lab", keep=True, finding_id="f-1", out_dir=str(tmp_path), log=lambda m: None) + e = _entries(tmp_path, "apply_malicious_user")[0] + assert e["finding_id"] == "f-1" and e["namespace"] == fake_xc.ns and e["lb"] == "lab" + + +def test_rate_limit_records_finding_and_namespace(monkeypatch, fake_xc, tmp_path): + _use(monkeypatch, fake_xc) + apply.apply_rate_limit("lab", requests=60, keep=True, finding_id="f-2", + out_dir=str(tmp_path), log=lambda m: None) + e = _entries(tmp_path, "apply_rate_limit")[0] + assert e["finding_id"] == "f-2" and e["namespace"] == fake_xc.ns + + +def test_bot_defense_records_finding_and_namespace(monkeypatch, fake_xc, tmp_path): + _use(monkeypatch, fake_xc) + apply.apply_bot_defense("lab", keep=True, finding_id="f-3", out_dir=str(tmp_path), log=lambda m: None) + e = _entries(tmp_path, "apply_bot_defense")[0] + assert e["finding_id"] == "f-3" and e["namespace"] == fake_xc.ns + + +def test_waf_records_finding_namespace_and_the_object_it_created(monkeypatch, fake_xc, tmp_path): + _use(monkeypatch, fake_xc) + monkeypatch.setattr(apply, "_run_validation", + lambda *a, **k: {"exploit_status": 403, "exploit_blocked": True, "legit_ok": True}) + apply.apply_waf("lab", target_url="http://x", keep=True, finding_id="f-4", + out_dir=str(tmp_path), log=lambda m: None) + applied = _entries(tmp_path, "apply_waf")[0] + created = _entries(tmp_path, "create_app_firewall")[0] + assert applied["finding_id"] == "f-4" and applied["namespace"] == fake_xc.ns + # "rolled back?" alone can't say the change is STILL LIVE — the trail has to record that + assert applied["kept"] is True + # the XC object creation is itself an auditable change and must be attributable too + assert created["finding_id"] == "f-4" and created["namespace"] == fake_xc.ns + + +def test_data_guard_records_finding_and_namespace(monkeypatch, fake_xc, tmp_path): + _use(monkeypatch, fake_xc) + apply.apply_data_guard("lab", keep=True, finding_id="f-5", out_dir=str(tmp_path), log=lambda m: None) + e = _entries(tmp_path, "apply_data_guard")[0] + assert e["finding_id"] == "f-5" and e["namespace"] == fake_xc.ns + + +def test_api_schema_records_finding_and_namespace(monkeypatch, fake_xc, tmp_path): + _use(monkeypatch, fake_xc) + monkeypatch.setattr(apply, "_run_validation", + lambda *a, **k: {"exploit_status": 403, "exploit_blocked": True, "legit_ok": True}) + apply.apply_api_schema("lab", target_url="http://x", keep=True, finding_id="f-6", + out_dir=str(tmp_path), log=lambda m: None) + for action in ("apply_api_schema", "create_api_definition"): + e = _entries(tmp_path, action)[0] + assert e["finding_id"] == "f-6" and e["namespace"] == fake_xc.ns, action + assert _entries(tmp_path, "apply_api_schema")[0]["kept"] is True + + +def test_retire_records_the_namespace_it_detached_from(monkeypatch, fake_xc, tmp_path): + """A detach is an LB change like any other — it has to name the tenant it happened in.""" + from vpcopilot import ledger, retire + monkeypatch.setattr(retire, "XC", lambda *a, **k: fake_xc) + ledger.mark_mitigated(str(tmp_path), "f-9", control="waf", policy_name="lab-waf", lb="lab") + retire.retire_finding(str(tmp_path), "f-9", force=True, log=lambda m: None) + e = _entries(tmp_path, "retire")[0] + assert e["finding_id"] == "f-9" and e["namespace"] == fake_xc.ns and e["lb"] == "lab" + + +def test_rollback_failure_is_attributable(fake_xc, tmp_path, noop_sleep): + """The single most important entry to attribute: the LB may be left in a changed state.""" + from vpcopilot.engine import ApplyContext, RollbackError, safe_rollback + ctx = ApplyContext(xc=fake_xc, lb="lab", out_dir=str(tmp_path), log=lambda m: None, + sleep=noop_sleep, finding_id="f-7").load() + ctx.put({"active_service_policies": {}}) + fake_xc.fail_put_lb = True + with pytest.raises(RollbackError): + safe_rollback(ctx, retries=2) + e = _entries(tmp_path, "rollback_failed")[0] + assert e["finding_id"] == "f-7" and e["namespace"] == fake_xc.ns and e["lb"] == "lab" + + +def test_apply_pins_the_finding_onto_the_context_so_a_rollback_can_name_it(monkeypatch, fake_xc, tmp_path): + """The apply handlers hand finding_id to ApplyContext, which is what makes the entry above + possible from deep inside the rollback spine.""" + _use(monkeypatch, fake_xc) + seen = {} + import vpcopilot.engine as engine + real_load = engine.ApplyContext.load + monkeypatch.setattr(engine.ApplyContext, "load", + lambda self: (seen.update(fid=self.finding_id), real_load(self))[1]) + apply.apply_malicious_user("lab", keep=True, finding_id="f-8", out_dir=str(tmp_path), log=lambda m: None) + assert seen["fid"] == "f-8" + + +# ---- run manifest ---- +def test_run_manifest_records_provenance_without_clobbering_the_run_id(tmp_path): + rid = runmeta.run_id(str(tmp_path)) + runmeta.write_manifest(str(tmp_path), repo="/src/app", models={"discover": "anthropic/x"}, + counts={"verified": 3}) + m = json.loads((tmp_path / "run.json").read_text()) + assert m["run_id"] == rid and m["repo"] == "/src/app" + assert m["models"]["discover"] == "anthropic/x" and m["actor"] and m["tool_version"] + + +def test_git_provenance_is_fail_soft_on_a_non_repo(tmp_path): + assert runmeta.git_provenance(str(tmp_path / "not-a-repo")) == {} diff --git a/tests/test_console_audit_export.py b/tests/test_console_audit_export.py new file mode 100644 index 0000000..4781e04 --- /dev/null +++ b/tests/test_console_audit_export.py @@ -0,0 +1,88 @@ +"""F3 — the Retire step's audit surface: /api/audit-events renders the trail on screen, and +/api/audit-export downloads the evidence bundle for this run or for every run on disk.""" +import io +import json +import zipfile +from pathlib import Path + +from fastapi.testclient import TestClient + +from vpcopilot import audit +from vpcopilot.console import app as A + + +def _client(): + return TestClient(A.app) + + +def _seed(out: Path): + out.mkdir(parents=True, exist_ok=True) + (out / "findings.json").write_text(json.dumps([ + {"id": "f-1", "title": "SQL injection in login", "vuln_class": "sqli", + "severity": "critical", "file": "api/login.js"}])) + audit.record(str(out), "apply_service_policy", finding_id="f-1", lb="lab", namespace="ns", + policy="deny-login-sqli", passed=True, kept=True) + + +def test_audit_events_are_shown_before_they_can_be_exported(tmp_path, monkeypatch): + _seed(tmp_path) + monkeypatch.setattr(A, "OUT", tmp_path) + body = _client().get("/api/audit-events").json() + assert body["out"] == str(tmp_path) and len(body["events"]) == 1 + e = body["events"][0] + assert e["finding_id"] == "f-1" and e["title"] == "SQL injection in login" + assert e["lb"] == "lab" and e["namespace"] == "ns" and e["outcome"] == "kept" + assert e["actor"] # who made the change + + +def test_audit_events_on_an_untouched_run_is_empty_not_an_error(tmp_path, monkeypatch): + monkeypatch.setattr(A, "OUT", tmp_path / "never-scanned") + r = _client().get("/api/audit-events") + assert r.status_code == 200 and r.json()["events"] == [] + + +def test_export_downloads_a_named_zip_for_this_run(tmp_path, monkeypatch): + _seed(tmp_path) + monkeypatch.setattr(A, "OUT", tmp_path) + r = _client().get("/api/audit-export?scope=run") + assert r.status_code == 200 and r.headers["content-type"] == "application/zip" + cd = r.headers["content-disposition"] + assert cd.startswith("attachment") and "vpcopilot-audit-" in cd and cd.rstrip('"').endswith(".zip") + z = zipfile.ZipFile(io.BytesIO(r.content)) + assert {"manifest.json", "audit.csv", "audit.log", "findings.json"} <= set(z.namelist()) + assert json.loads(z.read("manifest.json"))["events"] == 1 + + +def test_export_defaults_to_this_run(tmp_path, monkeypatch): + _seed(tmp_path) + monkeypatch.setattr(A, "OUT", tmp_path) + assert _client().get("/api/audit-export").status_code == 200 + + +def test_export_all_runs_bundles_each_under_its_own_folder(tmp_path, monkeypatch): + for name in ("out-a", "out-b"): + _seed(tmp_path / name) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(A, "OUT", tmp_path / "out-a") + r = _client().get("/api/audit-export?scope=all") + assert r.status_code == 200 + z = zipfile.ZipFile(io.BytesIO(r.content)) + assert "index.json" in z.namelist() + assert {run["folder"] for run in json.loads(z.read("index.json"))["runs"]} == {"out-a", "out-b"} + assert "vpcopilot-audit-all-" in r.headers["content-disposition"] + + +def test_unknown_export_scope_is_rejected(tmp_path, monkeypatch): + monkeypatch.setattr(A, "OUT", tmp_path) + assert _client().get("/api/audit-export?scope=everything").status_code == 400 + + +def test_runs_endpoint_lists_what_can_be_exported(tmp_path, monkeypatch): + _seed(tmp_path / "out-a") + (tmp_path / "out-empty").mkdir() + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(A, "OUT", tmp_path / "out-a") + body = _client().get("/api/runs").json() + names = {Path(r).name for r in body["runs"]} + assert "out-a" in names and "out-empty" not in names + assert body["current"] == str(tmp_path / "out-a") diff --git a/tests/test_console_log_stream.py b/tests/test_console_log_stream.py new file mode 100644 index 0000000..0af7137 --- /dev/null +++ b/tests/test_console_log_stream.py @@ -0,0 +1,63 @@ +"""Console log streaming: /api/scan and /api/action serve the FULL transcript with an incremental +`?since=` tail, so a long-running scan or apply can be scrolled end-to-end instead of being clipped +to a sliding window.""" +from fastapi.testclient import TestClient + +from vpcopilot.console import app as A + + +def _client(): + return TestClient(A.app) + + +# ---- scan log ---- +def test_scan_log_is_served_in_full(monkeypatch): + monkeypatch.setitem(A._scan, "log", [f"line {i}" for i in range(200)]) + monkeypatch.setitem(A._scan, "state", "running") + r = _client().get("/api/scan").json() + assert len(r["log"]) == 200 and r["log_total"] == 200 + assert r["log"][0] == "line 0" and r["log"][-1] == "line 199" + + +def test_scan_log_since_returns_only_the_new_tail(monkeypatch): + monkeypatch.setitem(A._scan, "log", [f"line {i}" for i in range(200)]) + r = _client().get("/api/scan?since=180").json() + assert r["log"] == [f"line {i}" for i in range(180, 200)] + assert r["log_total"] == 200 # the console advances its cursor from log_total, not len(log) + + +def test_scan_log_since_past_the_end_is_empty_not_an_error(monkeypatch): + monkeypatch.setitem(A._scan, "log", ["only"]) + r = _client().get("/api/scan?since=99") + assert r.status_code == 200 and r.json()["log"] == [] and r.json()["log_total"] == 1 + + +def test_scan_log_negative_since_yields_the_whole_log(monkeypatch): + monkeypatch.setitem(A._scan, "log", ["a", "b"]) + assert _client().get("/api/scan?since=-5").json()["log"] == ["a", "b"] + + +# ---- apply/action job log ---- +def test_action_log_is_served_in_full_and_supports_since(monkeypatch): + job = {"state": "running", "log": [f"step {i}" for i in range(120)], "result": None, + "error": None, "control": "waf", "finding_id": "f-1"} + monkeypatch.setitem(A._jobs, "job1234", job) + c = _client() + full = c.get("/api/action?job=job1234").json() + assert len(full["log"]) == 120 and full["log_total"] == 120 + tail = c.get("/api/action?job=job1234&since=118").json() + assert tail["log"] == ["step 118", "step 119"] and tail["job"] == "job1234" + + +def test_action_status_unknown_job_is_404(): + assert _client().get("/api/action?job=nope").status_code == 404 + + +# ---- bounded sink ---- +def test_log_sink_caps_growth_and_says_so_in_band(): + buf: list = [] + for i in range(A.LOG_MAX + 50): + A._append(buf, f"line {i}") + assert len(buf) == A.LOG_MAX + 1 # the cap, plus one explanatory line + assert "capped" in buf[-1] and str(A.LOG_MAX) in buf[-1] + assert buf[0] == "line 0" # keeps the HEAD (where a scan's setup/config lines live) diff --git a/tests/test_console_report.py b/tests/test_console_report.py new file mode 100644 index 0000000..6f5901a --- /dev/null +++ b/tests/test_console_report.py @@ -0,0 +1,73 @@ +"""GET /api/report — the shareable HTML report the Review step links to. It is rebuilt from the +console's current out dir on every request (so it is always the latest run), served inline for the +new-tab view, and as a stamped attachment when `download=1`.""" +import json +from pathlib import Path + +from fastapi.testclient import TestClient + +from vpcopilot.console import app as A + + +def _client(): + return TestClient(A.app) + + +def _seed(out: Path): + out.mkdir(parents=True, exist_ok=True) + (out / "summary.json").write_text(json.dumps({"candidates": 1, "verified": 1, "policies": [], + "no_bandaid": [], "code_fix_prs": []})) + (out / "findings.json").write_text(json.dumps([ + {"id": "a-001", "title": "SQLi login", "vuln_class": "sqli", "severity": "critical", + "file": "api/login.js", "line": 10, "description": "d", "exploit_sketch": "e", + "code_snippet": ""}])) + (out / "triage.json").write_text(json.dumps([ + {"finding_id": "a-001", "bandaids": [], "no_bandaid": True, "residual_risk": "", + "code_cure_required": True}])) + (out / "remediations.json").write_text(json.dumps([])) + + +def test_report_is_served_inline_for_the_new_tab_view(tmp_path, monkeypatch): + _seed(tmp_path) + monkeypatch.setattr(A, "OUT", tmp_path) + r = _client().get("/api/report") + assert r.status_code == 200 + assert r.headers["content-type"].startswith("text/html") + assert "attachment" not in r.headers.get("content-disposition", "") + assert "SQLi login" in r.text + + +def test_report_is_rebuilt_from_the_current_out_dir(tmp_path, monkeypatch): + """Review advertises 'the latest run' — so a changed findings.json must show up without a + restart, and repointing OUT must serve the other run.""" + _seed(tmp_path) + monkeypatch.setattr(A, "OUT", tmp_path) + assert "SQLi login" in _client().get("/api/report").text + (tmp_path / "findings.json").write_text(json.dumps([ + {"id": "a-001", "title": "Broken object auth", "vuln_class": "bola", "severity": "high", + "file": "api/order.js", "line": 3, "description": "d", "exploit_sketch": "", + "code_snippet": ""}])) + body = _client().get("/api/report").text + assert "Broken object auth" in body and "SQLi login" not in body + + +def test_report_download_sends_a_stamped_attachment(tmp_path, monkeypatch): + _seed(tmp_path) + monkeypatch.setattr(A, "OUT", tmp_path) + cd = _client().get("/api/report?download=1").headers["content-disposition"] + assert cd.startswith("attachment") + assert "vpcopilot-report-" in cd and cd.rstrip('"').endswith(".html") + + +def test_report_download_filename_carries_the_run_dir(tmp_path, monkeypatch): + """Several downloaded reports have to be distinguishable in one folder.""" + run = tmp_path / "out-claude-vampi" + _seed(run) + monkeypatch.setattr(A, "OUT", run) + assert "out-claude-vampi" in _client().get("/api/report?download=1").headers["content-disposition"] + + +def test_report_on_an_empty_run_dir_still_renders(tmp_path, monkeypatch): + monkeypatch.setattr(A, "OUT", tmp_path / "never-scanned") + r = _client().get("/api/report") + assert r.status_code == 200 and "= {"created", "pr_opened", "retired", "rollback_failed"} + + +def test_events_join_the_finding_that_justified_the_change(tmp_path): + out = str(tmp_path) + _seed(tmp_path) + audit.record(out, "apply_service_policy", finding_id="f-1", lb="lab", namespace="ns", + policy="deny-login-sqli", passed=True, kept=True) + e = export.build_audit_events(out)[0] + assert e["finding_id"] == "f-1" and e["title"] == "SQL injection in login" + assert e["severity"] == "critical" and e["vuln_class"] == "sqli" + assert e["ledger_state"] == "remediated" and e["control"] == "service_policy" + assert e["lb"] == "lab" and e["namespace"] == "ns" and e["outcome"] == "kept" + + +def test_legacy_entries_still_attribute_via_finding_key_and_policy_index(tmp_path): + """Logs written by older builds used `finding` on open_pr and recorded no finding_id at all on + apply_*; both must still resolve rather than exporting as anonymous changes.""" + out = tmp_path + _seed(out) + (out / "audit.log").write_text( + json.dumps({"ts": "2026-01-01T00:00:00+00:00", "action": "open_pr", "finding": "f-1", + "url": "https://github.com/o/r/pull/7"}) + "\n" + + json.dumps({"ts": "2026-01-01T00:00:01+00:00", "action": "apply_service_policy", + "policy": "deny-login-sqli", "lb": "lab", "passed": True}) + "\n") + events = export.build_audit_events(str(out)) + assert all(e["finding_id"] == "f-1" for e in events) + assert all(e["title"] == "SQL injection in login" for e in events) + + +def test_before_after_and_self_heal_are_flattened_for_review(tmp_path): + out = str(tmp_path) + _seed(tmp_path) + audit.record(out, "refine_apply", finding_id="f-1", control="service_policy", lb="lab", + policy="deny-login-sqli", passed=True, attempts=2, + before_after={"before": {"exploit_status": 200, "exploit_blocked": False}, + "after": {"exploit_status": 403, "exploit_blocked": True, "legit_ok": True}}) + e = export.build_audit_events(out)[0] + assert e["exploit_before"] == "200 allowed" and e["exploit_after"] == "403 blocked" + assert e["attempts"] == 2 and e["legit_ok"] is True + + +def test_outcome_coalesces_the_three_success_keys(tmp_path): + """apply_* disagree on passed / config_enabled / enabled, and the demo fixture uses yet another + mix — one column has to mean the same thing across all of them.""" + out = str(tmp_path) + for i, kw in enumerate(({"passed": True}, {"config_enabled": True}, {"enabled": True}, + {"passed": False}, {"rolled_back": True}, {"unfixable": True})): + audit.record(out, "apply_waf", finding_id=f"x-{i}", lb="lab", **kw) + got = {e["detail"].get("finding_id"): e["outcome"] for e in export.build_audit_events(out)} + assert got["x-0"] == got["x-1"] == got["x-2"] == "passed" + assert got["x-3"] == "failed" and got["x-4"] == "rolled_back" and got["x-5"] == "unfixable" + + +def test_events_are_newest_first(tmp_path): + out = str(tmp_path) + for i in range(3): + audit.record(out, "apply_waf", finding_id=f"f-{i}", lb="lab") + ts = [e["ts"] for e in export.build_audit_events(out)] + assert ts == sorted(ts, reverse=True) + + +def test_empty_run_dir_exports_cleanly(tmp_path): + assert export.build_audit_events(str(tmp_path / "never-scanned")) == [] + assert export.to_csv([]).strip() == ",".join(export.COLUMNS) + + +# ---- CSV ---- +def test_csv_has_every_column_and_one_row_per_event(tmp_path): + out = str(tmp_path) + _seed(tmp_path) + audit.record(out, "apply_waf", finding_id="f-1", lb="lab", namespace="ns", config_enabled=True) + audit.record(out, "retire", finding_id="f-1", control="waf", lb="lab", forced=True) + rows = export.to_csv(export.build_audit_events(out)).strip().splitlines() + assert rows[0] == ",".join(export.COLUMNS) and len(rows) == 3 + assert "SQL injection in login" in rows[1] + + +def test_csv_detail_keeps_the_raw_record(tmp_path): + """Flattening must not lose anything that was recorded.""" + out = str(tmp_path) + audit.record(out, "apply_rate_limit", finding_id="f-9", lb="lab", rate="5/MINUTE", + behavioral={"sent": 20, "limited": 15}) + body = export.to_csv(export.build_audit_events(out)) + assert "5/MINUTE" in body and "limited" in body + + +# ---- bundle ---- +def test_bundle_carries_the_evidence_and_a_verifiable_manifest(tmp_path): + out = tmp_path / "out-run" + _seed(out) + audit.record(str(out), "apply_service_policy", finding_id="f-1", lb="lab", namespace="ns", + policy="deny-login-sqli", passed=True, kept=True) + z = zipfile.ZipFile(io.BytesIO(export.bundle_bytes(str(out)))) + names = set(z.namelist()) + assert {"manifest.json", "audit.csv", "audit-events.json", "audit.log", "ledger.json", + "findings.json", "lb_snapshot.json", + "policies/service_policy.deny-login-sqli.json"} <= names + m = json.loads(z.read("manifest.json")) + assert m["kind"] == "vpcopilot-audit-bundle" and m["events"] == 1 + assert m["findings_touched"] == ["f-1"] and m["caveats"] + for name, meta in m["members"].items(): + assert hashlib.sha256(z.read(name)).hexdigest() == meta["sha256"] + + +def test_bundle_manifest_states_what_the_trail_omits(tmp_path): + """The one way this feature could mislead: implying it covers attempts it never recorded.""" + caveats = " ".join(export.build_manifest(str(tmp_path))["caveats"]).lower() + assert "dry run" in caveats and "apply_timing" in caveats + + +def test_bundle_includes_the_raw_log_verbatim(tmp_path): + out = tmp_path / "out-run" + audit.record(str(out), "apply_waf", finding_id="f-1", lb="lab") + z = zipfile.ZipFile(io.BytesIO(export.bundle_bytes(str(out)))) + assert z.read("audit.log") == (out / "audit.log").read_bytes() + + +def test_write_bundle_returns_its_path(tmp_path): + out = tmp_path / "out-run" + audit.record(str(out), "retire", finding_id="f-1", control="waf", lb="lab") + p = Path(export.write_bundle(str(out))) + assert p.exists() and p.name == "audit-bundle.zip" and zipfile.is_zipfile(p) + + +# ---- multi-run ---- +def test_find_runs_picks_up_run_dirs_with_something_to_export(tmp_path): + (tmp_path / "out-a").mkdir() + audit.record(str(tmp_path / "out-a"), "apply_waf", finding_id="f-1", lb="lab") + (tmp_path / "out-empty").mkdir() # no audit log, no findings — nothing to export + (tmp_path / "demo" / "out").mkdir(parents=True) + (tmp_path / "demo" / "out" / "findings.json").write_text("[]") + runs = {Path(r).name for r in export.find_runs(str(tmp_path))} + assert "out-a" in runs and "out" in runs and "out-empty" not in runs + + +def test_bundle_all_folders_each_run_under_an_index(tmp_path): + for name in ("out-a", "out-b"): + audit.record(str(tmp_path / name), "apply_waf", finding_id="f-1", lb="lab") + z = zipfile.ZipFile(io.BytesIO(export.bundle_all_bytes(str(tmp_path)))) + names = z.namelist() + assert "index.json" in names + assert any(n.startswith("out-a/") for n in names) and any(n.startswith("out-b/") for n in names) + idx = json.loads(z.read("index.json")) + assert {r["folder"] for r in idx["runs"]} == {"out-a", "out-b"} + assert all(r["events"] == 1 for r in idx["runs"]) diff --git a/tests/test_probe.py b/tests/test_probe.py index 4aeb641..077a6d8 100644 --- a/tests/test_probe.py +++ b/tests/test_probe.py @@ -18,7 +18,7 @@ def __enter__(self): def __exit__(self, *a): return False - def get(self, path): + def get(self, path, headers=None): return FakeResp(next(seq)) monkeypatch.setattr(probe.httpx, "Client", FakeClient)