diff --git a/.github/workflows/design-doc-check.yml b/.github/workflows/design-doc-check.yml new file mode 100644 index 00000000..0ec6a5bd --- /dev/null +++ b/.github/workflows/design-doc-check.yml @@ -0,0 +1,56 @@ +# Design Doc Check — issue #67 Phase 1 +# +# This workflow runs the design doc checker on every PR. It detects +# "new feature" PRs (by file pattern) and fails if no design doc is +# included in docs/design/. The check is bypassable via the +# 'skip-design-doc' PR label. +# +# The check is informational on first failure — the PR author can either +# add a design doc or add the bypass label, then re-push. + +name: Design Doc Check + +on: + pull_request: + branches: [main] + # Only run when files that could trigger the check are changed. This + # avoids wasting CI minutes on doc-only or test-only PRs. + paths: + - 'scripts/commands/**' + - 'scripts/formatters/**' + - 'scripts/parsers/**' + - 'scripts/*_engine.py' + - 'docs/design/**' + - '.github/workflows/design-doc-check.yml' + - 'scripts/check_design_doc.py' + +permissions: + contents: read + pull-requests: read + +jobs: + check: + name: Design Doc Required + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + # We need the full PR diff, not just the merge commit. fetch-depth 0 + # gets the full history so the check script can inspect the PR's + # changed files via the GitHub API instead. + fetch-depth: 1 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Run design doc check + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + python3 scripts/check_design_doc.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 69c714f4..a67b9da8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -183,11 +183,55 @@ When adding a new CLI command, create a new file in the `commands/` directory: ## Pull Request Process 1. **Update documentation** — README.md, SKILL.md, SKILL-QUICK.md, changelog.md -2. **Add tests** for new features -3. **Ensure all tests pass** — `python3 -m pytest tests/ -v` -4. **Follow the PR template** — describe changes, motivation, testing -5. **One PR per feature** — keep PRs focused and reviewable -6. **Update skill.json version** if adding new commands +2. **Add a design doc** for new features (see "Design Doc Requirement" below) +3. **Add tests** for new features +4. **Ensure all tests pass** — `python3 -m pytest tests/ -v` +5. **Follow the PR template** — describe changes, motivation, testing +6. **One PR per feature** — keep PRs focused and reviewable +7. **Update skill.json version** if adding new commands + +### Design Doc Requirement (issue #67 Phase 1) + +PRs that add a **new feature** must include a design doc in `docs/design/`. +The CI check (`.github/workflows/design-doc-check.yml`) automatically +detects new-feature PRs by file pattern and fails if no design doc is +included. + +**What counts as a "new feature"?** + +| Pattern | Example | Requires design doc? | +|---------|---------|---------------------| +| New file in `scripts/commands/` | `commands/yourfeature.py` | Yes | +| New `scripts/*_engine.py` | `yourfeature_engine.py` | Yes | +| New file in `scripts/formatters/` | `formatters/yourformat.py` | Yes | +| New parser (non-fallback) in `scripts/parsers/` | `parsers/yourlang_parser.py` | Yes | +| Fallback parser | `parsers/fallback_yourlang.py` | No (regex shadow of existing parser) | +| Bug fix (modified file) | any existing file | No | +| Test addition | `tests/test_*.py` | No | +| Documentation change | `*.md` | No | + +**How to write a design doc:** + +1. Copy `docs/design/template.md` to `docs/design/NNNN-feature-name.md` + - `NNNN` is the next available number (zero-padded to 4 digits) + - `feature-name` is a short kebab-case slug +2. Fill in the sections: Problem, Goal, Changes, Trade-offs, Open Questions, + Migration / Rollout +3. The **Trade-offs** section is the most important — document alternatives + considered and why they were rejected. This prevents future contributors + from re-litigating decisions without context. +4. See `docs/design/0001-taint-engine.md` through `0004-graph-model.md` for + retroactive examples documenting existing features. + +**Bypassing the check:** + +If a feature is genuinely trivial (e.g., a one-line command alias) and a +design doc would be pure overhead, add the `skip-design-doc` label to the +PR. Use this sparingly — the check exists to ensure design decisions are +recorded for future contributors. + +See `docs/README.md` for full details on the design doc and implementation +plan convention. ### PR Title Format diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..8969dcfd --- /dev/null +++ b/docs/README.md @@ -0,0 +1,108 @@ +# CodeLens Documentation + +This directory contains CodeLens design documents, implementation plans, and +process guides. + +## Structure + +``` +docs/ +├── README.md ← you are here +├── design/ ← design docs (why a feature exists) +│ ├── template.md ← copy this to start a new design doc +│ ├── 0001-taint-engine.md +│ ├── 0002-mcp-server.md +│ ├── 0003-plugin-system.md +│ └── 0004-graph-model.md +└── plans/ ← implementation plans (how/when to build) + └── template.md ← copy this to start a new plan +``` + +## When Do I Need a Design Doc? + +A design doc is **required** for PRs that add a new feature. The CI check +(`scripts/check_design_doc.py`, runs via `.github/workflows/design-doc-check.yml`) +automatically detects new-feature PRs by file pattern and fails if no design +doc is included. + +### What counts as a "new feature"? + +Any of these triggers the requirement: + +| Pattern | Example | Why | +|---------|---------|-----| +| New file in `scripts/commands/` | `commands/yourfeature.py` | New CLI command | +| New `scripts/*_engine.py` file | `yourfeature_engine.py` | New analysis engine | +| New file in `scripts/formatters/` | `formatters/yourformat.py` | New output format | +| New parser in `scripts/parsers/` (non-fallback) | `parsers/yourlang_parser.py` | New language support | + +### What does NOT require a design doc? + +- Bug fixes (modifications to existing files) +- Test additions +- Documentation changes +- Dependency updates +- Refactors that don't change behavior +- Fallback parsers (`parsers/fallback_*.py`) — these are regex versions of + existing tree-sitter parsers, not new features + +### Bypassing the check + +If a feature is genuinely trivial (e.g., a one-line command alias) and a +design doc would be pure overhead, add the `skip-design-doc` label to the PR. +Use this sparingly — the check exists to ensure design decisions are recorded +for future contributors. + +## How to Write a Design Doc + +1. Copy `docs/design/template.md` to `docs/design/NNNN-feature-name.md` + - `NNNN` is the next available number (zero-padded to 4 digits) + - `feature-name` is a short kebab-case slug +2. Fill in the sections: + - **Problem** — what pain exists today? Be concrete. + - **Goal** — what does "done" look like? User-visible outcome. + - **Changes** — concrete file changes, grouped by area + - **Trade-offs** — alternatives considered and why they were rejected + (the most important section — prevents re-litigating decisions) + - **Open Questions** — what's NOT yet decided, with owners + - **Migration / Rollout** — how users migrate, or "additive — no migration" +3. Reference the design doc in your PR description + +See `docs/design/0001-taint-engine.md` through `0004-graph-model.md` for +retroactive examples documenting existing features. + +## How to Write an Implementation Plan + +A plan is **recommended** (but not enforced) for multi-phase features. Copy +`docs/plans/template.md` to `docs/plans/NNNN-feature-name.md` and break the +work into independently-reviewable phases. + +Plans are living documents — update the checklist as you progress. When the +feature is complete, the plan can be archived or merged into the design doc's +"Changes" section. + +## Numbering Convention + +Design docs and plans use ADR-style numbering: `NNNN-feature-name.md` where +`NNNN` is a zero-padded sequential number. This ensures sort stability and +makes it easy to reference a doc by number (e.g., "see design doc 0003"). + +The numbering is per-directory — `docs/design/0001-foo.md` and +`docs/plans/0001-bar.md` are independent sequences. + +## Lifecycle + +``` +1. PROPOSE → create design doc in docs/design/ as part of your feature PR +2. ACCEPT → BOS reviews the design doc as part of PR review +3. IMPLEMENT → the design doc reflects the as-built design; update if the + implementation diverged from the proposal +4. SUPERSEDE → if a future PR replaces this feature, mark the doc as + "Superseded by NNNN" and create a new doc for the replacement +5. DEPRECATE → if the feature is removed, mark the doc as "Deprecated" + but keep it for historical reference +``` + +Design docs are **never deleted** — they form the historical record of why +the codebase is structured the way it is. Even deprecated docs remain so +future contributors can understand past decisions. diff --git a/docs/design/0001-taint-engine.md b/docs/design/0001-taint-engine.md new file mode 100644 index 00000000..d64aa664 --- /dev/null +++ b/docs/design/0001-taint-engine.md @@ -0,0 +1,141 @@ +# Design Doc 0001: Taint Analysis Engine + +> **Status:** Accepted +> **Date:** 2026-06-15 (retroactive — backfilled 2026-07-02) +> **Author:** Wolfvin +> **Related issues:** #49 (Phase 1 consolidation) +> **Related PRs:** #140 (consolidation), original implementation pre-#49 + +--- + +## Problem + +CodeLens needed taint analysis to detect source-to-sink data flow vulnerabilities +(SQL injection, XSS, SSRF, path traversal, command injection). The first +attempt — `semantic_engine.py` — used regex pattern matching on source code +strings. This produced unacceptable false positives: + +- String literals containing `request` or `query` were flagged as taint + sources even when they appeared in comments or unrelated variable names. +- No path sensitivity: if a sanitizer ran on one branch of an `if/else`, + the regex engine still flagged the other branch. +- No scope awareness: a variable named `user_input` in function A was + treated as tainted in function B even though they had no data dependency. +- No inter-procedural flow: `def f(x): return x` followed by + `f(request.args)` was not recognized as passing taint through `f`. + +Agents using CodeLens reported that taint findings were "noisy and +untrustworthy" — they had to manually re-verify every finding, which defeated +the purpose of automated analysis. + +## Goal + +Produce taint analysis with: +- <5% false-positive rate on standard vulnerable-app fixtures +- Full taint path rendering (source → intermediate → sink) so agents can + verify the finding without re-reading source code +- Path sensitivity (different taint states on if/else branches) +- Inter-procedural flow within a single file +- Confidence scores so agents can prioritize high-confidence findings first + +## Changes + +### Architecture + +Six-phase pipeline per file: + +1. **Parse** — tree-sitter produces an AST (language-aware, no regex) +2. **CFG construction** — basic blocks with branches and joins +3. **Source identification** — built-in patterns + YAML rule definitions +4. **Forward propagation** — taint flows through assignments, calls, returns +5. **Sink check** — does taint arrive at a known sink? +6. **Finding generation** — render full path + confidence score + +### New Files + +- `scripts/ast_taint_engine.py` — the engine itself (~3700 lines) +- `scripts/crossfile_taint_engine.py` — cross-file wrapper (Phase 1 of #49 + made this a thin compat layer over `ast_taint_engine.analyze_workspace(cross_file=True)`) +- `scripts/commands/taint.py` — CLI command +- `scripts/rules/python_security.yaml` — built-in taint rules +- `scripts/rules/javascript_security.yaml` — built-in taint rules + +### Modified Files + +- `scripts/codelens.py` — auto-registers `taint` command via `commands/__init__.py` +- `scripts/mcp_server.py` — `codelens_taint` MCP tool + +### Confidence Scoring + +| Score | Meaning | +|-------|---------| +| 0.95+ | Direct source→sink, no sanitizer, same scope | +| 0.80+ | Source→sink through function call, no sanitizer | +| 0.60+ | Source→sink with partial sanitizer | +| 0.40+ | Indirect taint, may be sanitized | + +## Trade-offs + +### Alternative A: Regex-based (`semantic_engine.py`) + +- **Pros:** Fast, no tree-sitter dependency, simple to add new patterns +- **Cons:** No path sensitivity, no scope awareness, high false-positive rate +- **Why rejected:** False positives made the feature unusable for agents. + `semantic_engine.py` is now deprecated (PR #140) and prints a warning on + every use. + +### Alternative B: LSP-based taint analysis + +- **Pros:** Uses language servers' type inference — would catch flows the + AST-only approach misses (e.g., dynamic dispatch) +- **Cons:** Requires a running LSP server per language, 10-30s startup time, + not all languages have LSP servers, results vary by LSP implementation +- **Why rejected:** Too heavy a dependency for the core engine. LSP + verification is available as an optional `--deep` enhancement via + `hybrid_engine.py`, not as the primary path. + +### Alternative C: Dataflow engine reuse + +- **Pros:** `dataflow_engine.py` already does some flow tracking +- **Cons:** It tracks variable assignments, not taint semantics (source/sink). + Reusing it would require grafting on taint-specific logic, producing a + Frankenstein engine. +- **Why rejected:** Separation of concerns — dataflow answers "where does + this value come from?", taint answers "is this a security vulnerability?". + +### Chosen approach: Tree-sitter AST + CFG + +- **Why:** Language-aware (no regex false positives), path-sensitive (CFG + tracks branches), inter-procedural (follows calls within a file), and + tree-sitter is already a CodeLens dependency for parsing. The cross-file + extension (#49 Phase 1) adds inter-procedural flow across files without + changing the per-file algorithm. + +## Open Questions + +- [x] Q1: How to handle library method approximation? — **Resolved** by + #49 Phase 4 (not yet implemented as of 2026-07-01; cross-file flow is + Phase 1, library approximation is Phase 4). +- [x] Q2: Should taint findings be persisted to SQLite? — **Resolved**: + yes, via `persistent_registry.store_scan_result()`. +- [ ] Q3: How to handle taint through async/await boundaries? — **Open**. + Current engine treats `await f()` as a synchronous call, which may miss + taint flow through event-loop-mediated callbacks. + +## Migration / Rollout + +The AST taint engine is additive — it does not replace `semantic_engine.py` +(which remains as a deprecated alias for backward compatibility). Users who +had `semantic_engine` in their CI scripts see a deprecation warning but +their scripts continue to work. + +No database migration — taint findings are stored in the existing +`scan_results` table via the standard `store_scan_result()` path. + +## References + +- Issue: #49 (taint analysis depth — multi-phase consolidation) +- PR: #140 (Phase 1 — cross-file consolidation) +- Prior art: Semgrep's taint mode, CodeQL's dataflow analysis +- Related design docs: [0003-plugin-system](0003-plugin-system.md) (taint + rules can be shipped as a `rule_pack` plugin) diff --git a/docs/design/0002-mcp-server.md b/docs/design/0002-mcp-server.md new file mode 100644 index 00000000..c0f77e99 --- /dev/null +++ b/docs/design/0002-mcp-server.md @@ -0,0 +1,151 @@ +# Design Doc 0002: MCP Server + +> **Status:** Accepted +> **Date:** 2026-06-10 (retroactive — backfilled 2026-07-02) +> **Author:** Wolfvin +> **Related issues:** #17 (compact format), #59 (graphml format) +> **Related PRs:** original implementation, #139 (formatters), #153 (graphml) + +--- + +## Problem + +AI agents (Claude Code, Cursor, Continue.dev, Cline, VS Code Copilot) need +to query code intelligence without spawning a new process for every command. +The CLI-only model had three problems: + +1. **Cold-start latency** — each `python3 codelens.py query ...` invocation + took 200-500ms just to import modules and load the registry. An agent + doing 20 queries paid 4-10s in pure overhead. +2. **No persistent state** — every CLI invocation re-loaded the registry + from disk. For a 30k-node codebase, that's ~1.5s per invocation. +3. **No standard protocol** — agents had to parse CLI stdout (JSON) and + construct shell commands, which is fragile and token-expensive. + +## Goal + +Provide a persistent server mode that: +- Speaks a standard protocol (MCP — Model Context Protocol) so any + MCP-compatible agent can connect without CodeLens-specific glue code +- Keeps the registry in memory after initial scan → sub-millisecond query + latency for subsequent calls +- Auto-discovers all CodeLens commands and exposes them as MCP tools — no + manual tool registration when a new command is added +- Supports background file watching so the registry stays fresh without + manual `scan` calls + +## Changes + +### Architecture + +``` +Agent (Claude/Cursor/etc.) + │ JSON-RPC 2.0 over stdio + ▼ +CodeLens MCP Server (single long-running process) + │ + ├── In-memory registry cache (loaded once on init) + ├── Tool registry (auto-discovered from COMMAND_REGISTRY) + ├── File watcher (optional, --watch flag) + └── Format dispatcher (ai / compact / json / markdown / sarif / graphml) +``` + +### New Files + +- `scripts/mcp_server.py` — the server (~2700 lines), implements: + - JSON-RPC 2.0 over stdio + - MCP `initialize` handshake with server capabilities + - `tools/list` — returns all CodeLens commands as MCP tools + - `tools/call` — executes a command and returns formatted result + - `resources/list` — exposes codebase registry as resources +- `scripts/commands/serve.py` — CLI command that starts the server + +### Protocol Details + +- **Transport:** stdio (JSON-RPC 2.0); optional HTTP/SSE via `--port` +- **Default format:** `ai` (normalized schema: `{status, stats, items, truncated, recommendations, metadata}`) +- **Token-efficient format:** `compact` (single-char keys, ~50% smaller than `json`) +- **Tool naming:** `codelens_` (e.g., `codelens_query`, `codelens_taint`) +- **Tool count:** 68 tools (50 statically-defined + 14 dynamically-discovered; + `watch` and `serve` excluded because they're long-running) + +### Format Enum + +Every tool accepts a `format` parameter with the enum: +`[json, markdown, ai, sarif, compact, graphml]` + +- `ai` (default) — normalized schema for agent consumption +- `compact` — token-efficient single-char keys (issue #17) +- `graphml` — GraphML XML for graph-producing commands (issue #59 Phase 3) +- `json`/`markdown`/`sarif` — legacy verbose forms + +### Auto-Discovery + +New CLI commands auto-appear as MCP tools — no manual registration. The +server's `_handle_tools_list` iterates `COMMAND_REGISTRY` at request time +and infers a JSON Schema from each command's argparse definition. This +means adding a new command (e.g., `commands/yourfeature.py`) immediately +makes `codelens_yourfeature` available to every connected agent. + +## Trade-offs + +### Alternative A: HTTP REST API + +- **Pros:** Language-agnostic, curl-testable, standard tooling +- **Cons:** Agents need to manage a server lifecycle (start/stop/port), + no standard schema for "what tools exist", every agent writes custom glue +- **Why rejected:** MCP is becoming the standard for agent-tool integration. + Building a custom REST API would mean every agent needs CodeLens-specific + integration code, defeating the "standard protocol" goal. + +### Alternative B: gRPC + +- **Pros:** Strongly-typed, binary protocol, bidirectional streaming +- **Cons:** Heavy dependency (protobuf compiler), overkill for the request/ + response pattern of code queries, no agent ecosystem support +- **Why rejected:** Over-engineering. The payload is JSON-shaped anyway + (command results are dicts); gRPC adds complexity without benefit. + +### Alternative C: CLI-only with shell wrapper + +- **Pros:** No server lifecycle to manage, simplest implementation +- **Cons:** Cold-start latency per invocation, no persistent state, agents + must parse stdout +- **Why rejected:** This is the status quo ante. The 200-500ms cold start + per query is unacceptable for interactive agent workflows. + +### Chosen approach: MCP over stdio + +- **Why:** Standard protocol (any MCP-compatible agent connects without + CodeLens-specific code), persistent process (no cold start), stdio + transport (no port management, works in sandboxed environments). The + optional `--port` flag adds HTTP/SSE for non-stdio consumers. + +## Open Questions + +- [x] Q1: How to handle long-running commands (`watch`, `serve` itself)? + — **Resolved**: exclude them from `tools/list`. +- [x] Q2: Should the server re-scan on file changes? — **Resolved**: yes, + via `--watch` flag (uses `watchdog` library). +- [ ] Q3: How to handle concurrent tool calls from multiple agents sharing + one server? — **Open**. Current implementation serializes calls; parallel + execution would need registry locking. + +## Migration / Rollout + +The MCP server is additive — the CLI continues to work unchanged. Users who +don't want a persistent server can keep using `python3 codelens.py ` +per invocation. The `mcp_config.json` file at repo root provides +configuration templates for Claude Desktop, Cursor, VS Code Copilot, +Continue.dev, and Cline. + +No database migration — the server uses the same `.codelens/codelens.db` +SQLite database as the CLI. + +## References + +- MCP specification: https://modelcontextprotocol.io/ +- Issue: #17 (compact format for token efficiency) +- Issue: #59 (graphml format for graph-producing commands) +- Related design docs: [0004-graph-model](0004-graph-model.md) (the graph + the server queries) diff --git a/docs/design/0003-plugin-system.md b/docs/design/0003-plugin-system.md new file mode 100644 index 00000000..51f2df10 --- /dev/null +++ b/docs/design/0003-plugin-system.md @@ -0,0 +1,147 @@ +# Design Doc 0003: Plugin System + +> **Status:** Accepted +> **Date:** 2026-06-12 (retroactive — backfilled 2026-07-02) +> **Author:** Wolfvin +> **Related issues:** #46 (Semgrep-compat YAML rule engine) +> **Related PRs:** original implementation + +--- + +## Problem + +CodeLens shipped with built-in rules for OWASP Top 10 (36 rules) and +compliance (PCI-DSS + HIPAA, 53 rules). But users wanted to add: + +- Custom security rules specific to their codebase (e.g., "flag every use + of our internal `LegacyAuth` class") +- New analysis engines (e.g., a license-compatibility checker) +- Custom output formatters (e.g., a Jira-ticket formatter for findings) +- New CLI commands (e.g., a `deploy-check` command that runs pre-deploy + quality gates) + +Without a plugin system, every custom need required forking CodeLens. This +was unsustainable — users couldn't share customizations, and every CodeLens +update required re-applying forks. + +## Goal + +Provide a plugin system that: +- Supports four plugin types: `rule_pack`, `engine`, `formatter`, `command` +- Discovers plugins from three locations (priority: project > user > built-in) +- Isolates plugin failures — a broken plugin never crashes CodeLens +- Uses a standard manifest (`plugin.yaml`) so plugins are self-describing +- Allows plugins to be installed from a zip archive (marketplace foundation) + +## Changes + +### Architecture + +``` +Plugin discovery (at startup): + 1. .codelens/plugins/ (project-specific, highest priority) + 2. ~/.codelens/plugins/ (user-wide) + 3. scripts/plugins/ (shipped with CodeLens, lowest priority) + +Each plugin has: + plugin.yaml manifest: + name: my-plugin + version: 1.0.0 + type: rule_pack | engine | formatter | command + entry: rules/my_rules.yaml (for rule_pack) + entry: my_engine.py (for engine/formatter/command) + description: ... +``` + +### New Files + +- `scripts/plugin_system.py` — plugin loader, marketplace foundation (~1460 lines) +- `scripts/commands/plugin.py` — CLI command (`codelens plugin list/install/enable/disable`) +- `scripts/plugins/owasp_top10/plugin.yaml` — built-in OWASP rule pack +- `scripts/plugins/owasp_top10/rules/owasp_top10.yaml` — 36 OWASP rules +- `scripts/plugins/compliance/plugin.yaml` — built-in compliance rule pack +- `scripts/plugins/compliance/rules/pci_dss.yaml` — PCI-DSS rules +- `scripts/plugins/compliance/rules/hipaa.yaml` — HIPAA rules + +### Plugin Types + +| Type | Entry point | What it does | +|------|-------------|--------------| +| `rule_pack` | YAML file with rules | Adds rules to the rule engine (Semgrep-compat syntax) | +| `engine` | Python module with `analyze()` function | Adds a new analysis engine | +| `formatter` | Python module with `format()` function | Adds a new output format | +| `command` | Python module with `add_args()` + `execute()` | Adds a new CLI command | + +### Isolation + +Each plugin runs in its own namespace. Exceptions are caught and logged — +a failing plugin produces a warning on stderr but never crashes CodeLens. +This is critical because plugins may be untrusted (installed from +third-party marketplaces). + +## Trade-offs + +### Alternative A: No plugins — all rules built-in + +- **Pros:** Simpler codebase, no plugin loading complexity +- **Cons:** Every custom need requires a fork; users can't share + customizations; CodeLens becomes a monolith +- **Why rejected:** Unsustainable for a community tool. The OWASP + HIPAA + rules are useful but every team has domain-specific rules. + +### Alternative B: Python entry points (setuptools) + +- **Pros:** Standard Python packaging, `pip install` integration +- **Cons:** Requires plugins to be pip-installable packages (too heavy for + a single YAML rule file), doesn't support project-local plugins + (`.codelens/plugins/`) +- **Why rejected:** Too heavyweight for the common case (a team wants to + add one custom YAML rule file without packaging it). + +### Alternative C: Dynamic import without manifests + +- **Pros:** Simplest implementation — just `importlib.import_module()` +- **Cons:** No metadata (version, description, author), no type safety + (can't tell if a module is a rule_pack vs engine without inspecting it), + no marketplace foundation +- **Why rejected:** The manifest (`plugin.yaml`) is what makes plugins + self-describing and enables future marketplace distribution. + +### Chosen approach: YAML manifest + three-tier discovery + +- **Why:** Supports the lightest case (drop a YAML file in + `.codelens/plugins/`) and the heaviest case (install a zip from a + marketplace). The manifest provides metadata for `codelens plugin list` + and future marketplace features. Three-tier discovery respects the + project > user > built-in priority chain. + +## Open Questions + +- [x] Q1: How to handle plugin versioning and updates? — **Resolved**: the + `plugin.yaml` has a `version` field; `codelens plugin list` shows + installed versions. Updates are manual (re-install) for now. +- [x] Q2: How to handle plugin dependencies? — **Resolved**: plugins + declare dependencies in `plugin.yaml` (`depends_on: [other-plugin]`); + the loader checks the dependency graph and warns on missing deps. +- [ ] Q3: Should there be a signed-plugin mechanism for untrusted + marketplaces? — **Open**. Current implementation trusts all plugins + (they run in-process). A sandboxed execution model (subprocess or WASM) + would be needed for untrusted plugins. + +## Migration / Rollout + +The plugin system is additive — existing built-in rules (OWASP, compliance) +were converted to built-in plugins in `scripts/plugins/`. Users who +previously had custom rules in `scripts/rules/` can continue using them +(the rule engine still loads that directory) or migrate them to a +`.codelens/plugins/my-rules/` plugin. + +No database migration — plugins are loaded at startup, not persisted. + +## References + +- Issue: #46 (Semgrep-compat YAML rule engine — Phase 1) +- Prior art: Semgrep's rule packs, ESLint's plugin system, pytest's plugin + discovery via entry points +- Related design docs: [0001-taint-engine](0001-taint-engine.md) (taint + rules can be shipped as a `rule_pack` plugin) diff --git a/docs/design/0004-graph-model.md b/docs/design/0004-graph-model.md new file mode 100644 index 00000000..3ae47bd2 --- /dev/null +++ b/docs/design/0004-graph-model.md @@ -0,0 +1,193 @@ +# Design Doc 0004: Graph Data Model + +> **Status:** Accepted +> **Date:** 2026-06-08 (retroactive — backfilled 2026-07-02) +> **Author:** Wolfvin +> **Related issues:** #8 (graph backend for trace), #59 (graphml export) +> **Related PRs:** original v8.2 implementation, #153 (graphml) + +--- + +## Problem + +CodeLens v8.0-v8.1 used a flat registry: two JSON files (`backend.json` and +`frontend.json`) containing arrays of nodes and edges. Every structural +query — "who calls this function?", "what's the blast radius of renaming +this class?", "are there circular dependencies?" — required iterating the +full edge list (O(n) where n = 495k edges on a large codebase). + +This had three problems: + +1. **Performance** — `trace` on a 30k-node codebase took 8-12s because + every hop scanned the full edge list. Agents timed out. +2. **No index** — repeated queries re-scanned the same data. There was no + way to ask "give me the adjacency list for node X" without building it + on the fly. +3. **No transactional updates** — incremental scans rewrote the entire + JSON file. A crash mid-write could corrupt the registry. + +## Goal + +Introduce a proper graph data model that: +- Backs structural queries (trace, impact, circular, dependents) with + indexed lookups → <100ms per hop on a 30k-node graph +- Lives alongside the flat registry (non-breaking) — all 70 existing + commands continue to work unchanged +- Supports incremental updates (only changed files re-parsed, graph + patched in a single transaction) +- Persists to SQLite for crash safety and concurrent read access + +## Changes + +### Architecture + +``` +scan (full or incremental) + │ + ▼ +Flat registry (backend.json / frontend.json) ← source of truth during scan + │ + ▼ (single bulk transaction) +SQLite database (.codelens/codelens.db) + ├── graph_nodes table (one row per symbol: file:line:fn) + └── graph_edges table (one row per call/import/define/inherit edge) + + indexes on source_id, target_id, edge_type +``` + +### Schema + +```sql +graph_nodes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + node_id TEXT NOT NULL UNIQUE, -- matches flat registry "id" (file:line:fn) + node_type TEXT NOT NULL, -- function|class|file|module|route|type|interface + name TEXT NOT NULL, -- symbol name (flat registry "fn") + file TEXT, + line INTEGER, + extra_json TEXT -- preserves original "type", "status", etc. +) + +graph_edges ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id TEXT NOT NULL, -- references graph_nodes.node_id + target_id TEXT, -- NULL for unresolved external calls + edge_type TEXT NOT NULL, -- CALLS|IMPORTS|DEFINES|INHERITS|IMPLEMENTS|USES_TYPE + file TEXT, -- file where the edge originates + line INTEGER, -- line where the edge originates + confidence REAL NOT NULL DEFAULT 1.0, + extra_json TEXT -- preserves "ipc", "via_self", "to_fn", etc. +) +``` + +### Indexes + +- `idx_graph_nodes_name` — for `find_nodes_by_name()` (symbol search) +- `idx_graph_edges_source` — for `query_callees()` (forward traversal) +- `idx_graph_edges_target` — for `query_callers()` (reverse traversal) +- `idx_graph_edges_type` — for filtering by edge type + +### New Files + +- `scripts/graph_model.py` — schema, population, incremental update, BFS + queries (~1100 lines) +- `scripts/edge_resolver.py` — cached adjacency index (O(1) caller/callee + lookups during BFS) +- `scripts/commands/graph_schema.py` — CLI command for schema introspection + +### Modified Files + +- `scripts/trace_engine.py` — uses graph backend by default, falls back to + flat registry if graph tables are empty (issue #8) +- `scripts/impact_engine.py` — uses graph for dependents lookup +- `scripts/circular_engine.py` — uses graph for cycle detection +- `scripts/persistent_registry.py` — manages the SQLite database lifecycle + +### Query API + +```python +# Find all nodes matching a name (case-insensitive, fuzzy) +find_nodes_by_name(name, db_path) -> List[Dict] + +# BFS traversal — callers (who calls this node?) +query_callers(node_id, db_path, max_depth=1) -> List[Dict] + +# BFS traversal — callees (what does this node call?) +query_callees(node_id, db_path, max_depth=1) -> List[Dict] + +# Stats for diagnostics +graph_stats(db_path) -> {"nodes": int, "edges": int} +``` + +## Trade-offs + +### Alternative A: Keep flat JSON, add in-memory index + +- **Pros:** No schema migration, no SQLite dependency +- **Cons:** Still need to load the full JSON into memory on every startup + (~1.5s for 30k nodes), no transactional updates, no concurrent access +- **Why rejected:** The fundamental problem was re-scanning on every + startup. An in-memory index doesn't solve persistence or crash safety. + +### Alternative B: NetworkX graph persisted via pickle + +- **Pros:** Rich graph algorithms (PageRank, centrality) for free +- **Cons:** Pickle is not crash-safe (corrupt on partial write), no + concurrent read access, NetworkX is a heavy dependency, algorithms + are O(n) in memory when SQLite indexes give O(log n) on disk +- **Why rejected:** Crash safety and concurrency are hard requirements. + Pickle fails both. NetworkX is also a large dependency for a feature + that only needs BFS. + +### Alternative C: Neo4j or external graph database + +- **Pros:** Purpose-built graph database, query language (Cypher) +- **Cons:** External process to manage, network latency, license + (Neo4j Community is GPL), overkill for the query patterns CodeLens needs +- **Why rejected:** CodeLens is a single-process CLI tool. Requiring users + to run Neo4j would massively raise the installation barrier. SQLite is + already a dependency (for `persistent_registry`) and is zero-config. + +### Chosen approach: SQLite tables with indexes + +- **Why:** SQLite is already a dependency, zero-config, crash-safe (WAL + mode), supports concurrent reads, and indexed lookups give O(log n) + performance. The flat registry remains the source of truth during scan; + the graph is populated from it in a single bulk transaction. This is + non-breaking by design — all 70 existing commands work unchanged, and + `trace_engine` falls back to the flat registry if the graph tables are + empty. + +## Open Questions + +- [x] Q1: Should the graph replace the flat registry entirely? — + **Resolved**: no. The flat registry is the scan output format (JSON, + human-readable, diff-friendly). The graph is a derived index for + structural queries. They serve different purposes. +- [x] Q2: How to handle incremental updates? — **Resolved**: issue #8 + Phase 2 added `incremental_graph_update()` which patches only changed + nodes/edges in a single transaction. +- [ ] Q3: Should we add a Cypher-like query language? — **Open**, tracked + in issue #9. A Cypher-subset engine was implemented in PR #149/#151 + but is not yet merged as of 2026-07-02. + +## Migration / Rollout + +The graph tables are additive — `graph_nodes` and `graph_edges` are +prefixed to avoid colliding with any existing table name. The flat +registry tables and JSON files are untouched. Users who never run `scan` +after upgrading to v8.2 see no change (the graph tables are simply empty; +`trace_engine` falls back to the flat registry). + +The first `scan` after upgrading to v8.2 populates the graph tables +automatically — no manual migration step. + +## References + +- Issue: #8 (graph backend for trace — bidirectional fallback) +- Issue: #59 (graphml export — Phase 3 reads from `graph_nodes`/`graph_edges`) +- Issue: #9 (Cypher-like query engine — not yet merged) +- Prior art: Sourcegraph's code graph, GitHub's code navigation (tree-sitter + + stack graphs) +- Related design docs: [0002-mcp-server](0002-mcp-server.md) (the server + queries this graph), [0001-taint-engine](0001-taint-engine.md) (taint + engine uses `edge_resolver` for inter-procedural flow) diff --git a/docs/design/template.md b/docs/design/template.md new file mode 100644 index 00000000..5f79ed5f --- /dev/null +++ b/docs/design/template.md @@ -0,0 +1,94 @@ +# Design Doc: [Feature Name] + +> **Status:** Proposed | Accepted | Superseded by [NNNN](NNNN-feature-name.md) | Deprecated +> **Date:** YYYY-MM-DD +> **Author:** [Your name / GitHub handle] +> **Related issues:** #NNN +> **Related PRs:** #NNN + +--- + +## Problem + +Describe the problem this design solves. What pain exists today? What can't +users/agents/developers do that they should be able to do? What regression or +scaling issue prompted this? + +Be concrete — cite specific commands, file paths, error messages, or user +reports. Avoid vague statements like "the system is slow" in favor of +"trace on a 30k-edge graph takes 45s; agents time out at 30s (issue #17)". + +## Goal + +State the desired outcome in one or two sentences. What does "done" look like? +This is not a list of changes — it's the user-visible result. + +Example: "Agents can trace a call chain across the full workspace in <2s, +including cross-file resolution, with results paginated to fit within 5k tokens." + +## Changes + +List the concrete changes this design introduces. Group by area if the change +is large. Each item should be specific enough that a reviewer can verify it +was implemented. + +### Architecture / Data Model +- ... + +### New Files +- `scripts/yourfeature_engine.py` — ... +- `scripts/commands/yourfeature.py` — ... + +### Modified Files +- `scripts/codelens.py` — add `--your-flag` to format choices (3 places) +- `README.md` — document the new flag + +### CLI / MCP Surface +- New command: `codelens yourfeature [args]` +- New MCP tool: `codelens_yourfeature` + +### Tests +- `tests/test_yourfeature.py` — covers X, Y, Z + +## Trade-offs + +Document the alternatives considered and why they were rejected. This is the +most important section — it prevents future contributors from re-litigating a +decision without knowing the context. + +### Alternative A: [Name] +- **Pros:** ... +- **Cons:** ... +- **Why rejected:** ... + +### Alternative B: [Name] +- **Pros:** ... +- **Cons:** ... +- **Why rejected:** ... + +### Chosen approach: [Name] +- **Why:** ... + +## Open Questions + +List anything that is NOT yet decided. Be honest — an open question is not a +weakness, it's a flag for reviewers. Each item should have an owner and a +decision deadline. + +- [ ] Q1: Should we cache the result? (Owner: @handle, decide by YYYY-MM-DD) +- [ ] Q2: How does this interact with `--deep` (LSP) mode? (Owner: @handle) + +## Migration / Rollout + +If this change is user-visible or breaks existing behavior, describe how users +migrate. Include deprecation timelines, fallback behavior, and how to detect +the old vs new behavior. + +If there is no migration concern, write "No migration impact — additive change." + +## References + +- Issue: #NNN +- PR: #NNN +- Prior art: [link to blog post, paper, or other project's docs] +- Related design docs: [NNNN](NNNN-feature-name.md) diff --git a/docs/plans/template.md b/docs/plans/template.md new file mode 100644 index 00000000..8de041da --- /dev/null +++ b/docs/plans/template.md @@ -0,0 +1,90 @@ +# Implementation Plan: [Feature Name] + +> **Design doc:** [NNNN-feature-name.md](../design/NNNN-feature-name.md) +> **Issue:** #NNN +> **PR:** #NNN +> **Status:** Not started | In progress | Complete | Blocked + +--- + +## Scope + +One-paragraph summary of what this plan covers. Reference the design doc for +the "why" — this plan is only the "how" and "when". + +## Phases + +Break the work into phases that can be independently reviewed and merged. +Each phase should leave the codebase in a working state (tests pass, no +half-finished features). Prefer vertical slices over horizontal layers. + +### Phase 1: [Name] — [estimated duration] + +**Goal:** [one sentence outcome] + +**Tasks:** +- [ ] Create `scripts/yourfeature_engine.py` with stub `analyze()` function +- [ ] Add `commands/yourfeature.py` with `add_args` + `execute` +- [ ] Register in CLI (auto-registered via `commands/__init__.py`) +- [ ] Add basic test: `tests/test_yourfeature.py::test_smoke` +- [ ] Run `python scripts/sync_command_count.py --apply` +- [ ] Update `README.md` + `SKILL-QUICK.md` command list + +**Acceptance:** +- `codelens yourfeature --help` works +- Smoke test passes +- Command count in docs matches `COMMAND_REGISTRY` + +**Dependencies:** None + +### Phase 2: [Name] — [estimated duration] + +**Goal:** [one sentence outcome] + +**Tasks:** +- [ ] Implement full analysis logic in `yourfeature_engine.py` +- [ ] Add edge-case tests: empty input, missing files, corrupt data +- [ ] Add `--format graphml` support (if applicable) +- [ ] Benchmark on `tests/fixtures/` — must complete <1s + +**Acceptance:** +- All tests pass +- No regression in `tests/test_cli.py` +- Benchmark target met + +**Dependencies:** Phase 1 + +### Phase 3: [Name] — [estimated duration] + +**Goal:** [one sentence outcome] + +**Tasks:** +- [ ] ... + +**Acceptance:** +- ... + +**Dependencies:** Phase 1, Phase 2 + +## Testing Strategy + +Describe how each phase will be verified. Include: +- Unit tests (which files, what scenarios) +- Integration tests (if applicable) +- Manual verification steps (commands to run, expected output) +- Regression checks (which existing tests must still pass) + +## Rollback Plan + +If this needs to be reverted after merge, what's the procedure? +- Is the change behind a feature flag? +- Can the new command be removed without breaking existing workflows? +- Are there database migrations to undo? + +If the change is purely additive (new command, new file), rollback is trivial: +revert the PR. If it modifies existing behavior, describe the fallback path. + +## Notes + +Anything that doesn't fit above — links to research, relevant Slack +discussions, gotchas discovered during implementation, etc. diff --git a/scripts/check_design_doc.py b/scripts/check_design_doc.py new file mode 100644 index 00000000..6472f72f --- /dev/null +++ b/scripts/check_design_doc.py @@ -0,0 +1,356 @@ +# @WHO: scripts/check_design_doc.py +# @WHAT: CI check — require a design doc in docs/design/ for new-feature PRs +# @PART: ci +# @ENTRY: main() +""" +Design Doc Checker — CI gate for issue #67 Phase 1. + +Detects "new feature" PRs by file pattern and fails if no design doc is +included in docs/design/. The check is bypassable via the ``skip-design-doc`` +PR label for genuinely trivial features. + +Usage (GitHub Actions): + The workflow at .github/workflows/design-doc-check.yml calls this script + with the environment variables GITHUB_TOKEN, GITHUB_REPOSITORY, and + PR_NUMBER set. The script fetches the PR's files and labels via the + GitHub API, runs the check, and exits 0 (pass) or 1 (fail). + +Usage (local testing): + python3 scripts/check_design_doc.py --files ... [--labels label1 label2] + This mode is for unit testing and local pre-flight checks. It does not + call the GitHub API. + +@FLOW: DESIGN_DOC_CHECK +@CALLS: check_pr() -> CheckResult +@CALLS: _fetch_pr_files() / _fetch_pr_labels() -> GitHub API (CI mode only) +@MUTATES: none (pure check — exits 0/1, prints to stdout) +""" + +import argparse +import json +import os +import sys +import urllib.request +from typing import Any, Dict, List, Optional, Set, Tuple + + +# ─── Configuration ─────────────────────────────────────────── + + +# File patterns that indicate a "new feature" PR. Each entry is a tuple of +# (directory_prefix, status_filter) where status_filter is "added" (only new +# files trigger) or "any" (any change triggers). +# +# Rationale: +# - scripts/commands/ + added → a new CLI command is a new feature +# - scripts/*_engine.py + added → a new engine is a new feature +# - scripts/formatters/ + added → a new output format is a new feature +# - scripts/parsers/ (non-fallback) + added → a new language parser is a feature +# - scripts/parsers/fallback_*.py is EXCLUDED — fallbacks are regex shadows +# of existing tree-sitter parsers, not new features +# - scripts/mcp_server.py is EXCLUDED from "added" (it always exists) but +# flagged on "any" modification IF a new tool definition is added. We +# can't reliably detect "new tool definition" from a diff, so we don't +# flag mcp_server.py changes. The contributor should add a design doc +# if they added a new MCP tool, but the CI check doesn't enforce it. +_FEATURE_PATTERNS: List[Tuple[str, str]] = [ + ("scripts/commands/", "added"), + ("scripts/formatters/", "added"), +] + +# Engine files live directly in scripts/ with the suffix _engine.py. +# We detect these separately because they're not in a subdirectory. +_ENGINE_SUFFIX = "_engine.py" + +# Parser files: scripts/parsers/_parser.py is a new feature, but +# scripts/parsers/fallback_.py is NOT (it's a regex fallback for an +# existing tree-sitter parser). +_PARSER_DIR = "scripts/parsers/" +_PARSER_SUFFIX = "_parser.py" +_FALLBACK_PREFIX = "fallback_" + +# PR label that bypasses the design doc requirement. +BYPASS_LABEL = "skip-design-doc" + +# Design doc directory. +DESIGN_DOC_DIR = "docs/design/" + +# Plan directory (recommended but not enforced). +PLAN_DIR = "docs/plans/" + + +# ─── Pure Logic (unit-testable) ────────────────────────────── + + +def is_feature_file(filename: str, status: str) -> bool: + """Return True if a file change represents a new feature. + + Args: + filename: Path to the file in the PR (relative to repo root). + status: One of "added", "modified", "removed", "renamed". + + Returns: + True if this file pattern + status combination indicates a new + feature that requires a design doc. + """ + # New CLI command + if filename.startswith("scripts/commands/") and status == "added": + return True + + # New formatter + if filename.startswith("scripts/formatters/") and status == "added": + return True + + # New engine (scripts/_engine.py, added) + if ( + filename.startswith("scripts/") + and filename.endswith(_ENGINE_SUFFIX) + and "/" not in filename[len("scripts/"):] # directly in scripts/, not a subdir + and status == "added" + ): + return True + + # New parser (scripts/parsers/_parser.py, added) — exclude fallbacks + if ( + filename.startswith(_PARSER_DIR) + and filename.endswith(_PARSER_SUFFIX) + and status == "added" + ): + basename = filename[len(_PARSER_DIR):] + if not basename.startswith(_FALLBACK_PREFIX): + return True + + return False + + +def check_pr( + pr_files: List[Dict[str, Any]], + pr_labels: List[str], +) -> Dict[str, Any]: + """Check whether a PR requires and includes a design doc. + + This is the pure-logic entry point — no I/O, no API calls. It takes the + PR's files (as returned by the GitHub API, with ``filename`` and + ``status`` keys) and labels (as a list of strings), and returns a + result dict describing whether the check passed and why. + + Args: + pr_files: List of dicts, each with at least ``filename`` and + ``status`` keys (matching the GitHub PR files API response). + pr_labels: List of label names on the PR. + + Returns: + Dict with keys: + - ``passed`` (bool): True if the PR passes the check + - ``reason`` (str): Human-readable explanation + - ``feature_files`` (list[str]): Files that triggered the + feature requirement + - ``design_docs`` (list[str]): Design docs added in this PR + - ``bypassed`` (bool): True if the check was bypassed via label + """ + # Detect feature files + feature_files = [ + f["filename"] for f in pr_files + if is_feature_file(f.get("filename", ""), f.get("status", "")) + ] + + # Detect design docs added in this PR + design_docs = [ + f["filename"] for f in pr_files + if f.get("filename", "").startswith(DESIGN_DOC_DIR) + and f.get("filename", "").endswith(".md") + and f.get("status") in ("added", "modified", "renamed") + ] + + # Bypass via label + if BYPASS_LABEL in pr_labels: + return { + "passed": True, + "reason": ( + f"PR has '{BYPASS_LABEL}' label — design doc requirement " + f"bypassed." + ), + "feature_files": feature_files, + "design_docs": design_docs, + "bypassed": True, + } + + # No feature files → check passes silently + if not feature_files: + return { + "passed": True, + "reason": ( + "PR does not add new feature files (no new commands, " + "engines, formatters, or parsers). Design doc not required." + ), + "feature_files": [], + "design_docs": design_docs, + "bypassed": False, + } + + # Feature files present → require a design doc + if design_docs: + return { + "passed": True, + "reason": ( + f"PR adds feature file(s) and includes design doc(s): " + f"{', '.join(design_docs)}." + ), + "feature_files": feature_files, + "design_docs": design_docs, + "bypassed": False, + } + + # Feature files present, no design doc → FAIL + return { + "passed": False, + "reason": ( + f"PR adds new feature file(s) ({', '.join(feature_files)}) but " + f"does not include a design doc in {DESIGN_DOC_DIR}. " + f"Copy docs/design/template.md to docs/design/NNNN-feature-name.md " + f"and describe the design decisions. " + f"If this is genuinely trivial, add the '{BYPASS_LABEL}' label " + f"to bypass this check." + ), + "feature_files": feature_files, + "design_docs": [], + "bypassed": False, + } + + +# ─── GitHub API I/O (CI mode only) ─────────────────────────── + + +def _github_api(url: str, token: str) -> Any: + """Fetch JSON from the GitHub API with authentication.""" + req = urllib.request.Request(url, headers={ + "Authorization": f"token {token}", + "Accept": "application/vnd.github+json", + "User-Agent": "design-doc-checker", + }) + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read().decode()) + + +def fetch_pr_files(repo: str, pr_number: int, token: str) -> List[Dict[str, Any]]: + """Fetch the list of files changed in a PR via the GitHub API. + + Handles pagination — PRs with >100 files require multiple requests. + """ + all_files: List[Dict[str, Any]] = [] + page = 1 + while True: + url = ( + f"https://api.github.com/repos/{repo}/pulls/{pr_number}/files" + f"?per_page=100&page={page}" + ) + batch = _github_api(url, token) + if not batch: + break + all_files.extend(batch) + if len(batch) < 100: + break + page += 1 + return all_files + + +def fetch_pr_labels(repo: str, pr_number: int, token: str) -> List[str]: + """Fetch the labels on a PR via the GitHub API.""" + url = f"https://api.github.com/repos/{repo}/pulls/{pr_number}" + data = _github_api(url, token) + return [label["name"] for label in data.get("labels", [])] + + +# ─── CLI Entry Point ───────────────────────────────────────── + + +def main() -> int: + """CLI entry point. + + Two modes: + 1. CI mode (default): reads GITHUB_TOKEN, GITHUB_REPOSITORY, PR_NUMBER + env vars and fetches PR data from the GitHub API. + 2. Local mode (--files): reads file list and labels from command-line + args, for unit testing and local pre-flight checks. + + Returns: + 0 if the check passes, 1 if it fails. + """ + parser = argparse.ArgumentParser( + description="Check whether a PR requires and includes a design doc.", + ) + parser.add_argument( + "--files", nargs="*", default=None, + help="Local mode: list of changed files (space-separated paths). " + "When provided, the script does not call the GitHub API.", + ) + parser.add_argument( + "--labels", nargs="*", default=[], + help="Local mode: PR labels (space-separated).", + ) + parser.add_argument( + "--status", nargs="*", default=None, + help="Local mode: status for each file (--files), in the same order. " + "Values: added, modified, removed, renamed. If omitted, all " + "files are assumed 'added'.", + ) + args = parser.parse_args() + + if args.files is not None: + # Local mode + statuses = args.status if args.status else ["added"] * len(args.files) + if len(statuses) != len(args.files): + print( + f"Error: --files has {len(args.files)} items but --status has " + f"{len(statuses)} items. They must be the same length.", + file=sys.stderr, + ) + return 2 + pr_files = [ + {"filename": f, "status": s} + for f, s in zip(args.files, statuses) + ] + pr_labels = args.labels + else: + # CI mode — read env vars + token = os.environ.get("GITHUB_TOKEN") + repo = os.environ.get("GITHUB_REPOSITORY") + pr_number_str = os.environ.get("PR_NUMBER") + + if not all([token, repo, pr_number_str]): + print( + "Error: CI mode requires GITHUB_TOKEN, GITHUB_REPOSITORY, " + "and PR_NUMBER environment variables. For local testing, " + "use --files ...", + file=sys.stderr, + ) + return 2 + + pr_number = int(pr_number_str) + print(f"Fetching PR #{pr_number} files and labels from {repo}...", + file=sys.stderr) + try: + pr_files = fetch_pr_files(repo, pr_number, token) + pr_labels = fetch_pr_labels(repo, pr_number, token) + except Exception as exc: + print(f"Error fetching PR data: {exc}", file=sys.stderr) + return 2 + + # Run the check + result = check_pr(pr_files, pr_labels) + + # Print result + status_word = "PASS" if result["passed"] else "FAIL" + print(f"[design-doc-check] {status_word}") + print(f"[design-doc-check] {result['reason']}") + + if result["feature_files"]: + print(f"[design-doc-check] Feature files: {', '.join(result['feature_files'])}") + if result["design_docs"]: + print(f"[design-doc-check] Design docs: {', '.join(result['design_docs'])}") + + return 0 if result["passed"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_design_doc_check.py b/tests/test_design_doc_check.py new file mode 100644 index 00000000..7200d778 --- /dev/null +++ b/tests/test_design_doc_check.py @@ -0,0 +1,399 @@ +"""Tests for scripts/check_design_doc.py — issue #67 Phase 1. + +Tests the pure-logic check_pr() function (no GitHub API calls) and the +is_feature_file() classifier. The CLI entry point (main()) is tested via +subprocess to verify env-var and arg parsing. +""" + +import os +import subprocess +import sys +import unittest + +SCRIPT_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "scripts") +sys.path.insert(0, SCRIPT_DIR) + +from check_design_doc import ( + BYPASS_LABEL, + DESIGN_DOC_DIR, + check_pr, + is_feature_file, +) + + +# ─── 1. is_feature_file classifier ─────────────────────────── + + +class TestIsFeatureFile(unittest.TestCase): + """Classify file + status pairs as feature or non-feature.""" + + def test_new_command_added_is_feature(self): + self.assertTrue(is_feature_file("scripts/commands/yourfeature.py", "added")) + + def test_new_engine_added_is_feature(self): + self.assertTrue(is_feature_file("scripts/yourfeature_engine.py", "added")) + + def test_new_formatter_added_is_feature(self): + self.assertTrue(is_feature_file("scripts/formatters/yourformat.py", "added")) + + def test_new_parser_added_is_feature(self): + self.assertTrue(is_feature_file("scripts/parsers/yourlang_parser.py", "added")) + + def test_fallback_parser_added_is_not_feature(self): + """Fallback parsers are regex shadows of existing tree-sitter parsers.""" + self.assertFalse(is_feature_file("scripts/parsers/fallback_yourlang.py", "added")) + + def test_modified_command_is_not_feature(self): + """Bug fixes / refactors modify existing files — not 'added'.""" + self.assertFalse(is_feature_file("scripts/commands/scan.py", "modified")) + + def test_modified_engine_is_not_feature(self): + self.assertFalse(is_feature_file("scripts/ast_taint_engine.py", "modified")) + + def test_removed_file_is_not_feature(self): + self.assertFalse(is_feature_file("scripts/commands/oldfeature.py", "removed")) + + def test_renamed_file_is_not_feature(self): + """Renaming a file is not a new feature.""" + self.assertFalse(is_feature_file("scripts/commands/renamed.py", "renamed")) + + def test_test_file_is_not_feature(self): + self.assertFalse(is_feature_file("tests/test_yourfeature.py", "added")) + + def test_doc_file_is_not_feature(self): + self.assertFalse(is_feature_file("README.md", "added")) + self.assertFalse(is_feature_file("docs/design/0005-foo.md", "added")) + + def test_engine_in_subdirectory_is_not_feature(self): + """scripts/parsers/fallback_foo_engine.py should not match _engine.py rule. + + The engine rule requires the file to be directly in scripts/, not in + a subdirectory. This prevents false positives on files like + scripts/sca_parsers/pipfile_engine.py (hypothetical). + """ + # A file in a subdirectory that ends with _engine.py should NOT match + # the engine rule (it's checked by the subdirectory rules instead). + self.assertFalse( + is_feature_file("scripts/sca_parsers/some_engine.py", "added") + ) + + def test_non_python_command_file_still_feature(self): + """Any new file in scripts/commands/ counts, regardless of extension.""" + self.assertTrue(is_feature_file("scripts/commands/yourfeature.js", "added")) + + def test_empty_filename_is_not_feature(self): + self.assertFalse(is_feature_file("", "added")) + + def test_empty_status_is_not_feature(self): + self.assertFalse(is_feature_file("scripts/commands/foo.py", "")) + + +# ─── 2. check_pr — no feature files ────────────────────────── + + +class TestCheckPrNoFeatureFiles(unittest.TestCase): + """PRs without feature files should pass silently.""" + + def test_bug_fix_passes(self): + files = [{"filename": "scripts/commands/scan.py", "status": "modified"}] + result = check_pr(files, []) + self.assertTrue(result["passed"]) + self.assertEqual(result["feature_files"], []) + self.assertFalse(result["bypassed"]) + + def test_test_only_pr_passes(self): + files = [{"filename": "tests/test_foo.py", "status": "added"}] + result = check_pr(files, []) + self.assertTrue(result["passed"]) + + def test_doc_only_pr_passes(self): + files = [{"filename": "README.md", "status": "modified"}] + result = check_pr(files, []) + self.assertTrue(result["passed"]) + + def test_empty_pr_passes(self): + result = check_pr([], []) + self.assertTrue(result["passed"]) + + def test_design_doc_only_pr_passes(self): + """A PR that only adds a design doc (no feature) passes.""" + files = [{"filename": "docs/design/0005-retroactive.md", "status": "added"}] + result = check_pr(files, []) + self.assertTrue(result["passed"]) + self.assertEqual(result["design_docs"], ["docs/design/0005-retroactive.md"]) + + +# ─── 3. check_pr — feature files without design doc ───────── + + +class TestCheckPrFeatureWithoutDesignDoc(unittest.TestCase): + """PRs with feature files but no design doc should FAIL.""" + + def test_new_command_without_design_doc_fails(self): + files = [{"filename": "scripts/commands/newcmd.py", "status": "added"}] + result = check_pr(files, []) + self.assertFalse(result["passed"]) + self.assertIn("scripts/commands/newcmd.py", result["feature_files"]) + self.assertEqual(result["design_docs"], []) + self.assertFalse(result["bypassed"]) + # Reason should mention the design doc requirement + self.assertIn("design doc", result["reason"].lower()) + self.assertIn("docs/design/", result["reason"]) + + def test_new_engine_without_design_doc_fails(self): + files = [{"filename": "scripts/newfeat_engine.py", "status": "added"}] + result = check_pr(files, []) + self.assertFalse(result["passed"]) + + def test_new_formatter_without_design_doc_fails(self): + files = [{"filename": "scripts/formatters/newfmt.py", "status": "added"}] + result = check_pr(files, []) + self.assertFalse(result["passed"]) + + def test_new_parser_without_design_doc_fails(self): + files = [{"filename": "scripts/parsers/newlang_parser.py", "status": "added"}] + result = check_pr(files, []) + self.assertFalse(result["passed"]) + + def test_reason_mentions_bypass_label(self): + """The failure reason should tell users about the bypass label.""" + files = [{"filename": "scripts/commands/newcmd.py", "status": "added"}] + result = check_pr(files, []) + self.assertIn(BYPASS_LABEL, result["reason"]) + + def test_reason_mentions_template(self): + """The failure reason should point users to the template.""" + files = [{"filename": "scripts/commands/newcmd.py", "status": "added"}] + result = check_pr(files, []) + self.assertIn("template", result["reason"].lower()) + + +# ─── 4. check_pr — feature files WITH design doc ──────────── + + +class TestCheckPrFeatureWithDesignDoc(unittest.TestCase): + """PRs with feature files and a design doc should pass.""" + + def test_new_command_with_design_doc_passes(self): + files = [ + {"filename": "scripts/commands/newcmd.py", "status": "added"}, + {"filename": "docs/design/0005-newcmd.md", "status": "added"}, + ] + result = check_pr(files, []) + self.assertTrue(result["passed"]) + self.assertIn("scripts/commands/newcmd.py", result["feature_files"]) + self.assertIn("docs/design/0005-newcmd.md", result["design_docs"]) + + def test_modified_design_doc_counts(self): + """If the PR modifies an existing design doc, that counts.""" + files = [ + {"filename": "scripts/commands/newcmd.py", "status": "added"}, + {"filename": "docs/design/0001-taint-engine.md", "status": "modified"}, + ] + result = check_pr(files, []) + self.assertTrue(result["passed"]) + + def test_renamed_design_doc_counts(self): + files = [ + {"filename": "scripts/commands/newcmd.py", "status": "added"}, + {"filename": "docs/design/0005-newcmd.md", "status": "renamed"}, + ] + result = check_pr(files, []) + self.assertTrue(result["passed"]) + + def test_multiple_feature_files_with_one_design_doc_passes(self): + files = [ + {"filename": "scripts/commands/newcmd.py", "status": "added"}, + {"filename": "scripts/newfeat_engine.py", "status": "added"}, + {"filename": "scripts/formatters/newfmt.py", "status": "added"}, + {"filename": "docs/design/0005-newfeat.md", "status": "added"}, + ] + result = check_pr(files, []) + self.assertTrue(result["passed"]) + self.assertEqual(len(result["feature_files"]), 3) + + def test_non_md_file_in_design_dir_does_not_count(self): + """A .txt or .yaml file in docs/design/ is not a design doc.""" + files = [ + {"filename": "scripts/commands/newcmd.py", "status": "added"}, + {"filename": "docs/design/notes.txt", "status": "added"}, + ] + result = check_pr(files, []) + self.assertFalse(result["passed"]) + self.assertEqual(result["design_docs"], []) + + +# ─── 5. check_pr — bypass label ────────────────────────────── + + +class TestCheckPrBypass(unittest.TestCase): + """The 'skip-design-doc' label bypasses the check.""" + + def test_bypass_label_allows_feature_without_design_doc(self): + files = [{"filename": "scripts/commands/newcmd.py", "status": "added"}] + result = check_pr(files, [BYPASS_LABEL]) + self.assertTrue(result["passed"]) + self.assertTrue(result["bypassed"]) + self.assertIn("bypassed", result["reason"].lower()) + + def test_bypass_label_with_design_doc_still_passes(self): + files = [ + {"filename": "scripts/commands/newcmd.py", "status": "added"}, + {"filename": "docs/design/0005-newcmd.md", "status": "added"}, + ] + result = check_pr(files, [BYPASS_LABEL]) + self.assertTrue(result["passed"]) + self.assertTrue(result["bypassed"]) + + def test_other_labels_do_not_bypass(self): + """Only the exact 'skip-design-doc' label bypasses.""" + files = [{"filename": "scripts/commands/newcmd.py", "status": "added"}] + result = check_pr(files, ["skip-design", "skip-design-docs", "no-design-doc"]) + self.assertFalse(result["passed"]) + self.assertFalse(result["bypassed"]) + + def test_bypass_label_case_sensitive(self): + """Label matching is case-sensitive (GitHub labels are case-insensitive + but we match exactly — the API returns the label's stored name).""" + files = [{"filename": "scripts/commands/newcmd.py", "status": "added"}] + result = check_pr(files, ["Skip-Design-Doc"]) + self.assertFalse(result["passed"]) + + +# ─── 6. Edge cases ─────────────────────────────────────────── + + +class TestEdgeCases(unittest.TestCase): + """Edge cases: missing keys, weird inputs, mixed PRs.""" + + def test_file_without_status_key_treated_as_not_feature(self): + """If status is missing, is_feature_file returns False (defensive).""" + files = [{"filename": "scripts/commands/newcmd.py"}] # no status + result = check_pr(files, []) + self.assertTrue(result["passed"]) # no feature detected + + def test_file_without_filename_key_skipped(self): + files = [{"status": "added"}] # no filename + result = check_pr(files, []) + self.assertTrue(result["passed"]) + + def test_mixed_pr_bug_fix_and_new_feature_without_doc_fails(self): + """A PR with both a bug fix and a new feature still needs a design doc.""" + files = [ + {"filename": "scripts/commands/scan.py", "status": "modified"}, # bug fix + {"filename": "scripts/commands/newcmd.py", "status": "added"}, # new feature + ] + result = check_pr(files, []) + self.assertFalse(result["passed"]) + self.assertIn("scripts/commands/newcmd.py", result["feature_files"]) + self.assertNotIn("scripts/commands/scan.py", result["feature_files"]) + + def test_fallback_parser_does_not_trigger_requirement(self): + """Adding a fallback parser (regex shadow) does not require a design doc.""" + files = [{"filename": "scripts/parsers/fallback_newlang.py", "status": "added"}] + result = check_pr(files, []) + self.assertTrue(result["passed"]) + self.assertEqual(result["feature_files"], []) + + def test_design_doc_in_wrong_directory_does_not_count(self): + """A design doc in docs/ (not docs/design/) does not satisfy the check.""" + files = [ + {"filename": "scripts/commands/newcmd.py", "status": "added"}, + {"filename": "docs/newcmd-design.md", "status": "added"}, + ] + result = check_pr(files, []) + self.assertFalse(result["passed"]) + self.assertEqual(result["design_docs"], []) + + def test_plan_doc_does_not_satisfy_design_doc_requirement(self): + """A plan in docs/plans/ does not satisfy the design doc requirement. + + Plans are recommended but not enforced — only docs/design/ counts. + """ + files = [ + {"filename": "scripts/commands/newcmd.py", "status": "added"}, + {"filename": "docs/plans/0005-newcmd.md", "status": "added"}, + ] + result = check_pr(files, []) + self.assertFalse(result["passed"]) + self.assertEqual(result["design_docs"], []) + + +# ─── 7. CLI entry point (subprocess) ───────────────────────── + + +class TestCliEntryPoint(unittest.TestCase): + """Test the main() CLI entry point via subprocess.""" + + def _run(self, args, env=None): + """Run check_design_doc.py with the given args, return (returncode, stdout, stderr).""" + full_env = os.environ.copy() + if env is not None: + full_env.update(env) + # Ensure GITHUB_TOKEN etc. are NOT set (so CI mode doesn't trigger) + for key in ("GITHUB_TOKEN", "GITHUB_REPOSITORY", "PR_NUMBER"): + full_env.pop(key, None) + proc = subprocess.run( + [sys.executable, os.path.join(SCRIPT_DIR, "check_design_doc.py")] + args, + capture_output=True, text=True, env=full_env, timeout=30, + ) + return proc.returncode, proc.stdout, proc.stderr + + def test_local_mode_bug_fix_passes(self): + rc, out, _ = self._run(["--files", "scripts/commands/scan.py", "--status", "modified"]) + self.assertEqual(rc, 0) + self.assertIn("PASS", out) + + def test_local_mode_new_command_without_doc_fails(self): + rc, out, _ = self._run(["--files", "scripts/commands/newcmd.py", "--status", "added"]) + self.assertEqual(rc, 1) + self.assertIn("FAIL", out) + self.assertIn("design doc", out.lower()) + + def test_local_mode_new_command_with_doc_passes(self): + rc, out, _ = self._run([ + "--files", "scripts/commands/newcmd.py", "docs/design/0005-newcmd.md", + "--status", "added", "added", + ]) + self.assertEqual(rc, 0) + self.assertIn("PASS", out) + + def test_local_mode_bypass_label_passes(self): + rc, out, _ = self._run([ + "--files", "scripts/commands/newcmd.py", + "--status", "added", + "--labels", BYPASS_LABEL, + ]) + self.assertEqual(rc, 0) + self.assertIn("PASS", out) + self.assertIn("bypassed", out.lower()) + + def test_local_mode_default_status_is_added(self): + """If --status is omitted, all files are assumed 'added'.""" + rc, out, _ = self._run(["--files", "scripts/commands/newcmd.py"]) + self.assertEqual(rc, 1) + self.assertIn("FAIL", out) + + def test_local_mode_mismatched_files_and_status_returns_2(self): + rc, _, err = self._run([ + "--files", "a.py", "b.py", + "--status", "added", # only 1 status for 2 files + ]) + self.assertEqual(rc, 2) + self.assertIn("same length", err) + + def test_ci_mode_without_env_vars_returns_2(self): + """CI mode (no --files) without env vars should return 2 with an error.""" + full_env = os.environ.copy() + for key in ("GITHUB_TOKEN", "GITHUB_REPOSITORY", "PR_NUMBER"): + full_env.pop(key, None) + proc = subprocess.run( + [sys.executable, os.path.join(SCRIPT_DIR, "check_design_doc.py")], + capture_output=True, text=True, env=full_env, timeout=30, + ) + self.assertEqual(proc.returncode, 2) + self.assertIn("CI mode requires", proc.stderr) + + +if __name__ == "__main__": + unittest.main()