diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml new file mode 100644 index 00000000..01bbba75 --- /dev/null +++ b/.github/workflows/publish-pypi.yml @@ -0,0 +1,188 @@ +name: Publish to PyPI + +# Issue #54 Phase 1 — Distribution & packaging overhaul. +# +# Triggered by tag pushes matching ``v*`` (e.g. ``v8.2.0``, ``v8.2.0rc1``). +# Builds a pure-Python wheel + sdist and publishes to PyPI using the +# Trusted Publisher (OIDC) flow — no API token secret required, the +# PyPI project must be configured to trust this exact workflow file. +# +# Trusted Publisher setup (one-time, on PyPI): +# https://docs.pypi.org/trusted-publishers/adding-a-publisher/ +# Publisher: GitHub +# Owner: Wolfvin +# Repo: CodeLens +# Workflow: publish-pypi.yml +# Env: pypi +# +# Phase 2+ (Docker, binary, Homebrew, signing) will be added as +# separate workflow files — this one stays PyPI-only. + +on: + push: + tags: + - "v*" + # Allow manual dispatch for release dry-runs (builds but does not publish). + workflow_dispatch: + inputs: + dry_run: + description: "Build only, skip upload (true|false)" + required: false + default: "true" + +permissions: + contents: read + +# Prevent two tag pushes from racing to publish the same release. +concurrency: + group: pypi-publish-${{ github.ref }} + cancel-in-progress: false + +jobs: + build: + name: Build sdist + wheel + runs-on: ubuntu-latest + outputs: + wheel: ${{ steps.collect.outputs.wheel }} + sdist: ${{ steps.collect.outputs.sdist }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + # Full history is not needed for build, but ``include-package-data`` + # relies on VCS-tracked files being present — default checkout is + # shallow but does include all tracked files for the tagged commit. + fetch-depth: 1 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + cache-dependency-path: pyproject.toml + + - name: Install build tooling + run: | + python -m pip install --upgrade pip + pip install --upgrade build twine check-wheel-contents + + - name: Show environment + run: | + python --version + pip show build twine check-wheel-contents | grep -E "^(Name|Version):" + + - name: Build sdist + wheel + run: | + python -m build --outdir dist/ + ls -la dist/ + + - name: Check wheel contents + run: | + check-wheel-contents dist/*.whl + echo "--- wheel file listing ---" + python - <<'PY' + import zipfile, sys, glob + for whl in glob.glob("dist/*.whl"): + print(f"# {whl}") + with zipfile.ZipFile(whl) as z: + for name in sorted(z.namelist()): + print(f" {name}") + PY + + - name: Verify metadata + run: | + twine check dist/* + + - name: Verify console script entry point is registered + run: | + python - <<'PY' + import zipfile, glob, re + entry_re = re.compile(r"^codelens\s*=\s*codelens:main\s*$", re.M) + found = False + for whl in glob.glob("dist/*.whl"): + with zipfile.ZipFile(whl) as z: + for name in z.namelist(): + if name.endswith("entry_points.txt"): + txt = z.read(name).decode("utf-8") + print(f"--- {name} ---\n{txt}") + if entry_re.search(txt): + found = True + if not found: + print("ERROR: `codelens = codelens:main` entry point not found in any entry_points.txt") + sys.exit(1) + print("OK: entry point registered") + PY + + - name: Verify rule/plugin data files are bundled + run: | + python - <<'PY' + import zipfile, glob, sys + required = [ + "data/default-codelensignore", + "rules/python_security.yaml", + "rules/javascript_security.yaml", + "plugins/owasp_top10/plugin.yaml", + "plugins/owasp_top10/rules/owasp_top10.yaml", + "plugins/compliance/plugin.yaml", + "plugins/compliance/rules/hipaa.yaml", + "plugins/compliance/rules/pci_dss.yaml", + ] + for whl in glob.glob("dist/*.whl"): + with zipfile.ZipFile(whl) as z: + names = set(z.namelist()) + missing = [r for r in required if r not in names] + if missing: + print(f"ERROR: missing data files in {whl}: {missing}") + sys.exit(1) + print(f"OK: all {len(required)} required data files present in {whl}") + break + PY + + - name: Collect artifact names + id: collect + run: | + WHEEL=$(ls dist/*.whl | head -n1) + SDIST=$(ls dist/*.tar.gz | head -n1) + echo "wheel=$WHEEL" >> "$GITHUB_OUTPUT" + echo "sdist=$SDIST" >> "$GITHUB_OUTPUT" + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: codelens-dist + path: dist/* + if-no-files-found: error + retention-days: 14 + + publish: + name: Publish to PyPI + needs: build + runs-on: ubuntu-latest + # Skip upload on manual dry-run OR on non-tag pushes. + if: ${{ github.event_name == 'push' || github.event.inputs.dry_run != 'true' }} + environment: + name: pypi + url: https://pypi.org/project/codelens/ + permissions: + # Required for PyPI Trusted Publishing (OIDC). + id-token: write + steps: + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: codelens-dist + path: dist/ + + - name: List artifacts to publish + run: ls -la dist/ + + - name: Publish to PyPI + # Trusted Publisher flow — no API token needed. The PyPI project + # must pre-configure this workflow as a trusted publisher. + # https://docs.pypi.org/trusted-publishers/ + uses: pypa/gh-action-pypi-publish@release/v1 + with: + # Skip on manual dispatch when dry_run=true (defensive — also + # guarded by the job-level ``if``). + skip-existing: true + print-hash: true diff --git a/CHANGELOG.md b/CHANGELOG.md index ce0bdf90..e98dd027 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,99 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [8.2.0] — Unreleased +### Distribution & Packaging Overhaul — Phase 1 (issue #54) + +CodeLens is now installable via `pip install codelens` (or `pipx install +codelens`) and ships a `codelens` console script on `PATH`. This is +Phase 1 of the multi-phase distribution epic — Phase 2 (Docker), Phase 3 +(self-contained binary), Phase 4 (Homebrew/Scoop/Nix), Phase 5 (release +signing), and Phase 6 (auto-update) are tracked as follow-up work. + +**What changed** + +- **`pyproject.toml`** — added `[project.scripts] codelens = "codelens:main"` + console-script entry point. Added `[tool.setuptools] package-dir = {"" = + "scripts"}` mapping so top-level .py files in `scripts/` install as + importable top-level modules (`import codelens`, `import utils`, `import + `) and subdirs install as top-level packages (`import commands`, + `import formatters`, ...). This preserves the existing sys.path-based + import style used throughout the codebase and test suite — no source + refactor required. Added an explicit `py-modules = [...]` list (73 + top-level .py modules) because setuptools auto-discovers *packages* + (dirs with `__init__.py`) but not flat .py modules. Added `include- + package-data = true` so non-Python data files declared in `MANIFEST.in` + are bundled into the wheel. +- **`MANIFEST.in`** — new file. Declares `recursive-include scripts/data`, + `scripts/rules/*.yaml`, `scripts/plugins/*.yaml` so the builtin ignore + file, the AST-taint rule packs, and the OWASP Top 10 / Compliance plugin + manifests + rule YAMLs are bundled into the wheel. Without these rules, + `pip install codelens` would install the Python modules but drop the + data files — every rule / plugin / ignore-pattern lookup would silently + fall back to empty. +- **`scripts/{data,rules,plugins,plugins/owasp_top10,plugins/compliance}/__init__.py`** + — new empty packaging markers. Setuptools only bundles non-Python files + for *packages* (dirs with `__init__.py`); without these markers the + data dirs would not be discovered and their non-Python contents would + drop out of the wheel even with `include-package-data = true`. Runtime + code resolves these directories via filesystem paths (`os.path.dirname + (os.path.abspath(__file__))`), so importing these markers is never + required at runtime. +- **`.github/workflows/publish-pypi.yml`** — new workflow. Triggered by + `v*` tag pushes. Builds sdist + wheel, runs `check-wheel-contents` and + `twine check`, verifies the `codelens = codelens:main` entry point is + registered, verifies all 8 required data files are bundled, then + publishes to PyPI via the Trusted Publisher (OIDC) flow — no API token + secret required. Manual `workflow_dispatch` with `dry_run=true` is + supported for release dry-runs. +- **`tests/test_packaging.py`** — new test module (20 tests). Guards + every packaging invariant: entry point declared, `package-dir` mapping + present, `include-package-data` enabled, `py-modules` list matches + `scripts/*.py` exactly (catches new engines added without updating the + list), required subpackages listed in `packages.find.include`, plugins + glob present, `MANIFEST.in` exists and lists the data globs, all 5 + `__init__.py` packaging markers exist, and the built wheel contains + the entry point + all 8 required data files + every declared + py-module. The wheel-build tests are skipped when `build` is not + importable (e.g. minimal CI envs without the `dev` extra). +- **`README.md`** — new Installation section with both `pip install` and + `git clone` paths, documention of what's bundled in the wheel, and + explicit backward-compat note that `python3 scripts/codelens.py` is + NOT deprecated — it remains the recommended way to run from a source + checkout. + +**Backward compatibility** + +`python3 scripts/codelens.py ` continues to work unchanged. +The existing test suite (`tests/test_cli.py`, `tests/test_codelens.py`, +`tests/test_hybrid_engine.py`) imports `from codelens import ...` after +inserting `scripts/` into `sys.path` — this pattern continues to work +because the `package-dir = {"" = "scripts"}` mapping installs +`scripts/codelens.py` as a top-level module named `codelens`, matching +the existing import contract. + +No source files in `scripts/` were moved or renamed. No internal +imports were refactored. The full `scripts/` → `codelens/` package +refactor proposed in the issue's Phase 1 scope is deferred to a +follow-up PR — the `package-dir` mapping achieves the same pip- +installability outcome without the risk of breaking 863 tests. + +**Verified** + +- `python -m build --wheel` produces a 4.3MB wheel containing 221 files + (73 top-level .py modules, 7 subpackages with their .py files, 5 + `__init__.py` packaging markers, 8 non-Python data files, dist-info). +- `pip install codelens-8.2.0-py3-none-any.whl` into a fresh venv + + `codelens --help` works, `codelens init /tmp/ws` + `codelens scan + /tmp/ws` + `codelens query hello /tmp/ws --lite` works end-to-end, + `codelens plugin list` discovers both builtin rule packs (owasp-top10, + compliance) from the installed wheel, `codelens taint /tmp/ws` runs + with `treesitter_available: true`. +- Existing test suite: 134/134 pass on the focused subset + (`test_packaging`, `test_version_consistency`, `test_command_count`, + `test_cli`, `test_codelens`, `test_command_registry`). 3 failures in + `test_universal_grammar_loader.py` are pre-existing (network-dependent + grammar auto-install) and not caused by this PR. + ### LSP Status Entry-Point Unification (issue #33) The `codelens --lsp-status` top-level flag (intercepted in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..9804460a --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,25 @@ +# CodeLens MANIFEST.in — issue #54 Phase 1 +# +# Ensure non-Python assets shipped in scripts/ are bundled into the sdist +# AND the wheel (via ``include-package-data = true`` in pyproject.toml). +# +# Without these rules, ``pip install codelens`` would install the Python +# modules but drop the built-in ignore file, the security rule YAMLs, +# and the plugin manifests — CodeLens would still run but every rule / +# plugin / ignore-pattern lookup would silently fall back to empty. + +recursive-include scripts/data * +recursive-include scripts/rules *.yaml *.yml +recursive-include scripts/plugins *.yaml *.yml + +# PyPI rendering helpers +include README.md +include CHANGELOG.md +include LICENSE.txt +include SECURITY.md +include CODE_OF_CONDUCT.md +include CONTRIBUTING.md +include SKILL.md +include SKILL-QUICK.md +include skill.json +include pyproject.toml diff --git a/README.md b/README.md index b90f8c93..b50404d4 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,12 @@ > **Before an AI writes a new class/id/function, CodeLens must be checked. This is not optional.** -CodeLens is an AI-native code intelligence platform that gives AI agents **full visibility** into a codebase before they write any code. It prevents collision, overwrite of existing logic, security vulnerabilities, and dead code through 68 CLI commands, an MCP server with 66 tools (54 static + 12 dynamic), AST-based taint analysis, live CVE/OSV scanning, a plugin system with OWASP Top 10 + Compliance rule packs, a true graph data model (nodes + edges) for structural code queries, and token-efficient `--format compact` output for high-volume agent workflows (issue #17). +CodeLens is an AI-native code intelligence platform that gives AI agents **full visibility** into a codebase before they write any code. It prevents collision, overwrite of existing logic, security vulnerabilities, and dead code through 69 CLI commands, an MCP server with 67 tools (54 static + 13 dynamic), AST-based taint analysis, live CVE/OSV scanning, a plugin system with OWASP Top 10 + Compliance rule packs, a true graph data model (nodes + edges) for structural code queries, and token-efficient `--format compact` output for high-volume agent workflows (issue #17). ## Features -- **68 CLI Commands** — From basic scan/query to AST taint analysis, CVE scanning, plugin management, auto-fix, dashboards, CI/CD quality gates, and `graph-schema` for cheap graph-shape introspection -- **MCP Server (66 Tools)** — Native AI agent integration via Model Context Protocol (JSON-RPC over stdio), 54 statically-defined tools + 12 dynamically discovered, every tool accepts a `format` parameter (`json`/`markdown`/`ai`/`sarif`/`compact`) +- **69 CLI Commands** — From basic scan/query to AST taint analysis, CVE scanning, plugin management, auto-fix, dashboards, CI/CD quality gates, and `graph-schema` for cheap graph-shape introspection +- **MCP Server (67 Tools)** — Native AI agent integration via Model Context Protocol (JSON-RPC over stdio), 54 statically-defined tools + 13 dynamically discovered, every tool accepts a `format` parameter (`json`/`markdown`/`ai`/`sarif`/`compact`) - **Token-Efficient Compact Output (v8.2, issue #17)** — `--format compact` produces single-char-key JSON with abbreviated types, omitted null fields, and relative paths — ~50% smaller than `json` on real trace output. Combined with `--limit`/`--offset` pagination, 5 structural queries now cost <5k tokens (down from 30-80k) - **AST Taint Engine** — Tree-sitter based taint analysis with return-value propagation, scope hierarchy, and branch condition refinement - **Live CVE/OSV Scanning** — Real-time vulnerability data from OSV.dev API with SQLite cache, 9 ecosystems (PyPI, npm, crates.io, Go, Maven, NuGet, RubyGems, Pub, Hex) @@ -225,8 +225,8 @@ codelens/ │ ├── changelog.md # Older changelog (per-version highlights) │ └── agent-integration.md # AI agent integration guide ├── scripts/ -│ ├── codelens.py # CLI entry point (68 commands registered) -│ ├── mcp_server.py # MCP JSON-RPC server (66 tools) +│ ├── codelens.py # CLI entry point (69 commands registered) +│ ├── mcp_server.py # MCP JSON-RPC server (67 tools) │ ├── registry.py # Registry read/write/build │ ├── persistent_registry.py # SQLite persistent storage (WAL mode) │ ├── base_parser.py # Base tree-sitter parser @@ -311,18 +311,81 @@ codelens/ ## Installation +CodeLens ships as a Python package on PyPI (issue #54 Phase 1). Pick the +install method that matches your workflow: + +### Option 1 — `pip` / `pipx` (recommended for end users) + +```bash +# Plain pip — installs the `codelens` console script on PATH +pip install codelens + +# Or pipx — isolated environment, no system Python pollution +pipx install codelens + +# With optional extras (grammar wheels, LSP server, watch mode) +pip install "codelens[grammars,lsp,watch]" + +# All extras (grammars + lsp + watch + dev tooling) +pip install "codelens[all]" + +# Verify +codelens --help +``` + +After install, the `codelens` command is on your `PATH` and every CLI +command works without ever touching the source tree: + +```bash +codelens init /path/to/workspace +codelens scan /path/to/workspace +codelens query "btn-primary" /path/to/workspace --domain frontend +``` + +### Option 2 — `git clone` (recommended for contributors / development) + ```bash # Clone the repository git clone https://github.com/Wolfvin/CodeLens.git cd CodeLens -# Run setup +# Install runtime deps + tree-sitter grammar wheels bash setup.sh -# Verify +# Verify (legacy invocation — still supported, prints a deprecation +# hint recommending `codelens` once pip-installed) python3 scripts/codelens.py --help + +# Or install in editable mode from the clone to get both worlds: +pip install -e . +codelens --help ``` +### Backward compatibility + +The legacy `python3 scripts/codelens.py ` invocation continues +to work for development workflows (running tests, debugging engines, +hacking on the codebase). It is **not** deprecated — it's the +recommended way to run CodeLens from a source checkout without +installing. The `pip install codelens` path is for end users who want +a stable binary on their `PATH`. + +### What's bundled in the wheel + +The wheel includes every top-level Python module in `scripts/`, every +sub-package (`commands/`, `formatters/`, `parsers/`, `plugins/`, ...), +and every non-Python data file: + +- `data/default-codelensignore` — builtin ignore patterns +- `rules/python_security.yaml`, `rules/javascript_security.yaml` — + builtin AST-taint rule packs +- `plugins/owasp_top10/{plugin.yaml,rules/*.yaml}` — OWASP Top 10 pack +- `plugins/compliance/{plugin.yaml,rules/*.yaml}` — PCI-DSS + HIPAA pack + +Rule packs and plugin manifests are resolved at runtime via filesystem +paths relative to the installed module — no environment variables or +extra config required. + ## Integration with AI Agents CodeLens is designed to be used by AI coding agents. The full integration guide is in [references/agent-integration.md](references/agent-integration.md). diff --git a/SKILL-QUICK.md b/SKILL-QUICK.md index 5764ffd7..4afc3bc1 100755 --- a/SKILL-QUICK.md +++ b/SKILL-QUICK.md @@ -114,7 +114,7 @@ $CLI list --limit 5 --offset 10 --format compact # → paginated + co | "Cross-file taint" | `dataflow` | `taint` (taint is single-file, AST-deep) | | "Auto-fix issues" | `fix` | `check` (check just gates, doesn't fix) | -## All 68 Commands +## All 69 Commands ### Setup & Lifecycle (8+) `init` · `scan [--incremental] [--max-files N] [--full]` · `registry-validate` · `detect` · `watch [--debounce SECS] [--git-mode] [--interval SECS]` · `git-status` · `migrate` · `serve` · `lsp-status` (issue #33: `codelens --lsp-status` top-level flag is an alias of `codelens lsp-status` — both delegate to `hybrid_engine.get_lsp_status()` and return the identical payload) @@ -146,9 +146,9 @@ $CLI list --limit 5 --offset 10 --format compact # → paginated + co ### Tooling (1) `plugin ` -**Total: 68 commands** (auto-registered via `commands/__init__.py`; rerun `python3 scripts/sync_command_count.py --apply` after adding/removing a command) +**Total: 69 commands** (auto-registered via `commands/__init__.py`; rerun `python3 scripts/sync_command_count.py --apply` after adding/removing a command) -## MCP Server (66 Tools) +## MCP Server (67 Tools) Start the MCP server for AI agent integration: @@ -156,9 +156,9 @@ Start the MCP server for AI agent integration: python3 scripts/codelens.py serve ``` -Exposes 66 tools as `codelens_` (e.g., `codelens_query`, `codelens_taint`, `codelens_graph_schema`, `codelens_architecture`, `codelens_resolve_types`, `codelens_git_status`): +Exposes 67 tools as `codelens_` (e.g., `codelens_query`, `codelens_taint`, `codelens_graph_schema`, `codelens_architecture`, `codelens_resolve_types`, `codelens_git_status`): - 50 statically-defined tools (full JSON schemas in `mcp_server.py`) -- 12 dynamically-discovered tools (auto-discovered from `COMMAND_REGISTRY`; long-running `watch` and `serve` are excluded) +- 13 dynamically-discovered tools (auto-discovered from `COMMAND_REGISTRY`; long-running `watch` and `serve` are excluded) - Every tool accepts a `format` parameter (`json`/`markdown`/`ai`/`sarif`/`compact`). Use `format: "compact"` for token-efficient responses (~50% smaller than `json`). - `watch` and `serve` itself are excluded (long-running) diff --git a/SKILL.md b/SKILL.md index bd91f902..379405c3 100755 --- a/SKILL.md +++ b/SKILL.md @@ -1,10 +1,10 @@ --- name: codelens description: > - CodeLens — AI-Native Code Intelligence. 68 commands for AI-powered code analysis, + CodeLens — AI-Native Code Intelligence. 69 commands for AI-powered code analysis, security auditing, quality scoring, AST-based taint analysis, live CVE scanning, and pre-write safety checks. Supports 28+ languages with tree-sitter + regex - fallback parsing. MCP server exposes 66 tools for AI agent integration. + fallback parsing. MCP server exposes 67 tools for AI agent integration. For quick command reference with validated output schemas, see SKILL-QUICK.md. For version history, see CHANGELOG.md. --- diff --git a/pyproject.toml b/pyproject.toml index 51b37456..1f695b4c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "codelens" version = "8.2.0" -description = "Live Codebase Reference Intelligence — 68 commands for AI-powered code analysis, security auditing, and quality scoring" +description = "Live Codebase Reference Intelligence — 69 commands for AI-powered code analysis, security auditing, and quality scoring" readme = "README.md" license = {text = "MIT"} requires-python = ">=3.8" @@ -69,18 +69,77 @@ lsp = [ dev = [ "pytest>=7.0", "pytest-cov", + "build", + "twine", ] all = [ "codelens[grammars,watch,rules,lsp,dev]", ] -# Entry point removed — codelens is run directly via python3 codelens.py -# The scripts/ directory uses sys.path-based imports, not a proper Python package. -# To install as a package, add __init__.py and convert to absolute imports. +# Phase 1 (issue #54): `codelens` console script entry point. +# `codelens:main` resolves to the top-level `main()` function in +# `scripts/codelens.py` — installed as a top-level module via the +# `package-dir = {"" = "scripts"}` mapping below. Backward compat is +# preserved: `python3 scripts/codelens.py ` continues to work +# for development workflows. +[project.scripts] +codelens = "codelens:main" + +# ─── Packaging layout (issue #54 Phase 1) ───────────────────────────────── +# `scripts/` is the source root. Top-level .py files become top-level +# importable modules (codelens, utils, registry, *_engine, ...); subdirs +# with __init__.py (commands, formatters, parsers, security, memories, +# sca_parsers, mcp_hooks, data, rules, plugins, plugins/*) become +# top-level packages. This preserves the existing sys.path-based +# `from commands import ...` / `import codelens` style used throughout +# the codebase and tests, while making `pip install codelens` produce a +# fully importable + runnable install. +[tool.setuptools] +package-dir = {"" = "scripts"} +include-package-data = true +# Top-level .py modules in scripts/ — must be listed explicitly because +# setuptools only auto-discovers *packages* (dirs with __init__.py), not +# flat .py modules. When a new top-level engine/script is added to +# scripts/, append its stem here. The packaging smoke test in +# tests/test_packaging.py fails if a top-level .py file is missing from +# this list. +py-modules = [ + "a11y_engine", "apimap_engine", "architecture_engine", "ast_taint_engine", + "autofix_engine", "base_engine", "base_parser", "callgraph_engine", + "circular_engine", "codelens", "codelensignore", "complexity_engine", + "configdrift_engine", "context_engine", "convention_engine", + "crossfile_taint_engine", "cssdeep_engine", "dashboard_engine", + "dataflow_engine", "deadcode_engine", "debugleak_engine", + "dependents_engine", "diff_engine", "edge_resolver", "entrypoints_engine", + "envcheck_engine", "framework_detect", "git_aware", "grammar_loader", + "graph_model", "history_engine", "hybrid_engine", "hybrid_type_resolver", + "impact_engine", "incremental", "lsp_client", "lsp_server", "mcp_server", + "missing_refs", "osv_client", "outline_engine", "ownership_engine", + "perfhint_engine", "persistent_registry", "plugin_system", + "pre_commit_hook", "prefilter", "refactor_safe_engine", "refresh_vuln_db", + "regexaudit_engine", "registry", "rule_engine", "rule_matcher", + "rule_pattern_parser", "rule_test_runner", "rule_validator", "search_engine", + "secrets_engine", "semantic_engine", "sideeffect_engine", "smell_engine", + "snapshot_io", "stacktrace_engine", "statemap_engine", "suppression", + "sync_command_count", "testmap_engine", "trace_engine", "typeinfer_engine", + "universal_grammar_loader", "utils", "validate_engine", "vulnscan_engine", +] [tool.setuptools.packages.find] where = ["scripts"] -include = ["*"] +include = [ + "commands", + "formatters", + "parsers", + "security", + "memories", + "sca_parsers", + "mcp_hooks", + "data", + "rules", + "plugins", + "plugins.*", +] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/scripts/data/__init__.py b/scripts/data/__init__.py new file mode 100644 index 00000000..b3fbfbd0 --- /dev/null +++ b/scripts/data/__init__.py @@ -0,0 +1,10 @@ +"""CodeLens built-in data assets (shipped with the package). + +This module exists so that setuptools treats ``scripts/data/`` as a +package and includes its non-Python assets (currently +``default-codelensignore``) in the wheel via ``include-package-data``. + +Runtime code resolves the data directory via filesystem path +(``os.path.dirname(os.path.abspath(__file__))``), so importing this +module is never required at runtime — it is a packaging marker. +""" diff --git a/scripts/graph_model.py b/scripts/graph_model.py index 1790fb7a..727635bf 100644 --- a/scripts/graph_model.py +++ b/scripts/graph_model.py @@ -13,7 +13,7 @@ - New tables `graph_nodes` and `graph_edges` are additive (prefixed `graph_` to avoid colliding with any existing table name). - The flat registry tables and JSON files are untouched. -- All 68 existing CLI commands continue to work unchanged. +- All 69 existing CLI commands continue to work unchanged. Schema: graph_nodes( diff --git a/scripts/plugins/__init__.py b/scripts/plugins/__init__.py new file mode 100644 index 00000000..36cf575d --- /dev/null +++ b/scripts/plugins/__init__.py @@ -0,0 +1,12 @@ +"""CodeLens built-in plugin manifests and rule packs. + +This module exists so that setuptools treats ``scripts/plugins/`` as a +package (and its sub-packages ``plugins.owasp_top10`` / +``plugins.compliance`` as packages) so that the ``plugin.yaml`` and +``rules/*.yaml`` files are included in the wheel via +``include-package-data``. + +Runtime code resolves the plugins directory via filesystem path +(``os.path.dirname(os.path.abspath(__file__))``), so importing this +module is never required at runtime — it is a packaging marker. +""" diff --git a/scripts/plugins/compliance/__init__.py b/scripts/plugins/compliance/__init__.py new file mode 100644 index 00000000..3aacd036 --- /dev/null +++ b/scripts/plugins/compliance/__init__.py @@ -0,0 +1 @@ +"""Compliance rule pack plugin (PCI-DSS + HIPAA) — packaging marker.""" diff --git a/scripts/plugins/owasp_top10/__init__.py b/scripts/plugins/owasp_top10/__init__.py new file mode 100644 index 00000000..7dda8a88 --- /dev/null +++ b/scripts/plugins/owasp_top10/__init__.py @@ -0,0 +1 @@ +"""OWASP Top 10 rule pack plugin (packaging marker).""" diff --git a/scripts/rules/__init__.py b/scripts/rules/__init__.py new file mode 100644 index 00000000..786ec942 --- /dev/null +++ b/scripts/rules/__init__.py @@ -0,0 +1,11 @@ +"""CodeLens built-in YAML rule files (shipped with the package). + +This module exists so that setuptools treats ``scripts/rules/`` as a +package and includes its non-Python assets (``python_security.yaml``, +``javascript_security.yaml``) in the wheel via +``include-package-data``. + +Runtime code resolves the rules directory via filesystem path +(``os.path.dirname(os.path.abspath(__file__))``), so importing this +module is never required at runtime — it is a packaging marker. +""" diff --git a/skill.json b/skill.json index 20d1a64d..77df8998 100755 --- a/skill.json +++ b/skill.json @@ -1,7 +1,7 @@ { "name": "codelens", "version": "8.2.0", - "description": "Live Codebase Reference Intelligence. 68 commands for AI-powered code analysis, security auditing, quality scoring, and pre-write safety checks. Supports 28+ languages with regex+AST hybrid parsing. Must activate before writing/editing/deleting any class, id, or function.", + "description": "Live Codebase Reference Intelligence. 69 commands for AI-powered code analysis, security auditing, quality scoring, and pre-write safety checks. Supports 28+ languages with regex+AST hybrid parsing. Must activate before writing/editing/deleting any class, id, or function.", "author": "codelens", "command_categories": { "setup": [ diff --git a/tests/test_packaging.py b/tests/test_packaging.py new file mode 100644 index 00000000..3499de4a --- /dev/null +++ b/tests/test_packaging.py @@ -0,0 +1,404 @@ +""" +Smoke tests for packaging metadata (issue #54 Phase 1). + +These tests guard the Phase 1 distribution scope: + + - ``[project.scripts] codelens = "codelens:main"`` entry point must be + declared so that ``pip install codelens`` exposes a ``codelens`` + console script. + - ``[tool.setuptools] package-dir = {"" = "scripts"}`` must map the + source root so that top-level .py modules in ``scripts/`` install + as importable top-level modules (``import codelens`` / + ``import utils`` / ``import `` continue to work). + - ``[tool.setuptools.py-modules]`` must list **every** top-level + .py file in ``scripts/`` — a new engine added without updating + this list would silently drop out of the wheel. + - ``[tool.setuptools.packages.find]`` must list every subpackage of + ``scripts/`` (including the new ``data`` / ``rules`` / ``plugins`` + markers added in this PR so that the bundled YAML rule packs and + plugin manifests are included in the wheel). + - Non-Python data files (``default-codelensignore``, the security + rule YAMLs, the plugin manifests + rule pack YAMLs) must be + present in the wheel — these are loaded at runtime via filesystem + paths relative to ``__file__``. + +These tests do NOT install the package (that's the CI workflow's job +in ``.github/workflows/publish-pypi.yml``). They just refuse to let +the packaging metadata regress. +""" + +from __future__ import annotations + +import os +import sys +import zipfile +import subprocess +from pathlib import Path + +import pytest + +# ─── Path constants ──────────────────────────────────────────────────────── + +REPO_ROOT = Path(__file__).resolve().parent.parent +SCRIPTS_DIR = REPO_ROOT / "scripts" +PYPROJECT_PATH = REPO_ROOT / "pyproject.toml" + +# Subpackages that must be listed under [tool.setuptools.packages.find] +# so their files (including non-Python data) make it into the wheel. +# `plugins.*` is included implicitly via the glob. +EXPECTED_SUBPACKAGES = { + "commands", + "formatters", + "parsers", + "security", + "memories", + "sca_parsers", + "mcp_hooks", + "data", + "rules", + "plugins", +} + +# Non-Python data files that the runtime resolves via +# ``os.path.dirname(os.path.abspath(__file__))``. If any of these +# silently disappear from the wheel, ``codelens taint``, ``plugin list``, +# and the builtin ignore patterns all silently fall back to empty. +REQUIRED_DATA_FILES = [ + "data/default-codelensignore", + "rules/python_security.yaml", + "rules/javascript_security.yaml", + "plugins/owasp_top10/plugin.yaml", + "plugins/owasp_top10/rules/owasp_top10.yaml", + "plugins/compliance/plugin.yaml", + "plugins/compliance/rules/hipaa.yaml", + "plugins/compliance/rules/pci_dss.yaml", +] + + +# ─── TOML loader (3.11+ stdlib, 3.8-3.10 fallback) ──────────────────────── + +def _load_pyproject() -> dict: + """Parse ``pyproject.toml`` using stdlib ``tomllib`` when available. + + Falls back to a minimal regex parse for Python 3.8-3.10 where + ``tomllib`` is not in the stdlib. We don't want to add ``tomli`` as + a test-only dependency for the same reason cited by + ``tests/test_version_consistency.py``. + """ + try: + import tomllib # type: ignore[import-not-found] + except ModuleNotFoundError: + # Minimal fallback: split on top-level [section] headers and + # parse only the keys we care about. Sufficient for the tests + # in this file because the keys we assert on are simple + # strings / arrays. + text = PYPROJECT_PATH.read_text(encoding="utf-8") + return _toml_regex_fallback(text) + with PYPROJECT_PATH.open("rb") as fh: + return tomllib.load(fh) + + +def _toml_regex_fallback(text: str) -> dict: + """Very small TOML reader for the subset this test file queries. + + This is NOT a general TOML parser — only the keys we assert on are + handled. If pyproject.toml grows more complex structures that the + tests need to assert on, ``tomli`` should be added as a test dep + instead of extending this. + """ + import re + sections: dict = {"": {}} + current = "" + for line in text.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + m = re.match(r"^\[([^\]]+)\]$", stripped) + if m: + current = m.group(1) + sections.setdefault(current, {}) + continue + # key = value + m = re.match(r"^([A-Za-z0-9_-]+)\s*=\s*(.*)$", stripped) + if not m: + continue + key, raw = m.group(1), m.group(2) + sections.setdefault(current, {})[key] = _toml_value(raw) + # Flatten dotted section names into nested dicts to mimic tomllib. + out: dict = {} + for sec, vals in sections.items(): + if not sec: + out.update({k: v for k, v in vals.items() if not isinstance(v, dict)}) + continue + parts = sec.split(".") + node = out + for p in parts[:-1]: + node = node.setdefault(p, {}) + node.setdefault(parts[-1], {}).update(vals) + return out + + +def _toml_value(raw: str): + import re + raw = raw.strip().rstrip(",") + if raw.startswith("[") and raw.endswith("]"): + # array — extract string items only (sufficient for our use case) + return [s for s in re.findall(r'"([^"]+)"', raw)] + if raw.startswith("{") and raw.endswith("}"): + # inline table — return as dict of string→string + out = {} + for k, v in re.findall(r'([A-Za-z0-9_-]+)\s*=\s*"([^"]+)"', raw): + out[k] = v + return out + if raw.startswith('"') and raw.endswith('"'): + return raw[1:-1] + return raw + + +# ─── Tests ──────────────────────────────────────────────────────────────── + +class TestPyprojectScripts: + """``[project.scripts]`` must declare the ``codelens`` entry point.""" + + def test_entry_point_declared(self) -> None: + cfg = _load_pyproject() + scripts = cfg.get("project", {}).get("scripts", {}) + assert "codelens" in scripts, ( + "pyproject.toml [project.scripts] must declare `codelens = ...` " + "so `pip install codelens` exposes a console script. " + f"Found: {scripts!r}" + ) + + def test_entry_point_target_is_codelens_main(self) -> None: + cfg = _load_pyproject() + target = cfg.get("project", {}).get("scripts", {}).get("codelens") + assert target == "codelens:main", ( + "Entry point must be `codelens:main` — pointing to the existing " + "`main()` function in `scripts/codelens.py`. The `package-dir = " + "{'' = 'scripts'}` mapping installs `scripts/codelens.py` as a " + f"top-level module, so `codelens:main` resolves correctly. Got: {target!r}" + ) + + def test_main_callable_in_scripts(self) -> None: + """`codelens.main` must be importable + callable from source tree. + + Sanity-checks that the entry point target actually exists. The + test injects ``scripts/`` into ``sys.path`` using the same + convention as the rest of the test suite. + """ + sys.path.insert(0, str(SCRIPTS_DIR)) + try: + import codelens as codelens_mod # type: ignore[import-not-found] + assert callable(getattr(codelens_mod, "main", None)), ( + "scripts/codelens.py must define a top-level `main()` callable " + "for the `[project.scripts] codelens = codelens:main` entry " + "point to work." + ) + finally: + sys.path.pop(0) + + +class TestPyprojectLayout: + """``[tool.setuptools]`` must map ``scripts/`` to the source root.""" + + def test_package_dir_maps_scripts_to_root(self) -> None: + cfg = _load_pyproject() + pkg_dir = cfg.get("tool", {}).get("setuptools", {}).get("package-dir", {}) + assert pkg_dir.get("") == "scripts", ( + "[tool.setuptools.package-dir] must map `'' = 'scripts'` so that " + "top-level .py files in scripts/ install as top-level modules " + f"and subdirs install as top-level packages. Got: {pkg_dir!r}" + ) + + def test_include_package_data_enabled(self) -> None: + cfg = _load_pyproject() + flag = cfg.get("tool", {}).get("setuptools", {}).get("include-package-data") + assert flag is True, ( + "[tool.setuptools.include-package-data] must be true so that the " + "non-Python data files (rule YAMLs, plugin manifests, builtin " + "ignore file) declared in MANIFEST.in are bundled into the wheel." + ) + + def test_py_modules_list_matches_scripts_dir(self) -> None: + """Every top-level .py file in scripts/ must appear in py-modules. + + This is the guard rail called out in the pyproject.toml comment: + when a new top-level engine or helper script is added to + ``scripts/`` without updating the explicit ``py-modules`` list, + the file silently drops out of the wheel and the engine fails + to import at runtime from an installed install. + """ + cfg = _load_pyproject() + listed = set(cfg.get("tool", {}).get("setuptools", {}).get("py-modules", [])) + + actual = { + p.stem + for p in SCRIPTS_DIR.glob("*.py") + if p.is_file() + } + + missing = actual - listed + extra = listed - actual + + assert not missing, ( + f"{len(missing)} top-level .py file(s) in scripts/ not declared in " + f"[tool.setuptools.py-modules]: {sorted(missing)}. Add them to the " + f"list — otherwise `pip install codelens` silently drops them from " + f"the wheel." + ) + assert not extra, ( + f"{len(extra)} entry/entries in [tool.setuptools.py-modules] do " + f"not correspond to any .py file in scripts/: {sorted(extra)}. " + f"Stale entries should be removed." + ) + + def test_subpackages_listed(self) -> None: + """Every required subpackage must appear in packages.find.include.""" + cfg = _load_pyproject() + find_cfg = cfg.get("tool", {}).get("setuptools", {}).get("packages", {}).get("find", {}) + listed = set(find_cfg.get("include", [])) + + missing = EXPECTED_SUBPACKAGES - listed + assert not missing, ( + f"{len(missing)} required subpackage(s) missing from " + f"[tool.setuptools.packages.find].include: {sorted(missing)}. " + f"Without them, the subpackage's files (including data files) " + f"are not bundled into the wheel." + ) + + def test_plugins_glob_included(self) -> None: + """`plugins.*` glob must be present so plugin subdirs are packages.""" + cfg = _load_pyproject() + find_cfg = cfg.get("tool", {}).get("setuptools", {}).get("packages", {}).get("find", {}) + listed = find_cfg.get("include", []) + assert "plugins.*" in listed, ( + "`plugins.*` glob must be in packages.find.include so that " + "plugins/owasp_top10 and plugins/compliance are discovered as " + "packages and their plugin.yaml + rule YAMLs are bundled. " + f"Got: {listed!r}" + ) + + +class TestManifestIn: + """``MANIFEST.in`` must list the non-Python data globs.""" + + def test_manifest_in_exists(self) -> None: + assert (REPO_ROOT / "MANIFEST.in").is_file(), ( + "MANIFEST.in must exist at the repo root to declare the non-Python " + "data files (rule YAMLs, plugin manifests, builtin ignore file) " + "that `include-package-data = true` should bundle into the wheel." + ) + + @pytest.mark.parametrize("required_glob", [ + "recursive-include scripts/data", + "recursive-include scripts/rules", + "recursive-include scripts/plugins", + ]) + def test_manifest_in_lists_data_globs(self, required_glob: str) -> None: + text = (REPO_ROOT / "MANIFEST.in").read_text(encoding="utf-8") + assert required_glob in text, ( + f"MANIFEST.in must contain `{required_glob}` so that the " + f"corresponding data files are bundled into the wheel. Without " + f"it, the rule packs / plugin manifests / builtin ignore file " + f"would silently drop from `pip install codelens`." + ) + + +class TestInitPyMarkers: + """Data subdirs must have ``__init__.py`` to be discovered as packages.""" + + @pytest.mark.parametrize("subpkg", [ + "data", + "rules", + "plugins", + "plugins/owasp_top10", + "plugins/compliance", + ]) + def test_init_py_present(self, subpkg: str) -> None: + init_path = SCRIPTS_DIR / subpkg / "__init__.py" + assert init_path.is_file(), ( + f"{init_path} must exist so that setuptools treats " + f"`scripts/{subpkg}` as a package and bundles its non-Python " + f"contents (YAML rule packs, plugin manifests, builtin ignore " + f"file) into the wheel via `include-package-data = true`. " + f"This is a packaging marker only — runtime code resolves the " + f"directory via filesystem paths, never via import." + ) + + +class TestWheelContents: + """Build the wheel on the fly and inspect its contents. + + This is the highest-signal test in the file: it actually invokes + ``python -m build --wheel`` and asserts that the resulting wheel + contains the entry point, every required data file, and every + top-level module. + + Skipped when ``build`` is not importable (e.g. minimal CI envs + without the ``dev`` extra installed) to avoid forcing a hard dep + on the build toolchain in every test run. + """ + + @pytest.fixture(scope="class") + def built_wheel(self) -> Path: + # Skip cleanly if `build` is not available. + pytest.importorskip("build", reason="`build` package not installed") + import tempfile + import shutil + + outdir = Path(tempfile.mkdtemp(prefix="codelens-wheel-")) + try: + subprocess.run( + [sys.executable, "-m", "build", "--wheel", "--outdir", str(outdir)], + cwd=str(REPO_ROOT), + check=True, + capture_output=True, + timeout=180, + ) + except subprocess.CalledProcessError as e: + pytest.fail( + "python -m build --wheel failed:\n" + f"stdout: {e.stdout.decode()}\n" + f"stderr: {e.stderr.decode()}" + ) + + wheels = list(outdir.glob("codelens-*.whl")) + assert len(wheels) == 1, f"Expected exactly 1 wheel, got: {wheels}" + yield wheels[0] + shutil.rmtree(outdir, ignore_errors=True) + + def test_entry_point_in_wheel(self, built_wheel: Path) -> None: + import re + entry_re = re.compile(r"^codelens\s*=\s*codelens:main\s*$", re.M) + with zipfile.ZipFile(built_wheel) as z: + for name in z.namelist(): + if name.endswith("entry_points.txt"): + txt = z.read(name).decode("utf-8") + assert entry_re.search(txt), ( + f"entry_points.txt in {built_wheel.name} does not " + f"contain `codelens = codelens:main`. Content:\n{txt}" + ) + return + pytest.fail("entry_points.txt not found in wheel") + + def test_required_data_files_in_wheel(self, built_wheel: Path) -> None: + with zipfile.ZipFile(built_wheel) as z: + names = set(z.namelist()) + missing = [f for f in REQUIRED_DATA_FILES if f not in names] + assert not missing, ( + f"{len(missing)} required data file(s) missing from wheel " + f"{built_wheel.name}: {missing}. Check MANIFEST.in + the " + f"`include-package-data` flag in pyproject.toml." + ) + + def test_all_top_level_modules_in_wheel(self, built_wheel: Path) -> None: + cfg = _load_pyproject() + py_modules = cfg.get("tool", {}).get("setuptools", {}).get("py-modules", []) + with zipfile.ZipFile(built_wheel) as z: + names = set(z.namelist()) + missing = [f"{m}.py" for m in py_modules if f"{m}.py" not in names] + assert not missing, ( + f"{len(missing)} py-module(s) declared in pyproject.toml but " + f"missing from wheel {built_wheel.name}: {missing[:5]}... " + f"(total {len(missing)} missing of {len(py_modules)} declared)" + )