feat(packaging): PyPI distribution Phase 1 — pip install codelens + console script + GH Actions publish (closes #54 phase-1)

.github/workflows/publish-pypi.yml

@@ -0,0 +1,188 @@
+name: Publish to PyPI
+
+# Issue #54 Phase 1 — Distribution & packaging overhaul.
+#
+# Triggered by tag pushes matching ``v*`` (e.g. ``v8.2.0``, ``v8.2.0rc1``).
+# Builds a pure-Python wheel + sdist and publishes to PyPI using the
+# Trusted Publisher (OIDC) flow — no API token secret required, the
+# PyPI project must be configured to trust this exact workflow file.
+#
+# Trusted Publisher setup (one-time, on PyPI):
+#   https://docs.pypi.org/trusted-publishers/adding-a-publisher/
+#     Publisher:  GitHub
+#     Owner:      Wolfvin
+#     Repo:       CodeLens
+#     Workflow:   publish-pypi.yml
+#     Env:        pypi
+#
+# Phase 2+ (Docker, binary, Homebrew, signing) will be added as
+# separate workflow files — this one stays PyPI-only.
+
+on:
+  push:
+    tags:
+      - "v*"
+  # Allow manual dispatch for release dry-runs (builds but does not publish).
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: "Build only, skip upload (true|false)"
+        required: false
+        default: "true"
+
+permissions:
+  contents: read
+
+# Prevent two tag pushes from racing to publish the same release.
+concurrency:
+  group: pypi-publish-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  build:
+    name: Build sdist + wheel
+    runs-on: ubuntu-latest
+    outputs:
+      wheel: ${{ steps.collect.outputs.wheel }}
+      sdist: ${{ steps.collect.outputs.sdist }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          # Full history is not needed for build, but ``include-package-data``
+          # relies on VCS-tracked files being present — default checkout is
+          # shallow but does include all tracked files for the tagged commit.
+          fetch-depth: 1
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install build tooling
+        run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade build twine check-wheel-contents
+
+      - name: Show environment
+        run: |
+          python --version
+          pip show build twine check-wheel-contents | grep -E "^(Name|Version):"
+
+      - name: Build sdist + wheel
+        run: |
+          python -m build --outdir dist/
+          ls -la dist/
+
+      - name: Check wheel contents
+        run: |
+          check-wheel-contents dist/*.whl
+          echo "--- wheel file listing ---"
+          python - <<'PY'
+          import zipfile, sys, glob
+          for whl in glob.glob("dist/*.whl"):
+              print(f"# {whl}")
+              with zipfile.ZipFile(whl) as z:
+                  for name in sorted(z.namelist()):
+                      print(f"  {name}")
+          PY
+
+      - name: Verify metadata
+        run: |
+          twine check dist/*
+
+      - name: Verify console script entry point is registered
+        run: |
+          python - <<'PY'
+          import zipfile, glob, re
+          entry_re = re.compile(r"^codelens\s*=\s*codelens:main\s*$", re.M)
+          found = False
+          for whl in glob.glob("dist/*.whl"):
+              with zipfile.ZipFile(whl) as z:
+                  for name in z.namelist():
+                      if name.endswith("entry_points.txt"):
+                          txt = z.read(name).decode("utf-8")
+                          print(f"--- {name} ---\n{txt}")
+                          if entry_re.search(txt):
+                              found = True
+          if not found:
+              print("ERROR: `codelens = codelens:main` entry point not found in any entry_points.txt")
+              sys.exit(1)
+          print("OK: entry point registered")
+          PY
+
+      - name: Verify rule/plugin data files are bundled
+        run: |
+          python - <<'PY'
+          import zipfile, glob, sys
+          required = [
+              "data/default-codelensignore",
+              "rules/python_security.yaml",
+              "rules/javascript_security.yaml",
+              "plugins/owasp_top10/plugin.yaml",
+              "plugins/owasp_top10/rules/owasp_top10.yaml",
+              "plugins/compliance/plugin.yaml",
+              "plugins/compliance/rules/hipaa.yaml",
+              "plugins/compliance/rules/pci_dss.yaml",
+          ]
+          for whl in glob.glob("dist/*.whl"):
+              with zipfile.ZipFile(whl) as z:
+                  names = set(z.namelist())
+              missing = [r for r in required if r not in names]
+              if missing:
+                  print(f"ERROR: missing data files in {whl}: {missing}")
+                  sys.exit(1)
+              print(f"OK: all {len(required)} required data files present in {whl}")
+              break
+          PY
+
+      - name: Collect artifact names
+        id: collect
+        run: |
+          WHEEL=$(ls dist/*.whl | head -n1)
+          SDIST=$(ls dist/*.tar.gz | head -n1)
+          echo "wheel=$WHEEL" >> "$GITHUB_OUTPUT"
+          echo "sdist=$SDIST" >> "$GITHUB_OUTPUT"
+
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: codelens-dist
+          path: dist/*
+          if-no-files-found: error
+          retention-days: 14
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    # Skip upload on manual dry-run OR on non-tag pushes.
+    if: ${{ github.event_name == 'push' || github.event.inputs.dry_run != 'true' }}
+    environment:
+      name: pypi
+      url: https://pypi.org/project/codelens/
+    permissions:
+      # Required for PyPI Trusted Publishing (OIDC).
+      id-token: write
+    steps:
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: codelens-dist
+          path: dist/
+
+      - name: List artifacts to publish
+        run: ls -la dist/
+
+      - name: Publish to PyPI
+        # Trusted Publisher flow — no API token needed. The PyPI project
+        # must pre-configure this workflow as a trusted publisher.
+        # https://docs.pypi.org/trusted-publishers/
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          # Skip on manual dispatch when dry_run=true (defensive — also
+          # guarded by the job-level ``if``).
+          skip-existing: true
+          print-hash: true

CHANGELOG.md

@@ -7,6 +7,99 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [8.2.0] — Unreleased
 
+### Distribution & Packaging Overhaul — Phase 1 (issue #54)
+
+CodeLens is now installable via `pip install codelens` (or `pipx install
+codelens`) and ships a `codelens` console script on `PATH`. This is
+Phase 1 of the multi-phase distribution epic — Phase 2 (Docker), Phase 3
+(self-contained binary), Phase 4 (Homebrew/Scoop/Nix), Phase 5 (release
+signing), and Phase 6 (auto-update) are tracked as follow-up work.
+
+**What changed**
+
+- **`pyproject.toml`** — added `[project.scripts] codelens = "codelens:main"`
+  console-script entry point. Added `[tool.setuptools] package-dir = {"" =
+  "scripts"}` mapping so top-level .py files in `scripts/` install as
+  importable top-level modules (`import codelens`, `import utils`, `import
+  <engine>`) and subdirs install as top-level packages (`import commands`,
+  `import formatters`, ...). This preserves the existing sys.path-based
+  import style used throughout the codebase and test suite — no source
+  refactor required. Added an explicit `py-modules = [...]` list (73
+  top-level .py modules) because setuptools auto-discovers *packages*
+  (dirs with `__init__.py`) but not flat .py modules. Added `include-
+  package-data = true` so non-Python data files declared in `MANIFEST.in`
+  are bundled into the wheel.
+- **`MANIFEST.in`** — new file. Declares `recursive-include scripts/data`,
+  `scripts/rules/*.yaml`, `scripts/plugins/*.yaml` so the builtin ignore
+  file, the AST-taint rule packs, and the OWASP Top 10 / Compliance plugin
+  manifests + rule YAMLs are bundled into the wheel. Without these rules,
+  `pip install codelens` would install the Python modules but drop the
+  data files — every rule / plugin / ignore-pattern lookup would silently
+  fall back to empty.
+- **`scripts/{data,rules,plugins,plugins/owasp_top10,plugins/compliance}/__init__.py`**
+  — new empty packaging markers. Setuptools only bundles non-Python files
+  for *packages* (dirs with `__init__.py`); without these markers the
+  data dirs would not be discovered and their non-Python contents would
+  drop out of the wheel even with `include-package-data = true`. Runtime
+  code resolves these directories via filesystem paths (`os.path.dirname
+  (os.path.abspath(__file__))`), so importing these markers is never
+  required at runtime.
+- **`.github/workflows/publish-pypi.yml`** — new workflow. Triggered by
+  `v*` tag pushes. Builds sdist + wheel, runs `check-wheel-contents` and
+  `twine check`, verifies the `codelens = codelens:main` entry point is
+  registered, verifies all 8 required data files are bundled, then
+  publishes to PyPI via the Trusted Publisher (OIDC) flow — no API token
+  secret required. Manual `workflow_dispatch` with `dry_run=true` is
+  supported for release dry-runs.
+- **`tests/test_packaging.py`** — new test module (20 tests). Guards
+  every packaging invariant: entry point declared, `package-dir` mapping
+  present, `include-package-data` enabled, `py-modules` list matches
+  `scripts/*.py` exactly (catches new engines added without updating the
+  list), required subpackages listed in `packages.find.include`, plugins
+  glob present, `MANIFEST.in` exists and lists the data globs, all 5
+  `__init__.py` packaging markers exist, and the built wheel contains
+  the entry point + all 8 required data files + every declared
+  py-module. The wheel-build tests are skipped when `build` is not
+  importable (e.g. minimal CI envs without the `dev` extra).
+- **`README.md`** — new Installation section with both `pip install` and
+  `git clone` paths, documention of what's bundled in the wheel, and
+  explicit backward-compat note that `python3 scripts/codelens.py` is
+  NOT deprecated — it remains the recommended way to run from a source
+  checkout.
+
+**Backward compatibility**
+
+`python3 scripts/codelens.py <command>` continues to work unchanged.
+The existing test suite (`tests/test_cli.py`, `tests/test_codelens.py`,
+`tests/test_hybrid_engine.py`) imports `from codelens import ...` after
+inserting `scripts/` into `sys.path` — this pattern continues to work
+because the `package-dir = {"" = "scripts"}` mapping installs
+`scripts/codelens.py` as a top-level module named `codelens`, matching
+the existing import contract.
+
+No source files in `scripts/` were moved or renamed. No internal
+imports were refactored. The full `scripts/` → `codelens/` package
+refactor proposed in the issue's Phase 1 scope is deferred to a
+follow-up PR — the `package-dir` mapping achieves the same pip-
+installability outcome without the risk of breaking 863 tests.
+
+**Verified**
+
+- `python -m build --wheel` produces a 4.3MB wheel containing 221 files
+  (73 top-level .py modules, 7 subpackages with their .py files, 5
+  `__init__.py` packaging markers, 8 non-Python data files, dist-info).
+- `pip install codelens-8.2.0-py3-none-any.whl` into a fresh venv +
+  `codelens --help` works, `codelens init /tmp/ws` + `codelens scan
+  /tmp/ws` + `codelens query hello /tmp/ws --lite` works end-to-end,
+  `codelens plugin list` discovers both builtin rule packs (owasp-top10,
+  compliance) from the installed wheel, `codelens taint /tmp/ws` runs
+  with `treesitter_available: true`.
+- Existing test suite: 134/134 pass on the focused subset
+  (`test_packaging`, `test_version_consistency`, `test_command_count`,
+  `test_cli`, `test_codelens`, `test_command_registry`). 3 failures in
+  `test_universal_grammar_loader.py` are pre-existing (network-dependent
+  grammar auto-install) and not caused by this PR.
+
 ### LSP Status Entry-Point Unification (issue #33)
 
 The `codelens --lsp-status` top-level flag (intercepted in

MANIFEST.in

@@ -0,0 +1,25 @@
+# CodeLens MANIFEST.in — issue #54 Phase 1
+#
+# Ensure non-Python assets shipped in scripts/ are bundled into the sdist
+# AND the wheel (via ``include-package-data = true`` in pyproject.toml).
+#
+# Without these rules, ``pip install codelens`` would install the Python
+# modules but drop the built-in ignore file, the security rule YAMLs,
+# and the plugin manifests — CodeLens would still run but every rule /
+# plugin / ignore-pattern lookup would silently fall back to empty.
+
+recursive-include scripts/data *
+recursive-include scripts/rules *.yaml *.yml
+recursive-include scripts/plugins *.yaml *.yml
+
+# PyPI rendering helpers
+include README.md
+include CHANGELOG.md
+include LICENSE.txt
+include SECURITY.md
+include CODE_OF_CONDUCT.md
+include CONTRIBUTING.md
+include SKILL.md
+include SKILL-QUICK.md
+include skill.json
+include pyproject.toml