feat(sca): 14 new lockfile parsers — pnpm, yarn, Pipfile, Gemfile, composer, NuGet, Dart, SwiftPM, Gradle, Maven, Mix, requirements.txt, pyproject.toml (closes #53)

scripts/sca_parsers/__init__.py

@@ -0,0 +1,180 @@
+"""
+CodeLens SCA (Software Composition Analysis) lockfile parsers.
+
+This package implements pluggable parsers for dependency lockfiles and
+manifests across many ecosystems. Each parser lives in its own module and
+exposes a ``parse(path: str) -> List[Dependency]`` function.
+
+Public API:
+    Dependency          — dataclass representing a single resolved dependency
+    parse_lockfile      — auto-detect format from filename and dispatch
+    PARSER_REGISTRY     — mapping of filename -> parser module
+
+Design rules (Issue #53):
+- Reimplemented from public format specs — no code copied from other
+  projects (LGPL/GPL incompatibility).
+- Graceful failure: a parser that errors must log a warning and return []
+  so the rest of the vuln-scan keeps working.
+- Pure parsing: no network calls, no subprocess, no VULN_DB lookups here.
+  vulnscan_engine.py is responsible for matching against VULN_DB.
+"""
+
+from __future__ import annotations
+
+import importlib
+import logging
+import os
+from dataclasses import dataclass, asdict
+from typing import Dict, List, Optional, Tuple
+
+logger = logging.getLogger("codelens.sca")
+
+
+# ─── Dependency dataclass ──────────────────────────────────────
+
+
+@dataclass
+class Dependency:
+    """A single resolved dependency extracted from a lockfile or manifest.
+
+    Attributes:
+        name:        Package name (e.g. "lodash", "serde", "rails").
+        version:     Resolved version string (e.g. "4.17.21"). For unpinned
+                     manifests the bare specifier is preserved; downstream
+                     code may treat "0.0.0"/"*" as "unknown".
+        ecosystem:   One of: pypi, npm, cargo, maven, gem, nuget, pub,
+                     hex, swiftpm, gradle, composer, mix, go.
+        source_file: Path of the lockfile/manifest this dep was parsed
+                     from (relative to workspace or absolute, as given).
+        transitivity: "direct" or "transitive". "direct" means the dep
+                     is declared at the top level of the manifest;
+                     "transitive" means it was pulled in as a sub-dep
+                     and only appears in the lockfile. Defaults to
+                     "direct" for manifests; lockfile parsers may set
+                     "transitive" when they can tell.
+    """
+
+    name: str
+    version: str
+    ecosystem: str
+    source_file: str
+    transitivity: str = "direct"
+
+    def to_dict(self) -> Dict[str, str]:
+        return asdict(self)
+
+
+# ─── Parser registry ───────────────────────────────────────────
+#
+# Mapping of canonical filename -> parser module name (relative to this
+# package). Multiple aliases can point to the same module.
+#
+# Keys are filenames (case-sensitive on POSIX). Lookup is also done by
+# basename so that nested lockfiles (e.g. packages/foo/Gemfile.lock) are
+# still recognised.
+
+PARSER_REGISTRY: Dict[str, str] = {
+    # JS / Node
+    "pnpm-lock.yaml": "pnpm_lock",
+    "yarn.lock": "yarn_lock",
+    # Python
+    "Pipfile.lock": "pipfile_lock",
+    "Pipfile": "pipfile",
+    "requirements.txt": "requirements_txt",
+    "pyproject.toml": "pyproject_toml",
+    # Ruby
+    "Gemfile.lock": "gemfile_lock",
+    # PHP
+    "composer.lock": "composer_lock",
+    # .NET
+    "packages.lock.json": "packages_lock",
+    # Dart
+    "pubspec.lock": "pubspec_lock",
+    # Swift
+    "Package.resolved": "package_resolved",
+    # Gradle / Maven
+    "gradle.lockfile": "gradle_lock",
+    "build.gradle": "gradle_lock",
+    "pom.xml": "pom_xml",
+    # Elixir
+    "mix.lock": "mix_lock",
+    # NOTE: poetry.lock is intentionally NOT registered here — it stays
+    # handled by the existing inline _parse_poetry_lock() in
+    # vulnscan_engine.py (it predates Issue #53 and is not in the
+    # "14 new parsers" list).
+}
+
+
+# Filename -> ecosystem, used by vulnscan_engine to know which VULN_DB
+# ecosystem key to use when matching. Only files handled by this
+# package are listed.
+ECOSYSTEM_BY_FILE: Dict[str, str] = {
+    "pnpm-lock.yaml": "npm",
+    "yarn.lock": "npm",
+    "Pipfile.lock": "pypi",
+    "Pipfile": "pypi",
+    "requirements.txt": "pypi",
+    "pyproject.toml": "pypi",
+    "Gemfile.lock": "gem",
+    "composer.lock": "composer",
+    "packages.lock.json": "nuget",
+    "pubspec.lock": "pub",
+    "Package.resolved": "swiftpm",
+    "gradle.lockfile": "gradle",
+    "build.gradle": "gradle",
+    "pom.xml": "maven",
+    "mix.lock": "hex",
+}
+
+
+def _load_parser(module_name: str):
+    """Import a parser module by name (cached by Python's import system)."""
+    full = f"sca_parsers.{module_name}"
+    try:
+        return importlib.import_module(full)
+    except Exception as exc:  # pragma: no cover - defensive
+        logger.warning("sca_parsers: failed to import %s: %s", full, exc)
+        return None
+
+
+def parse_lockfile(path: str) -> Tuple[List[Dependency], Optional[str]]:
+    """Auto-detect the format of ``path`` and parse it.
+
+    Returns:
+        (deps, ecosystem) where ``deps`` is a list of Dependency objects
+        (possibly empty) and ``ecosystem`` is the canonical ecosystem
+        string or None if the format is not recognised.
+
+        On parser error returns ([], None) and logs a warning — never
+        raises, so callers can keep scanning other files.
+    """
+    basename = os.path.basename(path)
+    module_name = PARSER_REGISTRY.get(basename)
+    if module_name is None:
+        return [], None
+
+    mod = _load_parser(module_name)
+    if mod is None:
+        return [], None
+
+    try:
+        deps = mod.parse(path)
+    except Exception as exc:
+        logger.warning("sca_parsers: %s.parse(%s) failed: %s", module_name, path, exc)
+        return [], None
+
+    # Defensive: parsers must return a list of Dependency objects.
+    if not isinstance(deps, list):
+        logger.warning("sca_parsers: %s.parse returned non-list: %r", module_name, type(deps))
+        return [], None
+
+    ecosystem = ECOSYSTEM_BY_FILE.get(basename)
+    return deps, ecosystem
+
+
+__all__ = [
+    "Dependency",
+    "PARSER_REGISTRY",
+    "ECOSYSTEM_BY_FILE",
+    "parse_lockfile",
+]

scripts/sca_parsers/composer_lock.py

@@ -0,0 +1,82 @@
+"""
+Parser for ``composer.lock`` (PHP / Composer).
+
+Format reference (public, reimplemented from spec):
+- Top-level JSON object with ``packages`` and ``packages-dev`` arrays.
+- Each entry has at minimum ``name`` (``vendor/package``) and ``version``
+  (e.g. ``"1.2.3"`` or ``"v1.2.3"`` or ``"dev-master"``).
+- The ``require`` and ``require-dev`` sub-fields inside each package
+  are NOT what we want — those are the deps of that sub-package. We
+  only take the top-level ``packages`` and ``packages-dev`` arrays.
+- Direct vs transitive: composer.lock does not natively distinguish,
+  but ``packages-dev`` are dev deps (treated as direct) and ``packages``
+  are runtime deps (also direct from the lockfile perspective).
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from typing import List
+
+from . import Dependency
+
+logger = logging.getLogger("codelens.sca.composer_lock")
+
+
+def _normalise_version(version: str) -> str:
+    """Strip the leading ``v`` and any ``dev-`` prefix from a Composer version."""
+    if not isinstance(version, str) or not version:
+        return ""
+    v = version.strip()
+    # Composer uses "v1.2.3" pretty-print form; strip leading v.
+    if v.startswith("v") and len(v) > 1 and v[1].isdigit():
+        v = v[1:]
+    return v
+
+
+def parse(path: str) -> List[Dependency]:
+    try:
+        with open(path, "r", encoding="utf-8", errors="replace") as f:
+            data = json.load(f)
+    except Exception as exc:
+        logger.warning("composer_lock: cannot read %s: %s", path, exc)
+        return []
+
+    if not isinstance(data, dict):
+        return []
+
+    deps: List[Dependency] = []
+    seen = set()
+
+    for section, dev_flag in (("packages", False), ("packages-dev", True)):
+        arr = data.get(section, []) or []
+        if not isinstance(arr, list):
+            continue
+        for entry in arr:
+            if not isinstance(entry, dict):
+                continue
+            name = entry.get("name", "")
+            version = _normalise_version(entry.get("version", ""))
+            if not name or not version:
+                continue
+            key = (name.lower(), version)
+            if key in seen:
+                continue
+            seen.add(key)
+            deps.append(
+                Dependency(
+                    name=name,
+                    version=version,
+                    ecosystem="composer",
+                    source_file=path,
+                    # Composer treats both sections as resolved runtime/dev deps.
+                    transitivity="direct",
+                )
+            )
+
+    return deps
+
+
+__all__ = ["parse"]

scripts/sca_parsers/gemfile_lock.py

@@ -0,0 +1,119 @@
+"""
+Parser for ``Gemfile.lock`` (Bundler / RubyGems).
+
+Format reference (public, reimplemented from spec):
+- Plain-text file generated by ``bundle lock``.
+- Top-level sections include ``GEM``, ``GIT``, ``PATH``, ``PLATFORMS``,
+  ``DEPENDENCIES``, ``RUBY VERSION`` and ``BUNDLED WITH``.
+- Inside ``GEM`` (and ``GIT`` / ``PATH``) blocks, lines are indented
+  with two spaces and look like ``  remote: https://rubygems.org/`` and
+  ``  specs:`` followed by lines like ``    rake (13.0.6)`` or
+  ``    nokogiri (1.13.10)`` (with optional platform suffix
+  ``-java`` / ``-x64-mingw32``).
+- ``DEPENDENCIES`` block lists top-level declared gems; we use it to
+  mark ``direct`` vs ``transitive``.
+
+Implementation notes:
+- We only extract the (name, version) pairs from the ``specs:`` blocks
+  inside ``GEM``, ``GIT`` and ``PATH`` sections.
+- A spec line looks like ``    name (version)`` or
+  ``    name (version platform)``.
+- Indentation is significant: spec lines have 4+ leading spaces, section
+  headers have 2.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import List, Set
+
+from . import Dependency
+
+logger = logging.getLogger("codelens.sca.gemfile_lock")
+
+
+# Match `    name (1.2.3)` or `    name (1.2.3-java)` or `    name (1.2.3 x64-mingw32)`
+_SPEC_RE = re.compile(r"^    ([A-Za-z0-9_.:-]+)\s+\(([^)]+)\)\s*$")
+# DEPENDENCIES block: `  name` or `  name (>= 1.0)` or `  name!`
+_DEP_LINE_RE = re.compile(r"^  ([A-Za-z0-9_.:-]+)(?:\s*\([^)]*\))?\s*!?\s*$")
+
+
+def parse(path: str) -> List[Dependency]:
+    try:
+        with open(path, "r", encoding="utf-8", errors="replace") as f:
+            content = f.read()
+    except Exception as exc:
+        logger.warning("gemfile_lock: cannot read %s: %s", path, exc)
+        return []
+
+    # Two-pass scan: in Gemfile.lock the DEPENDENCIES block appears
+    # AFTER the GEM/specs block, so we cannot know which packages are
+    # direct until we have read the whole file. Pass 1 collects direct
+    # names from the DEPENDENCIES block; pass 2 extracts (name, version)
+    # pairs from every specs: section and tags them.
+
+    direct_names: Set[str] = set()
+    in_deps_block = False
+    for raw_line in content.splitlines():
+        if not raw_line.startswith((" ", "\t")):
+            in_deps_block = raw_line.strip() == "DEPENDENCIES"
+            continue
+        if not in_deps_block:
+            continue
+        m = _DEP_LINE_RE.match(raw_line)
+        if m:
+            direct_names.add(m.group(1).lower())
+
+    deps: List[Dependency] = []
+    seen: Set[str] = set()
+    in_specs = False
+
+    for raw_line in content.splitlines():
+        # Top-level section headers toggle the specs state.
+        if not raw_line.startswith((" ", "\t")):
+            in_specs = False
+            continue
+
+        stripped = raw_line.strip()
+        if not stripped:
+            continue
+
+        if stripped == "specs:":
+            in_specs = True
+            continue
+        if stripped.startswith(("remote:", "revision:", "branch:", "tag:")):
+            continue
+
+        if not in_specs:
+            continue
+
+        m = _SPEC_RE.match(raw_line)
+        if not m:
+            continue
+        name = m.group(1)
+        # Version may include platform: "1.2.3", "1.2.3-java",
+        # "1.2.3 x64-mingw32" — take the bare version (everything up
+        # to first whitespace or dash-after-digits).
+        version_field = m.group(2).strip()
+        bare = re.split(r"[\s-]", version_field, maxsplit=1)[0]
+        if not name or not bare:
+            continue
+        key = (name.lower(), bare)
+        if key in seen:
+            continue
+        seen.add(key)
+        deps.append(
+            Dependency(
+                name=name,
+                version=bare,
+                ecosystem="gem",
+                source_file=path,
+                transitivity="direct" if name.lower() in direct_names else "transitive",
+            )
+        )
+
+    return deps
+
+
+__all__ = ["parse"]