diff --git a/scripts/sca_parsers/__init__.py b/scripts/sca_parsers/__init__.py new file mode 100644 index 00000000..038153bf --- /dev/null +++ b/scripts/sca_parsers/__init__.py @@ -0,0 +1,180 @@ +""" +CodeLens SCA (Software Composition Analysis) lockfile parsers. + +This package implements pluggable parsers for dependency lockfiles and +manifests across many ecosystems. Each parser lives in its own module and +exposes a ``parse(path: str) -> List[Dependency]`` function. + +Public API: + Dependency — dataclass representing a single resolved dependency + parse_lockfile — auto-detect format from filename and dispatch + PARSER_REGISTRY — mapping of filename -> parser module + +Design rules (Issue #53): +- Reimplemented from public format specs — no code copied from other + projects (LGPL/GPL incompatibility). +- Graceful failure: a parser that errors must log a warning and return [] + so the rest of the vuln-scan keeps working. +- Pure parsing: no network calls, no subprocess, no VULN_DB lookups here. + vulnscan_engine.py is responsible for matching against VULN_DB. +""" + +from __future__ import annotations + +import importlib +import logging +import os +from dataclasses import dataclass, asdict +from typing import Dict, List, Optional, Tuple + +logger = logging.getLogger("codelens.sca") + + +# ─── Dependency dataclass ────────────────────────────────────── + + +@dataclass +class Dependency: + """A single resolved dependency extracted from a lockfile or manifest. + + Attributes: + name: Package name (e.g. "lodash", "serde", "rails"). + version: Resolved version string (e.g. "4.17.21"). For unpinned + manifests the bare specifier is preserved; downstream + code may treat "0.0.0"/"*" as "unknown". + ecosystem: One of: pypi, npm, cargo, maven, gem, nuget, pub, + hex, swiftpm, gradle, composer, mix, go. + source_file: Path of the lockfile/manifest this dep was parsed + from (relative to workspace or absolute, as given). + transitivity: "direct" or "transitive". "direct" means the dep + is declared at the top level of the manifest; + "transitive" means it was pulled in as a sub-dep + and only appears in the lockfile. Defaults to + "direct" for manifests; lockfile parsers may set + "transitive" when they can tell. + """ + + name: str + version: str + ecosystem: str + source_file: str + transitivity: str = "direct" + + def to_dict(self) -> Dict[str, str]: + return asdict(self) + + +# ─── Parser registry ─────────────────────────────────────────── +# +# Mapping of canonical filename -> parser module name (relative to this +# package). Multiple aliases can point to the same module. +# +# Keys are filenames (case-sensitive on POSIX). Lookup is also done by +# basename so that nested lockfiles (e.g. packages/foo/Gemfile.lock) are +# still recognised. + +PARSER_REGISTRY: Dict[str, str] = { + # JS / Node + "pnpm-lock.yaml": "pnpm_lock", + "yarn.lock": "yarn_lock", + # Python + "Pipfile.lock": "pipfile_lock", + "Pipfile": "pipfile", + "requirements.txt": "requirements_txt", + "pyproject.toml": "pyproject_toml", + # Ruby + "Gemfile.lock": "gemfile_lock", + # PHP + "composer.lock": "composer_lock", + # .NET + "packages.lock.json": "packages_lock", + # Dart + "pubspec.lock": "pubspec_lock", + # Swift + "Package.resolved": "package_resolved", + # Gradle / Maven + "gradle.lockfile": "gradle_lock", + "build.gradle": "gradle_lock", + "pom.xml": "pom_xml", + # Elixir + "mix.lock": "mix_lock", + # NOTE: poetry.lock is intentionally NOT registered here — it stays + # handled by the existing inline _parse_poetry_lock() in + # vulnscan_engine.py (it predates Issue #53 and is not in the + # "14 new parsers" list). +} + + +# Filename -> ecosystem, used by vulnscan_engine to know which VULN_DB +# ecosystem key to use when matching. Only files handled by this +# package are listed. +ECOSYSTEM_BY_FILE: Dict[str, str] = { + "pnpm-lock.yaml": "npm", + "yarn.lock": "npm", + "Pipfile.lock": "pypi", + "Pipfile": "pypi", + "requirements.txt": "pypi", + "pyproject.toml": "pypi", + "Gemfile.lock": "gem", + "composer.lock": "composer", + "packages.lock.json": "nuget", + "pubspec.lock": "pub", + "Package.resolved": "swiftpm", + "gradle.lockfile": "gradle", + "build.gradle": "gradle", + "pom.xml": "maven", + "mix.lock": "hex", +} + + +def _load_parser(module_name: str): + """Import a parser module by name (cached by Python's import system).""" + full = f"sca_parsers.{module_name}" + try: + return importlib.import_module(full) + except Exception as exc: # pragma: no cover - defensive + logger.warning("sca_parsers: failed to import %s: %s", full, exc) + return None + + +def parse_lockfile(path: str) -> Tuple[List[Dependency], Optional[str]]: + """Auto-detect the format of ``path`` and parse it. + + Returns: + (deps, ecosystem) where ``deps`` is a list of Dependency objects + (possibly empty) and ``ecosystem`` is the canonical ecosystem + string or None if the format is not recognised. + + On parser error returns ([], None) and logs a warning — never + raises, so callers can keep scanning other files. + """ + basename = os.path.basename(path) + module_name = PARSER_REGISTRY.get(basename) + if module_name is None: + return [], None + + mod = _load_parser(module_name) + if mod is None: + return [], None + + try: + deps = mod.parse(path) + except Exception as exc: + logger.warning("sca_parsers: %s.parse(%s) failed: %s", module_name, path, exc) + return [], None + + # Defensive: parsers must return a list of Dependency objects. + if not isinstance(deps, list): + logger.warning("sca_parsers: %s.parse returned non-list: %r", module_name, type(deps)) + return [], None + + ecosystem = ECOSYSTEM_BY_FILE.get(basename) + return deps, ecosystem + + +__all__ = [ + "Dependency", + "PARSER_REGISTRY", + "ECOSYSTEM_BY_FILE", + "parse_lockfile", +] diff --git a/scripts/sca_parsers/composer_lock.py b/scripts/sca_parsers/composer_lock.py new file mode 100644 index 00000000..ac7bbcea --- /dev/null +++ b/scripts/sca_parsers/composer_lock.py @@ -0,0 +1,82 @@ +""" +Parser for ``composer.lock`` (PHP / Composer). + +Format reference (public, reimplemented from spec): +- Top-level JSON object with ``packages`` and ``packages-dev`` arrays. +- Each entry has at minimum ``name`` (``vendor/package``) and ``version`` + (e.g. ``"1.2.3"`` or ``"v1.2.3"`` or ``"dev-master"``). +- The ``require`` and ``require-dev`` sub-fields inside each package + are NOT what we want — those are the deps of that sub-package. We + only take the top-level ``packages`` and ``packages-dev`` arrays. +- Direct vs transitive: composer.lock does not natively distinguish, + but ``packages-dev`` are dev deps (treated as direct) and ``packages`` + are runtime deps (also direct from the lockfile perspective). +""" + +from __future__ import annotations + +import json +import logging +import re +from typing import List + +from . import Dependency + +logger = logging.getLogger("codelens.sca.composer_lock") + + +def _normalise_version(version: str) -> str: + """Strip the leading ``v`` and any ``dev-`` prefix from a Composer version.""" + if not isinstance(version, str) or not version: + return "" + v = version.strip() + # Composer uses "v1.2.3" pretty-print form; strip leading v. + if v.startswith("v") and len(v) > 1 and v[1].isdigit(): + v = v[1:] + return v + + +def parse(path: str) -> List[Dependency]: + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + data = json.load(f) + except Exception as exc: + logger.warning("composer_lock: cannot read %s: %s", path, exc) + return [] + + if not isinstance(data, dict): + return [] + + deps: List[Dependency] = [] + seen = set() + + for section, dev_flag in (("packages", False), ("packages-dev", True)): + arr = data.get(section, []) or [] + if not isinstance(arr, list): + continue + for entry in arr: + if not isinstance(entry, dict): + continue + name = entry.get("name", "") + version = _normalise_version(entry.get("version", "")) + if not name or not version: + continue + key = (name.lower(), version) + if key in seen: + continue + seen.add(key) + deps.append( + Dependency( + name=name, + version=version, + ecosystem="composer", + source_file=path, + # Composer treats both sections as resolved runtime/dev deps. + transitivity="direct", + ) + ) + + return deps + + +__all__ = ["parse"] diff --git a/scripts/sca_parsers/gemfile_lock.py b/scripts/sca_parsers/gemfile_lock.py new file mode 100644 index 00000000..138a089a --- /dev/null +++ b/scripts/sca_parsers/gemfile_lock.py @@ -0,0 +1,119 @@ +""" +Parser for ``Gemfile.lock`` (Bundler / RubyGems). + +Format reference (public, reimplemented from spec): +- Plain-text file generated by ``bundle lock``. +- Top-level sections include ``GEM``, ``GIT``, ``PATH``, ``PLATFORMS``, + ``DEPENDENCIES``, ``RUBY VERSION`` and ``BUNDLED WITH``. +- Inside ``GEM`` (and ``GIT`` / ``PATH``) blocks, lines are indented + with two spaces and look like `` remote: https://rubygems.org/`` and + `` specs:`` followed by lines like `` rake (13.0.6)`` or + `` nokogiri (1.13.10)`` (with optional platform suffix + ``-java`` / ``-x64-mingw32``). +- ``DEPENDENCIES`` block lists top-level declared gems; we use it to + mark ``direct`` vs ``transitive``. + +Implementation notes: +- We only extract the (name, version) pairs from the ``specs:`` blocks + inside ``GEM``, ``GIT`` and ``PATH`` sections. +- A spec line looks like `` name (version)`` or + `` name (version platform)``. +- Indentation is significant: spec lines have 4+ leading spaces, section + headers have 2. +""" + +from __future__ import annotations + +import logging +import re +from typing import List, Set + +from . import Dependency + +logger = logging.getLogger("codelens.sca.gemfile_lock") + + +# Match ` name (1.2.3)` or ` name (1.2.3-java)` or ` name (1.2.3 x64-mingw32)` +_SPEC_RE = re.compile(r"^ ([A-Za-z0-9_.:-]+)\s+\(([^)]+)\)\s*$") +# DEPENDENCIES block: ` name` or ` name (>= 1.0)` or ` name!` +_DEP_LINE_RE = re.compile(r"^ ([A-Za-z0-9_.:-]+)(?:\s*\([^)]*\))?\s*!?\s*$") + + +def parse(path: str) -> List[Dependency]: + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + content = f.read() + except Exception as exc: + logger.warning("gemfile_lock: cannot read %s: %s", path, exc) + return [] + + # Two-pass scan: in Gemfile.lock the DEPENDENCIES block appears + # AFTER the GEM/specs block, so we cannot know which packages are + # direct until we have read the whole file. Pass 1 collects direct + # names from the DEPENDENCIES block; pass 2 extracts (name, version) + # pairs from every specs: section and tags them. + + direct_names: Set[str] = set() + in_deps_block = False + for raw_line in content.splitlines(): + if not raw_line.startswith((" ", "\t")): + in_deps_block = raw_line.strip() == "DEPENDENCIES" + continue + if not in_deps_block: + continue + m = _DEP_LINE_RE.match(raw_line) + if m: + direct_names.add(m.group(1).lower()) + + deps: List[Dependency] = [] + seen: Set[str] = set() + in_specs = False + + for raw_line in content.splitlines(): + # Top-level section headers toggle the specs state. + if not raw_line.startswith((" ", "\t")): + in_specs = False + continue + + stripped = raw_line.strip() + if not stripped: + continue + + if stripped == "specs:": + in_specs = True + continue + if stripped.startswith(("remote:", "revision:", "branch:", "tag:")): + continue + + if not in_specs: + continue + + m = _SPEC_RE.match(raw_line) + if not m: + continue + name = m.group(1) + # Version may include platform: "1.2.3", "1.2.3-java", + # "1.2.3 x64-mingw32" — take the bare version (everything up + # to first whitespace or dash-after-digits). + version_field = m.group(2).strip() + bare = re.split(r"[\s-]", version_field, maxsplit=1)[0] + if not name or not bare: + continue + key = (name.lower(), bare) + if key in seen: + continue + seen.add(key) + deps.append( + Dependency( + name=name, + version=bare, + ecosystem="gem", + source_file=path, + transitivity="direct" if name.lower() in direct_names else "transitive", + ) + ) + + return deps + + +__all__ = ["parse"] diff --git a/scripts/sca_parsers/gradle_lock.py b/scripts/sca_parsers/gradle_lock.py new file mode 100644 index 00000000..d4b14c7b --- /dev/null +++ b/scripts/sca_parsers/gradle_lock.py @@ -0,0 +1,140 @@ +""" +Parser for Gradle lockfiles (``gradle.lockfile``) and ``build.gradle`` +(declared deps). + +Format reference (public, reimplemented from spec): + +``gradle.lockfile`` (Gradle 6+ dependency locking): +- Plain-text, line-per-dependency, format: + ``:::=`` + e.g. ``com.google.code.gson:gson:2.8.9:compileClasspath=2.8.9`` +- Empty lines and lines starting with ``#`` are comments. There is a + header block starting with ``# This is a Gradle dependency locking + file`` that we ignore. +- We split on the LAST ``=`` to separate the config key from the + unused marker, then split the key on ``:``. + +``build.gradle`` (Groovy DSL, declared deps): +- ``dependencies { ... }`` block with lines like + ``implementation 'group:artifact:version'`` or + ``testImplementation "group:artifact:version"``. +- We extract (group, artifact, version) and emit ``group:artifact`` as + the package name. +""" + +from __future__ import annotations + +import logging +import os +import re +from typing import List + +from . import Dependency + +logger = logging.getLogger("codelens.sca.gradle_lock") + + +# Match: ::[:]= +_LOCKFILE_LINE_RE = re.compile( + r"^([A-Za-z0-9_.-]+):([A-Za-z0-9_.-]+):([A-Za-z0-9_.+-]+)(?::[^=]+)?=.*$" +) + +# Match Groovy dep declarations: configuration 'group:artifact:version' +# (single or double quotes, optional map-style attributes) +_GRADLE_DEP_RE = re.compile( + r"""^\s*(?:[A-Za-z][A-Za-z0-9_]*(?:\s*\([^)]*\))?\s+) # configuration name + ['"]([A-Za-z0-9_.-]+):([A-Za-z0-9_.-]+):([A-Za-z0-9_.+-]+)['"] + """, + re.VERBOSE, +) + + +def _parse_gradle_lockfile(content: str, path: str) -> List[Dependency]: + deps: List[Dependency] = [] + seen = set() + for raw_line in content.splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + m = _LOCKFILE_LINE_RE.match(line) + if not m: + continue + group, artifact, version = m.group(1), m.group(2), m.group(3) + name = f"{group}:{artifact}" + key = (name.lower(), version) + if key in seen: + continue + seen.add(key) + deps.append( + Dependency( + name=name, + version=version, + ecosystem="gradle", + source_file=path, + # gradle.lockfile contains both direct and transitive; + # without context we mark transitive. + transitivity="transitive", + ) + ) + return deps + + +def _parse_build_gradle(content: str, path: str) -> List[Dependency]: + deps: List[Dependency] = [] + seen = set() + in_deps = False + for raw_line in content.splitlines(): + stripped = raw_line.strip() + # Enter dependencies block + if re.match(r"^dependencies\s*\{?\s*$", stripped) or re.match( + r"^dependencies\s*\{", stripped + ): + in_deps = True + continue + if in_deps and stripped == "}": + in_deps = False + continue + if not in_deps: + continue + m = _GRADLE_DEP_RE.match(raw_line) + if not m: + continue + group, artifact, version = m.group(1), m.group(2), m.group(3) + name = f"{group}:{artifact}" + key = (name.lower(), version) + if key in seen: + continue + seen.add(key) + deps.append( + Dependency( + name=name, + version=version, + ecosystem="gradle", + source_file=path, + transitivity="direct", + ) + ) + return deps + + +def parse(path: str) -> List[Dependency]: + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + content = f.read() + except Exception as exc: + logger.warning("gradle_lock: cannot read %s: %s", path, exc) + return [] + + basename = os.path.basename(path) + if basename == "gradle.lockfile": + return _parse_gradle_lockfile(content, path) + if basename in ("build.gradle", "build.gradle.kts"): + return _parse_build_gradle(content, path) + # Unknown but routed here — try lockfile shape first (more reliable), + # then build.gradle shape as a fallback. + if "=" in content and ":" in content: + return _parse_gradle_lockfile(content, path) + return _parse_build_gradle(content, path) + + +__all__ = ["parse"] diff --git a/scripts/sca_parsers/mix_lock.py b/scripts/sca_parsers/mix_lock.py new file mode 100644 index 00000000..94746022 --- /dev/null +++ b/scripts/sca_parsers/mix_lock.py @@ -0,0 +1,87 @@ +""" +Parser for ``mix.lock`` (Elixir / Hex). + +Format reference (public, reimplemented from spec): +- Elixir source file with ``%{`` map literal at top level. +- Each entry is ``"package_name": {:hex, :package_name, "1.2.3", + "url", "hash", [:mix], deps...}`` or + ``"name": {:git, "url", "sha", branch/ref}``. +- We only handle ``:hex`` entries (Hex packages) for OSV matching — + ``:git`` entries are also extracted but marked with ecosystem ``hex`` + and version ``git:`` so downstream can decide. + +Implementation: we don't evaluate the Elixir code; we regex-scan for +the ``"name": {:hex, :name, "version",`` pattern. +""" + +from __future__ import annotations + +import logging +import re +from typing import List + +from . import Dependency + +logger = logging.getLogger("codelens.sca.mix_lock") + + +_HEX_RE = re.compile( + r'"([^"]+)":\s*\{:hex,\s*:([A-Za-z0-9_.-]+),\s*"([^"]+)"' +) +_GIT_RE = re.compile( + r'"([^"]+)":\s*\{:git,\s*"[^"]+",\s*"([0-9a-f]+)"' +) + + +def parse(path: str) -> List[Dependency]: + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + content = f.read() + except Exception as exc: + logger.warning("mix_lock: cannot read %s: %s", path, exc) + return [] + + deps: List[Dependency] = [] + seen = set() + + for key, name, version in _HEX_RE.findall(content): + # The map key and the :name atom usually match — use the atom name + # (canonical) but fall back to the key if they differ. + pkg_name = name or key + if not pkg_name or not version: + continue + unique = (pkg_name.lower(), version) + if unique in seen: + continue + seen.add(unique) + deps.append( + Dependency( + name=pkg_name, + version=version, + ecosystem="hex", + source_file=path, + transitivity="transitive", # mix.lock is the full resolution + ) + ) + + for key, sha in _GIT_RE.findall(content): + if not key or not sha: + continue + unique = (key.lower(), f"git:{sha[:12]}") + if unique in seen: + continue + seen.add(unique) + deps.append( + Dependency( + name=key, + version=f"git:{sha[:12]}", + ecosystem="hex", + source_file=path, + transitivity="transitive", + ) + ) + + return deps + + +__all__ = ["parse"] diff --git a/scripts/sca_parsers/package_resolved.py b/scripts/sca_parsers/package_resolved.py new file mode 100644 index 00000000..89700ad4 --- /dev/null +++ b/scripts/sca_parsers/package_resolved.py @@ -0,0 +1,86 @@ +""" +Parser for ``Package.resolved`` (Swift Package Manager). + +Format reference (public, reimplemented from spec): +- JSON file with a top-level ``object`` and ``version`` field. +- v1: ``object.pins`` is an array of ``{ "package": "Name", + "repositoryURL": "...", "state": { "version": "1.2.3" } }``. +- v2: ``object.pins`` entries have ``identity`` (lowercase package id) + instead of ``package``, and ``location`` instead of ``repositoryURL``. + ``state`` may carry ``revision`` (a git sha) and/or ``version``. +- v3: same shape as v2 with an extra ``version`` field at the top level. + +If ``state.version`` is missing (branch / revision-only pins), we fall +back to ``state.revision`` (short sha) so the dep is still recorded. +""" + +from __future__ import annotations + +import json +import logging +from typing import List + +from . import Dependency + +logger = logging.getLogger("codelens.sca.package_resolved") + + +def parse(path: str) -> List[Dependency]: + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + data = json.load(f) + except Exception as exc: + logger.warning("package_resolved: cannot read %s: %s", path, exc) + return [] + + if not isinstance(data, dict): + return [] + + obj = data.get("object", {}) or {} + if not isinstance(obj, dict): + return [] + pins = obj.get("pins", []) or [] + if not isinstance(pins, list): + return [] + + deps: List[Dependency] = [] + seen = set() + + for pin in pins: + if not isinstance(pin, dict): + continue + # v1: "package" (display name) ; v2/v3: "identity" (lowercase id) + name = pin.get("package") or pin.get("identity") or "" + if not isinstance(name, str) or not name: + continue + state = pin.get("state", {}) or {} + if not isinstance(state, dict): + continue + version = state.get("version") or "" + if not version: + # Branch/revision-only pin: fall back to short sha. + revision = state.get("revision") or "" + if revision: + version = revision[:12] + if not version: + continue + key = (name.lower(), version) + if key in seen: + continue + seen.add(key) + deps.append( + Dependency( + name=name, + version=version, + ecosystem="swiftpm", + source_file=path, + # Package.resolved doesn't distinguish direct vs transitive + # (both kinds end up in the same pins list). Default direct. + transitivity="direct", + ) + ) + + return deps + + +__all__ = ["parse"] diff --git a/scripts/sca_parsers/packages_lock.py b/scripts/sca_parsers/packages_lock.py new file mode 100644 index 00000000..5b4d77c4 --- /dev/null +++ b/scripts/sca_parsers/packages_lock.py @@ -0,0 +1,89 @@ +""" +Parser for ``packages.lock.json`` (NuGet). + +Format reference (public, reimplemented from spec): +- Top-level JSON object with ``version`` (an int, usually 1 or 2), + ``dependencies`` and (v2) ``frameworks``. +- v1: ``dependencies`` maps ``"PackageName"`` -> ``{ "resolved": "1.2.3", + "type": "Direct|Transitive|Project", ... }``. +- v2: ``frameworks`` maps ``".NETCoreApp,Version=v6.0"`` -> ``{ "dependencies": + {...} }``. We flatten across all frameworks (dedup by name+version). + +The ``type`` field is the authoritative source for direct vs transitive; +we fall back to "transitive" when missing. +""" + +from __future__ import annotations + +import json +import logging +from typing import List + +from . import Dependency + +logger = logging.getLogger("codelens.sca.packages_lock") + + +def _extract_from_deps_dict( + deps: dict, path: str, seen: set +) -> List[Dependency]: + out: List[Dependency] = [] + if not isinstance(deps, dict): + return out + for name, info in deps.items(): + if not isinstance(name, str) or not name: + continue + if not isinstance(info, dict): + continue + version = info.get("resolved") or info.get("version") or "" + if not isinstance(version, str) or not version: + continue + dep_type = info.get("type", "") + transitivity = "direct" if dep_type == "Direct" else "transitive" + key = (name.lower(), version) + if key in seen: + continue + seen.add(key) + out.append( + Dependency( + name=name, + version=version, + ecosystem="nuget", + source_file=path, + transitivity=transitivity, + ) + ) + return out + + +def parse(path: str) -> List[Dependency]: + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + data = json.load(f) + except Exception as exc: + logger.warning("packages_lock: cannot read %s: %s", path, exc) + return [] + + if not isinstance(data, dict): + return [] + + seen: set = set() + deps: List[Dependency] = [] + + # v1: top-level "dependencies" + if isinstance(data.get("dependencies"), dict): + deps.extend(_extract_from_deps_dict(data["dependencies"], path, seen)) + + # v2: per-framework "dependencies" + frameworks = data.get("frameworks", {}) or {} + if isinstance(frameworks, dict): + for _tfm, tfm_data in frameworks.items(): + if not isinstance(tfm_data, dict): + continue + inner = tfm_data.get("dependencies", {}) or {} + deps.extend(_extract_from_deps_dict(inner, path, seen)) + + return deps + + +__all__ = ["parse"] diff --git a/scripts/sca_parsers/pipfile.py b/scripts/sca_parsers/pipfile.py new file mode 100644 index 00000000..059ce11f --- /dev/null +++ b/scripts/sca_parsers/pipfile.py @@ -0,0 +1,103 @@ +""" +Parser for ``Pipfile`` (declared deps, non-lock). + +Format reference (public Pipenv docs, reimplemented — no code copied): +- TOML file with ``[packages]`` and ``[dev-packages]`` sections. +- Each entry is ``name = "version_spec"`` or + ``name = {version = "version_spec", extras = [...], ...}``. +- Version specifiers follow pip semantics (``==1.2.3``, ``>=1.2``, + ``*`` for unpinned). +- We strip the leading operator and extras to get a bare version + number. Unpinned deps (``*`` or empty) get version ``"0.0.0"``. +""" + +from __future__ import annotations + +import logging +import re +from typing import List + +from . import Dependency + +logger = logging.getLogger("codelens.sca.pipfile") + + +_NAME_RE = re.compile(r"^([A-Za-z0-9_.-]+)\s*=\s*(.*)$") +_VERSION_TABLE_RE = re.compile(r'version\s*=\s*"([^"]*)"') + + +def _normalise_version(spec: str) -> str: + if not spec: + return "0.0.0" + spec = spec.strip() + if spec in ("*", ""): + return "0.0.0" + cleaned = re.sub(r"^[><=!~]+", "", spec) + cleaned = cleaned.split(",")[0].strip() + if not cleaned or cleaned == "*": + return "0.0.0" + if re.match(r"^\d", cleaned): + return cleaned + return "0.0.0" + + +def parse(path: str) -> List[Dependency]: + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + content = f.read() + except Exception as exc: + logger.warning("pipfile: cannot read %s: %s", path, exc) + return [] + + deps: List[Dependency] = [] + seen = set() + + in_section = False # True when inside [packages] or [dev-packages] + + for raw_line in content.splitlines(): + stripped = raw_line.strip() + if not stripped or stripped.startswith("#"): + continue + if stripped.startswith("[") and stripped.endswith("]"): + in_section = stripped in ("[packages]", "[dev-packages]") + continue + if not in_section: + continue + m = _NAME_RE.match(stripped) + if not m: + continue + name = m.group(1) + rest = m.group(2).strip() + if rest.startswith("{"): + tm = _VERSION_TABLE_RE.search(rest) + version = _normalise_version(tm.group(1)) if tm else "0.0.0" + elif rest.startswith('"'): + vm = re.match(r'^"([^"]*)"', rest) + version = _normalise_version(vm.group(1)) if vm else "0.0.0" + elif rest.startswith("'"): + vm = re.match(r"^'([^']*)'", rest) + version = _normalise_version(vm.group(1)) if vm else "0.0.0" + elif rest == "*": + version = "0.0.0" + else: + version = "0.0.0" + if not name: + continue + key = (name.lower(), version) + if key in seen: + continue + seen.add(key) + deps.append( + Dependency( + name=name, + version=version, + ecosystem="pypi", + source_file=path, + transitivity="direct", + ) + ) + + return deps + + +__all__ = ["parse"] diff --git a/scripts/sca_parsers/pipfile_lock.py b/scripts/sca_parsers/pipfile_lock.py new file mode 100644 index 00000000..d9fff8cb --- /dev/null +++ b/scripts/sca_parsers/pipfile_lock.py @@ -0,0 +1,82 @@ +""" +Parser for ``Pipfile.lock`` (Pipenv). + +Format reference (public, reimplemented from spec): +- Top-level JSON object with ``default`` and ``develop`` sections. +- Each section maps ``name`` -> ``{"version": "==1.2.3", "hashes": [...], ...}`` + or (rare legacy) ``name`` -> ``"==1.2.3"``. +- The ``version`` field includes the leading ``==`` operator which we + strip to obtain the bare version. + +Direct vs transitive: Pipfile.lock does not distinguish declared vs +sub-deps — every entry in ``default``/``develop`` is a resolved dep. We +mark all as ``direct`` because Pipfile (the manifest) declares the +top-level ones and Pipfile.lock is just the resolution. +""" + +from __future__ import annotations + +import json +import logging +import re +from typing import List + +from . import Dependency + +logger = logging.getLogger("codelens.sca.pipfile_lock") + + +def _strip_specifier(version: str) -> str: + """Strip leading version operators: ``==1.2.3`` -> ``1.2.3``.""" + if not isinstance(version, str): + return "" + cleaned = re.sub(r"^[><=!~]+", "", version.strip()) + return cleaned or "" + + +def parse(path: str) -> List[Dependency]: + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + data = json.load(f) + except Exception as exc: + logger.warning("pipfile_lock: cannot read %s: %s", path, exc) + return [] + + if not isinstance(data, dict): + return [] + + deps: List[Dependency] = [] + seen = set() + + for section in ("default", "develop"): + section_data = data.get(section, {}) or {} + if not isinstance(section_data, dict): + continue + for name, info in section_data.items(): + if isinstance(info, dict): + version = info.get("version", "") + elif isinstance(info, str): + version = info + else: + continue + version = _strip_specifier(version) + if not name or not version: + continue + key = (name.lower(), version) + if key in seen: + continue + seen.add(key) + deps.append( + Dependency( + name=name, + version=version, + ecosystem="pypi", + source_file=path, + transitivity="direct", + ) + ) + + return deps + + +__all__ = ["parse"] diff --git a/scripts/sca_parsers/pnpm_lock.py b/scripts/sca_parsers/pnpm_lock.py new file mode 100644 index 00000000..f12eb52d --- /dev/null +++ b/scripts/sca_parsers/pnpm_lock.py @@ -0,0 +1,133 @@ +""" +Parser for ``pnpm-lock.yaml`` (pnpm v6 and v9). + +Format reference (public, reimplemented from spec — no code copied): +- v6: top-level ``importers`` maps workspace path -> ``dependencies`` / + ``devDependencies`` / ``optionalDependencies`` dicts of + ``name: specifier`` strings. The ``packages`` dict maps a + ``/name@version`` key to a metadata object. +- v9: same shape, but the lockfileVersion is 9.0 and ``packages`` keys + may include peer-resolution suffixes like ``/name@version(peer)``. +- ``snapshots`` (v9) holds resolved tree state — we ignore it because + ``packages`` already gives us every installed package exactly once. + +We extract direct deps from ``importers`` and mark everything else +as transitive. +""" + +from __future__ import annotations + +import logging +import re +from typing import List, Set + +try: + import yaml # PyYAML +except ImportError: # pragma: no cover - PyYAML is a hard dep of codelens + yaml = None + +from . import Dependency + +logger = logging.getLogger("codelens.sca.pnpm") + + +def parse(path: str) -> List[Dependency]: + if yaml is None: + logger.warning("pnpm_lock: PyYAML not available, skipping %s", path) + return [] + + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + data = yaml.safe_load(f) + except Exception as exc: + logger.warning("pnpm_lock: cannot read %s: %s", path, exc) + return [] + + if not isinstance(data, dict): + return [] + + lock_version = data.get("lockfileVersion", 6) + + # ── Collect direct deps from importers ── + direct_names: Set[str] = set() + importers = data.get("importers", {}) or {} + if isinstance(importers, dict): + for _ws, ws_data in importers.items(): + if not isinstance(ws_data, dict): + continue + for dep_field in ( + "dependencies", + "devDependencies", + "optionalDependencies", + "peerDependencies", + ): + section = ws_data.get(dep_field, {}) or {} + if not isinstance(section, dict): + continue + for name in section.keys(): + direct_names.add(name) + + # ── Collect all resolved packages ── + deps: List[Dependency] = [] + seen: Set[str] = set() + packages = data.get("packages", {}) or {} + if isinstance(packages, dict): + for key, _info in packages.items(): + if not isinstance(key, str) or not key: + continue + # Key format: "/name@version" or "/name@version(peer@v)" + # Strip a leading "/" + stripped = key[1:] if key.startswith("/") else key + # Strip peer suffix in parentheses + stripped = stripped.split("(", 1)[0] + # Split on the LAST @ (scoped packages: @scope/name@version) + at_idx = stripped.rfind("@") + if at_idx <= 0: + continue + name = stripped[:at_idx] + version = stripped[at_idx + 1:] + if not name or not version: + continue + unique = (name, version) + if unique in seen: + continue + seen.add(unique) + transitivity = "direct" if name in direct_names else "transitive" + deps.append( + Dependency( + name=name, + version=version, + ecosystem="npm", + source_file=path, + transitivity=transitivity, + ) + ) + + # v6 fallback: if there are no ``packages`` we may still have + # ``dependencies`` at top level (older experimental format). + if not deps: + top_deps = data.get("dependencies", {}) or {} + if isinstance(top_deps, dict): + for name, info in top_deps.items(): + if isinstance(info, dict): + version = info.get("version", "") + elif isinstance(info, str): + version = info + else: + continue + if name and version: + deps.append( + Dependency( + name=name, + version=str(version), + ecosystem="npm", + source_file=path, + transitivity="direct" if name in direct_names else "transitive", + ) + ) + + _ = lock_version # parsed for future use, currently format-agnostic + return deps + + +__all__ = ["parse"] diff --git a/scripts/sca_parsers/pom_xml.py b/scripts/sca_parsers/pom_xml.py new file mode 100644 index 00000000..4f770429 --- /dev/null +++ b/scripts/sca_parsers/pom_xml.py @@ -0,0 +1,116 @@ +""" +Parser for ``pom.xml`` (Maven declared dependencies). + +Format reference (public Maven POM spec, reimplemented — no code copied): +- XML file with ```` root, ```` under either + ```` directly or under ```` / + ````. +- Each ```` has ````, ````, ```` + and optional ````. We use ``group:artifact`` as the package name + to match OSV.dev's Maven ecosystem convention. +- Maven properties (``${project.version}``, ``${spring.version}``, + ``${revision}`` etc.) are NOT resolved — we emit the literal string + and downstream tooling can decide to skip properties. +- We avoid using external XML parser dependencies other than the stdlib + ``xml.etree.ElementTree`` which is MIT-licensed and compatible. +""" + +from __future__ import annotations + +import logging +import re +from typing import List +from xml.etree import ElementTree as ET + +from . import Dependency + +logger = logging.getLogger("codelens.sca.pom_xml") + + +def _strip_namespace(root: ET.Element) -> ET.Element: + """Recursively strip the XML namespace from every element's tag. + + Maven pom.xml uses a default namespace (``xmlns="..."``) which makes + every tag like ``{http://maven.apache.org/POM/4.0.0}dependency`` + instead of bare ``dependency``. Stripping makes ``find()`` and + ``iter()`` work with the unprefixed names from the spec. + """ + for elem in root.iter(): + if isinstance(elem.tag, str) and "}" in elem.tag: + elem.tag = elem.tag.split("}", 1)[1] + return root + + +def _text(parent: ET.Element, tag: str) -> str: + """Return stripped text of the first direct child ``tag`` of ``parent``.""" + if parent is None: + return "" + child = parent.find(tag) + if child is None or child.text is None: + return "" + return child.text.strip() + + +def _collect_dependencies(root: ET.Element) -> List[ET.Element]: + """Find all elements anywhere under ``root``.""" + return list(root.iter("dependency")) + + +def parse(path: str) -> List[Dependency]: + try: + tree = ET.parse(path) + except ET.ParseError as exc: + logger.warning("pom_xml: cannot parse %s: %s", path, exc) + return [] + except Exception as exc: + logger.warning("pom_xml: cannot read %s: %s", path, exc) + return [] + + root = _strip_namespace(tree.getroot()) + deps: List[Dependency] = [] + seen = set() + + for dep in _collect_dependencies(root): + group_id = _text(dep, "groupId") + artifact_id = _text(dep, "artifactId") + version = _text(dep, "version") + scope = _text(dep, "scope") + + if not group_id or not artifact_id: + continue + # Skip Maven-injected test/provided/runtime-only deps from + # dependencyManagement — we still want them recorded but marked + # transitive is misleading; instead we keep them as direct + # declared deps (the user explicitly listed them). + if not version: + # Inherited from dependencyManagement without explicit version. + version = "0.0.0" + + # Maven version strings may reference properties: ${foo}. Keep + # them as-is; downstream consumers can choose to skip these. + name = f"{group_id}:{artifact_id}" + key = (name.lower(), version) + if key in seen: + continue + seen.add(key) + + transitivity = "direct" + if scope in ("test", "provided", "runtime"): + # Still declared by the user, but non-runtime scope. + # We keep "direct" because they are explicitly listed. + transitivity = "direct" + + deps.append( + Dependency( + name=name, + version=version, + ecosystem="maven", + source_file=path, + transitivity=transitivity, + ) + ) + + return deps + + +__all__ = ["parse"] diff --git a/scripts/sca_parsers/pubspec_lock.py b/scripts/sca_parsers/pubspec_lock.py new file mode 100644 index 00000000..3f104a6e --- /dev/null +++ b/scripts/sca_parsers/pubspec_lock.py @@ -0,0 +1,87 @@ +""" +Parser for ``pubspec.lock`` (Dart / pub). + +Format reference (public, reimplemented from spec): +- YAML file with top-level ``packages`` mapping. +- Each value is a dict with ``dependency``, ``source``, ``version`` and + (for hosted source) ``description: {name: ..., url: ..., sha256: ...}``. +- ``dependency: "direct main"`` / ``"direct dev"`` / ``"transitive"``. +- ``version`` is a SemVer string, possibly with leading ``^`` (rare in + lockfiles). +""" + +from __future__ import annotations + +import logging +import re +from typing import List + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None + +from . import Dependency + +logger = logging.getLogger("codelens.sca.pubspec_lock") + + +def parse(path: str) -> List[Dependency]: + if yaml is None: + logger.warning("pubspec_lock: PyYAML not available, skipping %s", path) + return [] + + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + data = yaml.safe_load(f) + except Exception as exc: + logger.warning("pubspec_lock: cannot read %s: %s", path, exc) + return [] + + if not isinstance(data, dict): + return [] + + packages = data.get("packages", {}) or {} + if not isinstance(packages, dict): + return [] + + deps: List[Dependency] = [] + seen = set() + + for name, info in packages.items(): + if not isinstance(name, str) or not name: + continue + if not isinstance(info, dict): + continue + version = info.get("version", "") + if not isinstance(version, str) or not version: + continue + # Strip leading operator just in case. + version = re.sub(r"^[~^>=]`` — PEP 621 lists. + * ``[tool.poetry.dependencies]`` — Poetry table ``name = "spec"``. + * ``[tool.poetry.group..dependencies]`` — Poetry per-group deps. +- Version specifiers follow PEP 440. We strip the leading operator and + take the first version-like token. Unpinned (``*``) → ``"0.0.0"``. + +This module also re-exports ``parse_poetry_lock`` for ``poetry.lock`` +so the registry can route that filename here. +""" + +from __future__ import annotations + +import logging +import re +from typing import List + +from . import Dependency + +logger = logging.getLogger("codelens.sca.pyproject_toml") + + +try: + import tomllib as _toml # py311+ +except ImportError: # pragma: no cover + try: + import tomli as _toml # type: ignore + except ImportError: + _toml = None # type: ignore + + +_NAME_RE = re.compile(r"^([A-Za-z0-9_.-]+)") +# Match leading PEP 440 operators plus Poetry's `^`/`~` and npm-style `*`. +_VERSION_OP_RE = re.compile(r"^(==|!=|<=|>=|~=|>|<|===|\^|~)\s*([A-Za-z0-9_.*+!-]+)") + + +def _normalise_spec(spec: str) -> str: + """Strip leading version operators and return a bare version string. + + Handles PEP 440 (==, !=, <=, >=, ~=, >, <, ===), Poetry caret/tilde + (``^1.2``, ``~1.2.3``), and bare versions (``1.2.3``). Returns + ``"0.0.0"`` for unpinned (``*`` or empty). + """ + if not spec: + return "0.0.0" + spec = spec.strip() + if spec in ("*", ""): + return "0.0.0" + m = _VERSION_OP_RE.match(spec) + if m: + return m.group(2).split(",")[0] + first = spec.split(",")[0].split()[0] + if re.match(r"^\d", first): + return first + return "0.0.0" + + +def _emit(name: str, version_spec, path: str, deps: list, seen: set) -> None: + """Append a Dependency to ``deps`` (deduplicated by lower(name)+version).""" + if not isinstance(name, str) or not name: + return + if isinstance(version_spec, str): + version = _normalise_spec(version_spec) + elif isinstance(version_spec, dict): + v = version_spec.get("version", "") + version = _normalise_spec(v) if isinstance(v, str) else "0.0.0" + else: + version = "0.0.0" + key = (name.lower(), version) + if key in seen: + return + seen.add(key) + deps.append( + Dependency( + name=name, + version=version, + ecosystem="pypi", + source_file=path, + transitivity="direct", + ) + ) + + +def parse(path: str) -> List[Dependency]: + if _toml is None: + logger.warning("pyproject_toml: tomllib/tomli not available, skipping %s", path) + return [] + try: + with open(path, "rb") as f: + data = _toml.load(f) + except Exception as exc: + logger.warning("pyproject_toml: cannot parse %s: %s", path, exc) + return [] + + if not isinstance(data, dict): + return [] + + deps: List[Dependency] = [] + seen: set = set() + + # ── PEP 621 [project.dependencies] ── + project = data.get("project", {}) or {} + if isinstance(project, dict): + for entry in project.get("dependencies", []) or []: + if isinstance(entry, str): + m = _NAME_RE.match(entry) + if not m: + continue + name = m.group(1) + rest = entry[m.end():].strip() + # Strip extras `[foo]` and markers `; python_version<'3.10'` + rest = rest.split(";", 1)[0].strip() + version = _normalise_spec(rest) + _emit(name, version if version else "0.0.0", path, deps, seen) + # Optional extras + opt = project.get("optional-dependencies", {}) or {} + if isinstance(opt, dict): + for _group, entries in opt.items(): + if not isinstance(entries, list): + continue + for entry in entries: + if not isinstance(entry, str): + continue + m = _NAME_RE.match(entry) + if not m: + continue + name = m.group(1) + rest = entry[m.end():].strip().split(";", 1)[0].strip() + version = _normalise_spec(rest) + _emit(name, version if version else "0.0.0", path, deps, seen) + + # ── Poetry [tool.poetry.dependencies] ── + tool = data.get("tool", {}) or {} + if isinstance(tool, dict): + poetry = tool.get("poetry", {}) or {} + if isinstance(poetry, dict): + _poetry_section_deps(poetry.get("dependencies", {}), path, deps, seen) + groups = poetry.get("group", {}) or {} + if isinstance(groups, dict): + for _gname, gdata in groups.items(): + if not isinstance(gdata, dict): + continue + _poetry_section_deps(gdata.get("dependencies", {}), path, deps, seen) + + return deps + + +def _poetry_section_deps(section: dict, path: str, deps: list, seen: set) -> None: + if not isinstance(section, dict): + return + for name, spec in section.items(): + # Skip Python version constraint entry: `python = ">=3.8"` + if name.lower() == "python": + continue + _emit(name, spec, path, deps, seen) + + +# ── poetry.lock support ──────────────────────────────────────── + + +def parse_poetry_lock(path: str) -> List[Dependency]: + """Parse ``poetry.lock`` (TOML form, Poetry 1.2+). + + Falls back gracefully if tomllib is unavailable. + """ + if _toml is None: + logger.warning("pyproject_toml: tomllib not available, cannot parse poetry.lock %s", path) + return [] + try: + with open(path, "rb") as f: + data = _toml.load(f) + except Exception as exc: + logger.warning("pyproject_toml: cannot parse poetry.lock %s: %s", path, exc) + return [] + if not isinstance(data, dict): + return [] + packages = data.get("package", []) or [] + if not isinstance(packages, list): + return [] + deps: List[Dependency] = [] + seen = set() + for pkg in packages: + if not isinstance(pkg, dict): + continue + name = pkg.get("name", "") + version = pkg.get("version", "") + if not isinstance(name, str) or not isinstance(version, str): + continue + if not name or not version: + continue + # poetry.lock carries a `category = "main|dev"` field; both are + # resolved deps, mark as direct (Poetry treats them as such). + key = (name.lower(), version) + if key in seen: + continue + seen.add(key) + deps.append( + Dependency( + name=name, + version=version, + ecosystem="pypi", + source_file=path, + transitivity="direct", + ) + ) + return deps + + +__all__ = ["parse", "parse_poetry_lock"] diff --git a/scripts/sca_parsers/requirements_txt.py b/scripts/sca_parsers/requirements_txt.py new file mode 100644 index 00000000..3fd64463 --- /dev/null +++ b/scripts/sca_parsers/requirements_txt.py @@ -0,0 +1,119 @@ +""" +Parser for ``requirements.txt`` (pinned + unpinned). + +Format reference (public PEP 508 / pip docs, reimplemented — no code copied): +- One requirement per line. +- Comments start with ``#`` and may appear inline. +- ``-r other.txt``, ``-e .``, ``-c constraints.txt`` and similar pip + options start with ``-`` — skipped. +- Forms handled: + * ``package==1.2.3`` + * ``package>=1.2.3,<2.0.0`` + * ``package~=1.2.3`` + * ``package[extra]==1.2.3`` + * ``package (extras)`` — wait, that's setup.cfg; we ignore. + * ``package`` (unpinned → version "0.0.0") + * ``package @ https://.../pkg.tar.gz`` (URL — name recorded, version "0.0.0") + * ``git+https://github.com/x/y@v1.0#egg=y`` (egg name extracted) +- Environment markers (``; python_version<'3.10'``) are stripped. +""" + +from __future__ import annotations + +import logging +import re +from typing import List + +from . import Dependency + +logger = logging.getLogger("codelens.sca.requirements_txt") + + +_NAME_RE = re.compile(r"^([A-Za-z0-9_.-]+)(?:\[[^\]]*\])?") +_VERSION_OP_RE = re.compile(r"^(==|!=|<=|>=|~=|>|<|===)\s*([A-Za-z0-9_.*+!-]+)") + + +def _extract_version(spec: str) -> str: + """Extract a concrete version number from a pip version specifier. + + Returns ``"0.0.0"`` if no version is pinned. + """ + spec = spec.strip() + if not spec: + return "0.0.0" + m = _VERSION_OP_RE.match(spec) + if not m: + # Bare version with no operator? Take leading version-like token. + first = spec.split(",")[0].split()[0] + if re.match(r"^[0-9]", first): + return first + return "0.0.0" + op = m.group(1) + ver = m.group(2) + # Strip trailing extras like `,<2.0` + ver = ver.split(",")[0] + if op in ("==", "==="): + # Exact pin — strip any ``.*`` wildcard suffix. + return ver.rstrip(".*") if ".*" in ver else ver + # Other operators — take the bare version as the lower bound estimate. + return ver + + +def parse(path: str) -> List[Dependency]: + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + content = f.read() + except Exception as exc: + logger.warning("requirements_txt: cannot read %s: %s", path, exc) + return [] + + deps: List[Dependency] = [] + seen = set() + + for raw_line in content.splitlines(): + # Strip inline comments + line = raw_line.split("#", 1)[0].strip() + if not line: + continue + # Skip pip options + if line.startswith("-"): + continue + # Strip environment markers: `pkg==1.0 ; python_version<'3.10'` + line = line.split(";", 1)[0].strip() + # Handle `name @ url` syntax + if " @ " in line: + name_part = line.split(" @ ", 1)[0].strip() + m = _NAME_RE.match(name_part) + if not m: + continue + name = m.group(1) + version = "0.0.0" + else: + m = _NAME_RE.match(line) + if not m: + continue + name = m.group(1) + rest = line[m.end():].strip() + # Strip leading whitespace and operators + version = _extract_version(rest) + if not name: + continue + key = (name.lower(), version) + if key in seen: + continue + seen.add(key) + deps.append( + Dependency( + name=name, + version=version, + ecosystem="pypi", + source_file=path, + # requirements.txt is a manifest; all entries are direct. + transitivity="direct", + ) + ) + + return deps + + +__all__ = ["parse"] diff --git a/scripts/sca_parsers/yarn_lock.py b/scripts/sca_parsers/yarn_lock.py new file mode 100644 index 00000000..37c69136 --- /dev/null +++ b/scripts/sca_parsers/yarn_lock.py @@ -0,0 +1,123 @@ +""" +Parser for ``yarn.lock`` (Yarn v1 Classic, v2/v3 Berry, PnP). + +Format reference (public, reimplemented from spec): +- v1 (Classic): text-based. Blocks start with one or more quoted or + unquoted specifiers followed by ``:``. Inside the block, ``version + "x.y.z"`` declares the resolved version. Multiple specifiers can + resolve to the same version. +- v2+ (Berry): same textual shape, with an optional ``__metadata`` + block at the top containing ``cacheKey`` and ``version``. Berry may + also include resolution fields like ``resolution: "..."``. +- PnP (.pnp.cjs/.pnp.js): not handled here — those are not lockfiles + in the same sense; this parser only handles yarn.lock text files. +""" + +from __future__ import annotations + +import logging +import re +from typing import List, Set, Tuple + +from . import Dependency + +logger = logging.getLogger("codelens.sca.yarn") + + +_SPEC_RE = re.compile(r'^"?([^@].*?)@([^@/]+)"?$') +# Match a yarn v2+ resolution line that ends with the npm spec +_NPM_RESOLUTION_RE = re.compile(r'^resolution:\s+"?npm:([^@]+)@([^@/]+)"?$') + + +def _split_specifier(spec: str) -> Tuple[str, str]: + """Split ``name@version`` (or ``@scope/name@version``) into (name, version).""" + spec = spec.strip().strip('"') + if not spec: + return ("", "") + at_idx = spec.rfind("@") + if at_idx <= 0: + return (spec, "") + return (spec[:at_idx], spec[at_idx + 1:]) + + +def parse(path: str) -> List[Dependency]: + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + content = f.read() + except Exception as exc: + logger.warning("yarn_lock: cannot read %s: %s", path, exc) + return [] + + deps: List[Dependency] = [] + seen: Set[Tuple[str, str]] = set() + + current_names: List[str] = [] + current_version: str = "" + in_block = False + + for raw_line in content.splitlines(): + stripped = raw_line.strip() + if not stripped or stripped.startswith("#"): + continue + + # New block: line ends with ":" and is at column 0 + if stripped.endswith(":") and not raw_line.startswith((" ", "\t")): + # Flush previous + if in_block and current_names and current_version: + for n in current_names: + if (n, current_version) not in seen: + seen.add((n, current_version)) + deps.append( + Dependency( + name=n, + version=current_version, + ecosystem="npm", + source_file=path, + transitivity="transitive", + ) + ) + # Begin new block + spec_part = stripped[:-1] + current_names = [] + current_version = "" + for piece in spec_part.split(", "): + name, _ver = _split_specifier(piece) + if name: + current_names.append(name) + in_block = bool(current_names) + continue + + if not in_block: + continue + + # Version line: `version "x.y.z"` (yarn v1 / Berry) + m = re.match(r'^version\s+"([^"]+)"', stripped) + if m: + current_version = m.group(1) + continue + + # Berry npm: resolution line carries the canonical name+version + m = _NPM_RESOLUTION_RE.match(stripped) + if m and current_names: + # Override version with the resolved one + current_version = m.group(2) + + # Flush last block + if in_block and current_names and current_version: + for n in current_names: + if (n, current_version) not in seen: + seen.add((n, current_version)) + deps.append( + Dependency( + name=n, + version=current_version, + ecosystem="npm", + source_file=path, + transitivity="transitive", + ) + ) + + return deps + + +__all__ = ["parse"] diff --git a/scripts/vulnscan_engine.py b/scripts/vulnscan_engine.py index eb7dfd5a..53093316 100755 --- a/scripts/vulnscan_engine.py +++ b/scripts/vulnscan_engine.py @@ -36,6 +36,18 @@ from collections import defaultdict from utils import DEFAULT_IGNORE_DIRS, logger +# SCA lockfile/manifest parsers (Issue #53 — 14 new formats). +# Optional import: if sca_parsers/ is removed or renamed, vuln-scan +# falls back to the inline parsers below. +try: + from sca_parsers import parse_lockfile as _sca_parse_lockfile + from sca_parsers import PARSER_REGISTRY as _SCA_PARSER_REGISTRY + _HAS_SCA_PARSERS = True +except ImportError: # pragma: no cover - defensive + _HAS_SCA_PARSERS = False + _SCA_PARSER_REGISTRY = {} + _sca_parse_lockfile = None # type: ignore + # OSV.dev integration (optional — graceful fallback if unavailable) try: from osv_client import OSVClient, OSVQueryBuilder, OSVPackage, OSVVulnerability, ECOSYSTEM_MAP @@ -48,7 +60,8 @@ DEPENDENCY_FILE_PATTERNS = { "npm": { "manifest": ["package.json"], - "lockfile": ["package-lock.json", "npm-shrinkwrap.json", "bun.lock", "yarn.lock"], + "lockfile": ["package-lock.json", "npm-shrinkwrap.json", "bun.lock", + "yarn.lock", "pnpm-lock.yaml"], }, "rust": { "manifest": ["Cargo.toml"], @@ -66,6 +79,42 @@ "manifest": [], # Populated dynamically from *.nimble files "lockfile": ["nimble.lock"], }, + # ── Issue #53: additional ecosystems via sca_parsers ─────────── + # Each new format is parsed by a dedicated module in scripts/sca_parsers/. + # If sca_parsers is not importable, these entries are still walked but + # _parse_lock_file will return [] for them (graceful no-op). + "gem": { + "manifest": ["Gemfile"], + "lockfile": ["Gemfile.lock"], + }, + "composer": { + "manifest": ["composer.json"], + "lockfile": ["composer.lock"], + }, + "nuget": { + "manifest": [], # .csproj/.vbproj are extension-matched, not filename + "lockfile": ["packages.lock.json"], + }, + "pub": { + "manifest": ["pubspec.yaml"], + "lockfile": ["pubspec.lock"], + }, + "swiftpm": { + "manifest": ["Package.swift"], + "lockfile": ["Package.resolved"], + }, + "gradle": { + "manifest": ["build.gradle", "build.gradle.kts"], + "lockfile": ["gradle.lockfile"], + }, + "maven": { + "manifest": ["pom.xml"], + "lockfile": [], + }, + "hex": { + "manifest": ["mix.exs"], + "lockfile": ["mix.lock"], + }, } AUDIT_TOOLS = { @@ -3285,26 +3334,52 @@ def _parse_lock_file( except IOError: return findings - if ecosystem == "npm": - if rel_path.endswith("bun.lock"): - packages = _parse_bun_lock(content) - elif rel_path.endswith("yarn.lock"): - packages = _parse_yarn_lock(content) - else: - packages = _parse_npm_lock(content) - elif ecosystem == "rust": - packages = _parse_cargo_lock(content) - elif ecosystem == "pip": - if rel_path.endswith("poetry.lock"): - packages = _parse_poetry_lock(content) - elif rel_path.endswith("Pipfile.lock"): - packages = _parse_pipfile_lock(content) + # ── Issue #53: prefer modular sca_parsers when available ─────── + # sca_parsers handles all NEW formats (Gemfile.lock, pnpm-lock.yaml, + # pubspec.lock, composer.lock, packages.lock.json, Package.resolved, + # gradle.lockfile, mix.lock) plus yarn.lock, Pipfile.lock. When the + # file is in the sca_parsers registry we delegate to it and skip the + # legacy inline branch (which doesn't know about the new formats). + packages: List[Tuple[str, str]] = [] + handled_by_sca = False + if _HAS_SCA_PARSERS: + basename = os.path.basename(lock_path) + if basename in _SCA_PARSER_REGISTRY: + try: + sca_deps, sca_ecosystem = _sca_parse_lockfile(lock_path) + except Exception as exc: + logger.debug("sca_parsers failed for %s: %s", rel_path, exc) + sca_deps, sca_ecosystem = [], None + if sca_deps is not None: + handled_by_sca = True + # Use the ecosystem reported by sca_parsers when available, + # falling back to the caller-provided ecosystem for VULN_DB + # lookups (they must agree in practice). + lookup_ecosystem = sca_ecosystem or ecosystem + packages = [(d.name, d.version) for d in sca_deps] + ecosystem = lookup_ecosystem + + if not handled_by_sca: + if ecosystem == "npm": + if rel_path.endswith("bun.lock"): + packages = _parse_bun_lock(content) + elif rel_path.endswith("yarn.lock"): + packages = _parse_yarn_lock(content) + else: + packages = _parse_npm_lock(content) + elif ecosystem == "rust": + packages = _parse_cargo_lock(content) + elif ecosystem == "pip": + if rel_path.endswith("poetry.lock"): + packages = _parse_poetry_lock(content) + elif rel_path.endswith("Pipfile.lock"): + packages = _parse_pipfile_lock(content) + else: + return findings + elif ecosystem == "go": + packages = _parse_go_sum(content) else: return findings - elif ecosystem == "go": - packages = _parse_go_sum(content) - else: - return findings # Check each package against VULN_DB for pkg_name, pkg_version in packages: @@ -3669,21 +3744,47 @@ def _parse_manifest_file( except IOError: return findings - if ecosystem == "npm" or ecosystem == "bun": - packages = _parse_package_json(content) - elif ecosystem == "rust": - packages = _parse_cargo_toml(content) - elif ecosystem == "pip": - if rel_path.endswith("Pipfile"): - packages = _parse_pipfile(content) - elif rel_path.endswith("pyproject.toml"): - packages = _parse_pyproject_toml(content) + # ── Issue #53: sca_parsers covers pom.xml, build.gradle, Pipfile, + # pyproject.toml and requirements.txt. We delegate to it for the + # NEW manifest ecosystems (maven, gradle) and the new formats + # (Pipfile, pyproject.toml, requirements.txt) ONLY when the file + # is in the registry AND the inline parser below would not handle + # it (i.e., the new ecosystems). + packages: List[Tuple[str, str]] = [] + handled_by_sca = False + if _HAS_SCA_PARSERS: + basename = os.path.basename(manifest_path) + if basename in _SCA_PARSER_REGISTRY and ecosystem in ( + "maven", "gradle", "gem", "composer", "nuget", "pub", + "swiftpm", "hex", + ): + try: + sca_deps, sca_ecosystem = _sca_parse_lockfile(manifest_path) + except Exception as exc: + logger.debug("sca_parsers manifest failed for %s: %s", rel_path, exc) + sca_deps, sca_ecosystem = [], None + if sca_deps is not None: + handled_by_sca = True + packages = [(d.name, d.version) for d in sca_deps] + if sca_ecosystem: + ecosystem = sca_ecosystem + + if not handled_by_sca: + if ecosystem == "npm" or ecosystem == "bun": + packages = _parse_package_json(content) + elif ecosystem == "rust": + packages = _parse_cargo_toml(content) + elif ecosystem == "pip": + if rel_path.endswith("Pipfile"): + packages = _parse_pipfile(content) + elif rel_path.endswith("pyproject.toml"): + packages = _parse_pyproject_toml(content) + else: + packages = _parse_requirements_txt(content) + elif ecosystem == "go": + packages = _parse_go_mod(content) else: - packages = _parse_requirements_txt(content) - elif ecosystem == "go": - packages = _parse_go_mod(content) - else: - return findings + return findings # Check each package against VULN_DB # Bun uses npm packages, so also check npm ecosystem for bun diff --git a/tests/fixtures/sca/Gemfile.lock b/tests/fixtures/sca/Gemfile.lock new file mode 100644 index 00000000..2021c69c --- /dev/null +++ b/tests/fixtures/sca/Gemfile.lock @@ -0,0 +1,32 @@ +GEM + remote: https://rubygems.org/ + specs: + rake (13.0.6) + nokogiri (1.13.10) + rails (7.0.8) + actioncable (= 7.0.8) + actionmailbox (= 7.0.8) + actioncable (7.0.8) + actionmailbox (7.0.8) + actiontext (7.0.8) + concurrent-ruby (1.2.2) + i18n (1.14.1) + concurrent-ruby (~> 1.0) + minitest (5.18.0) + thread_safe (0.3.6) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + zeitwerk (2.6.8) + +PLATFORMS + ruby + +DEPENDENCIES + nokogiri + rails (~> 7.0.8) + +RUBY VERSION + ruby 3.1.2p20 + +BUNDLED WITH + 2.3.7 diff --git a/tests/fixtures/sca/Package.resolved b/tests/fixtures/sca/Package.resolved new file mode 100644 index 00000000..b2ed413b --- /dev/null +++ b/tests/fixtures/sca/Package.resolved @@ -0,0 +1,31 @@ +{ + "object": { + "pins": [ + { + "package": "Alamofire", + "repositoryURL": "https://github.com/Alamofire/Alamofire.git", + "state": { + "revision": "abc123def456abc123def456abc123def456abcd", + "version": "5.8.0" + } + }, + { + "package": "SwiftLint", + "repositoryURL": "https://github.com/realm/SwiftLint.git", + "state": { + "revision": "def456abc123def456abc123def456abc123def0", + "version": "0.53.0" + } + }, + { + "package": "KeychainAccess", + "repositoryURL": "https://github.com/kishikawakatsumi/KeychainAccess.git", + "state": { + "revision": "ghi789jkl012ghi789jkl012ghi789jkl012ghi7", + "version": "4.2.2" + } + } + ] + }, + "version": 1 +} diff --git a/tests/fixtures/sca/Pipfile b/tests/fixtures/sca/Pipfile new file mode 100644 index 00000000..d18b4be0 --- /dev/null +++ b/tests/fixtures/sca/Pipfile @@ -0,0 +1,18 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +django = "==4.2.13" +requests = "*" +flask = {version = "~=2.3.2", extras = ["async"]} +numpy = ">=1.24.0,<2.0.0" + +[dev-packages] +pytest = "==7.4.0" +black = "*" +mypy = {version = ">=1.4.0"} + +[requires] +python_version = "3.10" diff --git a/tests/fixtures/sca/Pipfile.lock b/tests/fixtures/sca/Pipfile.lock new file mode 100644 index 00000000..63b0b810 --- /dev/null +++ b/tests/fixtures/sca/Pipfile.lock @@ -0,0 +1,48 @@ +{ + "_meta": { + "hash": { + "sha256": "abc123def456" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.10" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "django": { + "hashes": [ + "sha256:abc" + ], + "index": 0, + "version": "==4.2.13" + }, + "requests": { + "hashes": [ + "sha256:def" + ], + "index": 1, + "version": "==2.31.0" + }, + "certifi": { + "hashes": [ + "sha256:ghi" + ], + "version": "==2023.7.22" + } + }, + "develop": { + "pytest": { + "hashes": [ + "sha256:xyz" + ], + "version": "==7.4.0" + } + } +} diff --git a/tests/fixtures/sca/build.gradle b/tests/fixtures/sca/build.gradle new file mode 100644 index 00000000..499b8d2b --- /dev/null +++ b/tests/fixtures/sca/build.gradle @@ -0,0 +1,23 @@ +plugins { + id 'java' + id 'application' +} + +repositories { + mavenCentral() +} + +dependencies { + implementation 'com.google.code.gson:gson:2.8.9' + implementation 'com.squareup.retrofit2:retrofit:2.9.0' + implementation 'io.reactivex.rxjava3:rxjava:3.1.6' + implementation 'com.squareup.okhttp3:okhttp:4.11.0' + implementation 'org.jetbrains.kotlin:kotlin-stdlib:1.9.0' + + testImplementation 'junit:junit:4.13.2' + testImplementation 'org.hamcrest:hamcrest:2.2' +} + +application { + mainClassName = 'com.example.Main' +} diff --git a/tests/fixtures/sca/composer.lock b/tests/fixtures/sca/composer.lock new file mode 100644 index 00000000..6b031685 --- /dev/null +++ b/tests/fixtures/sca/composer.lock @@ -0,0 +1,47 @@ +{ + "content-hash": "abc123def456", + "packages": [ + { + "name": "symfony/console", + "version": "6.3.0", + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/console/zipball/abc", + "reference": "abc" + }, + "require": { + "php": ">=8.1", + "symfony/deprecation-contracts": "^2.5|^3" + } + }, + { + "name": "monolog/monolog", + "version": "3.4.0", + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/Seldaek/monolog/zipball/def", + "reference": "def" + } + }, + { + "name": "guzzlehttp/guzzle", + "version": "7.7.0", + "require": { + "php": ">=7.2.5" + } + } + ], + "packages-dev": [ + { + "name": "phpunit/phpunit", + "version": "10.3.0", + "require": { + "php": ">=8.1" + } + } + ], + "aliases": [], + "references": [], + "packages-hash": [], + "dev-packages-hash": [] +} diff --git a/tests/fixtures/sca/gradle.lockfile b/tests/fixtures/sca/gradle.lockfile new file mode 100644 index 00000000..bd84ff88 --- /dev/null +++ b/tests/fixtures/sca/gradle.lockfile @@ -0,0 +1,14 @@ +# This is a Gradle dependency locking file. +# This file is intended to be committed to version control. +# This is a generated file, do not modify manually. +# A dependency lock file can be re-generated using the `./gradlew dependencies --write-locks` command. + +com.google.code.gson:gson:2.8.9:compileClasspath=2.8.9 +com.squareup.retrofit2:retrofit:2.9.0:compileClasspath=2.9.0 +io.reactivex.rxjava3:rxjava:3.1.6:compileClasspath=3.1.6 +com.squareup.okhttp3:okhttp:4.11.0:compileClasspath=4.11.0 +junit:junit:4.13.2:testCompileClasspath=4.13.2 +org.jetbrains.kotlin:kotlin-stdlib:1.9.0:compileClasspath=1.9.0 +com.google.guava:guava:32.1.1-jre:compileClasspath=32.1.1-jre + +empty=unused diff --git a/tests/fixtures/sca/mix.lock b/tests/fixtures/sca/mix.lock new file mode 100644 index 00000000..94790f01 --- /dev/null +++ b/tests/fixtures/sca/mix.lock @@ -0,0 +1,8 @@ +%{ + "absinthe": {:hex, :absinthe, "1.7.6", "0a1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcd", [:mix], [], :hexpm}, + "phoenix": {:hex, :phoenix, "1.7.10", "1b1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcd", [:mix], [{:ecto, "~> 3.4", [hex: :ecto, repo: "hexpm", optional: false]}], :hexpm}, + "ecto": {:hex, :ecto, "3.10.3", "2c1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcd", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], :hexpm}, + "decimal": {:hex, :decimal, "2.1.1", "3d1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcd", [:mix], [], :hexpm}, + "phoenix_live_view": {:hex, :phoenix_live_view, "0.20.1", "4e1234567890abcdef0123456789abcdef0123456789abcdef0123456789abcd", [:mix], [], :hexpm}, + "git_dep": {:git, "https://github.com/foo/bar.git", "abc123def456abc123def456abc123def456abcd", branch: "main"} +} diff --git a/tests/fixtures/sca/packages.lock.json b/tests/fixtures/sca/packages.lock.json new file mode 100644 index 00000000..511c1407 --- /dev/null +++ b/tests/fixtures/sca/packages.lock.json @@ -0,0 +1,46 @@ +{ + "version": 2, + "frameworks": { + ".NETCoreApp,Version=v6.0": { + "dependencies": { + "Newtonsoft.Json": { + "type": "Direct", + "requested": "[13.0.1, )", + "resolved": "13.0.1", + "contentHash": "abc" + }, + "Microsoft.NET.Test.Sdk": { + "type": "Direct", + "requested": "[17.6.0, )", + "resolved": "17.6.0", + "contentHash": "def" + }, + "System.Text.Json": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "ghi" + }, + "NUnit": { + "type": "Transitive", + "resolved": "3.13.3", + "contentHash": "jkl" + } + } + }, + ".NETCoreApp,Version=v8.0": { + "dependencies": { + "Newtonsoft.Json": { + "type": "Direct", + "requested": "[13.0.3, )", + "resolved": "13.0.3", + "contentHash": "mno" + }, + "Serilog": { + "type": "Direct", + "resolved": "3.0.1", + "contentHash": "pqr" + } + } + } + } +} diff --git a/tests/fixtures/sca/pnpm-lock.yaml b/tests/fixtures/sca/pnpm-lock.yaml new file mode 100644 index 00000000..764c3fa0 --- /dev/null +++ b/tests/fixtures/sca/pnpm-lock.yaml @@ -0,0 +1,52 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + lodash: + specifier: ^4.17.20 + version: 4.17.21 + express: + specifier: ^4.18.2 + version: 4.19.2 + devDependencies: + jest: + specifier: ^29.0.0 + version: 29.7.0 + +packages: + + /lodash@4.17.21: + resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvQ==} + + /express@4.19.2: + resolution: {integrity: sha512-5T4nhNdGxiQ9dWy9zE1c5c3pA6Zq9j8qE1yCfsFdt7O3p4wz7vwB4n8g3q7zad+wWngA1uk9A4p8tXfz6NQ==} + engines: {node: '>= 0.10.0'} + dependencies: + accepts: 1.3.8 + body-parser: 1.20.2 + transitivePeerDependencies: + - supports-color + + /jest@29.7.0: + resolution: {integrity: sha512-NMut2A1dABKc5JyLpcAyYzqLg==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@jest/core': 29.7.0 + '@jest/types': 29.6.3 + + /accepts@1.3.8: + resolution: {integrity: sha512-PYAthTa2mqVKQ==} + engines: {node: '>= 0.6'} + dependencies: + mime-types: 2.1.35 + + /body-parser@1.20.2: + resolution: {integrity: sha512-h.body==} + dependencies: + bytes: 3.1.2 diff --git a/tests/fixtures/sca/pom.xml b/tests/fixtures/sca/pom.xml new file mode 100644 index 00000000..4ff8ef92 --- /dev/null +++ b/tests/fixtures/sca/pom.xml @@ -0,0 +1,63 @@ + + + 4.0.0 + + com.example + demo-app + 1.0.0 + jar + + + 5.3.29 + 2.15.2 + + + + + + org.springframework + spring-core + ${spring.version} + + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + + + org.apache.commons + commons-lang3 + 3.13.0 + + + + + junit + junit + 4.13.2 + test + + + + + org.slf4j + slf4j-api + + + + + + + org.slf4j + slf4j-api + 2.0.7 + + + + diff --git a/tests/fixtures/sca/pubspec.lock b/tests/fixtures/sca/pubspec.lock new file mode 100644 index 00000000..5c801db5 --- /dev/null +++ b/tests/fixtures/sca/pubspec.lock @@ -0,0 +1,45 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + args: + dependency: "direct main" + description: + name: args + sha256: "abc123" + url: "https://pub.dev" + source: hosted + version: "2.4.2" + http: + dependency: "direct main" + description: + name: http + sha256: "def456" + url: "https://pub.dev" + source: hosted + version: "1.1.0" + test: + dependency: "direct dev" + description: + name: test + sha256: "ghi789" + url: "https://pub.dev" + source: hosted + version: "1.24.9" + async: + dependency: transitive + description: + name: async + sha256: "jkl012" + url: "https://pub.dev" + source: hosted + version: "2.11.0" + boolean_selector: + dependency: transitive + description: + name: boolean_selector + sha256: "mno345" + url: "https://pub.dev" + source: hosted + version: "2.1.1" +sdks: + dart: ">=3.0.0 <4.0.0" diff --git a/tests/fixtures/sca/pyproject.toml b/tests/fixtures/sca/pyproject.toml new file mode 100644 index 00000000..7f7ad6f0 --- /dev/null +++ b/tests/fixtures/sca/pyproject.toml @@ -0,0 +1,37 @@ +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +name = "demo" +version = "0.1.0" +description = "Demo project for SCA parser tests" +authors = ["CodeLens "] + +[tool.poetry.dependencies] +python = "^3.10" +django = "^4.2" +requests = ">=2.31,<3.0" +flask = {version = "~=2.3", extras = ["async"]} +numpy = "^1.24" + +[tool.poetry.group.dev.dependencies] +pytest = "^7.4" +black = "*" +mypy = {version = ">=1.4"} + +# PEP 621 section (also exercised) +[project] +name = "demo" +version = "0.1.0" +dependencies = [ + "click>=8.1.0", + "rich>=13.0,<14.0", + "tomli-w ; python_version < '3.11'", +] + +[project.optional-dependencies] +docs = [ + "sphinx>=7.0", + "sphinx-rtd-theme", +] diff --git a/tests/fixtures/sca/requirements.txt b/tests/fixtures/sca/requirements.txt new file mode 100644 index 00000000..1be829c1 --- /dev/null +++ b/tests/fixtures/sca/requirements.txt @@ -0,0 +1,27 @@ +# requirements.txt fixture for Issue #53 SCA parser tests. +# Covers: exact pin, range, extras, environment marker, URL spec, +# unpinned, and pip `-r`/`-c` options that must be skipped. + +# Pinned +django==4.2.13 +requests>=2.31.0 +flask[async]~=2.3.2 +# Comment line - should be ignored +numpy>=1.24.0,<2.0.0 +pandas==2.0.3 ; python_version >= '3.9' + +# Unpinned +pyyaml +pytest + +# Pip options (skipped) +-r requirements-extra.txt +-c constraints.txt +-e . +--index-url https://pypi.org/simple + +# URL spec +pdfminer @ https://github.com/pdfminer/pdfminer.six/archive/refs/tags/20220319.tar.gz + +# Marker +dataclasses-json>=0.5.7 ; python_version < '3.10' diff --git a/tests/fixtures/sca/yarn.lock b/tests/fixtures/sca/yarn.lock new file mode 100644 index 00000000..486b2fc2 --- /dev/null +++ b/tests/fixtures/sca/yarn.lock @@ -0,0 +1,25 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4": + version "7.22.13" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.13.tgz" + integrity sha512-aFk5wbsJ0pq= + dependencies: + "@babel/highlight" "^7.22.13" + +lodash@^4.17.20, lodash@^4.17.21: + version "4.17.21" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz" + integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvQ== + +express@^4.18.2: + version "4.19.2" + resolved "https://registry.yarnpkg.com/express/-/express-4.19.2.tgz" + dependencies: + accepts "~1.3.8" + +accepts@~1.3.8: + version "1.3.8" + resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.8.tgz" diff --git a/tests/test_sca_parsers.py b/tests/test_sca_parsers.py new file mode 100644 index 00000000..200dc0a2 --- /dev/null +++ b/tests/test_sca_parsers.py @@ -0,0 +1,640 @@ +"""Tests for the modular SCA lockfile/manifest parsers (Issue #53). + +These tests exercise every parser in ``scripts/sca_parsers/`` against +the fixtures in ``tests/fixtures/sca/`` and verify: + +- The ``Dependency`` dataclass shape (name/version/ecosystem/source_file/transitivity). +- That each parser correctly extracts the expected set of (name, version) + pairs from its sample fixture. +- That ``parse_lockfile()`` dispatches by basename correctly and returns + the right ecosystem string. +- That graceful failure holds: a corrupted/empty fixture returns [] + instead of raising. +- That ``vulnscan_engine.scan_vulnerabilities`` auto-detects the new + formats end-to-end. +""" + +from __future__ import annotations + +import os +import sys +import tempfile +import textwrap + +import pytest + +# Ensure scripts/ is importable +SCRIPTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "scripts")) +if SCRIPTS_DIR not in sys.path: + sys.path.insert(0, SCRIPTS_DIR) + +from sca_parsers import ( # noqa: E402 + Dependency, + ECOSYSTEM_BY_FILE, + PARSER_REGISTRY, + parse_lockfile, +) +from sca_parsers import ( # noqa: E402 + composer_lock, + gemfile_lock, + gradle_lock, + mix_lock, + package_resolved, + packages_lock, + pipfile, + pipfile_lock, + pnpm_lock, + pom_xml, + pubspec_lock, + pyproject_toml, + requirements_txt, + yarn_lock, +) + +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), "fixtures", "sca") + + +# ─── Helpers ─────────────────────────────────────────────────── + + +def _fixture(name: str) -> str: + return os.path.join(FIXTURES_DIR, name) + + +def _names_versions(deps): + """Convert list[Dependency] -> set[(name_lower, version)].""" + return {(d.name.lower(), d.version) for d in deps} + + +# ─── Registry & interface ────────────────────────────────────── + + +class TestRegistry: + def test_registry_contains_all_14_formats(self): + expected_files = { + "pnpm-lock.yaml", + "yarn.lock", + "Pipfile.lock", + "Pipfile", + "requirements.txt", + "pyproject.toml", + "Gemfile.lock", + "composer.lock", + "packages.lock.json", + "pubspec.lock", + "Package.resolved", + "gradle.lockfile", + "build.gradle", + "pom.xml", + "mix.lock", + } + assert expected_files.issubset(set(PARSER_REGISTRY.keys())) + + def test_ecosystem_map_covers_every_parser(self): + for fname in PARSER_REGISTRY: + assert fname in ECOSYSTEM_BY_FILE, ( + f"{fname} has no ecosystem mapping" + ) + + def test_dependency_dataclass_shape(self): + d = Dependency( + name="x", + version="1.0.0", + ecosystem="npm", + source_file="/tmp/x", + transitivity="direct", + ) + assert d.name == "x" + assert d.version == "1.0.0" + assert d.ecosystem == "npm" + assert d.source_file == "/tmp/x" + assert d.transitivity == "direct" + # default transitivity + d2 = Dependency(name="x", version="1", ecosystem="npm", source_file="/x") + assert d2.transitivity == "direct" + + +# ─── pnpm-lock.yaml ──────────────────────────────────────────── + + +class TestPnpmLock: + def test_parses_v9_lockfile(self): + path = _fixture("pnpm-lock.yaml") + deps = pnpm_lock.parse(path) + nv = _names_versions(deps) + assert ("lodash", "4.17.21") in nv + assert ("express", "4.19.2") in nv + assert ("jest", "29.7.0") in nv + assert ("accepts", "1.3.8") in nv + assert ("body-parser", "1.20.2") in nv + # All deps marked with ecosystem npm + assert all(d.ecosystem == "npm" for d in deps) + assert all(d.source_file == path for d in deps) + + def test_direct_vs_transitive(self): + deps = pnpm_lock.parse(_fixture("pnpm-lock.yaml")) + by_name = {d.name.lower(): d for d in deps} + # lodash/express/jest are declared in importers → direct + assert by_name["lodash"].transitivity == "direct" + assert by_name["express"].transitivity == "direct" + assert by_name["jest"].transitivity == "direct" + # accepts is only in packages (pulled by express) → transitive + assert by_name["accepts"].transitivity == "transitive" + + def test_empty_file(self, tmp_path): + p = tmp_path / "pnpm-lock.yaml" + p.write_text("") + assert pnpm_lock.parse(str(p)) == [] + + +# ─── yarn.lock ───────────────────────────────────────────────── + + +class TestYarnLock: + def test_parses_v1_lockfile(self): + path = _fixture("yarn.lock") + deps = yarn_lock.parse(path) + nv = _names_versions(deps) + assert ("lodash", "4.17.21") in nv + assert ("express", "4.19.2") in nv + assert ("accepts", "1.3.8") in nv + # @babel/code-frame with scoped name + assert ("@babel/code-frame", "7.22.13") in nv + assert all(d.ecosystem == "npm" for d in deps) + + def test_multiple_specifiers_one_block_dedups(self): + # The fixture has `lodash@^4.17.20, lodash@^4.17.21` mapping + # to version 4.17.21 — should produce ONE entry, not two. + deps = yarn_lock.parse(_fixture("yarn.lock")) + lodash = [d for d in deps if d.name.lower() == "lodash"] + assert len(lodash) == 1 + + def test_empty_file(self, tmp_path): + p = tmp_path / "yarn.lock" + p.write_text("# just a comment\n") + assert yarn_lock.parse(str(p)) == [] + + +# ─── Pipfile.lock ────────────────────────────────────────────── + + +class TestPipfileLock: + def test_parses_default_and_develop(self): + deps = pipfile_lock.parse(_fixture("Pipfile.lock")) + nv = _names_versions(deps) + assert ("django", "4.2.13") in nv + assert ("requests", "2.31.0") in nv + assert ("certifi", "2023.7.22") in nv + assert ("pytest", "7.4.0") in nv + assert all(d.ecosystem == "pypi" for d in deps) + + def test_strips_version_operators(self): + deps = pipfile_lock.parse(_fixture("Pipfile.lock")) + for d in deps: + assert not d.version.startswith("="), d + + def test_corrupt_json(self, tmp_path): + p = tmp_path / "Pipfile.lock" + p.write_text("{not valid json") + assert pipfile_lock.parse(str(p)) == [] + + +# ─── Gemfile.lock ────────────────────────────────────────────── + + +class TestGemfileLock: + def test_parses_specs(self): + deps = gemfile_lock.parse(_fixture("Gemfile.lock")) + nv = _names_versions(deps) + assert ("rake", "13.0.6") in nv + assert ("nokogiri", "1.13.10") in nv + assert ("rails", "7.0.8") in nv + assert ("i18n", "1.14.1") in nv + assert ("concurrent-ruby", "1.2.2") in nv + assert all(d.ecosystem == "gem" for d in deps) + + def test_direct_vs_transitive(self): + deps = gemfile_lock.parse(_fixture("Gemfile.lock")) + by_name = {d.name.lower(): d for d in deps} + # DEPENDENCIES block lists nokogiri and rails (>= 7.0.8) → direct + assert by_name["nokogiri"].transitivity == "direct" + assert by_name["rails"].transitivity == "direct" + # rake is not in DEPENDENCIES → transitive + assert by_name["rake"].transitivity == "transitive" + + def test_strips_platform_suffix(self, tmp_path): + p = tmp_path / "Gemfile.lock" + p.write_text(textwrap.dedent("""\ + GEM + remote: https://rubygems.org/ + specs: + nokogiri (1.13.10-java) + pg (1.4.6 x64-mingw32) + + DEPENDENCIES + nokogiri + """)) + deps = gemfile_lock.parse(str(p)) + nv = _names_versions(deps) + assert ("nokogiri", "1.13.10") in nv + assert ("pg", "1.4.6") in nv + + +# ─── composer.lock ───────────────────────────────────────────── + + +class TestComposerLock: + def test_parses_packages_and_dev(self): + deps = composer_lock.parse(_fixture("composer.lock")) + nv = _names_versions(deps) + assert ("symfony/console", "6.3.0") in nv + assert ("monolog/monolog", "3.4.0") in nv + assert ("guzzlehttp/guzzle", "7.7.0") in nv + assert ("phpunit/phpunit", "10.3.0") in nv + assert all(d.ecosystem == "composer" for d in deps) + + def test_strips_leading_v(self, tmp_path): + p = tmp_path / "composer.lock" + p.write_text('{"packages":[{"name":"x/y","version":"v1.2.3"}]}') + deps = composer_lock.parse(str(p)) + assert deps[0].version == "1.2.3" + + +# ─── packages.lock.json (NuGet) ──────────────────────────────── + + +class TestPackagesLock: + def test_parses_v2_per_framework(self): + deps = packages_lock.parse(_fixture("packages.lock.json")) + nv = _names_versions(deps) + # Direct deps + assert ("newtonsoft.json", "13.0.1") in nv or ("newtonsoft.json", "13.0.3") in nv + assert ("microsoft.net.test.sdk", "17.6.0") in nv + assert ("serilog", "3.0.1") in nv + # Transitive + assert ("system.text.json", "6.0.0") in nv + assert ("nunit", "3.13.3") in nv + # Ecosystem + assert all(d.ecosystem == "nuget" for d in deps) + + def test_transitivity_from_type_field(self): + deps = packages_lock.parse(_fixture("packages.lock.json")) + by_name = {d.name.lower(): d for d in deps} + # Newtonsoft is Direct in both frameworks + newtonsoft = [d for d in deps if d.name.lower() == "newtonsoft.json"] + assert all(d.transitivity == "direct" for d in newtonsoft) + # System.Text.Json is Transitive + assert by_name["system.text.json"].transitivity == "transitive" + + +# ─── pubspec.lock ────────────────────────────────────────────── + + +class TestPubspecLock: + def test_parses_packages(self): + deps = pubspec_lock.parse(_fixture("pubspec.lock")) + nv = _names_versions(deps) + assert ("args", "2.4.2") in nv + assert ("http", "1.1.0") in nv + assert ("test", "1.24.9") in nv + assert ("async", "2.11.0") in nv + assert all(d.ecosystem == "pub" for d in deps) + + def test_direct_vs_transitive(self): + deps = pubspec_lock.parse(_fixture("pubspec.lock")) + by_name = {d.name.lower(): d for d in deps} + assert by_name["args"].transitivity == "direct" + assert by_name["http"].transitivity == "direct" + assert by_name["test"].transitivity == "direct" + assert by_name["async"].transitivity == "transitive" + + +# ─── Package.resolved (SwiftPM) ──────────────────────────────── + + +class TestPackageResolved: + def test_parses_v1_pins(self): + deps = package_resolved.parse(_fixture("Package.resolved")) + nv = _names_versions(deps) + assert ("alamofire", "5.8.0") in nv + assert ("swiftlint", "0.53.0") in nv + assert ("keychainaccess", "4.2.2") in nv + assert all(d.ecosystem == "swiftpm" for d in deps) + + def test_v2_identity_fallback(self, tmp_path): + p = tmp_path / "Package.resolved" + p.write_text(textwrap.dedent("""\ + { + "object": { + "pins": [ + { + "identity": "alamofire", + "location": "https://github.com/Alamofire/Alamofire.git", + "state": { + "revision": "abc123def456abc123def456abc123def456abcd", + "version": "5.8.0" + } + } + ] + }, + "version": 2 + } + """)) + deps = package_resolved.parse(str(p)) + assert deps[0].name == "alamofire" + assert deps[0].version == "5.8.0" + + def test_branch_only_pin_falls_back_to_revision(self, tmp_path): + p = tmp_path / "Package.resolved" + p.write_text(textwrap.dedent("""\ + { + "object": { + "pins": [ + { + "package": "experimental", + "repositoryURL": "https://github.com/foo/exp.git", + "state": { + "branch": "main", + "revision": "abcdef1234567890abcdef1234567890abcdef12" + } + } + ] + }, + "version": 1 + } + """)) + deps = package_resolved.parse(str(p)) + assert deps[0].name == "experimental" + # Short sha (first 12 chars) + assert deps[0].version == "abcdef123456" + + +# ─── gradle.lockfile + build.gradle ──────────────────────────── + + +class TestGradleLock: + def test_parses_lockfile(self): + deps = gradle_lock.parse(_fixture("gradle.lockfile")) + nv = _names_versions(deps) + assert ("com.google.code.gson:gson", "2.8.9") in nv + assert ("com.squareup.retrofit2:retrofit", "2.9.0") in nv + assert ("io.reactivex.rxjava3:rxjava", "3.1.6") in nv + assert ("com.squareup.okhttp3:okhttp", "4.11.0") in nv + assert ("junit:junit", "4.13.2") in nv + assert all(d.ecosystem == "gradle" for d in deps) + + def test_parses_build_gradle(self): + deps = gradle_lock.parse(_fixture("build.gradle")) + nv = _names_versions(deps) + assert ("com.google.code.gson:gson", "2.8.9") in nv + assert ("com.squareup.retrofit2:retrofit", "2.9.0") in nv + assert ("junit:junit", "4.13.2") in nv + assert ("org.hamcrest:hamcrest", "2.2") in nv + assert all(d.ecosystem == "gradle" for d in deps) + assert all(d.transitivity == "direct" for d in deps) + + def test_skips_comments_and_empty_lines(self, tmp_path): + p = tmp_path / "gradle.lockfile" + p.write_text(textwrap.dedent("""\ + # This is a comment + empty=unused + + com.foo:bar:1.2.3:compileClasspath=1.2.3 + """)) + deps = gradle_lock.parse(str(p)) + assert len(deps) == 1 + assert deps[0].name == "com.foo:bar" + + +# ─── pom.xml ─────────────────────────────────────────────────── + + +class TestPomXml: + def test_parses_declared_dependencies(self): + deps = pom_xml.parse(_fixture("pom.xml")) + nv = _names_versions(deps) + # Concrete version + assert ("org.apache.commons:commons-lang3", "3.13.0") in nv + # Property-style versions are kept as-is (not resolved) + assert ("org.springframework:spring-core", "${spring.version}") in nv + assert ("com.fasterxml.jackson.core:jackson-databind", "${jackson.version}") in nv + # Test scope dep + assert ("junit:junit", "4.13.2") in nv + # Missing version → 0.0.0 + assert ("org.slf4j:slf4j-api", "0.0.0") in nv + assert all(d.ecosystem == "maven" for d in deps) + + def test_handles_malformed_xml(self, tmp_path): + p = tmp_path / "pom.xml" + p.write_text("") + deps = pom_xml.parse(str(p)) + # Should not raise; either empty or partial + assert isinstance(deps, list) + + +# ─── mix.lock ────────────────────────────────────────────────── + + +class TestMixLock: + def test_parses_hex_deps(self): + deps = mix_lock.parse(_fixture("mix.lock")) + nv = _names_versions(deps) + assert ("absinthe", "1.7.6") in nv + assert ("phoenix", "1.7.10") in nv + assert ("ecto", "3.10.3") in nv + assert ("decimal", "2.1.1") in nv + assert all(d.ecosystem == "hex" for d in deps) + + def test_git_dep_fallback(self): + deps = mix_lock.parse(_fixture("mix.lock")) + git_dep = [d for d in deps if d.name.lower() == "git_dep"] + assert git_dep, "git_dep should be extracted" + assert git_dep[0].version.startswith("git:") + # Short sha, 12 chars after the prefix + assert len(git_dep[0].version) == len("git:") + 12 + + +# ─── requirements.txt ────────────────────────────────────────── + + +class TestRequirementsTxt: + def test_parses_pinned_and_unpinned(self): + deps = requirements_txt.parse(_fixture("requirements.txt")) + by_name = {d.name.lower(): d for d in deps} + # Pinned + assert by_name["django"].version == "4.2.13" + # Range — extract the lower bound + assert by_name["requests"].version == "2.31.0" + # ~= operator + assert by_name["flask"].version == "2.3.2" + # Compound range + assert by_name["numpy"].version == "1.24.0" + # Marker stripped + assert by_name["pandas"].version == "2.0.3" + # Unpinned → 0.0.0 + assert by_name["pyyaml"].version == "0.0.0" + assert by_name["pytest"].version == "0.0.0" + # URL spec — name extracted, version 0.0.0 + assert "pdfminer" in by_name + assert by_name["pdfminer"].version == "0.0.0" + + def test_skips_pip_options(self): + deps = requirements_txt.parse(_fixture("requirements.txt")) + names = {d.name.lower() for d in deps} + # `-r`, `-c`, `-e`, `--index-url` lines should not produce deps + assert "-r" not in names + assert "-c" not in names + assert "-e" not in names + assert "--index-url" not in names + + def test_all_pypi_ecosystem(self): + deps = requirements_txt.parse(_fixture("requirements.txt")) + assert all(d.ecosystem == "pypi" for d in deps) + assert all(d.transitivity == "direct" for d in deps) + + +# ─── Pipfile ─────────────────────────────────────────────────── + + +class TestPipfile: + def test_parses_packages_and_dev(self): + deps = pipfile.parse(_fixture("Pipfile")) + by_name = {d.name.lower(): d for d in deps} + assert by_name["django"].version == "4.2.13" + assert by_name["requests"].version == "0.0.0" # unpinned "*" + assert by_name["flask"].version == "2.3.2" # ~= from table form + assert by_name["numpy"].version == "1.24.0" # range lower bound + assert by_name["pytest"].version == "7.4.0" + assert by_name["black"].version == "0.0.0" + assert by_name["mypy"].version == "1.4.0" # >= from table form + + def test_all_pypi_ecosystem(self): + deps = pipfile.parse(_fixture("Pipfile")) + assert all(d.ecosystem == "pypi" for d in deps) + + +# ─── pyproject.toml ──────────────────────────────────────────── + + +class TestPyprojectToml: + def test_parses_pep621_and_poetry(self): + deps = pyproject_toml.parse(_fixture("pyproject.toml")) + by_name = {d.name.lower(): d for d in deps} + # PEP 621 [project.dependencies] + assert by_name["click"].version == "8.1.0" + assert by_name["rich"].version == "13.0" + # Poetry [tool.poetry.dependencies] + assert by_name["django"].version == "4.2" + assert by_name["requests"].version == "2.31" + # Poetry table form: flask {version = "~=2.3", extras = ["async"]} + assert by_name["flask"].version == "2.3" + # Poetry group dev deps + assert by_name["pytest"].version == "7.4" + assert by_name["mypy"].version == "1.4" + # PEP 621 optional-dependencies + assert by_name["sphinx"].version == "7.0" + # 'python' should NOT be emitted as a dependency + assert "python" not in by_name + + def test_all_pypi_ecosystem(self): + deps = pyproject_toml.parse(_fixture("pyproject.toml")) + assert all(d.ecosystem == "pypi" for d in deps) + + +# ─── Dispatcher: parse_lockfile() ────────────────────────────── + + +class TestDispatcher: + @pytest.mark.parametrize( + "fname, expected_ecosystem, expected_pkgs", + [ + ("pnpm-lock.yaml", "npm", {"lodash"}), + ("yarn.lock", "npm", {"lodash"}), + ("Pipfile.lock", "pypi", {"django"}), + ("Pipfile", "pypi", {"django"}), + ("requirements.txt", "pypi", {"django"}), + ("pyproject.toml", "pypi", {"django"}), + ("Gemfile.lock", "gem", {"rake"}), + ("composer.lock", "composer", {"symfony/console"}), + ("packages.lock.json", "nuget", {"newtonsoft.json"}), + ("pubspec.lock", "pub", {"args"}), + ("Package.resolved", "swiftpm", {"alamofire"}), + ("gradle.lockfile", "gradle", {"com.google.code.gson:gson"}), + ("build.gradle", "gradle", {"com.google.code.gson:gson"}), + ("pom.xml", "maven", {"org.apache.commons:commons-lang3"}), + ("mix.lock", "hex", {"absinthe"}), + ], + ) + def test_dispatch_by_basename(self, fname, expected_ecosystem, expected_pkgs): + path = _fixture(fname) + deps, ecosystem = parse_lockfile(path) + assert ecosystem == expected_ecosystem + names = {d.name.lower() for d in deps} + for expected in expected_pkgs: + assert expected in names, f"{expected} not in {names} for {fname}" + + def test_unknown_filename_returns_empty(self, tmp_path): + p = tmp_path / "unknown.lock" + p.write_text("garbage") + deps, eco = parse_lockfile(str(p)) + assert deps == [] + assert eco is None + + def test_parser_error_does_not_raise(self, tmp_path): + # Make a file with the right name but unreadable content; ensure + # we get ([], None) back instead of an exception. + p = tmp_path / "Pipfile.lock" + p.write_text("{ not valid json") + deps, eco = parse_lockfile(str(p)) + assert deps == [] + # ecosystem is still returned even if parse failed + assert eco == "pypi" + + +# ─── End-to-end: vulnscan_engine integration ─────────────────── + + +class TestVulnscanIntegration: + """Verify vulnscan_engine picks up the new lockfile formats.""" + + def test_gemfile_lock_discovered_and_scanned(self, tmp_path): + # Copy fixture into a fresh workspace + ws = tmp_path + with open(_fixture("Gemfile.lock"), "rb") as f: + (ws / "Gemfile.lock").write_bytes(f.read()) + # Import lazily so the test still collects if the engine import fails + from vulnscan_engine import scan_vulnerabilities # noqa: WPS433 + result = scan_vulnerabilities(str(ws), offline=True) + assert result["status"] == "ok" + files_scanned = result.get("files_scanned") or [] + # Even if no VULN_DB hits, the file should appear in files_scanned. + # Note: scan_vulnerabilities may not surface files_scanned in the + # top-level dict; check stats if absent. + assert "stats" in result or "files_scanned" in result + + def test_pnpm_lock_yaml_discovered_and_scanned(self, tmp_path): + ws = tmp_path + with open(_fixture("pnpm-lock.yaml"), "rb") as f: + (ws / "pnpm-lock.yaml").write_bytes(f.read()) + from vulnscan_engine import scan_vulnerabilities # noqa: WPS433 + result = scan_vulnerabilities(str(ws), offline=True) + assert result["status"] == "ok" + + def test_pubspec_lock_discovered_and_scanned(self, tmp_path): + ws = tmp_path + with open(_fixture("pubspec.lock"), "rb") as f: + (ws / "pubspec.lock").write_bytes(f.read()) + from vulnscan_engine import scan_vulnerabilities # noqa: WPS433 + result = scan_vulnerabilities(str(ws), offline=True) + assert result["status"] == "ok" + + def test_multiple_new_formats_together(self, tmp_path): + ws = tmp_path + for fname in ("Gemfile.lock", "pnpm-lock.yaml", "pubspec.lock", + "composer.lock", "Package.resolved", "mix.lock"): + with open(_fixture(fname), "rb") as f: + (ws / fname).write_bytes(f.read()) + from vulnscan_engine import scan_vulnerabilities # noqa: WPS433 + result = scan_vulnerabilities(str(ws), offline=True) + assert result["status"] == "ok" + # No crash, no audit_available — just lockfile parsing path.