diff --git a/backend/cliff/agents/triage_deep/integration.py b/backend/cliff/agents/triage_deep/integration.py index aeacb11d..60c5701f 100644 --- a/backend/cliff/agents/triage_deep/integration.py +++ b/backend/cliff/agents/triage_deep/integration.py @@ -75,7 +75,7 @@ async def maybe_deep_dive( knowledge = { name: mgr.read_artifact(repo.id, name) - for name in ("profile", "code_map", "threat") + for name in ("profile", "code_map", "threat", "dep_manifest") } active_runner = runner or DeepDiveRunner( build_tier_models(ai_env, model_full_id), diff --git a/backend/cliff/agents/triage_deep/runner.py b/backend/cliff/agents/triage_deep/runner.py index 479e47ea..11852757 100644 --- a/backend/cliff/agents/triage_deep/runner.py +++ b/backend/cliff/agents/triage_deep/runner.py @@ -272,6 +272,7 @@ async def _run( "profile": repo_knowledge.get("profile"), "code_map": repo_knowledge.get("code_map"), "threat": repo_knowledge.get("threat"), + "dep_manifest": repo_knowledge.get("dep_manifest"), } def prov(exit_stage: str) -> TriageProvenance: diff --git a/backend/cliff/agents/triage_dep_manifest.py b/backend/cliff/agents/triage_dep_manifest.py new file mode 100644 index 00000000..307d0827 --- /dev/null +++ b/backend/cliff/agents/triage_dep_manifest.py @@ -0,0 +1,139 @@ +"""Deterministic dependency-manifest resolver (SP1 / ADR-0053 dep_manifest). + +Clears a **dependency** finding (``location`` shape ``pkg@version``) as +``false_positive`` BEFORE the LLM Deep dive when the package provably does not +ship: every root reaching its ``(name, version)`` node is dev/test/docs/build +(no prod, no optional) AND it is not imported by any first-party shipping source. + +Pure — no LLM, no network, no filesystem — keyless and CI-testable. This is a +*structural* clear like ``code_map`` (facts from the lockfile graph, not a model +hunch), so it is tier-independent under ADR-0054. + +Safety (never clear a shipping dependency): the clear requires TWO independent +gates to both hold — non-prod scope AND zero shipping import sites — plus a +resolved import name. Transitive reachability of a *vulnerable API* is NOT +decided here (that is Lane B / the Deep dive); this resolver only clears the +"doesn't ship at all" case. +""" + +from __future__ import annotations + +import re +from typing import Any + +from cliff.agents.schemas import TriageCheck, TriageOutput, TriageProvenance + +_CONF_DEP_CLEAR = 0.9 + +#: The ONLY scopes that are safe to clear (allowlist, not denylist). A node may be +#: cleared only if *every* scope reaching it is one of these — any other value +#: ("prod", "optional", or an unrecognized/future scope) blocks the clear, so a new +#: shipping scope can never silently pass the gate. +_CLEARABLE_SCOPES = frozenset({"dev", "test", "docs", "build"}) + + +def _parse_location(loc: str) -> tuple[str, str] | None: + """``"pkg@version"`` -> ``(name, version)``; scoped npm ``@scope/name@ver`` handled. + + Splits on the LAST ``@`` so the leading ``@`` of a scoped name is preserved. + """ + s = loc.strip() + if "@" not in s: + return None + name, _, version = s.rpartition("@") + if not name or not version: + return None + return name, version + + +def _pep503(name: str) -> str: + """PEP 503 canonical form (lowercase; runs of -_. → single -).""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def _name_matches(node: dict[str, Any], raw_name: str) -> bool: + """The finding's scanner-reported name vs a node's stored name. + + pypi nodes are stored PEP-503-normalized (``pyyaml``) but scanners report raw + names (``PyYAML``, ``ruamel.yaml``) — normalize before comparing. npm names + are compared verbatim (they are not PEP-503-normalized; ``lodash.merge`` ≠ + ``lodash-merge``). + """ + node_name = node.get("name") + if node.get("ecosystem") == "pypi": + return isinstance(node_name, str) and _pep503(raw_name) == node_name + return raw_name == node_name + + +def _clear(*, detail: str) -> TriageOutput: + return TriageOutput( + verdict="false_positive", + confidence=_CONF_DEP_CLEAR, + checks=[ + TriageCheck( + eyebrow="Dependency does not ship", + result="dev/test-only dependency", + kind="pass", + detail=detail, + ) + ], + provenance=TriageProvenance( + steps_run=["dep_manifest_resolver"], + exit_stage="dep_manifest_resolver", + escalated=False, + ), + ) + + +def resolve_by_dep_manifest( + finding: dict[str, Any], dep_manifest: dict[str, Any] | None +) -> TriageOutput | None: + """Clear *finding* if its ``pkg@version`` is a dev/test-only, unimported + dependency; else ``None`` (fall through to Lane B / the Deep dive).""" + if not dep_manifest: + return None + loc = finding.get("location") + if not isinstance(loc, str): + return None + parsed = _parse_location(loc) + if parsed is None: + return None + name, version = parsed + + nodes = dep_manifest.get("nodes") + if not isinstance(nodes, list): + return None + matches = [ + n + for n in nodes + if isinstance(n, dict) and _name_matches(n, name) and n.get("version") == version + ] + if len(matches) != 1: + # not found, or a cross-ecosystem (name, version) collision → conservative + return None + node = matches[0] + + scopes = node.get("scopes") + if not isinstance(scopes, list) or not scopes: + return None # unknown reachability → cannot prove non-prod → don't clear + scope_set = {s for s in scopes if isinstance(s, str)} + if not scope_set <= _CLEARABLE_SCOPES: + return None # a prod/optional/unknown scope reaches it → don't clear (allowlist) + + # Import-site gate — fail CLOSED: only a present, empty list proves "not imported". + # A missing / non-list / non-empty import_sites blocks the clear (pitfall 1). + import_sites = node.get("import_sites") + if not isinstance(import_sites, list) or import_sites: + return None # imported in first-party shipping code, or untrustworthy → don't clear + + if not node.get("import_name"): + # import name unresolved → couldn't run the import gate → don't clear (pitfall 3) + return None + + return _clear( + detail=( + f"{name}@{version} ({node.get('ecosystem', '?')}) reaches only " + f"{sorted(scope_set)} roots and is imported by no shipping first-party " + f"source (declared_in={node.get('declared_in') or 'transitive'})" + ) + ) diff --git a/backend/cliff/agents/triage_runner.py b/backend/cliff/agents/triage_runner.py index d235bd7e..ff181a63 100644 --- a/backend/cliff/agents/triage_runner.py +++ b/backend/cliff/agents/triage_runner.py @@ -22,6 +22,7 @@ from cliff.agents.sidebar_mapper import map_and_upsert from cliff.agents.triage_codemap import resolve_by_code_map from cliff.agents.triage_deep.integration import maybe_deep_dive +from cliff.agents.triage_dep_manifest import resolve_by_dep_manifest from cliff.db.repo_agent_run import ( create_agent_run, list_latest_runs_by_workspace_ids, @@ -107,13 +108,15 @@ def _structured_output(runs: dict[str, Any], agent_type: str) -> dict[str, Any] return run.structured_output if run is not None else None -async def _load_code_map(db: aiosqlite.Connection, repo_url: str | None) -> dict[str, Any] | None: - """The repo's cached code_map, or None when there's no ready profile — the - same resolution the Deep dive uses (cliff/agents/triage_deep/integration.py). +async def _load_artifact( + db: aiosqlite.Connection, repo_url: str | None, name: str +) -> dict[str, Any] | None: + """The repo's cached profile artifact *name*, or None when there's no ready + profile — the same resolution the Deep dive uses + (cliff/agents/triage_deep/integration.py). - Returns None (falls through to Deep dive) on any read/parse failure, or - when the artifact is not a dict — triage must never crash on a corrupt - code_map. + Returns None (falls through to Deep dive) on any read/parse failure, or when + the artifact is not a dict — triage must never crash on a corrupt artifact. """ if not repo_url: return None @@ -121,15 +124,21 @@ async def _load_code_map(db: aiosqlite.Connection, repo_url: str | None) -> dict if repo is None or repo.profile_status != "ready": return None try: - result = default_repo_dir_manager().read_artifact(repo.id, "code_map") + result = default_repo_dir_manager().read_artifact(repo.id, name) except Exception: - logger.debug( - "code_map unreadable for repo %s — falling through to Deep dive", repo_url - ) - return None - if not isinstance(result, dict): + logger.debug("%s unreadable for repo %s — falling through to Deep dive", name, repo_url) return None - return result + return result if isinstance(result, dict) else None + + +async def _load_code_map(db: aiosqlite.Connection, repo_url: str | None) -> dict[str, Any] | None: + return await _load_artifact(db, repo_url, "code_map") + + +async def _load_dep_manifest( + db: aiosqlite.Connection, repo_url: str | None +) -> dict[str, Any] | None: + return await _load_artifact(db, repo_url, "dep_manifest") async def run_triage( @@ -262,6 +271,22 @@ async def _run_scanner_triage( await _persist_synthesis(db, workspace.id, cleared) return cleared + # Deterministic dep_manifest gate (SP1): clear a dependency finding whose + # package provably does not ship (dev/test-only, unimported). The dep finding + # carries pkg@version in raw_payload; code_map's file-path gate never matches + # it, so this runs only for type=="dependency" and is a pure structural clear. + if finding.type == "dependency": + rp = finding.raw_payload or {} + pkg, ver = rp.get("package"), rp.get("version") + if pkg and ver: + dep_manifest = await _load_dep_manifest(db, workspace.repo_url) + dep_cleared = resolve_by_dep_manifest( + {**finding_ctx, "location": f"{pkg}@{ver}"}, dep_manifest + ) + if dep_cleared is not None: + await _persist_synthesis(db, workspace.id, dep_cleared) + return dep_cleared + # Escalate to the agentic Deep dive when warranted (ADR-0052). Best-effort: # any failure keeps the cheap Quick-read verdict — triage never breaks. final = quick diff --git a/backend/cliff/repos/deps/__init__.py b/backend/cliff/repos/deps/__init__.py new file mode 100644 index 00000000..06ec2186 --- /dev/null +++ b/backend/cliff/repos/deps/__init__.py @@ -0,0 +1,16 @@ +"""Deterministic dependency-manifest builder (ADR-0053 §3, dep_manifest artifact). + +Parses lockfiles/manifests across all workspaces into a graph keyed by +``(name, version)``, propagates scopes (prod dominates), finds first-party import +sites, and emits a ``DepManifest``. Pure — no LLM, no network. + +The public entry point ``build_dep_manifest`` is assembled in ``build.py`` and +re-exported here (SP1 Task 7). +""" + +from __future__ import annotations + +from .build import build_dep_manifest +from .graph import DepGraph, Root + +__all__ = ["DepGraph", "Root", "build_dep_manifest"] diff --git a/backend/cliff/repos/deps/build.py b/backend/cliff/repos/deps/build.py new file mode 100644 index 00000000..b4fd5172 --- /dev/null +++ b/backend/cliff/repos/deps/build.py @@ -0,0 +1,104 @@ +"""Assemble the ``dep_manifest`` artifact (SP1 Task 7). + +Discover manifests → parse npm + pypi graphs (each ecosystem independently) → +propagate scopes → resolve import names → attach first-party import sites (one +walk per ecosystem) → emit a ``DepManifest`` dict. + +Robustness: this builder runs in-process at scan time and parses *untrusted* +lockfiles. It MUST NOT raise — an uncaught parser error would fail the whole +profile build (``profile_status='error'``), which silently disables the code_map +gate, the dep_manifest gate, AND the Deep dive for that repo. Any per-ecosystem +failure is caught and recorded in ``unresolved``; the artifact is still emitted. +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +from ..schemas import DepManifest, DepNode +from .imports import collect_import_sites +from .manifests import discover_manifests +from .npm import parse_npm +from .pypi import dist_to_import, parse_pypi + +logger = logging.getLogger(__name__) + + +def _import_name(name: str, ecosystem: str) -> str | None: + if ecosystem == "npm": + return name # npm import name == package name (scoped kept) + if ecosystem == "pypi": + return dist_to_import(name) + return None + + +def _lookup_key(iname: str, ecosystem: str) -> str: + # pypi import sites are bucketed by TOP-LEVEL module; a dotted import name + # (``google.protobuf``) must be looked up by its first component (``google``). + return iname.split(".")[0] if ecosystem == "pypi" else iname + + +def _ecosystem_nodes(root: Path, ms: list, parser, eco: str) -> tuple[list[dict], list[str]]: + """Parse one ecosystem into DepNode dicts + its unresolved manifests. Never raises.""" + try: + graph, unresolved = parser(root, ms) + node_map = graph.nodes + if not node_map: + return [], list(unresolved) + scopes = graph.propagate_scopes() + direct = graph.direct_nodes() + declared = graph.declared_in_map() + imports = collect_import_sites(root, eco) # one walk per ecosystem + nodes: list[dict] = [] + for (name, version), node_eco in node_map.items(): + iname = _import_name(name, node_eco) + sites = imports.get(_lookup_key(iname, node_eco), []) if iname else [] + nodes.append( + DepNode( + name=name, + version=version, + ecosystem=node_eco, + scopes=sorted(scopes.get((name, version), set())), + direct=(name, version) in direct, + declared_in=declared.get((name, version), []), + import_name=iname, + import_sites=sites, + ).model_dump() + ) + return nodes, list(unresolved) + except Exception: # noqa: BLE001 - a bad lockfile must not fail the whole profile + logger.warning( + "dep_manifest: %s parse failed — recording as unresolved", eco, exc_info=True + ) + return [], [f"<{eco}-parse-error>"] + + +def build_dep_manifest(root) -> dict: # noqa: ANN001 - Path | str + root = Path(root) + try: + manifests = discover_manifests(root) + except Exception: # noqa: BLE001 - never let discovery crash the profile build + logger.warning("dep_manifest: manifest discovery failed", exc_info=True) + return DepManifest(unresolved=[""]).model_dump() + + npm_ms = [m for m in manifests if m.ecosystem == "npm"] + py_ms = [m for m in manifests if m.ecosystem == "pypi"] + nodes: list[dict] = [] + unresolved: list[str] = [] + ecosystems: list[str] = [] + for ms, parser, eco in ((npm_ms, parse_npm, "npm"), (py_ms, parse_pypi, "pypi")): + if not ms: + continue + eco_nodes, eco_unresolved = _ecosystem_nodes(root, ms, parser, eco) + unresolved.extend(eco_unresolved) + if eco_nodes: + ecosystems.append(eco) + nodes.extend(eco_nodes) + + return DepManifest( + ecosystems=ecosystems, + workspaces=[m.path for m in manifests], + nodes=nodes, + unresolved=unresolved, + ).model_dump() diff --git a/backend/cliff/repos/deps/graph.py b/backend/cliff/repos/deps/graph.py new file mode 100644 index 00000000..a6f04f0f --- /dev/null +++ b/backend/cliff/repos/deps/graph.py @@ -0,0 +1,85 @@ +"""The resolved dependency graph — parser-agnostic structure + scope propagation. + +A node is a ``(name, version)`` pair. A *root* is a node that some workspace +manifest declares as a top-level dependency, tagged with the scope it was +declared in (prod/dev/test/…). Edges point from a package to each package it +depends on (from the lockfile's resolved tree). + +``propagate_scopes`` unions each root's scope onto every node reachable from it. +This is the load-bearing computation: a transitive node reachable from a prod +root ships (``prod`` in its scopes); one reachable only from dev roots does not. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +NV = tuple[str, str] # (name, version) + + +@dataclass(frozen=True) +class Root: + name: str + version: str + scope: str # prod | dev | test | docs | build | optional + declared_in: str # manifest path + + +class DepGraph: + def __init__(self) -> None: + self._nodes: dict[NV, str] = {} # (name, version) -> ecosystem + self._edges: dict[NV, set[NV]] = {} # (name, version) -> deps + self.roots: list[Root] = [] + + # ── construction ──────────────────────────────────────────────────────── + def add_node(self, name: str, version: str, ecosystem: str) -> None: + nv = (name, version) + self._nodes.setdefault(nv, ecosystem) + self._edges.setdefault(nv, set()) + + def add_root( + self, name: str, version: str, scope: str, declared_in: str, ecosystem: str + ) -> None: + self.add_node(name, version, ecosystem) + self.roots.append(Root(name=name, version=version, scope=scope, declared_in=declared_in)) + + def add_edge(self, frm: NV, to: NV) -> None: + self._edges.setdefault(frm, set()).add(to) + self._edges.setdefault(to, set()) + + # ── queries ────────────────────────────────────────────────────────────── + @property + def nodes(self) -> dict[NV, str]: + return self._nodes + + def direct_nodes(self) -> set[NV]: + return {(r.name, r.version) for r in self.roots} + + def declared_in_map(self) -> dict[NV, list[str]]: + out: dict[NV, list[str]] = {} + for r in self.roots: + out.setdefault((r.name, r.version), []) + if r.declared_in not in out[(r.name, r.version)]: + out[(r.name, r.version)].append(r.declared_in) + return out + + def propagate_scopes(self) -> dict[NV, set[str]]: + """Union each root's scope onto every node reachable from it (prod dominates + by set membership — the resolver checks ``'prod' in scopes``).""" + scopes: dict[NV, set[str]] = {nv: set() for nv in self._nodes} + for root in self.roots: + start: NV = (root.name, root.version) + if start not in self._nodes: + continue + stack = [start] + seen: set[NV] = set() + while stack: + nv = stack.pop() + if nv in seen: + continue + seen.add(nv) + scopes.setdefault(nv, set()).add(root.scope) + for dep in self._edges.get(nv, ()): # traverse resolved edges + if dep not in seen: + stack.append(dep) + return scopes diff --git a/backend/cliff/repos/deps/imports.py b/backend/cliff/repos/deps/imports.py new file mode 100644 index 00000000..ff2109e7 --- /dev/null +++ b/backend/cliff/repos/deps/imports.py @@ -0,0 +1,109 @@ +"""First-party import-site scan — the pitfall-1 repo-wide grep. + +``collect_import_sites`` does ONE walk of shipping (non-test) first-party source +and buckets every import by the imported *top-level* name. This is the second, +independent safety gate: even if a package's scope were mis-computed as dev-only, +a non-empty result blocks the clear. It scans by import name (version-agnostic), +so a package imported anywhere in shipping first-party is never dev-cleared. +""" + +from __future__ import annotations + +import os +import re + +from .manifests import _BUILD_DIRS, _DOCS_DIRS, _SKIP_DIRS, _TEST_DIRS + +_EXCLUDE_DIRS = _SKIP_DIRS | _TEST_DIRS | _DOCS_DIRS | _BUILD_DIRS + +_NPM_EXT = {".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".vue", ".svelte"} +_PY_EXT = {".py", ".pyi"} +# test/non-ship file basenames, regardless of directory +_TEST_FILE_RE = re.compile(r"(\.|_|-)(test|spec)\.|(^|/)(test|conftest)_|_test\.py$", re.I) +_MAX_SITES = 30 + +# Matches the module specifier in `import x from 'spec'`, `require('spec')`, +# `import('spec')`, and re-exports `export ... from 'spec'`. +_NPM_SPEC_RE = re.compile( + r"""(?:from|import)\s*\(?\s*['"]([^'"]+)['"]|require\(\s*['"]([^'"]+)['"]""" +) +_PY_FROM_RE = re.compile(r"^\s*from\s+([A-Za-z_]\w*)") +_PY_IMPORT_RE = re.compile(r"^\s*import\s+(.+)") + + +def _is_test_file(rel: str) -> bool: + return bool(_TEST_FILE_RE.search(rel.replace("\\", "/"))) + + +def _npm_spec_to_pkg(spec: str) -> str | None: + if not spec or spec.startswith(".") or spec.startswith("/"): + return None # relative / absolute — not a package + parts = spec.split("/") + if spec.startswith("@"): + return "/".join(parts[:2]) if len(parts) >= 2 else None # @scope/name + return parts[0] + + +def _py_import_names(line: str) -> list[str]: + """Top-level module names imported on a Python line. + + ``from a.b import c`` → ``["a"]``; ``import a.b, c as d`` → ``["a", "c"]``. + Buckets by the TOP-LEVEL package so a dist whose import name is dotted + (``protobuf`` → ``google.protobuf``) is found when callers look up ``google``. + """ + m = _PY_FROM_RE.match(line) + if m: + return [m.group(1)] + m = _PY_IMPORT_RE.match(line) + if not m: + return [] + names: list[str] = [] + for part in m.group(1).split(","): + top = part.strip().split()[0].split(".")[0] if part.strip() else "" + if top.isidentifier(): + names.append(top) + return names + + +def collect_import_sites(root, ecosystem: str) -> dict[str, list[str]]: # noqa: ANN001 - Path + """One walk of shipping first-party source → ``{package_or_import_name: [file:line, ...]}``. + + npm keys are package names (``lodash``, ``@scope/name``); pypi keys are + top-level import names (``yaml``, ``aiohttp``, ``google``). Callers match a + node's ``import_name`` (top-level component, for pypi) against these keys. + """ + root = os.fspath(root) + is_npm = ecosystem == "npm" + exts = _NPM_EXT if is_npm else _PY_EXT if ecosystem == "pypi" else set() + if not exts: + return {} + out: dict[str, list[str]] = {} + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [d for d in dirnames if d.lower() not in _EXCLUDE_DIRS] + for fn in filenames: + if os.path.splitext(fn)[1].lower() not in exts: + continue + rel = os.path.relpath(os.path.join(dirpath, fn), root) + if _is_test_file(rel): + continue + try: + with open(os.path.join(dirpath, fn), encoding="utf-8", errors="ignore") as fh: + for i, line in enumerate(fh, 1): + # prefilter: `from`/`export … from` re-exports have "from" + # but not "import"/"require". + if "import" not in line and "require" not in line and "from" not in line: + continue + if is_npm: + m = _NPM_SPEC_RE.search(line) + pkgs = [_npm_spec_to_pkg(m.group(1) or m.group(2) or "")] if m else [] + else: + pkgs = _py_import_names(line) + for pkg in pkgs: + if not pkg: + continue + bucket = out.setdefault(pkg, []) + if len(bucket) < _MAX_SITES: + bucket.append(f"{rel}:{i}") + except OSError: + continue + return out diff --git a/backend/cliff/repos/deps/manifests.py b/backend/cliff/repos/deps/manifests.py new file mode 100644 index 00000000..9db29ae6 --- /dev/null +++ b/backend/cliff/repos/deps/manifests.py @@ -0,0 +1,158 @@ +"""Manifest discovery + scope classification. + +``classify_scope`` maps a (manifest path, dependency section) to a scope. It is +**conservative**: a manifest living under a non-shipping directory (docs/, +examples/, test/, …) makes all its deps non-prod; an unrecognized section +defaults to ``prod`` (so we never *accidentally* clear something as dev-only). +This is safety-critical — a package wrongly scoped non-prod could be false-cleared +(the import-site gate is the second line of defence). +""" + +from __future__ import annotations + +import os +import re +from dataclasses import dataclass + +# Directory segments whose manifests are treated as non-shipping (docs sites, +# example apps). CAUTION — this set is *coupled*: it both (a) forces a manifest's +# deps to a non-prod scope here and (b) is reused as an import-scan exclusion in +# imports.py. So a dep unique to one of these workspaces fails BOTH the scope gate +# and the import gate together. Keep it to names that are almost never the main +# shipping app; bare "site"/"demo" (commonly a shipping app root) are deliberately +# excluded. A production app whose entire source lives under docs/ or website/ +# with deps declared only there is the known residual false-clear edge case. +_DOCS_DIRS = {"docs", "doc", "documentation", "website", "examples", "example"} +_TEST_DIRS = { + "test", "tests", "__tests__", "__mocks__", "spec", "specs", "e2e", "cypress", "playwright" +} +# NOTE: intentionally NOT including scripts/tools here — real shipping code lives +# in scripts/ dirs (e.g. worker entrypoints), so excluding them from the import +# scan would false-negative "not imported" and risk an unsafe clear. +_BUILD_DIRS = {"benchmark", "benchmarks", "bench", "storybook", ".storybook"} + +# Dependency section -> scope. Unknown -> "prod" (conservative). +_SECTION_SCOPE: dict[str, str] = { + # npm + "dependencies": "prod", + "devDependencies": "dev", + "optionalDependencies": "optional", + "peerDependencies": "prod", # conservative: consumer is expected to provide it + # python (normalized section keys) + "project.dependencies": "prod", + "project.optional-dependencies": "optional", + "dependency-groups": "dev", + "tool.poetry.dependencies": "prod", + "tool.poetry.group.dev.dependencies": "dev", + "tool.poetry.dev-dependencies": "dev", + "requirements.txt": "prod", + "requirements-dev.txt": "dev", + "requirements-doc.txt": "docs", + "requirements-docs.txt": "docs", + "requirements-examples.txt": "docs", + "requirements-example.txt": "docs", + "requirements-test.txt": "test", + "requirements-tests.txt": "test", +} + + +def _path_dir_scope(manifest_path: str) -> str | None: + """Non-prod scope implied by the manifest's *directory*, else None.""" + parts = manifest_path.replace("\\", "/").split("/")[:-1] # drop filename + for seg in parts: + s = seg.lower() + if s in _DOCS_DIRS: + return "docs" + if s in _TEST_DIRS: + return "test" + if s in _BUILD_DIRS: + return "build" + return None + + +#: Python ``[project.optional-dependencies]`` EXTRA names that denote non-shipping +#: dev/docs/test tooling (vs feature extras like ``anthropic``/``postgres`` which +#: DO ship when installed → ``optional``). Conservative: unknown extra → optional. +_DEV_EXTRA_NAMES: dict[str, str] = { + "dev": "dev", "develop": "dev", "development": "dev", "devel": "dev", + "lint": "dev", "linting": "dev", "typing": "dev", "types": "dev", "mypy": "dev", + "style": "dev", "format": "dev", "ci": "dev", "check": "dev", "checks": "dev", + "docs": "docs", "doc": "docs", "documentation": "docs", "test-docs": "docs", + "test": "test", "tests": "test", "testing": "test", + "benchmark": "build", "benchmarks": "build", "bench": "build", "build": "build", +} + + +def classify_extra(extra_name: str) -> str: + """Scope for a Python ``optional-dependencies`` extra by its NAME. + + A ``dev``/``docs``/``test`` extra is non-shipping tooling; a feature extra + (``anthropic``, ``postgres``, …) ships when installed → ``optional``. + """ + return _DEV_EXTRA_NAMES.get(extra_name.strip().lower(), "optional") + + +def classify_scope(manifest_path: str, section: str) -> str: + """Scope for a dependency declared in ``section`` of ``manifest_path``. + + Directory dominates (a dep in a docs/ manifest is docs-scope regardless of + section); otherwise the section maps per ``_SECTION_SCOPE``; unknown → prod. + """ + dir_scope = _path_dir_scope(manifest_path) + if dir_scope is not None: + return dir_scope + if section in _SECTION_SCOPE: + return _SECTION_SCOPE[section] + # poetry groups other than 'dev' (test/docs/typing/…): scope by the group name + # so a test/docs-only group is cleared, not defaulted to prod. + m = re.fullmatch(r"tool\.poetry\.group\.(.+)\.dependencies", section) + if m: + return classify_extra(m.group(1)) + return "prod" + + +@dataclass(frozen=True) +class Manifest: + path: str # relative to repo root + ecosystem: str # "npm" | "pypi" + kind: str # package.json | pyproject.toml | requirements | pnpm-workspace | setup.cfg | pipfile + + +_SKIP_DIRS = { + "node_modules", ".git", "dist", "build", ".venv", "venv", "site-packages", ".tox", ".next", + "__pycache__", "vendor", +} + +_NPM_NAMES = {"package.json"} +_PNPM_WS = {"pnpm-workspace.yaml", "pnpm-workspace.yml"} + + +def discover_manifests(root) -> list[Manifest]: # noqa: ANN001 - Path + """Every first-party manifest under ``root`` (node_modules/venv/etc excluded).""" + root = os.fspath(root) + out: list[Manifest] = [] + for dirpath, dirnames, filenames in os.walk(root): + # case-insensitive, consistent with imports.py's exclusion walk + dirnames[:] = [d for d in dirnames if d.lower() not in _SKIP_DIRS] + for fn in filenames: + rel = os.path.relpath(os.path.join(dirpath, fn), root) + low = fn.lower() + if fn in _NPM_NAMES: + out.append(Manifest(rel, "npm", "package.json")) + elif fn in _PNPM_WS: + out.append(Manifest(rel, "npm", "pnpm-workspace")) + elif fn == "pyproject.toml": + out.append(Manifest(rel, "pypi", "pyproject.toml")) + elif fn == "setup.cfg": + out.append(Manifest(rel, "pypi", "setup.cfg")) + elif fn == "Pipfile": + out.append(Manifest(rel, "pypi", "pipfile")) + elif (low.startswith("requirements") and low.endswith((".txt", ".in"))) or ( + # requirements/base.txt, requirements/prod.in, etc. — declaring + # prod deps only here must not be missed (a missed prod root risks + # a false clear of a shipping dependency). + os.path.basename(dirpath).lower() == "requirements" + and low.endswith((".txt", ".in")) + ): + out.append(Manifest(rel, "pypi", "requirements")) + return out diff --git a/backend/cliff/repos/deps/npm.py b/backend/cliff/repos/deps/npm.py new file mode 100644 index 00000000..ccf21f39 --- /dev/null +++ b/backend/cliff/repos/deps/npm.py @@ -0,0 +1,562 @@ +"""Deterministic npm/JS lockfile parser (ADR-0053 §3, dep_manifest artifact). + +Builds the npm portion of the dependency graph from lockfiles + workspace +``package.json`` files. Pure — no network, no LLM. Supports the three formats the +JS ecosystem actually ships: + +* **pnpm-lock.yaml** (v9): ``importers:`` gives each workspace's declared deps with + a pre-resolved ``version``; ``packages:`` is the deduplicated node set; ``snapshots:`` + carries the resolved dependency edges. Version strings can carry a peer suffix + ``1.2.3(peer@4)`` — stripped to ``1.2.3``. +* **yarn.lock** — classic v1 (``name@range:`` blocks with ``version "x"``, NOT valid + YAML → line parser) and berry v2+ (``__metadata`` header, ``@npm:`` descriptors, + valid YAML → PyYAML). +* **package-lock.json** (v2/v3): ``packages`` object keyed by ``""`` and + ``node_modules/`` install paths. + +SAFETY: a dependency is only cleared as dev-only noise if its ``(name, version)`` +node carries NO prod scope. **Under-connecting a prod edge is the dangerous +direction** — it can make a shipping package look dev-only and get false-cleared. +So when a range→version resolution or an edge is ambiguous, this parser prefers to +OVER-connect (add the edge / add every candidate version as a prod root) rather +than miss one. It never fabricates a node absent from the lockfile. +""" + +from __future__ import annotations + +import json +import os +import re +from pathlib import Path + +import yaml + +try: # fast C loader when available; identical semantics to SafeLoader + from yaml import CSafeLoader as _YamlLoader +except ImportError: # pragma: no cover - pure-python fallback + from yaml import SafeLoader as _YamlLoader + +from .graph import DepGraph +from .manifests import ( + _SKIP_DIRS, # noqa: PLC2701 - shared walk-exclusion set + Manifest, + classify_scope, + discover_manifests, +) + +NV = tuple[str, str] + +# npm dependency sections we read from a first-party package.json / importer. +_SECTIONS = ("dependencies", "devDependencies", "optionalDependencies", "peerDependencies") +# transitive edge sections we follow inside the lockfile (peer deps are provided +# by the consumer, so they are NOT transitive install edges). +_EDGE_SECTIONS = ("dependencies", "optionalDependencies") + +# version strings that are workspace/local/non-registry refs — not resolvable to a +# published (name, version) node, so they are not emitted as external roots. +_LOCAL_PREFIXES = ("link:", "file:", "workspace:", "portal:", "self:") + + +# ── name/version helpers ───────────────────────────────────────────────────── +def _strip_peer(version: str) -> str: + """Drop a pnpm/berry peer suffix: ``1.2.3(react@18)`` → ``1.2.3``.""" + return str(version).split("(", 1)[0].strip() + + +def _split_key(key: str) -> tuple[str, str]: + """Split a ``name@version`` lock key into ``(name, version)``, scope-aware. + + The leading ``@`` of a scoped name is NOT the separator: the version is what + follows the LAST ``@``. Any peer suffix is stripped first. + """ + base = _strip_peer(key) + at = base.rfind("@") + if at <= 0: # no separator (or leading-@ only) — not a resolvable node key + return base, "" + return base[:at], base[at + 1 :] + + +def _is_local(version: str) -> bool: + v = str(version) + return any(v.startswith(p) for p in _LOCAL_PREFIXES) + + +# ── range → version resolution (yarn/npm; pnpm resolves itself) ─────────────── +def _resolve_range( + name: str, + rng: str, + by_descriptor: dict[str, str], + versions_by_name: dict[str, set[str]], + descriptor_candidates: list[str], +) -> list[str]: + """Resolve a declared ``(name, range)`` to installed version(s). + + Tries the exact descriptor forms first (the common, unambiguous case). Falls + back — safely OVER-connecting — when the descriptor is not found: + * the range is itself an installed exact version → that version, + * the name has exactly one installed version → that version, + * else every installed version of the name (never miss the shipping one). + Returns ``[]`` only when the name is absent from the lockfile (never invent). + """ + for cand in descriptor_candidates: + if cand in by_descriptor: + return [by_descriptor[cand]] + versions = versions_by_name.get(name) + if not versions: + return [] + bare = _strip_peer(rng) + if bare in versions: # range was an exact pinned version + return [bare] + if len(versions) == 1: + return [next(iter(versions))] + return sorted(versions) # ambiguous → over-connect across all versions + + +# ── pnpm-lock.yaml (v9) ────────────────────────────────────────────────────── +def parse_pnpm_lock( + text: str, lock_dir: str +) -> tuple[set[NV], list[tuple[NV, NV]], list[tuple[str, str, str, str]]]: + """Return ``(nodes, edges, roots)`` for a pnpm-lock. + + ``roots`` items are ``(name, version, section, manifest_path)`` — manifest_path + is repo-relative (the importer dir joined onto ``lock_dir``). + """ + doc = yaml.load(text, Loader=_YamlLoader) or {} + nodes: set[NV] = set() + edges: list[tuple[NV, NV]] = [] + roots: list[tuple[str, str, str, str]] = [] + + # nodes: every deduplicated package identity + for key in doc.get("packages", {}) or {}: + name, ver = _split_key(key) + if name and ver: + nodes.add((name, ver)) + + # edges: from resolved snapshots (fall back to any deps carried on packages) + def _emit_edges(container: dict) -> None: + for key, body in (container or {}).items(): + if not isinstance(body, dict): + continue + frm = _split_key(key) + if not frm[0] or not frm[1]: + continue + nodes.add(frm) + for section in _EDGE_SECTIONS: + for dep_name, dep_ver in (body.get(section) or {}).items(): + to = (dep_name, _strip_peer(dep_ver)) + if to[1]: + nodes.add(to) + edges.append((frm, to)) + + _emit_edges(doc.get("snapshots", {})) + _emit_edges(doc.get("packages", {})) # older layouts inline deps on packages + + # roots: each importer (workspace) declaration, version pre-resolved by pnpm + for importer, body in sorted((doc.get("importers", {}) or {}).items()): + manifest_path = _join_manifest(lock_dir, importer) + if not isinstance(body, dict): + continue + for section in _SECTIONS: + for name, info in sorted((body.get(section) or {}).items()): + version = info.get("version") if isinstance(info, dict) else info + if version is None or _is_local(str(version)): + continue + ver = _strip_peer(str(version)) + if ver: + roots.append((name, ver, section, manifest_path)) + return nodes, edges, roots + + +def _join_manifest(lock_dir: str, importer: str) -> str: + """Repo-relative path of an importer's package.json.""" + if importer in (".", ""): + base = lock_dir + else: + base = os.path.join(lock_dir, importer) if lock_dir else importer + return os.path.normpath(os.path.join(base, "package.json")).replace("\\", "/") + + +# ── yarn.lock ──────────────────────────────────────────────────────────────── +def _is_berry(text: str) -> bool: + return "__metadata:" in text or bool(re.search(r"^\s+resolution:", text, re.M)) + + +def _parse_yarn_berry( + text: str, +) -> tuple[dict[str, str], dict[str, set[str]], set[NV], list[tuple[NV, str, str]]]: + """Berry v2+ yarn.lock is valid YAML. + + Returns ``(by_descriptor, versions_by_name, nodes, edge_specs)`` where + ``edge_specs`` are ``((from_name, from_ver), dep_name, dep_rangeval)``. + """ + doc = yaml.load(text, Loader=_YamlLoader) or {} + by_descriptor: dict[str, str] = {} + versions_by_name: dict[str, set[str]] = {} + nodes: set[NV] = set() + edge_specs: list[tuple[NV, str, str]] = [] + + for raw_key, body in doc.items(): + if raw_key == "__metadata" or not isinstance(body, dict): + continue + version = body.get("version") + if version is None: + continue + version = _strip_peer(str(version)) + descriptors = [d.strip() for d in str(raw_key).split(",")] + name = _descriptor_name(descriptors[0]) if descriptors else None + resolution = body.get("resolution") + if resolution: # canonical (name, version) from the resolution field + name = _descriptor_name(str(resolution)) or name + if not name: + continue + node = (name, version) + nodes.add(node) + versions_by_name.setdefault(name, set()).add(version) + for desc in descriptors: + by_descriptor[desc] = version + for section in _EDGE_SECTIONS: + for dep_name, dep_val in (body.get(section) or {}).items(): + edge_specs.append((node, dep_name, str(dep_val))) + return by_descriptor, versions_by_name, nodes, edge_specs + + +def _descriptor_name(descriptor: str) -> str: + """Name from a yarn descriptor/resolution (berry or classic): + ``@scope/x@npm:1.2`` → ``@scope/x``. The leading ``@`` of a scoped name is not + the separator.""" + at = descriptor.rfind("@") + return descriptor[:at] if at > 0 else descriptor + + +def _parse_yarn_classic( + text: str, +) -> tuple[dict[str, str], dict[str, set[str]], set[NV], list[tuple[NV, str, str]]]: + """Classic v1 yarn.lock — line parser (``version "x"`` is not valid YAML).""" + by_descriptor: dict[str, str] = {} + versions_by_name: dict[str, set[str]] = {} + nodes: set[NV] = set() + edge_specs: list[tuple[NV, str, str]] = [] + + blocks: list[tuple[list[str], str | None, list[tuple[str, str]]]] = [] + cur_keys: list[str] | None = None + cur_ver: str | None = None + cur_deps: list[tuple[str, str]] = [] + in_deps = False + + def flush() -> None: + if cur_keys is not None: + blocks.append((cur_keys, cur_ver, cur_deps)) + + for line in text.splitlines(): + if not line.strip() or line.lstrip().startswith("#"): + continue + indent = len(line) - len(line.lstrip()) + if indent == 0 and line.rstrip().endswith(":"): + flush() + header = line.rstrip()[:-1] + cur_keys = [_unquote(k.strip()) for k in header.split(",")] + cur_ver = None + cur_deps = [] + in_deps = False + continue + if cur_keys is None: + continue + stripped = line.strip() + if indent == 2: + in_deps = False + m = re.match(r'version:?\s+"?([^"\s]+)"?', stripped) + if m: + cur_ver = m.group(1) + elif stripped in ("dependencies:", "optionalDependencies:"): + in_deps = True + elif indent >= 4 and in_deps: + m = re.match(r'("?[^"\s]+"?)\s+"?([^"]+?)"?$', stripped) + if m: + cur_deps.append((_unquote(m.group(1)), _unquote(m.group(2)))) + flush() + + for keys, ver, deps in blocks: + if not ver: + continue + ver = _strip_peer(ver) + name = _descriptor_name(keys[0]) if keys else None + if not name: + continue + node = (name, ver) + nodes.add(node) + versions_by_name.setdefault(name, set()).add(ver) + for k in keys: + by_descriptor[k] = ver + for dep_name, dep_range in deps: + edge_specs.append((node, dep_name, dep_range)) + return by_descriptor, versions_by_name, nodes, edge_specs + + +def _unquote(s: str) -> str: + s = s.strip() + if len(s) >= 2 and s[0] == s[-1] and s[0] in "\"'": + return s[1:-1] + return s + + +def parse_yarn_lock( + text: str, manifests_here: list[tuple[str, str, str, str]] +) -> tuple[set[NV], list[tuple[NV, NV]], list[tuple[str, str, str, str]]]: + """Return ``(nodes, edges, roots)`` for a yarn.lock (classic or berry). + + ``manifests_here`` items are ``(name, range, section, manifest_path)`` — every + dependency declared by a first-party package.json governed by this lockfile. + """ + berry = _is_berry(text) + by_descriptor, versions_by_name, nodes, edge_specs = ( + _parse_yarn_berry(text) if berry else _parse_yarn_classic(text) + ) + + def candidates(name: str, rng: str) -> list[str]: + if berry: + return [f"{name}@{rng}", f"{name}@npm:{rng}"] + return [f"{name}@{rng}"] + + # edges: resolve each dep range against the descriptor map + edges: list[tuple[NV, NV]] = [] + for frm, dep_name, dep_val in edge_specs: + if _is_local(dep_val): + continue + for ver in _resolve_range( + dep_name, dep_val, by_descriptor, versions_by_name, candidates(dep_name, dep_val) + ): + to = (dep_name, ver) + nodes.add(to) + edges.append((frm, to)) + + # roots: resolve each first-party declaration + roots: list[tuple[str, str, str, str]] = [] + for name, rng, section, manifest_path in manifests_here: + if _is_local(rng): + continue + for ver in _resolve_range( + name, rng, by_descriptor, versions_by_name, candidates(name, rng) + ): + roots.append((name, ver, section, manifest_path)) + return nodes, edges, roots + + +# ── package-lock.json (v2/v3) ──────────────────────────────────────────────── +def _pl_name(key: str) -> str: + """``node_modules/@scope/x`` → ``@scope/x`` (last node_modules segment).""" + idx = key.rfind("node_modules/") + return key[idx + len("node_modules/") :] if idx != -1 else key + + +def _pl_resolve(pkgs: dict, from_key: str, dep: str) -> str | None: + """Node-style resolution of ``dep`` from the package installed at ``from_key``.""" + cur = from_key + while True: + cand = (cur + "/node_modules/" + dep) if cur else ("node_modules/" + dep) + entry = pkgs.get(cand) + if isinstance(entry, dict) and entry.get("version"): + return entry.get("version") + if not cur: + return None + idx = cur.rfind("/node_modules/") + cur = cur[:idx] if idx != -1 else "" + + +def parse_package_lock( + text: str, manifests_here: list[tuple[str, str, str, str, str]] +) -> tuple[set[NV], list[tuple[NV, NV]], list[tuple[str, str, str, str]]]: + """Return ``(nodes, edges, roots)`` for a package-lock.json (v2/v3). + + ``manifests_here`` items are ``(name, range, section, manifest_path, pkg_key)`` + where ``pkg_key`` is the manifest dir relative to the lockfile (``""`` = root), + used as the resolution origin in the install tree. + """ + doc = json.loads(text) + pkgs = doc.get("packages") + nodes: set[NV] = set() + edges: list[tuple[NV, NV]] = [] + roots: list[tuple[str, str, str, str]] = [] + if not isinstance(pkgs, dict): # v1 (no packages map) — caller marks unresolved + raise ValueError("package-lock has no 'packages' map (v1)") + + # nodes + edges: every installed package path + for key, entry in pkgs.items(): + if not key.startswith("node_modules/") or not isinstance(entry, dict): + continue + version = entry.get("version") + if not version: + continue + frm = (_pl_name(key), str(version)) + nodes.add(frm) + for section in _EDGE_SECTIONS + ("peerDependencies",): + for dep_name, _rng in (entry.get(section) or {}).items(): + dep_ver = _pl_resolve(pkgs, key, dep_name) + if dep_ver: + to = (dep_name, str(dep_ver)) + nodes.add(to) + edges.append((frm, to)) + + # roots: resolve each first-party declaration through the install tree + for name, rng, section, manifest_path, pkg_key in manifests_here: + if _is_local(rng): + continue + ver = _pl_resolve(pkgs, pkg_key, name) + if ver: + roots.append((name, str(ver), section, manifest_path)) + return nodes, edges, roots + + +# ── package.json reading ───────────────────────────────────────────────────── +def _read_package_json(path: Path) -> dict | None: + """Parsed package.json, or ``None`` if it is missing/unreadable/malformed. + + ``None`` is distinct from ``{}`` (a valid empty manifest): a *parse failure* + means we lost that workspace's declared scopes, so the caller records it in + ``unresolved`` rather than silently under-connecting its prod dependencies. + """ + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + return data if isinstance(data, dict) else None + + +def _declared_deps(pkg: dict) -> list[tuple[str, str, str]]: + """``(name, range, section)`` for every dependency section of a package.json.""" + out: list[tuple[str, str, str]] = [] + for section in _SECTIONS: + block = pkg.get(section) + if isinstance(block, dict): + for name, rng in block.items(): + out.append((name, str(rng), section)) + return out + + +# ── lockfile discovery ─────────────────────────────────────────────────────── +_LOCK_PRECEDENCE = ("pnpm-lock.yaml", "yarn.lock", "package-lock.json") + + +def _find_lockfiles(root: Path) -> dict[str, str]: + """Repo-relative dir → chosen lockfile name (one per dir, by precedence).""" + by_dir: dict[str, dict[str, str]] = {} + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [d for d in dirnames if d not in _SKIP_DIRS] + rel_dir = os.path.relpath(dirpath, root) + rel_dir = "" if rel_dir == "." else rel_dir.replace("\\", "/") + for fn in filenames: + if fn in _LOCK_PRECEDENCE: + by_dir.setdefault(rel_dir, {})[fn] = fn + chosen: dict[str, str] = {} + for rel_dir, found in by_dir.items(): + for name in _LOCK_PRECEDENCE: + if name in found: + chosen[rel_dir] = name + break + return chosen + + +def _nearest_lock_dir(manifest_dir: str, lock_dirs: set[str]) -> str | None: + """Deepest lockfile dir that is an ancestor of (or equal to) ``manifest_dir``.""" + best: str | None = None + for ld in lock_dirs: + # a root lock dir ("") governs everything; otherwise ld must be an ancestor + is_ancestor = (not ld) or manifest_dir == ld or manifest_dir.startswith(ld + "/") + if is_ancestor and (best is None or len(ld) > len(best)): + best = ld + return best + + +# ── orchestrator ───────────────────────────────────────────────────────────── +def parse_npm(root: Path, manifests: list[Manifest]) -> tuple[DepGraph, list[str]]: + """Build the npm portion of the dependency graph from lockfiles + package.json. + + Returns ``(graph, unresolved)`` — ``unresolved`` lists manifest/lockfile paths + that could not be parsed confidently (never invents resolutions for them). + """ + root = Path(root) + graph = DepGraph() + unresolved: list[str] = [] + + if not manifests: + manifests = discover_manifests(root) + npm_manifests = [m for m in manifests if m.ecosystem == "npm" and m.kind == "package.json"] + + lock_by_dir = _find_lockfiles(root) # rel_dir -> lockfile name + lock_dirs = set(lock_by_dir) + + def _add( + nodes: set[NV], edges: list[tuple[NV, NV]], roots: list[tuple[str, str, str, str]] + ) -> None: + for name, ver in nodes: + graph.add_node(name, ver, "npm") + for frm, to in edges: + graph.add_edge(frm, to) + for name, ver, section, manifest_path in roots: + graph.add_root(name, ver, classify_scope(manifest_path, section), manifest_path, "npm") + + # group first-party package.json manifests under the lockfile that governs them + manifest_dir_of: dict[str, str] = {} + for m in npm_manifests: + md = os.path.dirname(m.path).replace("\\", "/") + manifest_dir_of[m.path] = md + governed: dict[str, list[Manifest]] = {ld: [] for ld in lock_dirs} + for m in npm_manifests: + ld = _nearest_lock_dir(manifest_dir_of[m.path], lock_dirs) + if ld is None: + # a package.json with deps but no lockfile above it can't be resolved; + # a parse failure (pkg is None) is likewise recorded, not swallowed. + pkg = _read_package_json(root / m.path) + if pkg is None or _declared_deps(pkg): + unresolved.append(m.path) + else: + governed[ld].append(m) + + for lock_dir, lock_name in sorted(lock_by_dir.items()): + lock_path = (root / lock_dir / lock_name) if lock_dir else (root / lock_name) + rel_lock = os.path.join(lock_dir, lock_name).replace("\\", "/") if lock_dir else lock_name + try: + text = lock_path.read_text(encoding="utf-8") + except OSError: + unresolved.append(rel_lock) + continue + + try: + if lock_name == "pnpm-lock.yaml": + # pnpm resolves its own workspace declarations via importers + nodes, edges, roots = parse_pnpm_lock(text, lock_dir) + _add(nodes, edges, roots) + elif lock_name == "yarn.lock": + decls: list[tuple[str, str, str, str]] = [] + for m in governed.get(lock_dir, []): + pkg = _read_package_json(root / m.path) + if pkg is None: # malformed manifest → record, don't drop silently + unresolved.append(m.path) + continue + for name, rng, section in _declared_deps(pkg): + decls.append((name, rng, section, m.path)) + nodes, edges, roots = parse_yarn_lock(text, decls) + _add(nodes, edges, roots) + elif lock_name == "package-lock.json": + decls_pl: list[tuple[str, str, str, str, str]] = [] + for m in governed.get(lock_dir, []): + pkg = _read_package_json(root / m.path) + if pkg is None: # malformed manifest → record, don't drop silently + unresolved.append(m.path) + continue + md = manifest_dir_of[m.path] + pkg_key = ( + md[len(lock_dir) + 1 :] + if lock_dir and md != lock_dir + else ("" if md == lock_dir else md) + ) + for name, rng, section in _declared_deps(pkg): + decls_pl.append((name, rng, section, m.path, pkg_key)) + nodes, edges, roots = parse_package_lock(text, decls_pl) + _add(nodes, edges, roots) + else: # pragma: no cover - precedence list is closed + unresolved.append(rel_lock) + except (ValueError, yaml.YAMLError, KeyError): + unresolved.append(rel_lock) + + unresolved.sort() + return graph, unresolved diff --git a/backend/cliff/repos/deps/pypi.py b/backend/cliff/repos/deps/pypi.py new file mode 100644 index 00000000..00ff6659 --- /dev/null +++ b/backend/cliff/repos/deps/pypi.py @@ -0,0 +1,566 @@ +"""Deterministic PyPI lockfile/manifest parser + dist→import name map. + +Builds the ``pypi`` portion of the :class:`~cliff.repos.deps.graph.DepGraph`. + +Parse strategy (in preference order — a *resolved* lockfile wins because it +carries the full transitive graph **and** the resolved edges): + +1. ``uv.lock`` (TOML): ``[[package]]`` entries → ``(name, version)`` nodes; each + package's ``dependencies`` array → edges. Workspace members (``source`` is + ``editable``/``virtual``) are the project itself; their ``dependencies`` / + ``[package.optional-dependencies]`` / ``[package.dev-dependencies]`` become + roots, scoped via :func:`classify_scope` on the owning ``pyproject.toml``. +2. ``poetry.lock`` (TOML): ``[[package]]`` + ``[package.dependencies]`` (a table + of name→spec) → edges. Direct deps + scopes come from the sibling + ``pyproject.toml`` poetry sections; ``groups`` is used as a safety fallback. +3. ``Pipfile.lock`` (JSON): flat ``default`` / ``develop`` maps → roots (no + transitive edges exist in the format, so every entry is a root — conservative). +4. Declared-only fallback (no lock): ``pyproject.toml`` (PEP 621 + PEP 735 + + poetry sections), ``requirements*.txt``, ``setup.cfg``, ``Pipfile``. Roots + only, no transitive edges — a node simply has no children, which is the + conservative (never-false-clear) shape. + +Safety framing (ADR-0053): when an edge target is ambiguous (a bare name that +resolves to several locked versions) we **over-connect** to every candidate +rather than risk dropping a path that reaches production. We never fabricate a +node that is not present in the lock/manifest. Environment markers on edges are +ignored (kept) for the same reason — dropping a marked edge could hide a +shipping path on some interpreter. + +Pure / deterministic. No network, no LLM. ``tomllib`` for TOML, ``json`` for +Pipfile.lock. +""" + +from __future__ import annotations + +import configparser +import json +import os +import re +import tomllib +from pathlib import Path + +from .graph import DepGraph +from .manifests import Manifest, classify_extra, classify_scope + +NV = tuple[str, str] +_ECO = "pypi" + +# ── PEP 503 normalization ───────────────────────────────────────────────────── +_NORM_RE = re.compile(r"[-_.]+") + + +def normalize_dist(name: str) -> str: + """PEP 503 normalized dist name: lowercased, runs of ``-``/``_``/``.`` → ``-``.""" + return _NORM_RE.sub("-", name.strip()).lower() + + +# ── dist → import name ──────────────────────────────────────────────────────── +# Keys are PEP-503-normalized dist names; values are the top-level import name. +DIST_TO_IMPORT: dict[str, str] = { + # required exceptions (task spec) + "pyyaml": "yaml", + "beautifulsoup4": "bs4", + "pillow": "PIL", + "scikit-learn": "sklearn", + "opencv-python": "cv2", + "python-dateutil": "dateutil", + "msgpack-python": "msgpack", + "protobuf": "google.protobuf", + "pyjwt": "jwt", + "python-multipart": "multipart", + "typing-extensions": "typing_extensions", + "attrs": "attr", + "setuptools": "setuptools", + # common additional exceptions (identity would be wrong) + "opencv-python-headless": "cv2", + "opencv-contrib-python": "cv2", + "pillow-simd": "PIL", + "scikit-image": "skimage", + "python-dotenv": "dotenv", + "python-jose": "jose", + "python-ldap": "ldap", + "python-slugify": "slugify", + "python-magic": "magic", + "websocket-client": "websocket", + "psycopg2-binary": "psycopg2", + "mysqlclient": "MySQLdb", + "pymysql": "pymysql", + "pycryptodome": "Crypto", + "pycryptodomex": "Cryptodome", + "grpcio": "grpc", + "google-cloud-storage": "google.cloud.storage", + "faiss-cpu": "faiss", + "faiss-gpu": "faiss", + "sqlalchemy": "sqlalchemy", + "markdown": "markdown", + "html5lib": "html5lib", +} + +# Dists whose import name genuinely cannot be derived from the dist name +# (multiple top-level modules, or stub-only). Returning identity here would be +# wrong, so we return None and let the caller stay conservative. +_AMBIGUOUS: frozenset[str] = frozenset( + { + "pywin32", # exposes win32api, win32con, pythoncom, … — no single import + "backports", # namespace shim, no importable top-level of its own + } +) + + +def dist_to_import(dist_name: str) -> str | None: + """Import name for a PyPI dist. + + Identity (normalized, ``-``→``_``) for the 1:1 majority; the curated map for + known exceptions; ``None`` only for genuinely ambiguous/unmappable dists. + Callers must NOT treat ``None`` as "not imported" — it means "unknown". + """ + norm = normalize_dist(dist_name) + if not norm: + return None + if norm in _AMBIGUOUS: + return None + if norm in DIST_TO_IMPORT: + return DIST_TO_IMPORT[norm] + return norm.replace("-", "_") + + +# ── requirement-string parsing ──────────────────────────────────────────────── +_REQ_RE = re.compile(r"^([A-Za-z0-9][A-Za-z0-9._-]*)\s*(?:\[[^\]]*\])?\s*(.*)$") + + +def _extract_pin(spec: str) -> str: + """Return the pinned version if the spec contains ``==``/``===``, else ``""``.""" + for part in spec.split(","): + part = part.strip() + if part.startswith("=="): + return part.lstrip("=").strip() + return "" + + +def _parse_req_line(raw: str) -> tuple[str, str] | None: + """Parse one requirement line → ``(normalized_name, version)`` or ``None``. + + Version is the pinned ``==`` version if present, else ``""`` (unknown). + Comments, options (``-r``/``-e``/``--hash``/…), and markers are stripped. + """ + line = raw.strip() + if not line or line.startswith("#") or line.startswith("-"): + return None + # strip inline comment + if " #" in line: + line = line.split(" #", 1)[0].strip() + # strip environment marker + if ";" in line: + line = line.split(";", 1)[0].strip() + # 'name @ url' direct reference → keep the name portion + if "@" in line: + head = line.split("@", 1)[0].strip() + if head: + line = head + if not line: + return None + m = _REQ_RE.match(line) + if not m: + return None + return normalize_dist(m.group(1)), _extract_pin(m.group(2) or "") + + +# ── uv.lock ─────────────────────────────────────────────────────────────────── +def _resolve_targets(dep: object, name_versions: dict[str, list[str]]) -> list[NV]: + """Resolve a uv dependency entry to concrete ``(name, version)`` target(s). + + Bare name with a unique locked version → that version. Bare name with several + locked versions (uv normally disambiguates, but be safe) → *all* of them + (over-connect). Unknown name → no target (never fabricate a node).""" + if not isinstance(dep, dict): + return [] + name = normalize_dist(str(dep.get("name", ""))) + if not name: + return [] + ver = dep.get("version") + if ver is not None: + return [(name, str(ver))] + versions = name_versions.get(name, []) + if len(versions) == 1: + return [(name, versions[0])] + return [(name, v) for v in versions] # 0 → [], >1 → over-connect + + +def _root_manifest(src: dict, pyprojects: list[Manifest], root: Path) -> str: + """Path of the ``pyproject.toml`` owning a uv workspace member ``source``.""" + path = str(src.get("editable") or src.get("virtual") or src.get("directory") or ".").strip("/") + candidate = "pyproject.toml" if path in ("", ".") else f"{path}/pyproject.toml" + norm = {m.path.replace("\\", "/"): m.path for m in pyprojects} + if candidate in norm: + return norm[candidate] + if "pyproject.toml" in norm: + return norm["pyproject.toml"] + return candidate + + +def _parse_uv_lock( + g: DepGraph, root: Path, lock_path: Path, manifests: list[Manifest], unresolved: list[str] +) -> bool: + try: + data = tomllib.loads(lock_path.read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError) as exc: + unresolved.append( + f"{_rel(root, lock_path)}: unparseable uv.lock ({exc.__class__.__name__})" + ) + return False + packages = data.get("package") or [] + if not packages: + # A header-only (empty) but syntactically-valid lock is not a parse + # failure — uv emits one before first resolve. Defer to the next + # strategy (poetry.lock / declared-only) instead of flagging it. + return False + + name_versions: dict[str, list[str]] = {} + for pkg in packages: + n = normalize_dist(str(pkg.get("name", ""))) + if not n: + continue + v = str(pkg.get("version", "")) + g.add_node(n, v, _ECO) + name_versions.setdefault(n, []) + if v not in name_versions[n]: + name_versions[n].append(v) + + pyprojects = [m for m in manifests if m.kind == "pyproject.toml"] + for pkg in packages: + n = normalize_dist(str(pkg.get("name", ""))) + if not n: + continue + frm: NV = (n, str(pkg.get("version", ""))) + # edges from every dependency section of this package (transitive graph) + for dep in pkg.get("dependencies") or []: + for tgt in _resolve_targets(dep, name_versions): + g.add_edge(frm, tgt) + for grp in ("optional-dependencies", "dev-dependencies"): + for deps in (pkg.get(grp) or {}).values(): + for dep in deps or []: + for tgt in _resolve_targets(dep, name_versions): + g.add_edge(frm, tgt) + # roots: only the workspace members (the project itself) + src = pkg.get("source") + if isinstance(src, dict) and ("editable" in src or "virtual" in src): + declared_in = _root_manifest(src, pyprojects, root) + _uv_roots(g, pkg.get("dependencies") or [], name_versions, + classify_scope(declared_in, "project.dependencies"), declared_in) + for extra_name, deps in (pkg.get("optional-dependencies") or {}).items(): + # a dev/docs/test-named extra is non-shipping tooling; a feature + # extra (anthropic, postgres, …) ships when installed → optional. + _uv_roots(g, deps or [], name_versions, classify_extra(extra_name), declared_in) + for deps in (pkg.get("dev-dependencies") or {}).values(): + _uv_roots(g, deps or [], name_versions, + classify_scope(declared_in, "dependency-groups"), declared_in) + return True + + +def _uv_roots( + g: DepGraph, deps: list, name_versions: dict[str, list[str]], scope: str, declared_in: str +) -> None: + for dep in deps: + for (n, v) in _resolve_targets(dep, name_versions): + g.add_root(n, v, scope, declared_in, _ECO) + + +# ── poetry.lock ─────────────────────────────────────────────────────────────── +def _poetry_group_scope(group: str) -> str: + return {"main": "prod", "dev": "dev", "test": "test", "docs": "docs"}.get(group, "prod") + + +def _parse_poetry_lock( + g: DepGraph, root: Path, lock_path: Path, manifests: list[Manifest], unresolved: list[str] +) -> bool: + try: + data = tomllib.loads(lock_path.read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError) as exc: + unresolved.append( + f"{_rel(root, lock_path)}: unparseable poetry.lock ({exc.__class__.__name__})" + ) + return False + packages = data.get("package") or [] + if not packages: + return False # empty-but-valid lock → defer to next strategy + + name_versions: dict[str, list[str]] = {} + groups_of: dict[str, set[str]] = {} + for pkg in packages: + n = normalize_dist(str(pkg.get("name", ""))) + if not n: + continue + v = str(pkg.get("version", "")) + g.add_node(n, v, _ECO) + name_versions.setdefault(n, []) + if v not in name_versions[n]: + name_versions[n].append(v) + groups = pkg.get("groups") + if groups is None: # legacy poetry: single 'category' + cat = pkg.get("category") + groups = [cat] if cat else ["main"] + groups_of.setdefault(n, set()).update(str(x) for x in groups) + + # edges: poetry [package.dependencies] is a table name→spec + for pkg in packages: + n = normalize_dist(str(pkg.get("name", ""))) + if not n: + continue + frm: NV = (n, str(pkg.get("version", ""))) + deps = pkg.get("dependencies") + if isinstance(deps, dict): + for dep_name in deps: + dn = normalize_dist(dep_name) + for v in name_versions.get(dn, []): + g.add_edge(frm, (dn, v)) + + # roots: prefer the sibling pyproject's declared direct deps (accurate + # direct-set + declared_in); fall back to lock 'groups' when unavailable. + declared_in = _sibling_pyproject(lock_path, root, manifests) + added = _poetry_roots_from_pyproject(g, root, declared_in, name_versions) + if not added: + lock_rel = _rel(root, lock_path) + for n, groups in groups_of.items(): + for grp in groups: + for v in name_versions.get(n, []): + g.add_root(n, v, _poetry_group_scope(grp), lock_rel, _ECO) + return True + + +def _poetry_roots_from_pyproject( + g: DepGraph, root: Path, pyproject_rel: str | None, name_versions: dict[str, list[str]] +) -> int: + if not pyproject_rel: + return 0 + full = root / pyproject_rel + try: + data = tomllib.loads(full.read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError): + return 0 + count = 0 + for section, name, ver in _iter_pyproject_deps(data): + versions = name_versions.get(name) or ([ver] if ver else [""]) + for v in versions: + g.add_root(name, v, classify_scope(pyproject_rel, section), pyproject_rel, _ECO) + count += 1 + return count + + +def _sibling_pyproject(lock_path: Path, root: Path, manifests: list[Manifest]) -> str | None: + lock_dir = _rel(root, lock_path.parent).replace("\\", "/").strip("/") + want = "pyproject.toml" if lock_dir in ("", ".") else f"{lock_dir}/pyproject.toml" + for m in manifests: + if m.kind == "pyproject.toml" and m.path.replace("\\", "/") == want: + return m.path + return None + + +# ── Pipfile.lock (JSON, flat) ───────────────────────────────────────────────── +def _parse_pipfile_lock(g: DepGraph, root: Path, lock_path: Path, unresolved: list[str]) -> bool: + try: + data = json.loads(lock_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + unresolved.append( + f"{_rel(root, lock_path)}: unparseable Pipfile.lock ({exc.__class__.__name__})" + ) + return False + rel = _rel(root, lock_path) + found = False + for section, cscope_section in ( + ("default", "project.dependencies"), + ("develop", "dependency-groups"), + ): + for name, meta in (data.get(section) or {}).items(): + n = normalize_dist(name) + if not n: + continue + ver = "" + if isinstance(meta, dict): + vv = meta.get("version", "") + if isinstance(vv, str) and vv.startswith("=="): + ver = vv.lstrip("=").strip() + g.add_root(n, ver, classify_scope(rel, cscope_section), rel, _ECO) + found = True + return found # empty-but-valid lock → defer to declared-only Pipfile + + +# ── declared-only fallback ──────────────────────────────────────────────────── +def _poetry_spec_version(spec: object) -> str: + if isinstance(spec, str): + return _extract_pin(spec) + if isinstance(spec, dict): + v = spec.get("version", "") + return _extract_pin(v) if isinstance(v, str) else "" + return "" + + +def _iter_pyproject_deps(data: dict): + """Yield ``(classify_section, normalized_name, version)`` for every declared + dependency in a pyproject (PEP 621, PEP 735, and poetry sections).""" + proj = data.get("project") + if isinstance(proj, dict): + for req in proj.get("dependencies") or []: + p = _parse_req_line(req) if isinstance(req, str) else None + if p: + yield "project.dependencies", p[0], p[1] + for reqs in (proj.get("optional-dependencies") or {}).values(): + for req in reqs or []: + p = _parse_req_line(req) if isinstance(req, str) else None + if p: + yield "project.optional-dependencies", p[0], p[1] + for reqs in (data.get("dependency-groups") or {}).values(): + for req in reqs or []: + if isinstance(req, str): + p = _parse_req_line(req) + if p: + yield "dependency-groups", p[0], p[1] + # {include-group = "..."} entries carry no dist → skip + poetry = (data.get("tool") or {}).get("poetry") or {} + for name, spec in (poetry.get("dependencies") or {}).items(): + nn = normalize_dist(name) + if nn and nn != "python": + yield "tool.poetry.dependencies", nn, _poetry_spec_version(spec) + for name, spec in (poetry.get("dev-dependencies") or {}).items(): + nn = normalize_dist(name) + if nn and nn != "python": + yield "tool.poetry.group.dev.dependencies", nn, _poetry_spec_version(spec) + for grp, gdata in (poetry.get("group") or {}).items(): + for name, spec in ((gdata or {}).get("dependencies") or {}).items(): + nn = normalize_dist(name) + if nn and nn != "python": + yield f"tool.poetry.group.{grp}.dependencies", nn, _poetry_spec_version(spec) + + +def _parse_declared( + g: DepGraph, root: Path, manifests: list[Manifest], unresolved: list[str] +) -> None: + for m in manifests: + full = root / m.path + try: + text = full.read_text(encoding="utf-8") + except OSError: + unresolved.append(f"{m.path}: unreadable") + continue + if m.kind == "pyproject.toml": + try: + data = tomllib.loads(text) + except tomllib.TOMLDecodeError: + unresolved.append(f"{m.path}: unparseable pyproject.toml") + continue + for section, name, ver in _iter_pyproject_deps(data): + g.add_root(name, ver, classify_scope(m.path, section), m.path, _ECO) + elif m.kind == "requirements": + section = Path(m.path).name # classify_scope keys on the filename + for line in text.splitlines(): + p = _parse_req_line(line) + if p: + g.add_root(p[0], p[1], classify_scope(m.path, section), m.path, _ECO) + elif m.kind == "setup.cfg": + _parse_setup_cfg(g, m.path, text, unresolved) + elif m.kind == "pipfile": + _parse_pipfile_toml(g, m.path, text, unresolved) + + +def _parse_setup_cfg(g: DepGraph, path: str, text: str, unresolved: list[str]) -> None: + cp = configparser.ConfigParser() + try: + cp.read_string(text) + except configparser.Error: + unresolved.append(f"{path}: unparseable setup.cfg") + return + if cp.has_option("options", "install_requires"): + for line in cp.get("options", "install_requires").splitlines(): + p = _parse_req_line(line) + if p: + g.add_root(p[0], p[1], classify_scope(path, "project.dependencies"), path, _ECO) + if cp.has_section("options.extras_require"): + for _key, val in cp.items("options.extras_require"): + for line in val.splitlines(): + p = _parse_req_line(line) + if p: + g.add_root( + p[0], p[1], classify_scope(path, "project.optional-dependencies"), path, + _ECO, + ) + + +def _parse_pipfile_toml(g: DepGraph, path: str, text: str, unresolved: list[str]) -> None: + try: + data = tomllib.loads(text) + except tomllib.TOMLDecodeError: + unresolved.append(f"{path}: unparseable Pipfile") + return + for section, cscope_section in ( + ("packages", "project.dependencies"), + ("dev-packages", "dependency-groups"), + ): + for name, spec in (data.get(section) or {}).items(): + nn = normalize_dist(name) + if not nn: + continue + ver = "" + if isinstance(spec, str) and spec.startswith("=="): + ver = spec.lstrip("=").strip() + elif isinstance(spec, dict): + vv = spec.get("version", "") + if isinstance(vv, str) and vv.startswith("=="): + ver = vv.lstrip("=").strip() + g.add_root(nn, ver, classify_scope(path, cscope_section), path, _ECO) + + +# ── helpers ─────────────────────────────────────────────────────────────────── +def _rel(root: Path, p: Path) -> str: + try: + return str(Path(p).relative_to(root)) + except ValueError: + return str(p) + + +# ── entry point ─────────────────────────────────────────────────────────────── +def parse_pypi(root: Path, manifests: list[Manifest]) -> tuple[DepGraph, list[str]]: + """Build the pypi portion of the dependency graph. + + Prefers a resolved lockfile (uv.lock → poetry.lock → Pipfile.lock) for the + full transitive graph + edges; otherwise falls back to declared-only roots + from pyproject/requirements/setup.cfg/Pipfile. Returns ``(graph, unresolved)`` + where ``unresolved`` lists artifacts that could not be parsed confidently. + """ + root = Path(root) + g = DepGraph() + unresolved: list[str] = [] + py_manifests = [m for m in manifests if m.ecosystem == _ECO] + + # candidate lock directories: repo root + every pyproject/Pipfile directory + lock_dirs: set[Path] = {Path(".")} + for m in py_manifests: + if m.kind in ("pyproject.toml", "pipfile"): + lock_dirs.add(Path(m.path).parent) + + # PER-DIRECTORY lock precedence (uv > poetry > pipfile): a lock in dir D governs + # D and everything under it. A repo-global "resolved" flag would drop a lock-less + # sub-project's deps whenever any *other* sub-project had a lock — under-connecting + # (and risking a false clear of) that sub-project's shipping dependencies. + resolved_dirs: set[str] = set() + for d in sorted(lock_dirs, key=str): + base = root / d + d_rel = "" if str(d) == "." else str(d).replace("\\", "/") + if (base / "uv.lock").is_file() and _parse_uv_lock( + g, root, base / "uv.lock", py_manifests, unresolved + ) or (base / "poetry.lock").is_file() and _parse_poetry_lock( + g, root, base / "poetry.lock", py_manifests, unresolved + ) or (base / "Pipfile.lock").is_file() and _parse_pipfile_lock( + g, root, base / "Pipfile.lock", unresolved + ): + resolved_dirs.add(d_rel) + + # declared-only fallback for every manifest NOT governed by a resolved lock dir + def _covered(manifest_path: str) -> bool: + md = os.path.dirname(manifest_path).replace("\\", "/") + return any(ld == "" or md == ld or md.startswith(ld + "/") for ld in resolved_dirs) + + uncovered = [m for m in py_manifests if not _covered(m.path)] + if uncovered: + _parse_declared(g, root, uncovered, unresolved) + + return g, unresolved diff --git a/backend/cliff/repos/knowledge.py b/backend/cliff/repos/knowledge.py index 49d23c3f..a9f0829a 100644 --- a/backend/cliff/repos/knowledge.py +++ b/backend/cliff/repos/knowledge.py @@ -37,6 +37,7 @@ class RepoKnowledge: profile: dict | None = None code_map: dict | None = None threat: dict | None = None + dep_manifest: dict | None = None clone_dir: Path | None = None @@ -66,5 +67,8 @@ def load_repo_knowledge( profile=mgr.read_artifact(repo_id, "profile") if "profile" in selected else None, code_map=mgr.read_artifact(repo_id, "code_map") if "code_map" in selected else None, threat=mgr.read_artifact(repo_id, "threat") if "threat" in selected else None, + dep_manifest=( + mgr.read_artifact(repo_id, "dep_manifest") if "dep_manifest" in selected else None + ), clone_dir=mgr.clone_dir(repo_id) if include_clone else None, ) diff --git a/backend/cliff/repos/profile_agents.py b/backend/cliff/repos/profile_agents.py index ea12b6ca..a07c8f39 100644 --- a/backend/cliff/repos/profile_agents.py +++ b/backend/cliff/repos/profile_agents.py @@ -1,8 +1,10 @@ -"""The three Project-profile builder agents (ADR-0053 §3, Phase 1.6). +"""The Project-profile builders (ADR-0053 §3, Phase 1.6). -Each is an in-process Pydantic AI agent (ADR-0047) that reads the cached clone -(read-only) and emits one typed artifact. They satisfy the ``ProfileBuilder`` -shape (``async (clone_dir) -> dict``) so :class:`ProfileRunner` can drive them. +Three are in-process Pydantic AI agents (ADR-0047) that read the cached clone +(read-only) and emit one typed artifact. The fourth, ``dep_manifest``, is a +**deterministic** lockfile-graph parser (no LLM). All satisfy the +``ProfileBuilder`` shape (``async (clone_dir) -> dict``) so :class:`ProfileRunner` +can drive them uniformly. Deps reuse: profile builders run on a *clone*, not a finding workspace, but the ``read`` tool is keyed on ``WorkspaceDeps.workspace_dir`` — so we reuse @@ -16,6 +18,7 @@ from __future__ import annotations +import asyncio from typing import TYPE_CHECKING from pydantic_ai import Agent @@ -23,6 +26,7 @@ from cliff.agents.runtime.deps import WorkspaceDeps from cliff.agents.runtime.tools.read import read +from cliff.repos.deps import build_dep_manifest from cliff.repos.schemas import CodeMap, RepoProfile, ThreatHistory if TYPE_CHECKING: @@ -103,12 +107,26 @@ def make_threat_history(model: Model) -> ProfileBuilder: return _make_builder(model, ThreatHistory, _THREAT_PROMPT) +def make_dep_manifest(model: Model) -> ProfileBuilder: + """Deterministic lockfile-graph builder (no LLM). ``model`` is accepted for + signature-compatibility with the agent builders and ignored.""" + + async def _build(clone_dir: Path) -> dict: + # build_dep_manifest is fully synchronous (several os.walk passes + regex); + # offload to a thread so the whole-repo scan doesn't block the event loop. + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, build_dep_manifest, clone_dir) + + return _build + + def make_profile_builders(model: Model) -> dict[str, ProfileBuilder]: """The full builder set keyed by artifact name, ready for ProfileRunner.""" return { "profile": make_repo_profiler(model), "code_map": make_code_map(model), "threat": make_threat_history(model), + "dep_manifest": make_dep_manifest(model), } @@ -116,6 +134,7 @@ def make_profile_builders(model: Model) -> dict[str, ProfileBuilder]: "PROFILE_BUILDER_TOOLS", "PROFILE_REQUEST_LIMIT", "make_code_map", + "make_dep_manifest", "make_profile_builders", "make_repo_profiler", "make_threat_history", diff --git a/backend/cliff/repos/repo_dir_manager.py b/backend/cliff/repos/repo_dir_manager.py index 4c9412b8..5fb6ad16 100644 --- a/backend/cliff/repos/repo_dir_manager.py +++ b/backend/cliff/repos/repo_dir_manager.py @@ -31,8 +31,9 @@ SCHEMA_VERSION = 1 #: The Project-profile artifacts (ADR-0053 §3). The clone lives alongside in -#: ``repo/`` (see :meth:`clone_dir`). -ARTIFACTS = ("profile", "code_map", "threat") +#: ``repo/`` (see :meth:`clone_dir`). ``dep_manifest`` is the deterministic +#: lockfile-resolved dependency graph (no LLM). +ARTIFACTS = ("profile", "code_map", "threat", "dep_manifest") class RepoDirManager: diff --git a/backend/cliff/repos/schemas.py b/backend/cliff/repos/schemas.py index 8fc4587a..b48f4b34 100644 --- a/backend/cliff/repos/schemas.py +++ b/backend/cliff/repos/schemas.py @@ -63,6 +63,47 @@ class CodeMap(BaseModel): classified: list[PathClass] = [] +# ── Map the dependencies ───────────────────────────────────────────────────── + +DepScope = Literal["prod", "dev", "test", "docs", "build", "optional"] +Ecosystem = Literal["npm", "pypi"] + + +class DepNode(BaseModel): + """One resolved ``(name, version)`` node in the dependency graph. + + ``scopes`` is the union of the scope of every top-level root that can reach + this node in the lockfile graph — ``prod`` dominates (if any prod root + reaches it, it ships). ``import_sites`` are first-party *shipping* source + lines that import it (repo-wide, non-test). See ADR-0053 §3. + """ + + model_config = {"extra": "allow"} + name: str + version: str + ecosystem: Ecosystem + scopes: list[DepScope] = [] + direct: bool = False # declared as a top-level dep in some manifest + declared_in: list[str] = [] # manifest paths declaring it (empty for pure transitive) + import_name: str | None = None # resolved import name (None = could not resolve → don't clear) + import_sites: list[str] = [] # first-party shipping "file:line" that import it + + +class DepManifest(BaseModel): + """``dep_manifest.json`` — the lockfile-resolved dependency graph. + + Deterministic (lockfile/manifest parse, no LLM). Keyed by ``(name, version)`` + so a direct-prod ``undici@6.23.0`` and a transitive ``undici@7.16.0`` in the + same lockfile are distinct nodes. + """ + + model_config = {"extra": "allow"} + ecosystems: list[Ecosystem] = [] + workspaces: list[str] = [] # manifest paths discovered + nodes: list[DepNode] = [] + unresolved: list[str] = [] # manifests/lockfiles we could not parse confidently + + # ── Review past issues ─────────────────────────────────────────────────────── FixCoverage = Literal["instance", "class", "none"] diff --git a/backend/tests/agents/test_deep_dive_integration.py b/backend/tests/agents/test_deep_dive_integration.py index e1b54907..cda64a6c 100644 --- a/backend/tests/agents/test_deep_dive_integration.py +++ b/backend/tests/agents/test_deep_dive_integration.py @@ -98,7 +98,7 @@ async def test_escalates_and_runs_deep_dive(db, tmp_path): assert runner.called # The runner received the repo knowledge + the SHA-pinned clone. assert runner.kwargs["traced_sha"] == "a" * 40 - assert set(runner.kwargs["repo_knowledge"]) == {"profile", "code_map", "threat"} + assert set(runner.kwargs["repo_knowledge"]) == {"profile", "code_map", "threat", "dep_manifest"} async def test_no_ai_skips(db, tmp_path): diff --git a/backend/tests/agents/test_triage_dep_manifest.py b/backend/tests/agents/test_triage_dep_manifest.py new file mode 100644 index 00000000..4d6e125f --- /dev/null +++ b/backend/tests/agents/test_triage_dep_manifest.py @@ -0,0 +1,144 @@ +"""SP1 Task 8 — resolve_by_dep_manifest (pure, safety-critical).""" + +from __future__ import annotations + +from cliff.agents.triage_dep_manifest import _parse_location, resolve_by_dep_manifest + + +def _mf(*nodes: dict) -> dict: + return {"ecosystems": ["npm"], "workspaces": [], "nodes": list(nodes), "unresolved": []} + + +def _node( + name, version, scopes, *, import_name="x", import_sites=None, ecosystem="npm", declared_in=None +): + return { + "name": name, + "version": version, + "ecosystem": ecosystem, + "scopes": scopes, + "direct": False, + "declared_in": declared_in or [], + "import_name": import_name, + "import_sites": import_sites or [], + } + + +def _finding(location: str) -> dict: + return {"type": "dependency", "location": location, "scanner": "trivy"} + + +# ── the clear case ──────────────────────────────────────────────────────────── +def test_dev_only_unimported_clears() -> None: + mf = _mf(_node("webpack-dev-server", "4.15.2", ["dev"], import_sites=[])) + out = resolve_by_dep_manifest(_finding("webpack-dev-server@4.15.2"), mf) + assert out is not None + assert out.verdict == "false_positive" + assert out.provenance.exit_stage == "dep_manifest_resolver" + + +def test_docs_and_build_scopes_clear() -> None: + mf = _mf(_node("sphinx-thing", "1.0.0", ["docs", "build"])) + assert resolve_by_dep_manifest(_finding("sphinx-thing@1.0.0"), mf) is not None + + +# ── the block (never-clear) cases ───────────────────────────────────────────── +def test_prod_scope_blocks() -> None: + mf = _mf(_node("axios", "1.13.2", ["prod"])) + assert resolve_by_dep_manifest(_finding("axios@1.13.2"), mf) is None + + +def test_both_dev_and_prod_blocks() -> None: + mf = _mf(_node("shared", "2.0.0", ["dev", "prod"])) + assert resolve_by_dep_manifest(_finding("shared@2.0.0"), mf) is None + + +def test_optional_scope_blocks() -> None: + mf = _mf(_node("fsevents", "2.3.0", ["optional"])) + assert resolve_by_dep_manifest(_finding("fsevents@2.3.0"), mf) is None + + +def test_unknown_scope_blocks() -> None: + # allowlist: an unrecognized/future scope must NOT pass the gate + mf = _mf(_node("mystery", "1.0.0", ["dev", "runtime"])) + assert resolve_by_dep_manifest(_finding("mystery@1.0.0"), mf) is None + + +def test_imported_in_firstparty_blocks() -> None: + mf = _mf(_node("yaml", "2.8.0", ["dev"], import_sites=["src/app.ts:12"])) + assert resolve_by_dep_manifest(_finding("yaml@2.8.0"), mf) is None + + +def test_import_name_none_blocks() -> None: + mf = _mf(_node("some-dist", "1.0.0", ["dev"], import_name=None)) + assert resolve_by_dep_manifest(_finding("some-dist@1.0.0"), mf) is None + + +def test_empty_scopes_blocks() -> None: + mf = _mf(_node("mystery", "1.0.0", [])) + assert resolve_by_dep_manifest(_finding("mystery@1.0.0"), mf) is None + + +def test_package_not_in_manifest_falls_through() -> None: + mf = _mf(_node("other", "1.0.0", ["dev"])) + assert resolve_by_dep_manifest(_finding("absent@9.9.9"), mf) is None + + +def test_version_mismatch_falls_through() -> None: + # the undici split: 6.23.0 ships (prod); the finding is 7.16.0 (dev) — but if + # only the prod node is present, the 7.16.0 finding must not match it. + mf = _mf(_node("undici", "6.23.0", ["prod"])) + assert resolve_by_dep_manifest(_finding("undici@7.16.0"), mf) is None + + +def test_cross_ecosystem_collision_blocks() -> None: + mf = _mf( + _node("yaml", "2.8.0", ["dev"], ecosystem="npm"), + _node("yaml", "2.8.0", ["dev"], ecosystem="pypi"), + ) + # two nodes match (name, version) → ambiguous → conservative + assert resolve_by_dep_manifest(_finding("yaml@2.8.0"), mf) is None + + +def test_none_manifest() -> None: + assert resolve_by_dep_manifest(_finding("x@1.0"), None) is None + + +# ── location parsing ────────────────────────────────────────────────────────── +def test_parse_scoped_location() -> None: + assert _parse_location("@xmldom/xmldom@0.8.0") == ("@xmldom/xmldom", "0.8.0") + assert _parse_location("yaml@2.8.0") == ("yaml", "2.8.0") + assert _parse_location("noversion") is None + + +def test_scoped_package_clears() -> None: + mf = _mf(_node("@xmldom/xmldom", "0.8.0", ["dev"], ecosystem="npm")) + assert resolve_by_dep_manifest(_finding("@xmldom/xmldom@0.8.0"), mf) is not None + + +# ── pypi PEP-503 name normalization (scanner reports raw names) ──────────────── +def test_pypi_raw_name_matches_normalized_node() -> None: + # node stored PEP-503-normalized; scanner reports "PyYAML" / "ruamel.yaml" + mf = _mf(_node("pyyaml", "6.0", ["dev"], ecosystem="pypi")) + assert resolve_by_dep_manifest(_finding("PyYAML@6.0"), mf) is not None + mf2 = _mf(_node("ruamel-yaml", "0.18.6", ["docs"], ecosystem="pypi")) + assert resolve_by_dep_manifest(_finding("ruamel.yaml@0.18.6"), mf2) is not None + + +def test_npm_name_not_pep503_normalized() -> None: + # npm names are compared verbatim — 'lodash.merge' must NOT match 'lodash-merge' + mf = _mf(_node("lodash-merge", "1.0.0", ["dev"], ecosystem="npm")) + assert resolve_by_dep_manifest(_finding("lodash.merge@1.0.0"), mf) is None + + +# ── import-site gate fails CLOSED on bad data ───────────────────────────────── +def test_import_sites_none_fails_closed() -> None: + node = _node("x", "1.0.0", ["dev"], import_sites=[]) + node["import_sites"] = None # bad/absent data → must NOT clear + assert resolve_by_dep_manifest(_finding("x@1.0.0"), _mf(node)) is None + + +def test_import_sites_nonlist_fails_closed() -> None: + node = _node("x", "1.0.0", ["dev"]) + node["import_sites"] = "src/app.ts:12" # a bare string, not a list → must NOT clear + assert resolve_by_dep_manifest(_finding("x@1.0.0"), _mf(node)) is None diff --git a/backend/tests/agents/test_triage_runner.py b/backend/tests/agents/test_triage_runner.py index 42291db5..7ebd3c49 100644 --- a/backend/tests/agents/test_triage_runner.py +++ b/backend/tests/agents/test_triage_runner.py @@ -300,6 +300,130 @@ async def _fake_deep(*a, **k): # noqa: ANN002, ANN003 assert len(synth) == 1 and synth[0].structured_output["verdict"] == "false_positive" +async def test_dep_manifest_gate_clears_dev_only_dependency(monkeypatch, db) -> None: + """A dependency finding whose package is dev/test-only + unimported is cleared + false_positive by the dep_manifest gate, WITHOUT invoking the Deep dive.""" + import cliff.agents.triage_runner as tr + + async def _no_code_map(db, repo_url): # noqa: ANN001 + return None + + async def _fake_load_dep_manifest(db, repo_url): # noqa: ANN001 + return { + "nodes": [ + { + "name": "webpack-dev-server", + "version": "4.15.2", + "ecosystem": "npm", + "scopes": ["dev"], + "direct": True, + "declared_in": ["package.json"], + "import_name": "webpack-dev-server", + "import_sites": [], + } + ] + } + + called = {"deep": False} + + async def _fake_deep(*a, **k): # noqa: ANN002, ANN003 + called["deep"] = True + return None + + monkeypatch.setattr(tr, "_load_code_map", _no_code_map) + monkeypatch.setattr(tr, "_load_dep_manifest", _fake_load_dep_manifest) + monkeypatch.setattr(tr, "maybe_deep_dive", _fake_deep) + + finding = await create_finding( + db, + FindingCreate( + source_type="trivy", + source_id="webpack-dev-server@4.15.2:CVE-2026-9", + title="CVE-2026-9 in webpack-dev-server", + type="dependency", + raw_payload={"package": "webpack-dev-server", "version": "4.15.2"}, + ), + ) + ws = await create_workspace( + db, WorkspaceCreate(finding_id=finding.id, repo_url="https://github.com/test/repo") + ) + now = datetime.now(UTC).isoformat() + await db.execute( + "UPDATE workspace SET workspace_dir = ?, updated_at = ? WHERE id = ?", + (f"/tmp/ws/{ws.id}", now, ws.id), + ) + await db.commit() + executor = _StubExecutor( + {"finding_enricher": _ENRICH_REAL, "exposure_analyzer": _EXPOSURE_REAL} + ) + + triage = await run_triage(executor, db, ws, env_vars={}) + + assert triage is not None + assert triage.verdict == "false_positive" + assert called["deep"] is False + + +async def test_dep_manifest_gate_prod_dependency_falls_through(monkeypatch, db) -> None: + """A prod dependency is NOT cleared by the gate → it escalates to the Deep dive.""" + import cliff.agents.triage_runner as tr + + async def _no_code_map(db, repo_url): # noqa: ANN001 + return None + + async def _fake_load_dep_manifest(db, repo_url): # noqa: ANN001 + return { + "nodes": [ + { + "name": "axios", + "version": "1.13.2", + "ecosystem": "npm", + "scopes": ["prod"], + "direct": True, + "declared_in": ["apps/web/package.json"], + "import_name": "axios", + "import_sites": ["apps/web/src/api.ts:3"], + } + ] + } + + called = {"deep": False} + + async def _fake_deep(*a, **k): # noqa: ANN002, ANN003 + called["deep"] = True + return None + + monkeypatch.setattr(tr, "_load_code_map", _no_code_map) + monkeypatch.setattr(tr, "_load_dep_manifest", _fake_load_dep_manifest) + monkeypatch.setattr(tr, "maybe_deep_dive", _fake_deep) + + finding = await create_finding( + db, + FindingCreate( + source_type="trivy", + source_id="axios@1.13.2:CVE-2026-8", + title="CVE-2026-8 in axios", + type="dependency", + raw_payload={"package": "axios", "version": "1.13.2"}, + ), + ) + ws = await create_workspace( + db, WorkspaceCreate(finding_id=finding.id, repo_url="https://github.com/test/repo") + ) + now = datetime.now(UTC).isoformat() + await db.execute( + "UPDATE workspace SET workspace_dir = ?, updated_at = ? WHERE id = ?", + (f"/tmp/ws/{ws.id}", now, ws.id), + ) + await db.commit() + executor = _StubExecutor( + {"finding_enricher": _ENRICH_REAL, "exposure_analyzer": _EXPOSURE_REAL} + ) + + await run_triage(executor, db, ws, env_vars={}) + assert called["deep"] is True # prod dep → not cleared → Deep dive runs + + async def test_codemap_gate_no_match_runs_deep_dive(monkeypatch, db) -> None: """When nothing matches, the Deep dive still runs (gate is transparent).""" import cliff.agents.triage_runner as tr diff --git a/backend/tests/repos/deps/__init__.py b/backend/tests/repos/deps/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/tests/repos/deps/fixtures/npm/package-lock/package-lock.json b/backend/tests/repos/deps/fixtures/npm/package-lock/package-lock.json new file mode 100644 index 00000000..36af456f --- /dev/null +++ b/backend/tests/repos/deps/fixtures/npm/package-lock/package-lock.json @@ -0,0 +1,44 @@ +{ + "name": "nicegui-slice", + "lockfileVersion": 3, + "requires": true, + "comment": "SMALL real slice of nicegui/package-lock.json (v3). dompurify is a prod dep; @antfu/install-pkg is a dev dep that pulls package-manager-detector + tinyexec (dev-only transitives).", + "packages": { + "": { + "name": "nicegui-slice", + "dependencies": { + "dompurify": "^3.4.0" + }, + "devDependencies": { + "@antfu/install-pkg": "^1.1.0" + } + }, + "node_modules/@antfu/install-pkg": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@antfu/install-pkg/-/install-pkg-1.1.0.tgz", + "integrity": "sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ==", + "dev": true, + "dependencies": { + "package-manager-detector": "^1.3.0", + "tinyexec": "^1.0.1" + } + }, + "node_modules/dompurify": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.0.tgz", + "integrity": "sha512-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaAA==" + }, + "node_modules/package-manager-detector": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.6.0.tgz", + "integrity": "sha512-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbBB==", + "dev": true + }, + "node_modules/tinyexec": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.0.1.tgz", + "integrity": "sha512-cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccCC==", + "dev": true + } + } +} diff --git a/backend/tests/repos/deps/fixtures/npm/package-lock/package.json b/backend/tests/repos/deps/fixtures/npm/package-lock/package.json new file mode 100644 index 00000000..468dd9d1 --- /dev/null +++ b/backend/tests/repos/deps/fixtures/npm/package-lock/package.json @@ -0,0 +1,10 @@ +{ + "name": "nicegui-slice", + "private": true, + "dependencies": { + "dompurify": "^3.4.0" + }, + "devDependencies": { + "@antfu/install-pkg": "^1.1.0" + } +} diff --git a/backend/tests/repos/deps/fixtures/npm/pnpm/pnpm-lock.yaml b/backend/tests/repos/deps/fixtures/npm/pnpm/pnpm-lock.yaml new file mode 100644 index 00000000..888c5ecd --- /dev/null +++ b/backend/tests/repos/deps/fixtures/npm/pnpm/pnpm-lock.yaml @@ -0,0 +1,58 @@ +# SMALL real slice of karakeep's pnpm-lock.yaml (v9). Real package identities: +# - undici version-split: @6.23.0 is a PROD dep of packages/shared; @7.16.0 is a +# transitive of cheerio, which here is a DEV-only dep of the root workspace. +# - '@xmldom/xmldom@0.8.0' exercises scoped-name parsing. +# - 'openai@6.34.0(ws@8.18.3)(zod@4.3.6)' exercises peer-suffix stripping. +lockfileVersion: '9.0' + +importers: + + .: + devDependencies: + cheerio: + specifier: ^1.1.2 + version: 1.1.2 + + packages/shared: + dependencies: + undici: + specifier: ^6.23.0 + version: 6.23.0 + '@xmldom/xmldom': + specifier: ^0.8.0 + version: 0.8.0 + openai: + specifier: ^6.0.0 + version: 6.34.0(ws@8.18.3)(zod@4.3.6) + +packages: + + '@xmldom/xmldom@0.8.0': + resolution: {integrity: sha512-aAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==} + + cheerio@1.1.2: + resolution: {integrity: sha512-bBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB==} + + openai@6.34.0: + resolution: {integrity: sha512-cCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC==} + + undici@6.23.0: + resolution: {integrity: sha512-dDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD==} + + undici@7.16.0: + resolution: {integrity: sha512-eEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE==} + +snapshots: + + '@xmldom/xmldom@0.8.0': {} + + cheerio@1.1.2: + dependencies: + undici: 7.16.0 + + 'openai@6.34.0(ws@8.18.3)(zod@4.3.6)': + dependencies: {} + + undici@6.23.0: {} + + undici@7.16.0: {} diff --git a/backend/tests/repos/deps/fixtures/npm/yarn-berry/apps/web/package.json b/backend/tests/repos/deps/fixtures/npm/yarn-berry/apps/web/package.json new file mode 100644 index 00000000..9f744cf2 --- /dev/null +++ b/backend/tests/repos/deps/fixtures/npm/yarn-berry/apps/web/package.json @@ -0,0 +1,12 @@ +{ + "name": "@linkwarden/web", + "private": true, + "dependencies": { + "axios": "^1.5.1", + "next": "15.3.9", + "dompurify": "^3.2.4" + }, + "devDependencies": { + "@types/react": "^19.1.0" + } +} diff --git a/backend/tests/repos/deps/fixtures/npm/yarn-berry/yarn.lock b/backend/tests/repos/deps/fixtures/npm/yarn-berry/yarn.lock new file mode 100644 index 00000000..c29f575c --- /dev/null +++ b/backend/tests/repos/deps/fixtures/npm/yarn-berry/yarn.lock @@ -0,0 +1,58 @@ +# This file is generated by running "yarn install" inside your project. +# Manual changes might be lost - proceed with caution! +# SMALL real slice of linkwarden/yarn.lock (yarn berry v8 — valid YAML). +# axios/next/dompurify are PROD deps of apps/web; dompurify -> @types/trusted-types +# is a real edge; '@types/react' block has a COMMA-JOINED descriptor key and is a +# dev-only dep. + +__metadata: + version: 8 + cacheKey: 10c0 + +"@types/react@npm:*, @types/react@npm:^19.1.0": + version: 19.2.14 + resolution: "@types/react@npm:19.2.14" + dependencies: + csstype: "npm:^3.0.2" + checksum: 10c0/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + languageName: node + linkType: hard + +"@types/trusted-types@npm:^2.0.7": + version: 2.0.7 + resolution: "@types/trusted-types@npm:2.0.7" + checksum: 10c0/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + languageName: node + linkType: hard + +"axios@npm:^1.5.1": + version: 1.13.2 + resolution: "axios@npm:1.13.2" + dependencies: + follow-redirects: "npm:^1.15.6" + form-data: "npm:^4.0.4" + proxy-from-env: "npm:^1.1.0" + checksum: 10c0/cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc + languageName: node + linkType: hard + +"dompurify@npm:^3.2.4": + version: 3.3.1 + resolution: "dompurify@npm:3.3.1" + dependencies: + "@types/trusted-types": "npm:^2.0.7" + dependenciesMeta: + "@types/trusted-types": + optional: true + checksum: 10c0/dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd + languageName: node + linkType: hard + +"next@npm:15.3.9": + version: 15.3.9 + resolution: "next@npm:15.3.9" + dependencies: + "@next/env": "npm:15.3.9" + checksum: 10c0/eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee + languageName: node + linkType: hard diff --git a/backend/tests/repos/deps/fixtures/npm/yarn-classic/package.json b/backend/tests/repos/deps/fixtures/npm/yarn-classic/package.json new file mode 100644 index 00000000..5ed83c27 --- /dev/null +++ b/backend/tests/repos/deps/fixtures/npm/yarn-classic/package.json @@ -0,0 +1,11 @@ +{ + "name": "@mealie/frontend-slice", + "private": true, + "dependencies": { + "axios": "^1.8.1", + "@mdi/js": "^7.4.47" + }, + "devDependencies": { + "@types/node": "^25.5.2" + } +} diff --git a/backend/tests/repos/deps/fixtures/npm/yarn-classic/yarn.lock b/backend/tests/repos/deps/fixtures/npm/yarn-classic/yarn.lock new file mode 100644 index 00000000..c08be132 --- /dev/null +++ b/backend/tests/repos/deps/fixtures/npm/yarn-classic/yarn.lock @@ -0,0 +1,33 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 +# SMALL real slice of mealie/frontend/yarn.lock (classic v1). Real blocks: +# axios (prod) -> follow-redirects (transitive edge), @mdi/js (prod, scoped), +# @types/node (dev-only). + + +"@mdi/js@^7.4.47": + version "7.4.47" + resolved "https://registry.yarnpkg.com/@mdi/js/-/js-7.4.47.tgz#7d8a4edc9631bffeed80d1ec784f9beae559a76a" + integrity sha512-KPnNOtm5i2pMabqZxpUz7iQf+mfrYZyKCZ8QNz85czgEt7cuHcGorWfdzUMWYA0SD+a6Hn4FmJ+YhzzzjkTZrQ== + +"@types/node@^25.5.2": + version "25.9.2" + resolved "https://registry.yarnpkg.com/@types/node/-/node-25.9.2.tgz#fc8958e757994b71fee516f9634bdb03d1b19e9f" + integrity sha512-G05zqtJhcDLb8uslf5EjCxXg9G1KQxiV8OS0R26IC//Eoyitzqe8z37I7cqvnZlrlSfgocQRfSn/AHBZJJFyGw== + dependencies: + undici-types ">=7.24.0 <7.24.7" + +axios@^1.8.1: + version "1.17.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.17.0.tgz#ae5a1164a4f719942cd73c67e6a3f62d3ccb8f2b" + integrity sha512-J8SwNxprqqpbfenehxWYXE7CW+wM1BB4w3+N+g+/Wx40xM4rsLrfPmHHxSWIxJLYDgSY/HqlFPIYb2/S3rxafw== + dependencies: + follow-redirects "^1.16.0" + form-data "^4.0.5" + https-proxy-agent "^5.0.1" + proxy-from-env "^2.1.0" + +follow-redirects@^1.16.0: + version "1.16.0" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.16.0.tgz#28474a159d3b9d11ef62050a14ed60e4df6d61bc" + integrity sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw== diff --git a/backend/tests/repos/deps/fixtures/pypi/reqdoc/requirements-doc.txt b/backend/tests/repos/deps/fixtures/pypi/reqdoc/requirements-doc.txt new file mode 100644 index 00000000..9365a3b8 --- /dev/null +++ b/backend/tests/repos/deps/fixtures/pypi/reqdoc/requirements-doc.txt @@ -0,0 +1,6 @@ +# Tiny declared-only docs requirements slice (no lockfile present). +sphinx==7.2.6 +mkdocs +mkdocstrings-python # inline comment is stripped +furo>=2024.1.0 ; python_version >= "3.9" +-r other-constraints.txt diff --git a/backend/tests/repos/deps/fixtures/pypi/uvslice/uv.lock b/backend/tests/repos/deps/fixtures/pypi/uvslice/uv.lock new file mode 100644 index 00000000..bd8052b1 --- /dev/null +++ b/backend/tests/repos/deps/fixtures/pypi/uvslice/uv.lock @@ -0,0 +1,57 @@ +version = 1 +revision = 3 +requires-python = ">=3.9, <4.0" + +# A small, structurally-faithful slice of instructor's uv.lock: +# workspace root 'demo' (editable) directly depends on aiohttp + urllib3; +# aiohttp pulls two transitive packages, exercising the resolved-edge path. + +[[package]] +name = "demo" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "aiohttp" }, + { name = "urllib3" }, +] + +[package.optional-dependencies] +docs = [ + { name = "mkdocs" }, +] + +[package.metadata] +requires-dist = [ + { name = "aiohttp", specifier = ">=3.9.1,<4.0.0" }, + { name = "urllib3", specifier = ">=2.0.0" }, + { name = "mkdocs", marker = "extra == 'docs'", specifier = ">=1.6.1" }, +] + +[[package]] +name = "aiohttp" +version = "3.13.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohappyeyeballs" }, + { name = "frozenlist" }, +] + +[[package]] +name = "aiohappyeyeballs" +version = "2.6.1" +source = { registry = "https://pypi.org/simple" } + +[[package]] +name = "frozenlist" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } + +[[package]] +name = "urllib3" +version = "2.6.3" +source = { registry = "https://pypi.org/simple" } + +[[package]] +name = "mkdocs" +version = "1.6.1" +source = { registry = "https://pypi.org/simple" } diff --git a/backend/tests/repos/deps/test_graph.py b/backend/tests/repos/deps/test_graph.py new file mode 100644 index 00000000..28bead6f --- /dev/null +++ b/backend/tests/repos/deps/test_graph.py @@ -0,0 +1,52 @@ +"""SP1 Task 3 — DepGraph scope propagation (prod dominates).""" + +from __future__ import annotations + +from cliff.repos.deps.graph import DepGraph + + +def test_undici_version_split() -> None: + """karakeep reality: undici@6.23.0 is direct prod (packages/shared); + undici@7.16.0 is transitive, pulled only via a dev tool → does NOT ship.""" + g = DepGraph() + # direct prod dep resolves to 6.23.0 + g.add_root("undici", "6.23.0", "prod", "packages/shared/package.json", "npm") + # a dev tool root pulls undici@7.16.0 transitively + g.add_root("some-dev-tool", "1.0.0", "dev", "package.json", "npm") + g.add_node("undici", "7.16.0", "npm") + g.add_edge(("some-dev-tool", "1.0.0"), ("undici", "7.16.0")) + + scopes = g.propagate_scopes() + assert scopes[("undici", "6.23.0")] == {"prod"} + assert scopes[("undici", "7.16.0")] == {"dev"} # the finding's version — safe + assert g.direct_nodes() == {("undici", "6.23.0"), ("some-dev-tool", "1.0.0")} + + +def test_both_dev_and_prod_yields_prod() -> None: + g = DepGraph() + g.add_root("prod-app", "1.0", "prod", "apps/web/package.json", "npm") + g.add_root("dev-tool", "1.0", "dev", "package.json", "npm") + g.add_node("shared", "2.0", "npm") + g.add_edge(("prod-app", "1.0"), ("shared", "2.0")) + g.add_edge(("dev-tool", "1.0"), ("shared", "2.0")) + scopes = g.propagate_scopes() + assert scopes[("shared", "2.0")] == {"prod", "dev"} + assert "prod" in scopes[("shared", "2.0")] # prod dominates → treated as ships + + +def test_pure_transitive_via_prod_ships() -> None: + g = DepGraph() + g.add_root("prod-app", "1.0", "prod", "package.json", "npm") + g.add_node("yaml", "2.8.0", "npm") + g.add_edge(("prod-app", "1.0"), ("yaml", "2.8.0")) + scopes = g.propagate_scopes() + assert scopes[("yaml", "2.8.0")] == {"prod"} + assert ("yaml", "2.8.0") not in g.direct_nodes() # transitive, not declared + + +def test_declared_in_map() -> None: + g = DepGraph() + g.add_root("axios", "1.13.2", "prod", "apps/web/package.json", "npm") + g.add_root("axios", "1.13.2", "prod", "apps/worker/package.json", "npm") + m = g.declared_in_map() + assert set(m[("axios", "1.13.2")]) == {"apps/web/package.json", "apps/worker/package.json"} diff --git a/backend/tests/repos/deps/test_imports.py b/backend/tests/repos/deps/test_imports.py new file mode 100644 index 00000000..3cafae34 --- /dev/null +++ b/backend/tests/repos/deps/test_imports.py @@ -0,0 +1,79 @@ +"""SP1 Task 6 — first-party import-site scan (the safety grep, collect_import_sites).""" + +from __future__ import annotations + +from cliff.repos.deps.imports import collect_import_sites + + +def test_npm_shipping_not_test_and_subpath(tmp_path) -> None: + src = tmp_path / "src" + src.mkdir() + (src / "app.ts").write_text("import { parse } from 'yaml'\nconst m = require('lodash/merge')\n") + tests = tmp_path / "src" / "__tests__" + tests.mkdir() + (tests / "app.test.ts").write_text("import { parse } from 'yaml'\n") # test → excluded + got = collect_import_sites(tmp_path, "npm") + assert got.get("yaml") == ["src/app.ts:1"] # test file not counted + assert got.get("lodash") == ["src/app.ts:2"] # subpath → package + + +def test_npm_scoped_and_relative(tmp_path) -> None: + (tmp_path / "b.js").write_text("const z = require('@xmldom/xmldom')\n") + (tmp_path / "rel.ts").write_text("import './local'\n") # relative — ignored + got = collect_import_sites(tmp_path, "npm") + assert got.get("@xmldom/xmldom") == ["b.js:1"] + assert "." not in got and "./local" not in got + + +def test_npm_no_false_match_on_substring(tmp_path) -> None: + # 'undici-types' must NOT be bucketed as 'undici' + (tmp_path / "x.ts").write_text("import type { X } from 'undici-types'\n") + got = collect_import_sites(tmp_path, "npm") + assert "undici" not in got + assert got.get("undici-types") == ["x.ts:1"] + + +def test_npm_barrel_reexport_counted(tmp_path) -> None: + # `export … from 'pkg'` is a real shipping import — must be recorded (it has no + # 'import'/'require' token, so the prefilter must also allow 'from'). + (tmp_path / "index.ts").write_text( + "export { Foo } from 'some-lib'\nexport * from 'other-lib'\n" + ) + got = collect_import_sites(tmp_path, "npm") + assert got.get("some-lib") == ["index.ts:1"] + assert got.get("other-lib") == ["index.ts:2"] + + +def test_py_import_forms_and_test_excluded(tmp_path) -> None: + pkg = tmp_path / "instructor" / "cli" + pkg.mkdir(parents=True) + (pkg / "usage.py").write_text("import aiohttp\n") + (tmp_path / "other.py").write_text("from aiohttp.client import ClientSession\n") + t = tmp_path / "tests" + t.mkdir() + (t / "test_x.py").write_text("import aiohttp\n") # test → excluded + got = collect_import_sites(tmp_path, "pypi") + assert set(got.get("aiohttp", [])) == {"instructor/cli/usage.py:1", "other.py:1"} + + +def test_py_dotted_import_bucketed_by_top_level(tmp_path) -> None: + # protobuf's import name is 'google.protobuf'; the collector must bucket by the + # TOP-LEVEL 'google' so build.py's top-level lookup matches. + (tmp_path / "m.py").write_text( + "from google.protobuf import message\nimport google.cloud.storage\n" + ) + got = collect_import_sites(tmp_path, "pypi") + assert got.get("google") == ["m.py:1", "m.py:2"] + + +def test_py_multi_import_all_captured(tmp_path) -> None: + # `import json, vulnerable_pkg` must capture BOTH, not just the first. + (tmp_path / "m.py").write_text("import json, vulnerable_pkg as vp\n") + got = collect_import_sites(tmp_path, "pypi") + assert got.get("vulnerable_pkg") == ["m.py:1"] + assert got.get("json") == ["m.py:1"] + + +def test_py_not_imported(tmp_path) -> None: + (tmp_path / "m.py").write_text("import requests\n") + assert collect_import_sites(tmp_path, "pypi").get("urllib3") is None diff --git a/backend/tests/repos/deps/test_manifests.py b/backend/tests/repos/deps/test_manifests.py new file mode 100644 index 00000000..31cf4365 --- /dev/null +++ b/backend/tests/repos/deps/test_manifests.py @@ -0,0 +1,79 @@ +"""SP1 Task 2 — manifest discovery + scope classification.""" + +from __future__ import annotations + +import pytest + +from cliff.repos.deps.manifests import classify_extra, classify_scope, discover_manifests + + +@pytest.mark.parametrize( + "extra,expected", + [ + ("dev", "dev"), ("Dev", "dev"), ("lint", "dev"), ("typing", "dev"), + ("docs", "docs"), ("test-docs", "docs"), ("test", "test"), ("tests", "test"), + ("benchmark", "build"), + # feature extras ship when installed → optional (conservative) + ("anthropic", "optional"), ("postgres", "optional"), ("all", "optional"), + ], +) +def test_classify_extra(extra: str, expected: str) -> None: + assert classify_extra(extra) == expected + + +@pytest.mark.parametrize( + "path,section,expected", + [ + ("package.json", "dependencies", "prod"), + ("package.json", "devDependencies", "dev"), + ("package.json", "optionalDependencies", "optional"), + ("package.json", "peerDependencies", "prod"), + ("apps/web/package.json", "dependencies", "prod"), + # directory dominates section + ("website/package.json", "dependencies", "docs"), + ("docs/package.json", "dependencies", "docs"), + ("examples/foo/package.json", "dependencies", "docs"), + ("e2e/package.json", "dependencies", "test"), + ("packages/api/__tests__/package.json", "dependencies", "test"), + # python + ("pyproject.toml", "project.dependencies", "prod"), + ("pyproject.toml", "dependency-groups", "dev"), + ("pyproject.toml", "project.optional-dependencies", "optional"), + ("requirements.txt", "requirements.txt", "prod"), + ("requirements-dev.txt", "requirements-dev.txt", "dev"), + ("requirements-doc.txt", "requirements-doc.txt", "docs"), + ("requirements-examples.txt", "requirements-examples.txt", "docs"), + # unknown section defaults to prod (conservative) + ("package.json", "someFutureSection", "prod"), + ], +) +def test_classify_scope(path: str, section: str, expected: str) -> None: + assert classify_scope(path, section) == expected + + +def test_discover_manifests(tmp_path) -> None: + (tmp_path / "package.json").write_text("{}") + (tmp_path / "pnpm-workspace.yaml").write_text("packages:\n - packages/*\n") + pkgs = tmp_path / "packages" / "api" + pkgs.mkdir(parents=True) + (pkgs / "package.json").write_text("{}") + py = tmp_path / "sdk" + py.mkdir() + (py / "pyproject.toml").write_text("[project]\n") + (tmp_path / "requirements-doc.txt").write_text("sphinx\n") + # excluded + nm = tmp_path / "node_modules" / "x" + nm.mkdir(parents=True) + (nm / "package.json").write_text("{}") + + found = discover_manifests(tmp_path) + paths = {m.path for m in found} + assert "package.json" in paths + assert "packages/api/package.json" in paths + assert "sdk/pyproject.toml" in paths + assert "requirements-doc.txt" in paths + assert "pnpm-workspace.yaml" in paths + # node_modules excluded + assert not any("node_modules" in p for p in paths) + ecos = {m.ecosystem for m in found} + assert ecos == {"npm", "pypi"} diff --git a/backend/tests/repos/deps/test_npm.py b/backend/tests/repos/deps/test_npm.py new file mode 100644 index 00000000..f87c9982 --- /dev/null +++ b/backend/tests/repos/deps/test_npm.py @@ -0,0 +1,161 @@ +"""SP1 — npm/JS lockfile parser (pnpm / yarn-classic / yarn-berry / package-lock). + +Fixtures under ``fixtures/npm/`` are SMALL real slices of the actual lockfiles in +the triage clones (karakeep, mealie, linkwarden, nicegui). The final test runs the +parser over the FULL real karakeep clone. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from cliff.repos.deps.manifests import discover_manifests +from cliff.repos.deps.npm import ( + _split_key, + _strip_peer, + parse_npm, +) + +FIX = Path(__file__).parent / "fixtures" / "npm" +KARAKEEP = Path("/private/tmp/cliff-triage-work/clones/karakeep-9879eb5d36") + + +def _build(fixture_dir: Path): + """Run parse_npm over a fixture mini-repo → (graph, unresolved, scopes).""" + manifests = discover_manifests(fixture_dir) + graph, unresolved = parse_npm(fixture_dir, manifests) + return graph, unresolved, graph.propagate_scopes() + + +# ── unit: name/version splitting ───────────────────────────────────────────── +@pytest.mark.parametrize( + "key,name,ver", + [ + ("undici@7.16.0", "undici", "7.16.0"), + ("@xmldom/xmldom@0.8.0", "@xmldom/xmldom", "0.8.0"), # scoped: @ in name + ("@aws-sdk/client-s3@3.1014.0", "@aws-sdk/client-s3", "3.1014.0"), + ("openai@6.34.0(ws@8.18.3)(zod@4.3.6)", "openai", "6.34.0"), # peer suffix + ], +) +def test_split_key(key: str, name: str, ver: str) -> None: + assert _split_key(key) == (name, ver) + + +def test_strip_peer() -> None: + assert _strip_peer("6.34.0(ws@8.18.3)(zod@4.3.6)") == "6.34.0" + assert _strip_peer("6.23.0") == "6.23.0" + + +# ── pnpm (the primary format) ──────────────────────────────────────────────── +def test_pnpm_undici_version_split() -> None: + graph, unresolved, scopes = _build(FIX / "pnpm") + assert unresolved == [] + + undici_620 = ("undici", "6.23.0") + undici_716 = ("undici", "7.16.0") + # both versions are real resolved nodes + assert undici_620 in graph.nodes + assert undici_716 in graph.nodes + # 6.23.0 is declared prod in packages/shared -> carries prod + assert "prod" in scopes[undici_620] + # 7.16.0 is only reachable via cheerio (a DEV dep of the root) -> NOT prod + assert "prod" not in scopes[undici_716] + assert scopes[undici_716] == {"dev"} + + +def test_pnpm_scoped_name_and_root() -> None: + graph, _unresolved, scopes = _build(FIX / "pnpm") + scoped = ("@xmldom/xmldom", "0.8.0") + assert scoped in graph.nodes # scoped name parsed correctly + assert "prod" in scopes[scoped] + # the prod root is declared in the shared workspace's package.json + assert ("@xmldom/xmldom", "0.8.0") in graph.direct_nodes() + decl = graph.declared_in_map()[("undici", "6.23.0")] + assert decl == ["packages/shared/package.json"] + + +def test_pnpm_peer_suffix_stripped_on_root() -> None: + graph, _unresolved, scopes = _build(FIX / "pnpm") + # importer version was openai@6.34.0(ws@8.18.3)(zod@4.3.6) -> resolves to 6.34.0 + assert ("openai", "6.34.0") in graph.direct_nodes() + assert "prod" in scopes[("openai", "6.34.0")] + + +def test_pnpm_dev_only_not_prod() -> None: + _graph, _unresolved, scopes = _build(FIX / "pnpm") + # cheerio is a dev-only workspace dep -> never carries prod + assert scopes[("cheerio", "1.1.2")] == {"dev"} + + +# ── yarn classic v1 ────────────────────────────────────────────────────────── +def test_yarn_classic() -> None: + graph, unresolved, scopes = _build(FIX / "yarn-classic") + assert unresolved == [] + # axios is a prod dep; its range ^1.8.1 resolves to 1.17.0 + assert ("axios", "1.17.0") in graph.direct_nodes() + assert "prod" in scopes[("axios", "1.17.0")] + # transitive edge axios -> follow-redirects propagates prod + assert "prod" in scopes[("follow-redirects", "1.16.0")] + # scoped prod dep parses + assert ("@mdi/js", "7.4.47") in graph.nodes + assert "prod" in scopes[("@mdi/js", "7.4.47")] + # dev-only dep does not carry prod + assert "prod" not in scopes[("@types/node", "25.9.2")] + + +# ── yarn berry v2+ ─────────────────────────────────────────────────────────── +def test_yarn_berry() -> None: + graph, unresolved, scopes = _build(FIX / "yarn-berry") + assert unresolved == [] + # prod deps of apps/web resolve through @npm: descriptors + assert "prod" in scopes[("axios", "1.13.2")] + assert "prod" in scopes[("next", "15.3.9")] + assert "prod" in scopes[("dompurify", "3.3.1")] + # real transitive edge dompurify -> @types/trusted-types propagates prod + assert "prod" in scopes[("@types/trusted-types", "2.0.7")] + # comma-joined scoped descriptor key resolves; dev-only -> not prod + assert ("@types/react", "19.2.14") in graph.nodes + assert "prod" not in scopes[("@types/react", "19.2.14")] + + +# ── package-lock v3 ────────────────────────────────────────────────────────── +def test_package_lock() -> None: + graph, unresolved, scopes = _build(FIX / "package-lock") + assert unresolved == [] + # prod dep resolves via the install tree + assert "prod" in scopes[("dompurify", "3.4.0")] + # scoped dev dep parses + assert ("@antfu/install-pkg", "1.1.0") in graph.nodes + assert "prod" not in scopes[("@antfu/install-pkg", "1.1.0")] + # dev-only transitive stays non-prod + assert ("package-manager-detector", "1.6.0") in graph.nodes + assert "prod" not in scopes[("package-manager-detector", "1.6.0")] + + +# ── absent lockfile ────────────────────────────────────────────────────────── +def test_absent_lockfile_is_unresolved(tmp_path: Path) -> None: + (tmp_path / "package.json").write_text( + json.dumps({"name": "x", "dependencies": {"axios": "^1.0.0"}}) + ) + graph, unresolved = parse_npm(tmp_path, discover_manifests(tmp_path)) + # no lockfile => cannot resolve => flagged, and nothing invented + assert "package.json" in unresolved + assert graph.nodes == {} + + +# ── integration: FULL real karakeep clone ──────────────────────────────────── +@pytest.mark.skipif(not KARAKEEP.exists(), reason="karakeep triage clone not present") +def test_karakeep_full_clone_integration() -> None: + """Reads the FULL karakeep clone (pnpm-lock v9, 27 workspaces).""" + manifests = discover_manifests(KARAKEEP) + graph, unresolved = parse_npm(KARAKEEP, manifests) # (a) no exception + # (b) the transitive undici@7.16.0 is a real resolved node + assert ("undici", "7.16.0") in graph.nodes + assert ("undici", "6.23.0") in graph.nodes + # (c) the pnpm-lock parsed cleanly — nothing left unresolved + assert unresolved == [] + # sanity: the prod root undici@6.23.0 is declared in packages/shared + assert "packages/shared/package.json" in graph.declared_in_map()[("undici", "6.23.0")] diff --git a/backend/tests/repos/deps/test_pypi.py b/backend/tests/repos/deps/test_pypi.py new file mode 100644 index 00000000..cd2015b8 --- /dev/null +++ b/backend/tests/repos/deps/test_pypi.py @@ -0,0 +1,157 @@ +"""SP1 — PyPI lockfile/manifest parser + dist→import map. + +Unit tests run against small committed fixtures; one integration test runs the +parser over the FULL real ``instructor`` clone (uv.lock) when it is available. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from cliff.repos.deps.manifests import Manifest, discover_manifests +from cliff.repos.deps.pypi import dist_to_import, normalize_dist, parse_pypi + +FIXTURES = Path(__file__).parent / "fixtures" / "pypi" +INSTRUCTOR = Path("/private/tmp/cliff-triage-work/clones/instructor-36394b835f") + + +# ── dist_to_import ──────────────────────────────────────────────────────────── +@pytest.mark.parametrize( + "dist,expected", + [ + ("aiohttp", "aiohttp"), # 1:1 identity + ("urllib3", "urllib3"), # 1:1 identity (digits preserved) + ("requests", "requests"), + ("PyYAML", "yaml"), # known exception, case-insensitive + ("pyyaml", "yaml"), + ("beautifulsoup4", "bs4"), + ("Pillow", "PIL"), + ("scikit-learn", "sklearn"), + ("opencv-python", "cv2"), + ("python-dateutil", "dateutil"), + ("typing_extensions", "typing_extensions"), # normalizes then maps + ("attrs", "attr"), + ("PyJWT", "jwt"), + ("protobuf", "google.protobuf"), + ("some-dashed-lib", "some_dashed_lib"), # 1:1: dashes → underscores + ], +) +def test_dist_to_import(dist: str, expected: str) -> None: + assert dist_to_import(dist) == expected + + +def test_dist_to_import_unknown_ambiguous_is_none() -> None: + # genuinely ambiguous dist (multiple top-level modules) → None, not identity + assert dist_to_import("pywin32") is None + assert dist_to_import("") is None + + +# ── normalize_dist (PEP 503) ────────────────────────────────────────────────── +@pytest.mark.parametrize( + "raw,expected", + [ + ("PyYAML", "pyyaml"), + ("typing_extensions", "typing-extensions"), + ("Flask-SQLAlchemy", "flask-sqlalchemy"), + ("zope.interface", "zope-interface"), + ("a__b--c..d", "a-b-c-d"), + ], +) +def test_normalize_dist(raw: str, expected: str) -> None: + assert normalize_dist(raw) == expected + + +# ── uv.lock slice: resolved graph + edges ───────────────────────────────────── +def test_uv_lock_slice_nodes_and_edges() -> None: + root = FIXTURES / "uvslice" + manifests = [Manifest("pyproject.toml", "pypi", "pyproject.toml")] + g, unresolved = parse_pypi(root, manifests) + + assert unresolved == [] + # nodes resolved with versions + assert ("aiohttp", "3.13.5") in g.nodes + assert ("urllib3", "2.6.3") in g.nodes + assert ("aiohappyeyeballs", "2.6.1") in g.nodes + + # direct deps of the workspace root are roots (prod) + assert ("aiohttp", "3.13.5") in g.direct_nodes() + assert ("urllib3", "2.6.3") in g.direct_nodes() + + scopes = g.propagate_scopes() + assert scopes[("aiohttp", "3.13.5")] == {"prod"} + # transitive edge aiohttp → aiohappyeyeballs propagates prod (proves edges exist) + assert scopes[("aiohappyeyeballs", "2.6.1")] == {"prod"} + assert scopes[("frozenlist", "1.8.0")] == {"prod"} + # a docs-named extra is non-shipping tooling → scoped docs, never prod + assert scopes[("mkdocs", "1.6.1")] == {"docs"} + + +# ── declared-only fallback (no lockfile) ────────────────────────────────────── +def test_declared_only_requirements_doc_scope() -> None: + root = FIXTURES / "reqdoc" + manifests = [Manifest("requirements-doc.txt", "pypi", "requirements")] + g, unresolved = parse_pypi(root, manifests) + + assert unresolved == [] + roots = {r.name: r for r in g.roots} + # every declared dep became a docs-scoped root + assert roots["sphinx"].scope == "docs" + assert roots["sphinx"].version == "7.2.6" # pinned version parsed + assert roots["mkdocs"].scope == "docs" + assert roots["mkdocs"].version == "" # unpinned → unknown version + assert roots["mkdocstrings-python"].scope == "docs" # inline comment stripped + assert roots["furo"].scope == "docs" # env marker stripped + # '-r other-constraints.txt' include line is ignored (not fabricated) + assert "other-constraints" not in roots + + scopes = g.propagate_scopes() + assert scopes[("sphinx", "7.2.6")] == {"docs"} + + +# ── integration: full real instructor clone via uv.lock ─────────────────────── +@pytest.mark.skipif(not INSTRUCTOR.exists(), reason="instructor clone not present") +def test_integration_instructor_full_clone() -> None: + manifests = discover_manifests(INSTRUCTOR) + g, unresolved = parse_pypi(INSTRUCTOR, manifests) + + # uv.lock parsed cleanly → nothing unresolved + assert unresolved == [] + + names = {n for (n, _v) in g.nodes} + # direct prod dep declared in pyproject [project.dependencies] + assert "aiohttp" in names + assert "urllib3" in names # transitive via requests, multi-version in lock + assert "requests" in names + + # aiohttp is a direct prod root and ships + scopes = g.propagate_scopes() + aiohttp_nvs = [(n, v) for (n, v) in g.nodes if n == "aiohttp"] + assert aiohttp_nvs, "aiohttp node missing" + assert any("prod" in scopes.get(nv, set()) for nv in aiohttp_nvs) + assert any(nv in g.direct_nodes() for nv in aiohttp_nvs) + + # the resolved graph has real transitive edges (not just declared roots) + assert any("prod" in s for nv, s in scopes.items() if nv[0] == "yarl") + + +def test_lockless_subproject_not_dropped_when_sibling_has_lock(tmp_path) -> None: + """Per-directory precedence: subA has a uv.lock, subB has only a pyproject. + subB's declared prod dep must still be parsed (a repo-global 'resolved' flag + would drop it and risk a false clear of subB's shipping deps).""" + (tmp_path / "subA").mkdir() + (tmp_path / "subA" / "pyproject.toml").write_text('[project]\nname = "a"\nversion = "0.1"\n') + (tmp_path / "subA" / "uv.lock").write_text( + 'version = 1\n[[package]]\nname = "a"\nversion = "0.1"\nsource = { editable = "." }\n' + ) + (tmp_path / "subB").mkdir() + (tmp_path / "subB" / "pyproject.toml").write_text( + '[project]\nname = "b"\nversion = "0.1"\ndependencies = ["subb-prod-dep"]\n' + ) + manifests = [ + Manifest("subA/pyproject.toml", "pypi", "pyproject.toml"), + Manifest("subB/pyproject.toml", "pypi", "pyproject.toml"), + ] + g, _unresolved = parse_pypi(tmp_path, manifests) + assert "subb-prod-dep" in {r.name for r in g.roots} # subB not dropped diff --git a/backend/tests/repos/test_dep_schemas.py b/backend/tests/repos/test_dep_schemas.py new file mode 100644 index 00000000..f8fd32d6 --- /dev/null +++ b/backend/tests/repos/test_dep_schemas.py @@ -0,0 +1,41 @@ +"""SP1 Task 1 — DepNode / DepManifest schema.""" + +from __future__ import annotations + +from cliff.repos.schemas import DepManifest, DepNode + + +def test_dep_node_roundtrips() -> None: + n = DepNode( + name="undici", + version="7.16.0", + ecosystem="npm", + scopes=["dev"], + direct=False, + declared_in=[], + import_name="undici", + import_sites=[], + ) + d = n.model_dump() + assert d["name"] == "undici" + assert d["scopes"] == ["dev"] + assert DepNode(**d) == n + + +def test_dep_manifest_roundtrips_and_allows_extra() -> None: + m = DepManifest( + ecosystems=["npm"], + workspaces=["package.json"], + nodes=[DepNode(name="yaml", version="2.8.0", ecosystem="npm", scopes=["prod"])], + unresolved=[], + ) + d = m.model_dump() + assert d["nodes"][0]["version"] == "2.8.0" + # extra="allow": a newer builder may add a field + m2 = DepManifest(ecosystems=["npm"], nodes=[], future_field=123) + assert m2.model_dump()["future_field"] == 123 + + +def test_import_name_none_is_representable() -> None: + n = DepNode(name="some-dist", version="1.0", ecosystem="pypi", import_name=None) + assert n.import_name is None diff --git a/backend/tests/repos/test_profile_agents.py b/backend/tests/repos/test_profile_agents.py index db2ac947..a931a14d 100644 --- a/backend/tests/repos/test_profile_agents.py +++ b/backend/tests/repos/test_profile_agents.py @@ -69,7 +69,7 @@ async def test_threat_history_defaults_to_empty(clone): assert parsed.prior_issues == [] -def test_make_profile_builders_has_all_three(): +def test_make_profile_builders_has_all_artifacts(): builders = make_profile_builders(TestModel()) - assert set(builders) == {"profile", "code_map", "threat"} + assert set(builders) == {"profile", "code_map", "threat", "dep_manifest"} assert all(callable(b) for b in builders.values()) diff --git a/backend/tests/repos/test_service.py b/backend/tests/repos/test_service.py index a77ca1d1..c6fe0cd5 100644 --- a/backend/tests/repos/test_service.py +++ b/backend/tests/repos/test_service.py @@ -25,7 +25,7 @@ async def token(): ) assert runner._sync_clone is git_ops.sync_clone assert runner._head_sha is git_ops.git_head_sha - assert set(runner._builders) == {"profile", "code_map", "threat"} + assert set(runner._builders) == {"profile", "code_map", "threat", "dep_manifest"} async def test_end_to_end_with_real_git_adapters(db, bare_repo, tmp_path):