XRAY-156190 - Implement all packages blocked in packument fallback lo…

commands/curation/curationaudit.go

@@ -105,6 +105,14 @@ const (
 
 	// hfUnresolvedReportKey is used when an HF scan found only dynamic references — no table, just warnings.
 	hfUnresolvedReportKey = "huggingface (unresolved references)"
+
+	// npmLogPartialReportWarning is shown when npm install crashed mid-resolution (e.g. a
+	// fully-blocked package's packument returned 403) and runNpmLogFallback recovered a
+	// partial report from npm's own debug log instead of aborting with no output.
+	npmLogPartialReportWarning = "npm install could not fully resolve the dependency tree because one or more packages " +
+		"are blocked by the curation policy. This report was reconstructed from npm's debug log after the installation " +
+		"failed, so it may be incomplete. Dependencies of blocked packages could not be analyzed because their metadata " +
+		"was unavailable."
 )
 
 var CurationOutputFormats = []string{string(outFormat.Table), string(outFormat.Json)}
@@ -289,6 +297,10 @@ type CurationReport struct {
 	// Kept separate from isPartial, which also triggers the CVS-specific warning
 	// and skips the waiver flow.
 	hfPartial bool
+	// npmLogPartial marks a report produced by runNpmLogFallback. Set alongside isPartial
+	// (to reuse its waiver-skip gate), but kept distinct for its own warning text and
+	// partialReason.
+	npmLogPartial bool
 	// warnings holds user-facing messages from tree-build (e.g. unresolved HF references).
 	warnings []string
 	// huggingFaceReport marks reports produced by the Hugging Face audit path (including
@@ -426,7 +438,11 @@ func (ca *CurationAuditCommand) Run() (err error) {
 	}
 	for projectPath, report := range results {
 		if report.isPartial {
-			log.Warn(fmt.Sprintf("[%s] %s", projectPath, cvsPartialReportWarning))
+			warningText := cvsPartialReportWarning
+			if report.npmLogPartial {
+				warningText = npmLogPartialReportWarning
+			}
+			log.Warn(fmt.Sprintf("[%s] %s", projectPath, warningText))
 		}
 	}
 
@@ -481,6 +497,11 @@ func convertResultsToSummary(results map[string]*CurationReport) formats.Results
 		}
 		var partialReason string
 		switch {
+		// npmLogPartial is checked before isPartial: an npm-log-fallback report sets both
+		// (isPartial to reuse the existing waiver-skip gate below), so npmLogPartial must
+		// win here or it would incorrectly report "cvs_fallback".
+		case packagesStatus.npmLogPartial:
+			partialReason = "npm_log_fallback"
 		case packagesStatus.isPartial:
 			partialReason = "cvs_fallback"
 		case packagesStatus.hfPartial:
@@ -949,6 +970,26 @@ func (ca *CurationAuditCommand) auditTree(tech techutils.Technology, results map
 	if err != nil {
 		return err
 	}
+	// For npm, discover the user's own cache dir and logs-max setting before installing —
+	// --cache is never overridden, only used to locate the debug log runNpmLogFallback reads.
+	// If logs-max was already 0, our forced --logs-max=10 is the only reason a log exists,
+	// so we delete just that file afterward.
+	var npmCacheDir string
+	var npmLogsMaxWasZero bool
+	if tech == techutils.Npm {
+		workDir, wdErr := osGetwd()
+		if wdErr != nil {
+			return errorutils.CheckErrorf("failed to determine working directory for npm config lookup: %v", wdErr)
+		}
+		if npmCacheDir, err = npmtech.GetNpmConfigValue(workDir, "cache"); err != nil {
+			return errorutils.CheckErrorf("failed to determine npm cache directory: %v", err)
+		}
+		logsMaxValue, logsMaxErr := npmtech.GetNpmConfigValue(workDir, "logs-max")
+		if logsMaxErr != nil {
+			return errorutils.CheckErrorf("failed to determine npm logs-max config: %v", logsMaxErr)
+		}
+		npmLogsMaxWasZero = logsMaxValue == "0"
+	}
 	depTreeResult, err := buildinfo.GetTechDependencyTree(params, serverDetails, tech)
 	if err != nil {
 		// When CVS strips a pinned version from the simple index, pip can't
@@ -959,6 +1000,11 @@ func (ca *CurationAuditCommand) auditTree(tech techutils.Technology, results map
 		if (tech == techutils.Pip || tech == techutils.Poetry) && errors.As(err, &cvsErr) {
 			return ca.runCvsFallback(cvsErr, tech, results)
 		}
+		// npm's install error is a plain opaque error, nothing to type-match on — trigger the
+		// debug-log fallback unconditionally on any npm tree-build failure.
+		if tech == techutils.Npm {
+			return ca.runNpmLogFallback(npmCacheDir, tech, results, err, npmLogsMaxWasZero)
+		}
 		return err
 	}
 	// Validate the graph isn't empty.
@@ -1724,6 +1770,201 @@ func (ca *CurationAuditCommand) runCvsFallback(cvsErr *python.CvsBlockedError, t
 	return nil
 }
 
+// runNpmLogFallback reconstructs a partial curation report from npm's debug log
+// (cacheDir/_logs/) when the npm tree-build install crashes, since buildDeps completes
+// before reify can fail. Falls back to originalErr unchanged if nothing is recoverable.
+// See NPM_CURATION_DEBUGLOG_FALLBACK_DESIGN for the full design.
+//
+// logsMaxWasZero means our forced --logs-max=10 is the only reason a debug log exists for
+// this run, so the log file read here is removed afterward; otherwise npm's own rotation
+// manages it like any other run.
+func (ca *CurationAuditCommand) runNpmLogFallback(cacheDir string, tech techutils.Technology, results map[string]*CurationReport, originalErr error, logsMaxWasZero bool) error {
+	entries, blockedPackages, logFilePath, parseErr := parseNpmDebugLog(cacheDir)
+	if logsMaxWasZero && logFilePath != "" {
+		defer func() {
+			_ = os.Remove(logFilePath)
+		}()
+	}
+	if parseErr != nil {
+		log.Debug(fmt.Sprintf("npm curation debug-log fallback: failed to parse debug log (%v); original error: %s", parseErr, originalErr.Error()))
+		return originalErr
+	}
+	if len(entries) == 0 {
+		// Nothing recoverable — install likely failed before buildDeps ever started.
+		return originalErr
+	}
+
+	rtManager, serverDetails, err := ca.getRtManagerAndAuth(tech)
+	if err != nil {
+		return fmt.Errorf("npm curation debug-log fallback: failed to get Artifactory manager (%w); %s error: %w", err, tech, originalErr)
+	}
+	rtAuth, err := serverDetails.CreateArtAuthConfig()
+	if err != nil {
+		return fmt.Errorf("npm curation debug-log fallback: failed to create auth config (%w); %s error: %w", err, tech, originalErr)
+	}
+
+	workPath, wdErr := osGetwd()
+	if wdErr != nil {
+		log.Warn(fmt.Sprintf("npm curation debug-log fallback: could not determine working directory (%v) — reporting under fallback key", wdErr))
+		workPath = "unknown-project"
+	}
+	rootName, rootVersion, pkgErr := readNpmProjectNameVersion(workPath)
+	if pkgErr != nil {
+		// Same convention auditTree itself uses when a project doesn't declare a name.
+		rootName = filepath.Base(workPath)
+	}
+
+	analyzer := treeAnalyzer{
+		rtManager:            rtManager,
+		extractPoliciesRegex: ca.extractPoliciesRegex,
+		rtAuth:               rtAuth,
+		httpClientDetails:    rtAuth.CreateHttpClientDetails(),
+		url:                  rtAuth.GetUrl(),
+		repo:                 ca.PackageManagerConfig.TargetRepo(),
+		tech:                 tech,
+	}
+
+	// Used later to decide whether a whole-package-blocked row's recommendation should be
+	// "remove and replace" (direct) or name the real direct dependency (transitive).
+	directDepNames := map[string]bool{}
+	for _, entry := range entries {
+		if entry.ParentName == rootName && entry.ParentVersion == rootVersion {
+			directDepNames[entry.Name] = true
+		}
+	}
+
+	preProcessMap := &sync.Map{}
+	var advisoryWarnings []string
+	seenKeys := map[string]bool{}
+	for _, entry := range sortedNpmLogEntries(entries) {
+		category := classifyBlankVersionEntry(entry, blockedPackages)
+		key, keyErr := npmPackageKey(tech, analyzer.url, analyzer.repo, entry.Name, entry.Version)
+		if keyErr != nil || seenKeys[key] {
+			continue
+		}
+		seenKeys[key] = true
+
+		switch category {
+		case npmEntryResolved:
+			pkgStatus, probeErr := analyzer.getBlockedPackageDetails(key, entry.Name, entry.Version)
+			if probeErr != nil || pkgStatus == nil {
+				continue // network/auth error, or genuinely clean — no row either way
+			}
+			preProcessMap.Store(key, pkgStatus)
+		case npmEntryWholePackageBlocked:
+			preProcessMap.Store(key, wholePackageBlockedStatus(entry, blockedPackages[entry.Name], tech))
+		case npmEntryNonRegistrySpecifier:
+			preProcessMap.Store(key, nonRegistrySpecifierStatus(entry, tech))
+		case npmEntryUnresolvableRange:
+			advisoryWarnings = append(advisoryWarnings, fmt.Sprintf(
+				"Package '%s' (requested as '%s') could not be resolved to a specific version, so its curation status could not be checked. This is not necessarily a curation block — verify the dependency resolves correctly.",
+				entry.Name, entry.Specifier))
+		case npmEntryETARGET:
+			etargetKey, etargetKeyErr := npmPackageKey(tech, analyzer.url, analyzer.repo, entry.Name, entry.Specifier)
+			if etargetKeyErr != nil {
+				continue
+			}
+			pkgStatus, probeErr := analyzer.getBlockedPackageDetails(etargetKey, entry.Name, entry.Specifier)
+			if probeErr != nil {
+				continue
+			}
+			if pkgStatus == nil {
+				// Plain 404, no curation signal — not curation-relevant, advisory only.
+				advisoryWarnings = append(advisoryWarnings, fmt.Sprintf(
+					"Package '%s@%s' could not be resolved (no matching version found in the registry) — this is not a curation policy block. Verify the version in package.json.",
+					entry.Name, entry.Specifier))
+				continue
+			}
+			preProcessMap.Store(etargetKey, pkgStatus)
+		}
+	}
+
+	graph := buildGraphFromLogEntries(entries, rootName, rootVersion)
+	var packagesStatus []*PackageStatus
+	analyzer.GraphsRelations([]*xrayUtils.GraphNode{graph}, preProcessMap, &packagesStatus)
+
+	// Recommendation depends on ParentName, only known after GraphsRelations runs.
+	for _, ps := range packagesStatus {
+		if ps.PackageVersion == allVersionsBlockedText {
+			applyTransitiveAwareRecommendation(ps, directDepNames[ps.PackageName])
+		}
+	}
+
+	if len(packagesStatus) == 0 && len(advisoryWarnings) == 0 {
+		return originalErr
+	}
+
+	results[uniqueReportKey(results, filepath.Base(workPath), tech)] = &CurationReport{
+		packagesStatus:        packagesStatus,
+		totalNumberOfPackages: len(packagesStatus),
+		isPartial:             true,
+		npmLogPartial:         true,
+		warnings:              advisoryWarnings,
+	}
+	return nil
+}
+
+// allVersionsBlockedText also doubles as the marker applyTransitiveAwareRecommendation uses
+// to identify whole-package-blocked rows after the graph walk.
+const allVersionsBlockedText = "All versions blocked"
+
+// wholePackageBlockedStatus synthesizes a *PackageStatus straight from the log's notice+403
+// evidence — no HEAD-check needed. Recommendation is filled in later by
+// applyTransitiveAwareRecommendation, once the graph walk knows the parent.
+func wholePackageBlockedStatus(entry npmLogEntry, info npmBlockedInfo, tech techutils.Technology) *PackageStatus {
+	return &PackageStatus{
+		PackageName:    entry.Name,
+		PackageVersion: allVersionsBlockedText,
+		Action:         blocked,
+		BlockingReason: BlockingReasonPolicy,
+		PkgType:        string(tech),
+		Policy: []Policy{{
+			Policy:      info.Policy,
+			Condition:   info.Condition,
+			Explanation: allVersionsBlockedText,
+			// Recommendation is filled in by applyTransitiveAwareRecommendation.
+		}},
+	}
+}
+
+// applyTransitiveAwareRecommendation: "remove and replace" is only actionable when the
+// blocked package is itself a direct dependency; otherwise name the real direct dependency
+// so the user has an actual lever (waiver, or replace that dependency).
+func applyTransitiveAwareRecommendation(ps *PackageStatus, isDirectDependency bool) {
+	for i := range ps.Policy {
+		if isDirectDependency {
+			ps.Policy[i].Recommendation = "Remove this package from your project and replace with an alternate package"
+		} else {
+			ps.Policy[i].Recommendation = fmt.Sprintf(
+				"%s is a transitive dependency of %s and cannot be removed directly. Apply a waiver if acceptable, or replace %s with an alternative that doesn't depend on %s.",
+				ps.PackageName, ps.ParentName, ps.ParentName, ps.PackageName)
+		}
+	}
+}
+
+// nonRegistrySpecifierStatus reports a git URL/local-reference/npm-alias dependency as its
+// own row with "—" Policy/Condition — a third state distinct from clean/blocked, since npm
+// never contacts the registry for these at all and curation's enforcement never applies.
+// Never makes a network call on this package's behalf.
+func nonRegistrySpecifierStatus(entry npmLogEntry, tech techutils.Technology) *PackageStatus {
+	return &PackageStatus{
+		PackageName:    entry.Name,
+		PackageVersion: fmt.Sprintf("%s (not evaluated)", entry.Specifier),
+		Action:         blocked,
+		BlockingReason: BlockingReasonUnknown,
+		PkgType:        string(tech),
+		Policy: []Policy{{
+			Policy:    "—",
+			Condition: "—",
+			Explanation: "Not evaluated — this dependency's specifier is not a registry version (e.g. a git URL, local " +
+				"reference, or npm alias). npm never contacts Artifactory for it at all (no packument fetch, no tarball " +
+				"download), so curation's enforcement never applies.",
+			Recommendation: "If policy compliance is required for this dependency, use the equivalent published npm " +
+				"package instead of this reference, if one exists.",
+		}},
+	}
+}
+
 // lookupPypiAllVersions calls the Artifactory PyPI metadata API for a package
 // name (no version — returns all releases) and returns all available version
 // strings. This endpoint is NOT filtered by CVS, so it includes versions that