From 3b7b42101c2fb17d0efc826fcdf83f9991b2a041 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Thu, 30 Jul 2026 19:25:48 +0900 Subject: [PATCH 1/3] Estimate index dry-run mutations (#4893) --- DEVELOPER_GUIDE.md | 2 + README.md | 32 + changelog.d/unreleased/4893.changed.md | 20 + .../Cli/IndexCommandRunner.DryRun.cs | 574 +++++++++++++++--- src/CodeIndex/Cli/IndexCommandRunner.cs | 1 + src/CodeIndex/Cli/JsonOutputContracts.cs | 16 +- .../IndexCommandRunnerDryRunTests.cs | 241 +++++++- 7 files changed, 794 insertions(+), 92 deletions(-) create mode 100644 changelog.d/unreleased/4893.changed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index a10bb1476..032f84713 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -33,6 +33,7 @@ Development contracts: |---|---| | Formatting and warnings | CI enforces repository formatting with `.editorconfig` and treats compiler warnings as errors through `Directory.Build.props`, so local changes should pass the format check before opening a PR. Existing trim-analysis warnings are explicitly listed in `WarningsNotAsErrors` until they are fixed without blocking ordinary compiler-warning enforcement, and ILLink keeps reporting trim warnings without failing trimmed publish smoke tests. | | CLI help | `cdidx --help` stays brief, `cdidx --help-all` prints the full command/flag/example reference, `cdidx --help-flags` prints only shared flag tables, and `cdidx --help` prints one command's usage line. Keep new commands visible in the brief summary only when they are a primary user workflow; every command must remain present in the full help and command-specific usage table. | +| `index --dry-run` mutation estimates | Dry-run remains strictly non-mutating: it does not create or lock the database and does not alter sources or index artifacts. It compares checksums to project unchanged skips; projects updates, content-policy skips, deletes, purges, and symbol/reference cap hits; and performs the normal chunk/symbol/reference/content-diagnostic extraction for at most 100 planned update files. `estimated_table_mutations` counts projected delete-and-insert row operations, not final table cardinality. Its nullable values must stay synchronized with `estimated_table_mutation_details`: `source` identifies filesystem planning or parse-only plus index-snapshot input, `confidence` is `exact`, `estimate`, or `unknown`, and stable `unknown_reasons` cover candidate truncation, parse-estimate truncation/failure, and partial-index table absence. A numeric zero is reserved for measured no work and remains qualified by its confidence. Parse-only estimates intentionally omit post-extraction hook mutations and cross-file C# workspace augmentation, so child-table values remain estimates. | | `cdidx validate` | This is the user-facing integrity scan for indexed content issues such as replacement characters, BOMs, NUL bytes, mixed line endings, UTF-16 BOMs, and likely non-UTF8 content. Keep its CLI usage, README entry, and help summary in sync when adding validation issue kinds or filters. | | `cdidx doctor` | This is the copy-pasteable environment summary for support requests. Keep it redacted by default: secret-like `CDIDX_*` values must not be printed, and new diagnostic fields should be stable enough for issue triage. Full environment inventory filters (`--env-domain`, `--env-category`, and `--env-sensitivity`) use case-insensitive exact values and compose with AND; filtered JSON summaries describe the returned inventory rather than the global catalog. `--max-json-bytes` is valid only with `--json --env-inventory=full`, counts the serialized UTF-8 document plus its newline, and returns a structured usage error rather than an oversized successful document. The `github` block reports `proxy_default_credentials` as `enabled` / `disabled` and the bounded `max_request_timeout_s`; never print proxy credential material or raw secret values. `license --json` returns the versioned `license`, `commercial_use`, `trademark`, and controlling `documents` contract. | | Exception diagnostics | User-facing CLI, JSON, MCP, file-issue, and local diagnostic output must not echo raw `ex.Message` directly. Route exception prose through `CommandErrorWriter.FormatSanitizedExceptionMessage`, `DiagnosticSanitizer.ForMessage`, or an existing bounded `DiagnosticRedactor` helper, and use stable error codes/categories when the message is not needed for recovery. Intentional broad catches should match the `risky-code/broad-exception-catch` taxonomy and normalize to bounded diagnostics, private best-effort suppression, or a documented fallback. | @@ -3388,6 +3389,7 @@ net9 CI lane に合わせる場合は `FRAMEWORK=net9.0 make test` を使いま |---|---| | formatting と warning | CI は `.editorconfig` による repository formatting を強制し、`Directory.Build.props` により compiler warning を error として扱います。ローカル変更は PR 前に formatting check を通してください。既存の trim 解析警告は、通常の警告エラー化を止めずに修正を進められるよう `WarningsNotAsErrors` に明示列挙されています。ILLink は trimmed publish の smoke test を失敗させずに trim warning を報告し続けます。 | | CLI help | `cdidx --help` は短い概要、`cdidx --help-all` は全コマンド・flag・例の一覧、`cdidx --help-flags` は共有 flag table のみ、`cdidx --help` は 1 コマンドの usage line を出します。新しいコマンドは主要な user workflow である場合だけ簡易概要に載せ、full help とコマンド固有の usage table には必ず載せてください。 | +| `index --dry-run` の mutation 推定 | dry-run は厳密に非変更のままにし、database の作成・lock や source / index artifact の変更を行いません。checksum で project の未変更 skip を判定し、update、内容に対する policy skip、delete、purge、symbol / reference 上限到達を予測し、update 予定 file のうち最大100件に通常の chunk / symbol / reference / content diagnostic 抽出を適用します。`estimated_table_mutations` は最終 table row 数ではなく、delete と insert の予測 row operation 数です。nullable な値は `estimated_table_mutation_details` と常に同期させます。`source` は filesystem plan または parse-only と index snapshot の入力元、`confidence` は `exact` / `estimate` / `unknown`、安定した `unknown_reasons` は candidate の切り詰め、parse 推定の切り詰め・失敗、partial index の table 不足を表します。数値のゼロは、処理がないと計測できた場合だけに使い、その信頼度も併記します。parse-only 推定では extraction 後の hook mutation と C# の cross-file workspace augmentation を意図的に省くため、child table の値は estimate のままです。 | | `cdidx validate` | replacement character、BOM、NUL byte、混在改行、UTF-16 BOM、非 UTF-8 らしい内容など、indexed content の問題を user-facing に検査する integrity scan です。validation issue の種別や filter を追加する場合は、CLI usage、README entry、help summary を同期してください。 | | `cdidx doctor` | support request 向けにコピーしやすい environment summary です。既定では redacted に保ち、secret 風の `CDIDX_*` 値は出力しないでください。新しい diagnostic field は issue triage に使える程度に安定したものだけにします。full environment inventory の filter（`--env-domain`、`--env-category`、`--env-sensitivity`）は大文字小文字を区別しない完全一致で AND 合成し、filtered JSON summary は global catalog ではなく返却 inventory を表します。`--max-json-bytes` は `--json --env-inventory=full` とだけ組み合わせ、serialize した UTF-8 文書と改行を数え、上限を超える成功文書の代わりに structured usage error を返します。`github` block は `proxy_default_credentials` を `enabled` / `disabled` として出力し、bounded な `max_request_timeout_s` も出します。proxy credential material や raw secret value は出力しないでください。`license --json` は version 付きの `license`、`commercial_use`、`trademark`、controlling `documents` contract を返します。 | | 例外診断 | user-facing な CLI / JSON / MCP / file issue / local diagnostic output では raw `ex.Message` を直接 echo しないでください。例外の prose は `CommandErrorWriter.FormatSanitizedExceptionMessage`、`DiagnosticSanitizer.ForMessage`、または既存の bounded な `DiagnosticRedactor` helper を通し、回復に message が不要な場合は安定した error code/category を使ってください。意図的に残す broad catch は `risky-code/broad-exception-catch` taxonomy に沿い、bounded diagnostic、private な best-effort suppression、または documented fallback に正規化してください。 | diff --git a/README.md b/README.md index c4dfa5061..e618ea64f 100644 --- a/README.md +++ b/README.md @@ -227,6 +227,23 @@ stderr. Human mode keeps the corresponding `Error`, `Hint`, and `Usage` lines. See the [Developer Guide](DEVELOPER_GUIDE.md#cli-recoverable-error-format) for the stable code/category mapping. +## Index Dry-Run Mutation Estimates + +`cdidx index --dry-run --json` reports update, unchanged-skip, +content-policy-skip, delete, purge, symbol-cap, and reference-cap projections +without creating a database, acquiring the index lock, or changing source or +index artifacts. The first 100 planned update files receive parse-only +estimates for `chunks`, `symbols`, `symbol_references`, `reference_lines`, and +`file_issues`. The nullable +`estimated_table_mutations` values are paired with +`estimated_table_mutation_details`, whose `source`, `confidence`, and +`unknown_reasons` distinguish an estimated or exact zero from unavailable +work. Candidate-path truncation, the parse-estimate cap, parse failures, and +missing tables in a partial index therefore produce `null` plus a stable reason +instead of a misleading zero. See the +[Developer Guide](DEVELOPER_GUIDE.md#build--test) for the implementation +contract and limitations. + ## Status JSON Contract `cdidx status --json` exposes trust, freshness, compatibility, and remediation @@ -681,6 +698,21 @@ prose を stderr に混ぜず、envelope を stdout に出力します。human m category 対応は [開発者ガイド](DEVELOPER_GUIDE.md#cli-の回復可能エラー形式) を参照してください。 +## index dry-run の mutation 推定 + +`cdidx index --dry-run --json` は database を作成せず、index lock を +取得せず、source / index artifact を変更せずに、update、未変更 skip、内容に +対する policy skip、delete、purge、symbol 上限到達、reference 上限到達の予測を +返します。update 予定 file の先頭100件について `chunks`、`symbols`、 +`symbol_references`、`reference_lines`、`file_issues` を parse-only で推定します。 +nullable な +`estimated_table_mutations` の各値には `estimated_table_mutation_details` が対応し、 +`source`、`confidence`、`unknown_reasons` により、推定値または正確なゼロと +計測不能を区別します。そのため candidate path の切り詰め、parse 推定の上限、 +parse failure、partial index の table 不足は、誤解を招くゼロではなく安定した +理由付きの `null` になります。実装上の契約と制限は +[開発者ガイド](DEVELOPER_GUIDE.md#ビルドテスト) を参照してください。 + ## Status JSON 契約 `cdidx status --json` は script、MCP client、release check 向けに trust、 diff --git a/changelog.d/unreleased/4893.changed.md b/changelog.d/unreleased/4893.changed.md new file mode 100644 index 000000000..d9f65c76a --- /dev/null +++ b/changelog.d/unreleased/4893.changed.md @@ -0,0 +1,20 @@ +--- +category: changed +issues: + - 4893 +affected: + - src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs + - src/CodeIndex/Cli/IndexCommandRunner.cs + - src/CodeIndex/Cli/JsonOutputContracts.cs + - tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs + - README.md + - DEVELOPER_GUIDE.md +--- + +## English + +- **Index dry-run now reports useful mutation estimates and explicit unknowns (#4893)** — New and changed files receive bounded parse-only chunk, symbol, reference, reference-line, and issue estimates; unchanged, deleted, policy-skipped, symbol-capped, and reference-capped files expose their projected outcomes; and truncation, parse failures, or partial index schemas return nullable metrics with source, confidence, and stable unknown reasons instead of misleading zeros, while preserving dry-run's non-mutation guarantee. + +## 日本語 + +- **index dry-run が有用な mutation 推定と明示的な unknown を返すようになりました (#4893)** — 新規・変更 file には上限付きの parse-only による chunk、symbol、reference、reference-line、issue 推定を返し、未変更、削除、policy skip、symbol 上限、reference 上限の予測結果も公開します。切り詰め、parse failure、partial index schema の場合は誤解を招くゼロではなく、入力元、信頼度、安定した unknown reason を持つ nullable metric を返し、dry-run の非変更保証を維持します。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs b/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs index 71e0d2222..20b81829e 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs @@ -2,6 +2,7 @@ using System.Text.Json; using CodeIndex.Database; using CodeIndex.Indexer; +using CodeIndex.Models; using Microsoft.Data.Sqlite; namespace CodeIndex.Cli; @@ -12,6 +13,7 @@ public static partial class IndexCommandRunner internal const int DryRunLanguageDetectionLimit = 100; internal const int DryRunWarningSampleLimit = 100; internal const int DryRunErrorSampleLimit = 100; + internal const int DryRunParseEstimateFileLimit = 100; internal const int DefaultDryRunPathLimit = 100_000; internal const int MaxDryRunPathLimit = 1_000_000; private const int DryRunScanErrorKeyLimit = 2048; @@ -48,7 +50,7 @@ private static int RunDryRun( var warningCount = 0; var dryScanErrorKeys = new HashSet(StringComparer.Ordinal); DryRunScanMetadata dryScanMetadata; - var dbSnapshot = ReadDryRunDbSnapshot(resolvedDbPath, options.SymbolKindFilter); + var dbSnapshot = ReadDryRunDbSnapshot(resolvedDbPath); if (options.MemoryTrace) memorySamples.Add(CaptureMemorySample("snapshot", stopwatch)); var normalizedProjectRoot = Path.GetFullPath(projectPath); @@ -59,10 +61,19 @@ private static int RunDryRun( var retainedRelativePaths = new HashSet(StringComparer.Ordinal); var projectedDeletePaths = new HashSet(StringComparer.Ordinal); var projectedPurgePaths = new HashSet(StringComparer.Ordinal); - var estimatedTableMutations = CreateEmptyEstimatedTableMutations(); + var mutationEstimates = new DryRunMutationEstimateAccumulator(); var estimatedSymbolsDroppedByKindFilter = 0L; + var projectedFileUpdates = 0; + var projectedFileSkips = 0; + var projectedPolicySkips = 0; + var projectedSymbolCapHits = 0; + var projectedReferenceCapHits = 0; + var parseEstimateFilesProcessed = 0; + var parseEstimateFilesTruncated = false; var unsupportedTotal = 0; var unknownExtensionTotal = 0; + using var symbolExtractionWorker = new LazyDisposable( + () => new SymbolExtractionWorkerClient(options.MaxFileSizeBytes)); var normalizedUpdatePaths = options.UpdateFiles.Count > 0 ? NormalizeUpdateFileTargets(projectPath, options.UpdateFiles, options.Json) : []; @@ -178,6 +189,36 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) ? FileIndexer.GetReusableDetectedLanguage(f, dryScanMetadata.FileLanguages) : null; var probe = ProbeDryRunFile(dryIndexer, f, displayRelativePath, knownLanguage); + if (probe.PolicySkipped) + { + dryFileCount++; + projectedFileUpdates++; + projectedPolicySkips++; + retainedRelativePaths.Add(dbRelativePath); + AddEstimatedExistingUpdateMutations( + mutationEstimates, + dbSnapshot, + dbRelativePath); + mutationEstimates.AddParsedEstimate(new DryRunParsedMutationEstimate( + 0, + 0, + 0, + 0, + 1, + 0, + SymbolCapHit: false, + ReferenceCapHit: false)); + if (dryFileSamples.Count < DryRunFileSampleLimit) + dryFileSamples.Add(displayRelativePath); + langCounts[probe.Language] = langCounts.GetValueOrDefault(probe.Language) + 1; + if (probe.Error != null) + { + RecordDryRunError(displayRelativePath, probe.Error); + if (!options.Json && !options.Quiet) + ConsoleUi.PrintWarning($"{displayRelativePath}: {probe.Error}"); + } + continue; + } if (!probe.Supported) { if (probe.UnknownExtension) @@ -238,7 +279,62 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) dbRelativePath); } } - estimatedSymbolsDroppedByKindFilter += AddEstimatedUpdateMutation(estimatedTableMutations, dbSnapshot, dbRelativePath); + var projectedSkip = !options.Rebuild + && dbSnapshot.Files.TryGetValue(dbRelativePath, out var existingRows) + && !string.IsNullOrEmpty(probe.Checksum) + && string.Equals(existingRows.Checksum, probe.Checksum, StringComparison.Ordinal); + if (projectedSkip) + { + projectedFileSkips++; + } + else + { + projectedFileUpdates++; + AddEstimatedExistingUpdateMutations( + mutationEstimates, + dbSnapshot, + dbRelativePath); + if (parseEstimateFilesProcessed >= DryRunParseEstimateFileLimit) + { + parseEstimateFilesTruncated = true; + mutationEstimates.MarkParseUnknown("parse_estimate_file_limit_reached"); + } + else + { + parseEstimateFilesProcessed++; + try + { + var injectedFailure = DryRunParseEstimateFailureForTesting?.Invoke(displayRelativePath); + if (injectedFailure != null) + throw injectedFailure; + var parsedEstimate = BuildDryRunParsedMutationEstimate( + options, + dryIndexer, + probe.Loaded!.Value, + f, + projectPath, + symbolExtractionWorker.Value, + cancellationToken); + mutationEstimates.AddParsedEstimate(parsedEstimate); + estimatedSymbolsDroppedByKindFilter += parsedEstimate.SymbolsDroppedByKindFilter; + if (parsedEstimate.SymbolCapHit) + projectedSymbolCapHits++; + if (parsedEstimate.ReferenceCapHit) + projectedReferenceCapHits++; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return WriteDryRunInterrupted(options, jsonOptions); + } + catch (Exception ex) + { + mutationEstimates.MarkParseUnknown("parse_estimation_failed"); + RecordDryRunError( + displayRelativePath, + $"Parse-only mutation estimate unavailable: {CommandErrorWriter.FormatSanitizedExceptionMessage(ex)}"); + } + } + } if (dryFileSamples.Count < DryRunFileSampleLimit) dryFileSamples.Add(displayRelativePath); langCounts[probe.Language] = langCounts.GetValueOrDefault(probe.Language) + 1; @@ -273,9 +369,15 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) var projectedPurges = projectedPurgePaths.Count; foreach (var relativePath in projectedDeletePaths) - AddEstimatedDeleteMutation(estimatedTableMutations, dbSnapshot, relativePath); + AddEstimatedDeleteMutation(mutationEstimates, dbSnapshot, relativePath); foreach (var relativePath in projectedPurgePaths) - AddEstimatedDeleteMutation(estimatedTableMutations, dbSnapshot, relativePath); + AddEstimatedDeleteMutation(mutationEstimates, dbSnapshot, relativePath); + + if (candidatePathsTruncated) + mutationEstimates.MarkAllUnknown("candidate_path_limit_reached"); + + var estimatedTableMutations = mutationEstimates.BuildValues(); + var estimatedTableMutationDetails = mutationEstimates.BuildDetails(); if (options.MemoryTrace) { @@ -292,16 +394,24 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) Status = "dry_run", FilesTotal = dryFileCount, Estimates = true, - ProjectedFileUpdates = dryFileCount, + ProjectedFileUpdates = projectedFileUpdates, + ProjectedFileSkips = projectedFileSkips, + ProjectedPolicySkips = projectedPolicySkips, ProjectedFileDeletes = projectedDeletes, ProjectedFilePurges = projectedPurges, + ProjectedSymbolCapHits = projectedSymbolCapHits, + ProjectedReferenceCapHits = projectedReferenceCapHits, UnsupportedTotal = unsupportedTotal, UnknownExtensionTotal = unknownExtensionTotal, CandidatePathLimit = dryRunPathLimit, CandidatePathsProcessed = candidatePathsProcessed, CandidatePathsTruncated = candidatePathsTruncated, TotalsLowerBound = candidatePathsTruncated, + ParseEstimateFileLimit = DryRunParseEstimateFileLimit, + ParseEstimateFilesProcessed = parseEstimateFilesProcessed, + ParseEstimateFilesTruncated = parseEstimateFilesTruncated, EstimatedTableMutations = estimatedTableMutations, + EstimatedTableMutationDetails = estimatedTableMutationDetails, SymbolsDroppedByKindFilter = estimatedSymbolsDroppedByKindFilter, SymbolKindFilter = options.SymbolKindFilter.ToJsonResult(), FileSamples = dryFileSamples.Count > 0 ? dryFileSamples : null, @@ -326,11 +436,28 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) else { var lowerBound = candidatePathsTruncated ? " (truncated; totals are lower bounds)" : string.Empty; - CommandOutputWriter.WriteLine($"Dry run: {dryFileCount} files would be indexed{lowerBound}"); + CommandOutputWriter.WriteLine($"Dry run: {dryFileCount} indexable files inspected{lowerBound}"); if (candidatePathsTruncated) CommandOutputWriter.WriteLine($" candidate paths processed {candidatePathsProcessed.ToString("N0", System.Globalization.CultureInfo.InvariantCulture)} of limit {dryRunPathLimit.ToString("N0", System.Globalization.CultureInfo.InvariantCulture)}"); + CommandOutputWriter.WriteLine($" projected updates {projectedFileUpdates,6}"); + CommandOutputWriter.WriteLine($" projected skips {projectedFileSkips,6}"); + CommandOutputWriter.WriteLine($" projected policy skips {projectedPolicySkips,6}"); CommandOutputWriter.WriteLine($" projected deletes {projectedDeletes,6}"); CommandOutputWriter.WriteLine($" projected purges {projectedPurges,6}"); + CommandOutputWriter.WriteLine($" projected symbol cap hits {projectedSymbolCapHits,6}"); + CommandOutputWriter.WriteLine($" projected reference cap hits {projectedReferenceCapHits,6}"); + foreach (var metric in DryRunMutationEstimateAccumulator.MetricNames) + { + var estimate = estimatedTableMutationDetails[metric]; + var value = estimate.Value?.ToString("N0", System.Globalization.CultureInfo.InvariantCulture) ?? "unknown"; + var reasons = estimate.UnknownReasons.Count == 0 + ? string.Empty + : $"; reason {string.Join(",", estimate.UnknownReasons)}"; + CommandOutputWriter.WriteLine( + $" estimated {metric,-17} {value,10} ({estimate.Source}, {estimate.Confidence}{reasons})"); + } + if (parseEstimateFilesTruncated) + CommandOutputWriter.WriteLine($" parse estimates capped at {DryRunParseEstimateFileLimit.ToString("N0", System.Globalization.CultureInfo.InvariantCulture)} update files"); foreach (var (lang, count) in langCounts.OrderByDescending(kv => kv.Value)) CommandOutputWriter.WriteLine($" {lang,-12} {count,6}"); foreach (var detection in languageDetectionSamples) @@ -703,7 +830,29 @@ private static DryRunFileProbe ProbeDryRunFile( Unsupported: false, UnknownExtension: false, DetectionSource: loaded.LanguageDetection.DetectionSource, - DetectionConfidence: loaded.LanguageDetection.Confidence); + DetectionConfidence: loaded.LanguageDetection.Confidence, + Loaded: loaded, + PolicySkipped: false); + } + catch (FileIndexer.FileTooLargeSkippedException ex) + { + var skipped = indexer.BuildSkippedFileRecord( + absolutePath, + relativePath, + reusableLanguage); + return DryRunFileProbe.FromPolicySkip( + skipped, + CommandErrorWriter.FormatSanitizedExceptionMessage(ex)); + } + catch (FileIndexer.BinaryFileSkippedException ex) + { + var skipped = indexer.BuildSkippedFileRecord( + absolutePath, + relativePath, + reusableLanguage); + return DryRunFileProbe.FromPolicySkip( + skipped, + CommandErrorWriter.FormatSanitizedExceptionMessage(ex)); } catch (Exception ex) { @@ -711,55 +860,216 @@ private static DryRunFileProbe ProbeDryRunFile( } } - private static Dictionary CreateEmptyEstimatedTableMutations() - => new(StringComparer.Ordinal) + private static DryRunParsedMutationEstimate BuildDryRunParsedMutationEstimate( + IndexCommandOptions options, + FileIndexer indexer, + LoadedFileRecord loaded, + string absolutePath, + string projectRoot, + SymbolExtractionWorkerClient symbolExtractionWorker, + CancellationToken cancellationToken) + { + var record = loaded.Record; + var chunks = ChunkSplitter.SplitNormalized( + 0, + loaded.Content, + loaded.HasOversizeLine, + record.Lines); + var generatedSuppressionIssue = indexer.IsGeneratedCodeExtractionSuppressed(record.Path) + ? indexer.BuildGeneratedCodeExtractionSkippedIssue(record.Path) + : null; + if (generatedSuppressionIssue != null) + { + var generatedIssues = AppendIssueIfMissing( + FileIndexer.ValidateContent( + record.Path, + loaded.RawBytes, + loaded.Content, + record.Lang, + loaded.Inspection, + loaded.HasOversizeLine, + loaded.ConflictMarkerLine), + generatedSuppressionIssue); + return new DryRunParsedMutationEstimate( + chunks.Count, + 0, + 0, + 0, + generatedIssues.Count, + 0, + SymbolCapHit: false, + ReferenceCapHit: false); + } + + var symbolExtraction = ExtractSymbolsWithStallTimeout( + 0, + record.Lang, + loaded.Content, + absolutePath, + projectRoot, + record.Path, + FormatIndexPhasePath(record.Path, "dry_run_symbols"), + true, + loaded.HasOversizeLine, + loaded.ConflictMarkerLine, + symbolExtractionWorker, + cancellationToken); + var symbols = symbolExtraction.Symbols; + if (symbols.Count > options.MaxSymbolsPerFile) + { + var issueCount = symbolExtraction.RegexTimeoutIssue == null ? 1 : 2; + return new DryRunParsedMutationEstimate( + 0, + 0, + 0, + 0, + issueCount, + 0, + SymbolCapHit: true, + ReferenceCapHit: false); + } + + SymbolExtractor.ApplyFamilyScope( + symbols, + indexer.GetFamilyScopeKey(absolutePath, record.Lang)); + var symbolsDroppedByKindFilter = options.SymbolKindFilter.Apply(symbols); + if (symbols.Count > options.MaxSymbolsPerFile) + { + var issueCount = symbolExtraction.RegexTimeoutIssue == null ? 1 : 2; + return new DryRunParsedMutationEstimate( + 0, + 0, + 0, + 0, + issueCount, + symbolsDroppedByKindFilter, + SymbolCapHit: true, + ReferenceCapHit: false); + } + + FileIndexer.ValidateSymbolLineRanges(record, symbols); + List references; + FileIssue? referenceRegexTimeoutIssue = null; + ReferenceExtractionResult? referenceExtraction = null; + if (options.SymbolsOnly) + { + references = []; + } + else + { + using var regexTimeouts = BoundedRegex.CaptureTimeouts( + record.Lang, + "reference_extraction"); + referenceExtraction = ReferenceExtractor.ExtractDetailedNormalized( + 0, + record.Lang, + loaded.Content, + loaded.HasOversizeLine, + symbols, + record.Path, + workspaceSymbols: null, + cancellationToken, + maxReferenceCount: options.MaxReferencesPerFile + 1, + conflictMarkerLine: loaded.ConflictMarkerLine, + workspaceRoot: projectRoot, + csharpStaticInterfaceMemberLookups: null); + references = referenceExtraction.References; + referenceRegexTimeoutIssue = BuildRegexTimeoutIssue(record.Path, regexTimeouts); + } + + var extractedReferenceCount = references.Count; + var referenceCapHit = extractedReferenceCount > options.MaxReferencesPerFile; + if (referenceCapHit) + references = []; + + IReadOnlyList issues = FileIndexer.ValidateContent( + record.Path, + loaded.RawBytes, + loaded.Content, + record.Lang, + loaded.Inspection, + loaded.HasOversizeLine, + loaded.ConflictMarkerLine); + if (symbolExtraction.RegexTimeoutIssue != null) + issues = AppendIssue(issues, symbolExtraction.RegexTimeoutIssue); + if (referenceRegexTimeoutIssue != null) + issues = AppendIssue(issues, referenceRegexTimeoutIssue); + if (referenceExtraction != null) { - ["files"] = 0, - ["chunks"] = 0, - ["symbols"] = 0, - ["symbol_references"] = 0, - ["reference_lines"] = 0, - ["file_issues"] = 0, - }; + issues = AppendReferenceExtractionDiagnosticIssues( + issues, + record.Path, + referenceExtraction.Diagnostics); + } + if (referenceCapHit) + { + issues = AppendIssue( + issues, + BuildReferenceCountExceededIssue( + record.Path, + extractedReferenceCount, + options.MaxReferencesPerFile)); + } - private static long AddEstimatedUpdateMutation( - Dictionary mutations, + var referenceLines = references + .Select(reference => (reference.Line, reference.Context)) + .Distinct() + .Count(); + return new DryRunParsedMutationEstimate( + chunks.Count, + symbols.Count, + references.Count, + referenceLines, + issues.Count, + symbolsDroppedByKindFilter, + SymbolCapHit: false, + ReferenceCapHit: referenceCapHit); + } + + private static void AddEstimatedExistingUpdateMutations( + DryRunMutationEstimateAccumulator mutations, DryRunDbSnapshot snapshot, string relativePath) { - mutations["files"]++; + mutations.Add("files", 1); if (!snapshot.Files.TryGetValue(relativePath, out var rows)) - return 0; + return; - AddExistingChildRows(mutations, rows, rows.FilteredSymbols); - return rows.Symbols - rows.FilteredSymbols; + AddExistingChildRows(mutations, snapshot, rows, rows.Symbols); } private static void AddEstimatedDeleteMutation( - Dictionary mutations, + DryRunMutationEstimateAccumulator mutations, DryRunDbSnapshot snapshot, string relativePath) { if (!snapshot.Files.TryGetValue(relativePath, out var rows)) return; - mutations["files"]++; - AddExistingChildRows(mutations, rows, rows.Symbols); + mutations.Add("files", 1); + AddExistingChildRows(mutations, snapshot, rows, rows.Symbols); } private static void AddExistingChildRows( - Dictionary mutations, + DryRunMutationEstimateAccumulator mutations, + DryRunDbSnapshot snapshot, DryRunExistingFileRows rows, long symbols) { - mutations["chunks"] += rows.Chunks; - mutations["symbols"] += symbols; - mutations["symbol_references"] += rows.SymbolReferences; - mutations["reference_lines"] += rows.ReferenceLines; - mutations["file_issues"] += rows.FileIssues; + mutations.AddExisting("chunks", rows.Chunks, snapshot.ChunksAvailable); + mutations.AddExisting("symbols", symbols, snapshot.SymbolsAvailable); + mutations.AddExisting( + "symbol_references", + rows.SymbolReferences, + snapshot.SymbolReferencesAvailable); + mutations.AddExisting( + "reference_lines", + rows.ReferenceLines, + snapshot.ReferenceLinesAvailable); + mutations.AddExisting("file_issues", rows.FileIssues, snapshot.FileIssuesAvailable); } - private static DryRunDbSnapshot ReadDryRunDbSnapshot(string dbPath, SymbolKindFilter symbolKindFilter) + private static DryRunDbSnapshot ReadDryRunDbSnapshot(string dbPath) { try { @@ -786,15 +1096,11 @@ private static DryRunDbSnapshot ReadDryRunDbSnapshot(string dbPath, SymbolKindFi var hasFileIssues = DryRunTableExists(connection, "file_issues"); using var command = connection.CreateCommand(); - var filteredSymbolsExpression = hasSymbols && symbolKindFilter.IsActive - ? BuildDryRunFilteredSymbolCountExpression(command, symbolKindFilter) - : "0"; command.CommandText = $""" SELECT f.path, f.checksum, {(hasChunks ? "(SELECT COUNT(*) FROM chunks c WHERE c.file_id = f.id)" : "0")} AS chunks_count, {(hasSymbols ? "(SELECT COUNT(*) FROM symbols s WHERE s.file_id = f.id)" : "0")} AS symbols_count, - {filteredSymbolsExpression} AS filtered_symbols_count, {(hasSymbolReferences ? "(SELECT COUNT(*) FROM symbol_references r WHERE r.file_id = f.id)" : "0")} AS symbol_references_count, {(hasReferenceLines ? "(SELECT COUNT(*) FROM reference_lines l WHERE l.file_id = f.id)" : "0")} AS reference_lines_count, {(hasFileIssues ? "(SELECT COUNT(*) FROM file_issues i WHERE i.file_id = f.id)" : "0")} AS file_issues_count @@ -805,18 +1111,23 @@ FROM files f using var reader = command.ExecuteReader(); while (reader.Read()) { - var symbols = reader.GetInt64(3); files[reader.GetString(0)] = new DryRunExistingFileRows( reader.IsDBNull(1) ? null : reader.GetString(1), reader.GetInt64(2), - symbols, - symbolKindFilter.IsActive ? reader.GetInt64(4) : symbols, + reader.GetInt64(3), + reader.GetInt64(4), reader.GetInt64(5), - reader.GetInt64(6), - reader.GetInt64(7)); + reader.GetInt64(6)); } - return new DryRunDbSnapshot(files, indexedProjectRoot); + return new DryRunDbSnapshot( + files, + indexedProjectRoot, + hasChunks, + hasSymbols, + hasSymbolReferences, + hasReferenceLines, + hasFileIssues); } catch (SqliteException) { @@ -832,39 +1143,6 @@ FROM files f } } - private static string BuildDryRunFilteredSymbolCountExpression( - SqliteCommand command, - SymbolKindFilter symbolKindFilter) - { - var conditions = new List(); - if (symbolKindFilter.Include.Count > 0) - { - var parameters = AddDryRunSymbolKindParameters(command, "include", symbolKindFilter.Include); - conditions.Add($"s.kind IS NOT NULL AND trim(s.kind) <> '' AND s.kind COLLATE NOCASE IN ({parameters})"); - } - if (symbolKindFilter.Exclude.Count > 0) - { - var parameters = AddDryRunSymbolKindParameters(command, "exclude", symbolKindFilter.Exclude); - conditions.Add($"(s.kind IS NULL OR trim(s.kind) = '' OR s.kind COLLATE NOCASE NOT IN ({parameters}))"); - } - - return $"(SELECT COUNT(*) FROM symbols s WHERE s.file_id = f.id AND {string.Join(" AND ", conditions)})"; - } - - private static string AddDryRunSymbolKindParameters( - SqliteCommand command, - string prefix, - IReadOnlyList values) - { - var names = new string[values.Count]; - for (var i = 0; i < values.Count; i++) - { - names[i] = $"@{prefix}{i}"; - command.Parameters.AddWithValue(names[i], values[i]); - } - return string.Join(", ", names); - } - private static bool DryRunTableExists(SqliteConnection connection, string tableName) { using var command = connection.CreateCommand(); @@ -892,16 +1170,131 @@ private static int WriteDryRunInterrupted(IndexCommandOptions options, JsonSeria "Rerun `cdidx index --dry-run` when you are ready to inspect the candidate files again.", CommandErrorCodes.Interrupted); - private sealed record DryRunDbSnapshot(IReadOnlyDictionary Files, string? IndexedProjectRoot) + private readonly record struct DryRunParsedMutationEstimate( + long Chunks, + long Symbols, + long SymbolReferences, + long ReferenceLines, + long FileIssues, + long SymbolsDroppedByKindFilter, + bool SymbolCapHit, + bool ReferenceCapHit); + + private sealed class DryRunMutationEstimateAccumulator + { + internal static readonly string[] MetricNames = + [ + "files", + "chunks", + "symbols", + "symbol_references", + "reference_lines", + "file_issues", + ]; + + private readonly Dictionary values = MetricNames.ToDictionary( + static metric => metric, + static _ => 0L, + StringComparer.Ordinal); + private readonly Dictionary> unknownReasons = MetricNames.ToDictionary( + static metric => metric, + static _ => new SortedSet(StringComparer.Ordinal), + StringComparer.Ordinal); + + internal void Add(string metric, long value) + => values[metric] += value; + + internal void AddExisting(string metric, long value, bool available) + { + if (!available) + { + MarkUnknown(metric, "existing_table_unavailable"); + return; + } + + Add(metric, value); + } + + internal void AddParsedEstimate(DryRunParsedMutationEstimate estimate) + { + Add("chunks", estimate.Chunks); + Add("symbols", estimate.Symbols); + Add("symbol_references", estimate.SymbolReferences); + Add("reference_lines", estimate.ReferenceLines); + Add("file_issues", estimate.FileIssues); + } + + internal void MarkParseUnknown(string reason) + { + foreach (var metric in MetricNames) + { + if (metric != "files") + MarkUnknown(metric, reason); + } + } + + internal void MarkAllUnknown(string reason) + { + foreach (var metric in MetricNames) + MarkUnknown(metric, reason); + } + + internal Dictionary BuildValues() + => MetricNames.ToDictionary( + static metric => metric, + metric => unknownReasons[metric].Count == 0 ? (long?)values[metric] : null, + StringComparer.Ordinal); + + internal Dictionary BuildDetails() + => MetricNames.ToDictionary( + static metric => metric, + metric => + { + var reasons = unknownReasons[metric].ToList(); + var value = reasons.Count == 0 ? (long?)values[metric] : null; + var source = metric == "files" + ? "filesystem_plan" + : "parse_only_and_index_snapshot"; + var confidence = reasons.Count > 0 + ? "unknown" + : metric == "files" + ? "exact" + : "estimate"; + return new IndexDryRunEstimateJsonResult( + value, + source, + confidence, + reasons); + }, + StringComparer.Ordinal); + + private void MarkUnknown(string metric, string reason) + => unknownReasons[metric].Add(reason); + } + + private sealed record DryRunDbSnapshot( + IReadOnlyDictionary Files, + string? IndexedProjectRoot, + bool ChunksAvailable, + bool SymbolsAvailable, + bool SymbolReferencesAvailable, + bool ReferenceLinesAvailable, + bool FileIssuesAvailable) { - public static DryRunDbSnapshot Empty { get; } = new(new Dictionary(StringComparer.Ordinal), null); + public static DryRunDbSnapshot Empty { get; } = new( + new Dictionary(StringComparer.Ordinal), + null, + false, + false, + false, + false, + false); } private readonly record struct DryRunExistingFileRows( string? Checksum, long Chunks, long Symbols, - long FilteredSymbols, long SymbolReferences, long ReferenceLines, long FileIssues); @@ -946,10 +1339,25 @@ private readonly record struct DryRunFileProbe( bool Unsupported, bool UnknownExtension, string? DetectionSource, - FileIndexer.LanguageDetectionConfidence? DetectionConfidence) + FileIndexer.LanguageDetectionConfidence? DetectionConfidence, + LoadedFileRecord? Loaded, + bool PolicySkipped) { - public static DryRunFileProbe FromError(string message) => new(false, string.Empty, null, message, Unsupported: false, UnknownExtension: false, null, null); - public static DryRunFileProbe FromUnsupported() => new(false, string.Empty, null, null, Unsupported: true, UnknownExtension: false, null, null); - public static DryRunFileProbe FromUnknownExtension() => new(false, string.Empty, null, null, Unsupported: false, UnknownExtension: true, null, null); + public static DryRunFileProbe FromError(string message) => new(false, string.Empty, null, message, Unsupported: false, UnknownExtension: false, null, null, null, PolicySkipped: false); + public static DryRunFileProbe FromUnsupported() => new(false, string.Empty, null, null, Unsupported: true, UnknownExtension: false, null, null, null, PolicySkipped: false); + public static DryRunFileProbe FromUnknownExtension() => new(false, string.Empty, null, null, Unsupported: false, UnknownExtension: true, null, null, null, PolicySkipped: false); + public static DryRunFileProbe FromPolicySkip( + FileRecord record, + string message) => new( + false, + record.Lang ?? "unknown", + record.Checksum, + message, + Unsupported: false, + UnknownExtension: false, + null, + null, + null, + PolicySkipped: true); } } diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index cecfb4e81..a75504d44 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -104,6 +104,7 @@ public void Dispose() internal static Action? UpdateSkippedFileRecordBuiltForTesting { get; set; } internal static Action? UpdateFileCommittedForTesting { get; set; } internal static Func? IndexExtractionStallTimeoutForTesting { get; set; } + internal static Func? DryRunParseEstimateFailureForTesting { get; set; } internal static Action? HotspotFamilyUpdateRestampReadyForCommitForTesting { get; set; } internal static Action? WriteScanCheckpointForTesting { get; set; } internal static Action? DeleteScanCheckpointForTesting { get; set; } diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index 14b2641a0..def9ef010 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -820,6 +820,12 @@ internal sealed record IndexLanguageDetectionJsonResult( [property: JsonPropertyName("source")] string Source, [property: JsonPropertyName("confidence")] string Confidence); +internal sealed record IndexDryRunEstimateJsonResult( + [property: JsonPropertyName("value")] long? Value, + [property: JsonPropertyName("source")] string Source, + [property: JsonPropertyName("confidence")] string Confidence, + [property: JsonPropertyName("unknown_reasons")] List UnknownReasons); + internal sealed class IndexDryRunJsonResult : IVersionedJsonResult { public string ApiVersion { get; init; } = JsonOutputContract.ApiVersion; @@ -827,15 +833,23 @@ internal sealed class IndexDryRunJsonResult : IVersionedJsonResult public int FilesTotal { get; init; } public bool Estimates { get; init; } public int ProjectedFileUpdates { get; init; } + public int ProjectedFileSkips { get; init; } + public int ProjectedPolicySkips { get; init; } public int ProjectedFileDeletes { get; init; } public int ProjectedFilePurges { get; init; } + public int ProjectedSymbolCapHits { get; init; } + public int ProjectedReferenceCapHits { get; init; } public int UnsupportedTotal { get; init; } public int UnknownExtensionTotal { get; init; } public int CandidatePathLimit { get; init; } public int CandidatePathsProcessed { get; init; } public bool CandidatePathsTruncated { get; init; } public bool TotalsLowerBound { get; init; } - public Dictionary EstimatedTableMutations { get; init; } = new(); + public int ParseEstimateFileLimit { get; init; } + public int ParseEstimateFilesProcessed { get; init; } + public bool ParseEstimateFilesTruncated { get; init; } + public Dictionary EstimatedTableMutations { get; init; } = new(); + public Dictionary EstimatedTableMutationDetails { get; init; } = new(); public long SymbolsDroppedByKindFilter { get; init; } public IndexSymbolKindFilterJsonResult SymbolKindFilter { get; init; } = new(); public List? FileSamples { get; init; } diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs index c775a7257..96edb561b 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs @@ -34,6 +34,20 @@ public void Run_DryRun_ReadOnlyUriDbPath_ReturnsDryRunSummary() Assert.Equal("dry_run", json.GetProperty("status").GetString()); Assert.Equal(1, json.GetProperty("files_total").GetInt32()); Assert.Equal(1, json.GetProperty("languages").GetProperty("csharp").GetInt32()); + Assert.Equal(1, json.GetProperty("projected_file_updates").GetInt32()); + Assert.Equal(0, json.GetProperty("projected_file_skips").GetInt32()); + Assert.Equal(1, json.GetProperty("parse_estimate_files_processed").GetInt32()); + Assert.False(json.GetProperty("parse_estimate_files_truncated").GetBoolean()); + var mutations = json.GetProperty("estimated_table_mutations"); + Assert.True(mutations.GetProperty("chunks").GetInt64() > 0); + Assert.True(mutations.GetProperty("symbols").GetInt64() > 0); + Assert.Equal(0, mutations.GetProperty("symbol_references").GetInt64()); + var symbolDetail = json + .GetProperty("estimated_table_mutation_details") + .GetProperty("symbols"); + Assert.Equal("parse_only_and_index_snapshot", symbolDetail.GetProperty("source").GetString()); + Assert.Equal("estimate", symbolDetail.GetProperty("confidence").GetString()); + Assert.Empty(symbolDetail.GetProperty("unknown_reasons").EnumerateArray()); } finally { @@ -249,6 +263,19 @@ public void Run_DryRun_JsonCapsFileSamples() Assert.Equal(IndexCommandRunner.DryRunFileSampleLimit, json.GetProperty("file_samples").GetArrayLength()); Assert.Equal(0, json.GetProperty("errors_total").GetInt32()); Assert.False(json.GetProperty("errors_truncated").GetBoolean()); + Assert.Equal(IndexCommandRunner.DryRunParseEstimateFileLimit, json.GetProperty("parse_estimate_file_limit").GetInt32()); + Assert.Equal(IndexCommandRunner.DryRunParseEstimateFileLimit, json.GetProperty("parse_estimate_files_processed").GetInt32()); + Assert.True(json.GetProperty("parse_estimate_files_truncated").GetBoolean()); + Assert.Equal( + JsonValueKind.Null, + json.GetProperty("estimated_table_mutations").GetProperty("chunks").ValueKind); + Assert.Contains( + "parse_estimate_file_limit_reached", + json.GetProperty("estimated_table_mutation_details") + .GetProperty("chunks") + .GetProperty("unknown_reasons") + .EnumerateArray() + .Select(value => value.GetString())); } finally { @@ -278,6 +305,16 @@ public void Run_DryRun_PathLimitTruncatesCandidateProcessing() Assert.True(json.GetProperty("totals_lower_bound").GetBoolean()); Assert.True(json.GetProperty("file_samples_truncated").GetBoolean()); Assert.Equal(0, json.GetProperty("errors_total").GetInt32()); + Assert.Equal( + JsonValueKind.Null, + json.GetProperty("estimated_table_mutations").GetProperty("symbols").ValueKind); + var detail = json + .GetProperty("estimated_table_mutation_details") + .GetProperty("symbols"); + Assert.Equal("unknown", detail.GetProperty("confidence").GetString()); + Assert.Contains( + "candidate_path_limit_reached", + detail.GetProperty("unknown_reasons").EnumerateArray().Select(value => value.GetString())); } finally { @@ -299,16 +336,201 @@ public void Run_DryRun_JsonCapsErrorSamples() Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Equal("dry_run", json.GetProperty("status").GetString()); - Assert.Equal(0, json.GetProperty("files_total").GetInt32()); + Assert.Equal(fileCount, json.GetProperty("files_total").GetInt32()); + Assert.Equal(fileCount, json.GetProperty("projected_file_updates").GetInt32()); + Assert.Equal(fileCount, json.GetProperty("projected_policy_skips").GetInt32()); + Assert.Equal(0, json.GetProperty("parse_estimate_files_processed").GetInt32()); + var mutations = json.GetProperty("estimated_table_mutations"); + Assert.Equal(fileCount, mutations.GetProperty("files").GetInt32()); + Assert.Equal(0, mutations.GetProperty("chunks").GetInt32()); + Assert.Equal(0, mutations.GetProperty("symbols").GetInt32()); + Assert.Equal(0, mutations.GetProperty("symbol_references").GetInt32()); + Assert.Equal(0, mutations.GetProperty("reference_lines").GetInt32()); + Assert.Equal(fileCount, mutations.GetProperty("file_issues").GetInt32()); Assert.Equal(fileCount, json.GetProperty("errors_total").GetInt32()); Assert.Equal(IndexCommandRunner.DryRunErrorSampleLimit, json.GetProperty("error_limit").GetInt32()); Assert.True(json.GetProperty("errors_truncated").GetBoolean()); Assert.Equal(IndexCommandRunner.DryRunErrorSampleLimit, json.GetProperty("errors").GetArrayLength()); Assert.Equal(IndexCommandRunner.DryRunFileSampleLimit, json.GetProperty("file_sample_limit").GetInt32()); - Assert.False(json.GetProperty("file_samples_truncated").GetBoolean()); + Assert.True(json.GetProperty("file_samples_truncated").GetBoolean()); + } + finally + { + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_DryRun_EstimatesParseOnlyMutationsAndCapOutcomes_Issue4893() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "app.cs"), + """ + public class App + { + public void First() + { + Second(); + Third(); + } + + public void Second() { } + public void Third() { } + } + """); + + var (humanExitCode, humanOutput, _) = RunAndCaptureStreams([projectRoot, "--dry-run"]); + Assert.Equal(CommandExitCodes.Success, humanExitCode); + Assert.Contains("projected updates", humanOutput); + Assert.Contains("estimated chunks", humanOutput); + Assert.Contains("estimated symbols", humanOutput); + Assert.Contains("parse_only_and_index_snapshot", humanOutput); + Assert.Contains("estimate", humanOutput); + + var (normalExitCode, normal) = RunAndCaptureJson([projectRoot, "--dry-run", "--json"]); + Assert.Equal(CommandExitCodes.Success, normalExitCode); + Assert.Equal(1, normal.GetProperty("projected_file_updates").GetInt32()); + Assert.Equal(0, normal.GetProperty("projected_file_skips").GetInt32()); + Assert.Equal(0, normal.GetProperty("projected_symbol_cap_hits").GetInt32()); + Assert.Equal(0, normal.GetProperty("projected_reference_cap_hits").GetInt32()); + var normalMutations = normal.GetProperty("estimated_table_mutations"); + Assert.True(normalMutations.GetProperty("chunks").GetInt64() > 0); + Assert.True(normalMutations.GetProperty("symbols").GetInt64() > 1); + Assert.True(normalMutations.GetProperty("symbol_references").GetInt64() > 1); + Assert.True(normalMutations.GetProperty("reference_lines").GetInt64() > 0); + + var (symbolCapExitCode, symbolCap) = RunAndCaptureJson([ + projectRoot, + "--dry-run", + "--max-symbols-per-file", + "1", + "--json", + ]); + Assert.Equal(CommandExitCodes.Success, symbolCapExitCode); + Assert.Equal(1, symbolCap.GetProperty("projected_symbol_cap_hits").GetInt32()); + var symbolCapMutations = symbolCap.GetProperty("estimated_table_mutations"); + Assert.Equal(0, symbolCapMutations.GetProperty("chunks").GetInt32()); + Assert.Equal(0, symbolCapMutations.GetProperty("symbols").GetInt32()); + Assert.Equal(0, symbolCapMutations.GetProperty("symbol_references").GetInt32()); + Assert.True(symbolCapMutations.GetProperty("file_issues").GetInt32() > 0); + + var (referenceCapExitCode, referenceCap) = RunAndCaptureJson([ + projectRoot, + "--dry-run", + "--max-references-per-file", + "1", + "--json", + ]); + Assert.Equal(CommandExitCodes.Success, referenceCapExitCode); + Assert.Equal(1, referenceCap.GetProperty("projected_reference_cap_hits").GetInt32()); + var referenceCapMutations = referenceCap.GetProperty("estimated_table_mutations"); + Assert.Equal(0, referenceCapMutations.GetProperty("symbol_references").GetInt32()); + Assert.Equal(0, referenceCapMutations.GetProperty("reference_lines").GetInt32()); + Assert.True(referenceCapMutations.GetProperty("file_issues").GetInt32() > 0); + + Assert.False(Directory.Exists(Path.Combine(projectRoot, ".cdidx"))); + } + finally + { + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_DryRun_ParseEstimateFailureReturnsExplicitUnknown_Issue4893() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText(Path.Combine(projectRoot, "app.cs"), "public class App { }\n"); + IndexCommandRunner.DryRunParseEstimateFailureForTesting = static _ => + new InvalidOperationException("injected parse-only estimate failure"); + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--dry-run", "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(1, json.GetProperty("parse_estimate_files_processed").GetInt32()); + Assert.Equal( + JsonValueKind.Null, + json.GetProperty("estimated_table_mutations").GetProperty("symbols").ValueKind); + var detail = json + .GetProperty("estimated_table_mutation_details") + .GetProperty("symbols"); + Assert.Equal(JsonValueKind.Null, detail.GetProperty("value").ValueKind); + Assert.Equal("unknown", detail.GetProperty("confidence").GetString()); + Assert.Contains( + "parse_estimation_failed", + detail.GetProperty("unknown_reasons").EnumerateArray().Select(value => value.GetString())); + Assert.Equal(1, json.GetProperty("errors_total").GetInt32()); + Assert.Contains( + "Parse-only mutation estimate unavailable", + json.GetProperty("errors")[0].GetProperty("message").GetString()); + Assert.False(Directory.Exists(Path.Combine(projectRoot, ".cdidx"))); } finally { + IndexCommandRunner.DryRunParseEstimateFailureForTesting = null; + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_DryRun_PartialIndexSchemaReturnsExplicitUnknownWithoutMutation_Issue4893() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText(Path.Combine(projectRoot, "app.cs"), "public class App { }\n"); + var indexDirectory = Path.Combine(projectRoot, ".cdidx"); + Directory.CreateDirectory(indexDirectory); + var dbPath = Path.Combine(indexDirectory, "codeindex.db"); + using (var connection = new SqliteConnection($"Data Source={dbPath}")) + { + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = """ + CREATE TABLE files ( + id INTEGER PRIMARY KEY, + path TEXT NOT NULL, + checksum TEXT + ); + INSERT INTO files(id, path, checksum) VALUES (1, 'app.cs', 'stale'); + """; + command.ExecuteNonQuery(); + } + SqliteConnection.ClearAllPools(); + var before = File.ReadAllBytes(dbPath); + + var (exitCode, json) = RunAndCaptureJson([ + projectRoot, + "--db", + dbPath, + "--dry-run", + "--json", + ]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(1, json.GetProperty("projected_file_updates").GetInt32()); + Assert.Equal(1, json.GetProperty("estimated_table_mutations").GetProperty("files").GetInt32()); + Assert.Equal( + JsonValueKind.Null, + json.GetProperty("estimated_table_mutations").GetProperty("chunks").ValueKind); + var detail = json + .GetProperty("estimated_table_mutation_details") + .GetProperty("chunks"); + Assert.Equal("unknown", detail.GetProperty("confidence").GetString()); + Assert.Contains( + "existing_table_unavailable", + detail.GetProperty("unknown_reasons").EnumerateArray().Select(value => value.GetString())); + SqliteConnection.ClearAllPools(); + Assert.Equal(before, File.ReadAllBytes(dbPath)); + } + finally + { + SqliteConnection.ClearAllPools(); DeleteDirectory(projectRoot); } } @@ -417,10 +639,11 @@ public void Run_DryRun_WithFiles_NormalizesUnicodeDbPathForEstimates() Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Equal("dry_run", json.GetProperty("status").GetString()); Assert.Equal(1, json.GetProperty("files_total").GetInt32()); - Assert.Equal(1, json.GetProperty("projected_file_updates").GetInt32()); + Assert.Equal(0, json.GetProperty("projected_file_updates").GetInt32()); + Assert.Equal(1, json.GetProperty("projected_file_skips").GetInt32()); Assert.Equal(0, json.GetProperty("projected_file_deletes").GetInt32()); Assert.Equal(0, json.GetProperty("projected_file_purges").GetInt32()); - Assert.True(json.GetProperty("estimated_table_mutations").GetProperty("chunks").GetInt64() > 0); + Assert.Equal(0, json.GetProperty("estimated_table_mutations").GetProperty("chunks").GetInt64()); Assert.Equal(1, CountRows(dbPath, "files")); } finally @@ -441,9 +664,10 @@ public void Run_DryRun_SymbolKindFiltersAdjustSymbolEstimateAndExposeResolvedPol var (initialExitCode, _) = RunAndCaptureJson([projectRoot, "--json"]); Assert.Equal(CommandExitCodes.Success, initialExitCode); - var (_, unfiltered) = RunAndCaptureJson([projectRoot, "--dry-run", "--json"]); + var (_, unfiltered) = RunAndCaptureJson([projectRoot, "--rebuild", "--dry-run", "--json"]); var (exitCode, filtered) = RunAndCaptureJson([ projectRoot, + "--rebuild", "--dry-run", "--include-symbol-kind", "method,class,method", @@ -457,8 +681,8 @@ public void Run_DryRun_SymbolKindFiltersAdjustSymbolEstimateAndExposeResolvedPol var filteredMutations = filtered.GetProperty("estimated_table_mutations"); Assert.True(filteredMutations.GetProperty("symbols").GetInt64() < unfilteredMutations.GetProperty("symbols").GetInt64()); Assert.Equal( - unfilteredMutations.GetProperty("symbol_references").GetInt64(), - filteredMutations.GetProperty("symbol_references").GetInt64()); + JsonValueKind.Number, + filteredMutations.GetProperty("symbol_references").ValueKind); Assert.True(filtered.GetProperty("symbols_dropped_by_kind_filter").GetInt64() > 0); var policy = filtered.GetProperty("symbol_kind_filter"); Assert.Equal(["class", "method"], policy.GetProperty("include").EnumerateArray().Select(value => value.GetString()).ToArray()); @@ -595,11 +819,12 @@ public void Run_DryRun_FullScan_ReportsProjectedPurgesWithoutWriting() Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Equal("dry_run", json.GetProperty("status").GetString()); Assert.Equal(1, json.GetProperty("files_total").GetInt32()); + Assert.Equal(1, json.GetProperty("projected_file_skips").GetInt32()); Assert.Equal(0, json.GetProperty("projected_file_deletes").GetInt32()); Assert.Equal(1, json.GetProperty("projected_file_purges").GetInt32()); Assert.Equal(1, json.GetProperty("unknown_extension_total").GetInt32()); Assert.True(json.TryGetProperty("unsupported_total", out _)); - Assert.True(json.GetProperty("estimated_table_mutations").GetProperty("files").GetInt64() >= 2); + Assert.Equal(1, json.GetProperty("estimated_table_mutations").GetProperty("files").GetInt64()); Assert.Equal(2, CountRows(dbPath, "files")); } finally From 6ece823f88ff6182962456ad60a0a16490a64c7b Mon Sep 17 00:00:00 2001 From: Widthdom Date: Thu, 30 Jul 2026 19:53:42 +0900 Subject: [PATCH 2/3] Align dry-run reuse projections (#4893) --- DEVELOPER_GUIDE.md | 4 +- .../Cli/IndexCommandRunner.DryRun.cs | 554 ++++++++++++++++-- .../IndexCommandRunnerDryRunTests.cs | 209 +++++++ 3 files changed, 731 insertions(+), 36 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 032f84713..bf237fb6e 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -33,7 +33,7 @@ Development contracts: |---|---| | Formatting and warnings | CI enforces repository formatting with `.editorconfig` and treats compiler warnings as errors through `Directory.Build.props`, so local changes should pass the format check before opening a PR. Existing trim-analysis warnings are explicitly listed in `WarningsNotAsErrors` until they are fixed without blocking ordinary compiler-warning enforcement, and ILLink keeps reporting trim warnings without failing trimmed publish smoke tests. | | CLI help | `cdidx --help` stays brief, `cdidx --help-all` prints the full command/flag/example reference, `cdidx --help-flags` prints only shared flag tables, and `cdidx --help` prints one command's usage line. Keep new commands visible in the brief summary only when they are a primary user workflow; every command must remain present in the full help and command-specific usage table. | -| `index --dry-run` mutation estimates | Dry-run remains strictly non-mutating: it does not create or lock the database and does not alter sources or index artifacts. It compares checksums to project unchanged skips; projects updates, content-policy skips, deletes, purges, and symbol/reference cap hits; and performs the normal chunk/symbol/reference/content-diagnostic extraction for at most 100 planned update files. `estimated_table_mutations` counts projected delete-and-insert row operations, not final table cardinality. Its nullable values must stay synchronized with `estimated_table_mutation_details`: `source` identifies filesystem planning or parse-only plus index-snapshot input, `confidence` is `exact`, `estimate`, or `unknown`, and stable `unknown_reasons` cover candidate truncation, parse-estimate truncation/failure, and partial-index table absence. A numeric zero is reserved for measured no work and remains qualified by its confidence. Parse-only estimates intentionally omit post-extraction hook mutations and cross-file C# workspace augmentation, so child-table values remain estimates. | +| `index --dry-run` mutation estimates | Dry-run remains strictly non-mutating: it does not create or lock the database and does not alter sources or index artifacts. It uses read-only stat/checksum data together with the production cap-issue, symbol-filter, extractor-version, and graph-contract reuse gates to project unchanged skips; projects updates, content-policy skips, deletes, purges, and symbol/reference cap hits; and performs the normal chunk/symbol/reference/content-diagnostic extraction for at most 100 planned update files. `estimated_table_mutations` counts projected delete-and-insert row operations, not final table cardinality. Its nullable values must stay synchronized with `estimated_table_mutation_details`: `source` identifies filesystem planning or parse-only plus index-snapshot input, `confidence` is `exact`, `estimate`, or `unknown`, and stable `unknown_reasons` cover candidate truncation, parse-estimate truncation/failure, and partial-index table absence. A numeric zero is reserved for measured no work and remains qualified by its confidence. Parse-only estimates intentionally omit post-extraction hook mutations and cross-file C# workspace augmentation, so child-table values remain estimates. | | `cdidx validate` | This is the user-facing integrity scan for indexed content issues such as replacement characters, BOMs, NUL bytes, mixed line endings, UTF-16 BOMs, and likely non-UTF8 content. Keep its CLI usage, README entry, and help summary in sync when adding validation issue kinds or filters. | | `cdidx doctor` | This is the copy-pasteable environment summary for support requests. Keep it redacted by default: secret-like `CDIDX_*` values must not be printed, and new diagnostic fields should be stable enough for issue triage. Full environment inventory filters (`--env-domain`, `--env-category`, and `--env-sensitivity`) use case-insensitive exact values and compose with AND; filtered JSON summaries describe the returned inventory rather than the global catalog. `--max-json-bytes` is valid only with `--json --env-inventory=full`, counts the serialized UTF-8 document plus its newline, and returns a structured usage error rather than an oversized successful document. The `github` block reports `proxy_default_credentials` as `enabled` / `disabled` and the bounded `max_request_timeout_s`; never print proxy credential material or raw secret values. `license --json` returns the versioned `license`, `commercial_use`, `trademark`, and controlling `documents` contract. | | Exception diagnostics | User-facing CLI, JSON, MCP, file-issue, and local diagnostic output must not echo raw `ex.Message` directly. Route exception prose through `CommandErrorWriter.FormatSanitizedExceptionMessage`, `DiagnosticSanitizer.ForMessage`, or an existing bounded `DiagnosticRedactor` helper, and use stable error codes/categories when the message is not needed for recovery. Intentional broad catches should match the `risky-code/broad-exception-catch` taxonomy and normalize to bounded diagnostics, private best-effort suppression, or a documented fallback. | @@ -3389,7 +3389,7 @@ net9 CI lane に合わせる場合は `FRAMEWORK=net9.0 make test` を使いま |---|---| | formatting と warning | CI は `.editorconfig` による repository formatting を強制し、`Directory.Build.props` により compiler warning を error として扱います。ローカル変更は PR 前に formatting check を通してください。既存の trim 解析警告は、通常の警告エラー化を止めずに修正を進められるよう `WarningsNotAsErrors` に明示列挙されています。ILLink は trimmed publish の smoke test を失敗させずに trim warning を報告し続けます。 | | CLI help | `cdidx --help` は短い概要、`cdidx --help-all` は全コマンド・flag・例の一覧、`cdidx --help-flags` は共有 flag table のみ、`cdidx --help` は 1 コマンドの usage line を出します。新しいコマンドは主要な user workflow である場合だけ簡易概要に載せ、full help とコマンド固有の usage table には必ず載せてください。 | -| `index --dry-run` の mutation 推定 | dry-run は厳密に非変更のままにし、database の作成・lock や source / index artifact の変更を行いません。checksum で project の未変更 skip を判定し、update、内容に対する policy skip、delete、purge、symbol / reference 上限到達を予測し、update 予定 file のうち最大100件に通常の chunk / symbol / reference / content diagnostic 抽出を適用します。`estimated_table_mutations` は最終 table row 数ではなく、delete と insert の予測 row operation 数です。nullable な値は `estimated_table_mutation_details` と常に同期させます。`source` は filesystem plan または parse-only と index snapshot の入力元、`confidence` は `exact` / `estimate` / `unknown`、安定した `unknown_reasons` は candidate の切り詰め、parse 推定の切り詰め・失敗、partial index の table 不足を表します。数値のゼロは、処理がないと計測できた場合だけに使い、その信頼度も併記します。parse-only 推定では extraction 後の hook mutation と C# の cross-file workspace augmentation を意図的に省くため、child table の値は estimate のままです。 | +| `index --dry-run` の mutation 推定 | dry-run は厳密に非変更のままにし、database の作成・lock や source / index artifact の変更を行いません。read-only な stat / checksum と production の cap issue、symbol filter、extractor version、graph contract の再利用条件を組み合わせて未変更 skip を判定し、update、内容に対する policy skip、delete、purge、symbol / reference 上限到達を予測し、update 予定 file のうち最大100件に通常の chunk / symbol / reference / content diagnostic 抽出を適用します。`estimated_table_mutations` は最終 table row 数ではなく、delete と insert の予測 row operation 数です。nullable な値は `estimated_table_mutation_details` と常に同期させます。`source` は filesystem plan または parse-only と index snapshot の入力元、`confidence` は `exact` / `estimate` / `unknown`、安定した `unknown_reasons` は candidate の切り詰め、parse 推定の切り詰め・失敗、partial index の table 不足を表します。数値のゼロは、処理がないと計測できた場合だけに使い、その信頼度も併記します。parse-only 推定では extraction 後の hook mutation と C# の cross-file workspace augmentation を意図的に省くため、child table の値は estimate のままです。 | | `cdidx validate` | replacement character、BOM、NUL byte、混在改行、UTF-16 BOM、非 UTF-8 らしい内容など、indexed content の問題を user-facing に検査する integrity scan です。validation issue の種別や filter を追加する場合は、CLI usage、README entry、help summary を同期してください。 | | `cdidx doctor` | support request 向けにコピーしやすい environment summary です。既定では redacted に保ち、secret 風の `CDIDX_*` 値は出力しないでください。新しい diagnostic field は issue triage に使える程度に安定したものだけにします。full environment inventory の filter（`--env-domain`、`--env-category`、`--env-sensitivity`）は大文字小文字を区別しない完全一致で AND 合成し、filtered JSON summary は global catalog ではなく返却 inventory を表します。`--max-json-bytes` は `--json --env-inventory=full` とだけ組み合わせ、serialize した UTF-8 文書と改行を数え、上限を超える成功文書の代わりに structured usage error を返します。`github` block は `proxy_default_credentials` を `enabled` / `disabled` として出力し、bounded な `max_request_timeout_s` も出します。proxy credential material や raw secret value は出力しないでください。`license --json` は version 付きの `license`、`commercial_use`、`trademark`、controlling `documents` contract を返します。 | | 例外診断 | user-facing な CLI / JSON / MCP / file issue / local diagnostic output では raw `ex.Message` を直接 echo しないでください。例外の prose は `CommandErrorWriter.FormatSanitizedExceptionMessage`、`DiagnosticSanitizer.ForMessage`、または既存の bounded な `DiagnosticRedactor` helper を通し、回復に message が不要な場合は安定した error code/category を使ってください。意図的に残す broad catch は `risky-code/broad-exception-catch` taxonomy に沿い、bounded diagnostic、private な best-effort suppression、または documented fallback に正規化してください。 | diff --git a/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs b/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs index 20b81829e..b551fbbee 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs @@ -50,7 +50,24 @@ private static int RunDryRun( var warningCount = 0; var dryScanErrorKeys = new HashSet(StringComparer.Ordinal); DryRunScanMetadata dryScanMetadata; - var dbSnapshot = ReadDryRunDbSnapshot(resolvedDbPath); + var dbSnapshot = ReadDryRunDbSnapshot(resolvedDbPath, options); + var scopedUpdateSymbolKindFilterMatchesPrior = string.Equals( + dbSnapshot.SymbolKindFilterSignature, + options.SymbolKindFilter.Signature, + StringComparison.Ordinal) + || (dbSnapshot.SymbolKindFilterSignature == null + && !options.SymbolKindFilter.IsActive); + if (IsUpdateMode(options) + && !scopedUpdateSymbolKindFilterMatchesPrior) + { + return WriteCommandError( + options.Json, + jsonOptions, + "symbol-kind filter policy cannot change during a scoped update because existing files would keep symbols from the prior index policy", + CommandExitCodes.UsageError, + "Run a full index refresh without --files, --commits, or --changed-between when changing --include-symbol-kind or --exclude-symbol-kind.", + CommandErrorCodes.UsageError); + } if (options.MemoryTrace) memorySamples.Add(CaptureMemorySample("snapshot", stopwatch)); var normalizedProjectRoot = Path.GetFullPath(projectPath); @@ -192,9 +209,29 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) if (probe.PolicySkipped) { dryFileCount++; + retainedRelativePaths.Add(dbRelativePath); + if (dryFileSamples.Count < DryRunFileSampleLimit) + dryFileSamples.Add(displayRelativePath); + langCounts[probe.Language] = langCounts.GetValueOrDefault(probe.Language) + 1; + var projectedBinarySkip = probe.PolicySkipKind == DryRunPolicySkipKind.Binary + && IsDryRunStatReusable( + options, + dbSnapshot, + dbRelativePath, + probe.Language, + probe.Size, + probe.Modified, + dryIndexer.IsGeneratedCodeExtractionSuppressed(dbRelativePath), + authoritativeFullScan, + normalizedProjectRoot); + if (projectedBinarySkip) + { + projectedFileSkips++; + continue; + } + projectedFileUpdates++; projectedPolicySkips++; - retainedRelativePaths.Add(dbRelativePath); AddEstimatedExistingUpdateMutations( mutationEstimates, dbSnapshot, @@ -208,9 +245,6 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) 0, SymbolCapHit: false, ReferenceCapHit: false)); - if (dryFileSamples.Count < DryRunFileSampleLimit) - dryFileSamples.Add(displayRelativePath); - langCounts[probe.Language] = langCounts.GetValueOrDefault(probe.Language) + 1; if (probe.Error != null) { RecordDryRunError(displayRelativePath, probe.Error); @@ -279,10 +313,14 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) dbRelativePath); } } - var projectedSkip = !options.Rebuild - && dbSnapshot.Files.TryGetValue(dbRelativePath, out var existingRows) - && !string.IsNullOrEmpty(probe.Checksum) - && string.Equals(existingRows.Checksum, probe.Checksum, StringComparison.Ordinal); + var projectedSkip = IsDryRunLoadedFileReusable( + options, + dbSnapshot, + dbRelativePath, + probe.Loaded!.Value.Record, + dryIndexer.IsGeneratedCodeExtractionSuppressed(dbRelativePath), + authoritativeFullScan, + normalizedProjectRoot); if (projectedSkip) { projectedFileSkips++; @@ -832,7 +870,10 @@ private static DryRunFileProbe ProbeDryRunFile( DetectionSource: loaded.LanguageDetection.DetectionSource, DetectionConfidence: loaded.LanguageDetection.Confidence, Loaded: loaded, - PolicySkipped: false); + PolicySkipped: false, + DryRunPolicySkipKind.None, + record.Size, + record.Modified); } catch (FileIndexer.FileTooLargeSkippedException ex) { @@ -842,7 +883,8 @@ private static DryRunFileProbe ProbeDryRunFile( reusableLanguage); return DryRunFileProbe.FromPolicySkip( skipped, - CommandErrorWriter.FormatSanitizedExceptionMessage(ex)); + CommandErrorWriter.FormatSanitizedExceptionMessage(ex), + DryRunPolicySkipKind.FileTooLarge); } catch (FileIndexer.BinaryFileSkippedException ex) { @@ -852,7 +894,8 @@ private static DryRunFileProbe ProbeDryRunFile( reusableLanguage); return DryRunFileProbe.FromPolicySkip( skipped, - CommandErrorWriter.FormatSanitizedExceptionMessage(ex)); + CommandErrorWriter.FormatSanitizedExceptionMessage(ex), + DryRunPolicySkipKind.Binary); } catch (Exception ex) { @@ -1026,6 +1069,193 @@ private static DryRunParsedMutationEstimate BuildDryRunParsedMutationEstimate( ReferenceCapHit: referenceCapHit); } + private static bool IsDryRunLoadedFileReusable( + IndexCommandOptions options, + DryRunDbSnapshot snapshot, + string relativePath, + FileRecord record, + bool generatedExtractionSuppressed, + bool authoritativeFullScan, + string projectRoot) + { + if (!snapshot.Files.TryGetValue(relativePath, out var existing) + || !IsDryRunReuseAllowed( + options, + snapshot, + record.Lang, + authoritativeFullScan, + projectRoot) + || !existing.ContentReuseEligible + || existing.GeneratedExtractionSuppressed + != generatedExtractionSuppressed) + { + return false; + } + + var statMatches = existing.StatReuseEligible + && existing.ModifiedUtc == record.Modified + && existing.Size == record.Size + && string.Equals( + existing.Language, + record.Lang, + StringComparison.Ordinal); + var checksumMatches = !string.IsNullOrEmpty(record.Checksum) + && string.Equals( + existing.Checksum, + record.Checksum, + StringComparison.Ordinal) + && existing.Lines == record.Lines; + return statMatches || checksumMatches; + } + + private static bool IsDryRunStatReusable( + IndexCommandOptions options, + DryRunDbSnapshot snapshot, + string relativePath, + string language, + long? size, + DateTime? modified, + bool generatedExtractionSuppressed, + bool authoritativeFullScan, + string projectRoot) + { + if (!size.HasValue + || !modified.HasValue + || !snapshot.Files.TryGetValue(relativePath, out var existing) + || !existing.StatReuseEligible + || existing.GeneratedExtractionSuppressed + != generatedExtractionSuppressed + || !IsDryRunReuseAllowed( + options, + snapshot, + language, + authoritativeFullScan, + projectRoot)) + { + return false; + } + + return existing.Size == size.Value + && existing.ModifiedUtc == modified.Value + && string.Equals( + existing.Language, + language, + StringComparison.Ordinal); + } + + private static bool IsDryRunReuseAllowed( + IndexCommandOptions options, + DryRunDbSnapshot snapshot, + string? language, + bool authoritativeFullScan, + string projectRoot) + { + if (options.Rebuild + || string.IsNullOrWhiteSpace(language) + || !string.Equals( + snapshot.SymbolKindFilterSignature, + options.SymbolKindFilter.Signature, + StringComparison.Ordinal) + || !DryRunExtractorContractsMatchCurrent(snapshot, language)) + { + return false; + } + + if (authoritativeFullScan && snapshot.SymbolsOnlyGraphOmitted) + return false; + + if (language == "csharp") + { + var indexedProjectRoot = snapshot.IndexedProjectRoot; + if (!string.IsNullOrWhiteSpace(indexedProjectRoot) + && !PathsEqual( + Path.GetFullPath(indexedProjectRoot), + projectRoot)) + { + return false; + } + + var currentContract = DbContext.CSharpSymbolNameContractVersion + .ToString(System.Globalization.CultureInfo.InvariantCulture); + if (!string.Equals( + snapshot.GetMeta( + DbContext.CSharpSymbolNameContractVersionMetaKey), + currentContract, + StringComparison.Ordinal) + || snapshot.CSharpStaticInterfaceSourceEvidence == true) + { + return false; + } + } + + if (language == "sql") + { + var currentContract = DbContext.SqlGraphContractVersion + .ToString(System.Globalization.CultureInfo.InvariantCulture); + if (!string.Equals( + snapshot.GetMeta(DbContext.SqlGraphContractVersionMetaKey), + currentContract, + StringComparison.Ordinal)) + { + return false; + } + } + + if (language is "verilog" or "systemverilog" or "vhdl") + { + var currentContract = DbContext.HdlGraphContractVersion + .ToString(System.Globalization.CultureInfo.InvariantCulture); + if (!string.Equals( + snapshot.GetMeta(DbContext.HdlGraphContractVersionMetaKey), + currentContract, + StringComparison.Ordinal)) + { + return false; + } + } + + return true; + } + + private static bool DryRunExtractorContractsMatchCurrent( + DryRunDbSnapshot snapshot, + string language) + { + var storedExtractorVersion = snapshot.GetMeta( + DbContext.GetSymbolExtractorVersionMetaKey(language)); + if (storedExtractorVersion != null) + { + var currentExtractorVersion = SymbolExtractor + .GetContractVersion(language) + .ToString( + System.Globalization.CultureInfo.InvariantCulture); + if (!string.Equals( + storedExtractorVersion, + currentExtractorVersion, + StringComparison.Ordinal)) + { + return false; + } + } + + if (!SymbolExtractor + .RequiresExplicitReferenceGraphContractStamp(language)) + { + return true; + } + + var storedGraphContract = snapshot.GetMeta( + DbContext.GetDynamicReferenceGraphContractVersionMetaKey( + language)); + var currentGraphContract = SymbolExtractor + .GetReferenceGraphContractVersion(language) + .ToString(System.Globalization.CultureInfo.InvariantCulture); + return string.Equals( + storedGraphContract, + currentGraphContract, + StringComparison.Ordinal); + } + private static void AddEstimatedExistingUpdateMutations( DryRunMutationEstimateAccumulator mutations, DryRunDbSnapshot snapshot, @@ -1069,7 +1299,9 @@ private static void AddExistingChildRows( mutations.AddExisting("file_issues", rows.FileIssues, snapshot.FileIssuesAvailable); } - private static DryRunDbSnapshot ReadDryRunDbSnapshot(string dbPath) + private static DryRunDbSnapshot ReadDryRunDbSnapshot( + string dbPath, + IndexCommandOptions options) { try { @@ -1088,22 +1320,96 @@ private static DryRunDbSnapshot ReadDryRunDbSnapshot(string dbPath) if (!DryRunTableExists(connection, "files")) return DryRunDbSnapshot.Empty; - var indexedProjectRoot = DryRunReadMetaString(connection, DbContext.IndexedProjectRootMetaKey); + var metadata = DryRunReadMetadata(connection); + metadata.TryGetValue( + DbContext.IndexedProjectRootMetaKey, + out var indexedProjectRoot); var hasChunks = DryRunTableExists(connection, "chunks"); var hasSymbols = DryRunTableExists(connection, "symbols"); var hasSymbolReferences = DryRunTableExists(connection, "symbol_references"); var hasReferenceLines = DryRunTableExists(connection, "reference_lines"); var hasFileIssues = DryRunTableExists(connection, "file_issues"); + var hasChecksum = DryRunColumnExists( + connection, + "files", + "checksum"); + var hasLanguage = DryRunColumnExists(connection, "files", "lang"); + var hasSize = DryRunColumnExists(connection, "files", "size"); + var hasModified = DryRunColumnExists( + connection, + "files", + "modified"); + var hasLines = DryRunColumnExists(connection, "files", "lines"); + var hasIssueKind = hasFileIssues + && DryRunColumnExists( + connection, + "file_issues", + "kind"); + var hasIssueOrigin = hasFileIssues + && DryRunColumnExists( + connection, + "file_issues", + "origin"); + var hasIssueSeverity = hasFileIssues + && DryRunColumnExists( + connection, + "file_issues", + "severity"); + var hasCurrentIssueSchema = hasIssueKind + && hasIssueOrigin + && hasIssueSeverity; + + string IssueExists(string kind) => hasIssueKind + ? $""" + EXISTS ( + SELECT 1 + FROM file_issues i + WHERE i.file_id = f.id + AND i.kind = '{kind}' + ) + """ + : "0"; + var staleIssueMetadata = hasCurrentIssueSchema + ? """ + EXISTS ( + SELECT 1 + FROM file_issues i + WHERE i.file_id = f.id + AND ( + (i.kind IN ( + 'replacement_char', + 'non_utf8_likely', + 'bom', + 'utf16_bom') + AND ( + i.origin IS NULL + OR i.severity IS NULL)) + OR ( + i.kind = 'bom' + AND f.path LIKE '%.sln') + ) + ) + """ + : "0"; using var command = connection.CreateCommand(); command.CommandText = $""" SELECT f.path, - f.checksum, + {(hasChecksum ? "f.checksum" : "NULL")} AS checksum, + {(hasLanguage ? "f.lang" : "NULL")} AS lang, + {(hasSize ? "f.size" : "NULL")} AS size, + {(hasModified ? "f.modified" : "NULL")} AS modified, + {(hasLines ? "f.lines" : "NULL")} AS lines, {(hasChunks ? "(SELECT COUNT(*) FROM chunks c WHERE c.file_id = f.id)" : "0")} AS chunks_count, {(hasSymbols ? "(SELECT COUNT(*) FROM symbols s WHERE s.file_id = f.id)" : "0")} AS symbols_count, {(hasSymbolReferences ? "(SELECT COUNT(*) FROM symbol_references r WHERE r.file_id = f.id)" : "0")} AS symbol_references_count, {(hasReferenceLines ? "(SELECT COUNT(*) FROM reference_lines l WHERE l.file_id = f.id)" : "0")} AS reference_lines_count, - {(hasFileIssues ? "(SELECT COUNT(*) FROM file_issues i WHERE i.file_id = f.id)" : "0")} AS file_issues_count + {(hasFileIssues ? "(SELECT COUNT(*) FROM file_issues i WHERE i.file_id = f.id)" : "0")} AS file_issues_count, + {IssueExists(FileIndexer.GeneratedCodeExtractionSkippedIssueKind)} AS generated_suppressed, + {IssueExists("symbol_count_exceeded")} AS symbol_cap_issue, + {IssueExists("reference_count_exceeded")} AS reference_cap_issue, + {IssueExists("file_too_large")} AS file_too_large_issue, + {staleIssueMetadata} AS stale_issue_metadata FROM files f """; @@ -1111,18 +1417,72 @@ FROM files f using var reader = command.ExecuteReader(); while (reader.Read()) { + var language = reader.IsDBNull(2) + ? null + : reader.GetString(2); + var size = reader.IsDBNull(3) + || reader.GetValue(3) is not long rawSize + || rawSize < 0 + ? null + : (long?)rawSize; + var modifiedUtc = reader.GetValue(4) is not string rawModified + || !DateTime.TryParse( + rawModified, + System.Globalization.CultureInfo.InvariantCulture, + System.Globalization.DateTimeStyles.AssumeUniversal + | System.Globalization.DateTimeStyles.AdjustToUniversal, + out var parsedModifiedUtc) + ? null + : (DateTime?)parsedModifiedUtc; + var lines = reader.IsDBNull(5) + || reader.GetValue(5) is not long rawLines + || rawLines < 0 + ? null + : (long?)rawLines; + var symbols = reader.GetInt64(7); + var references = reader.GetInt64(8); + var generatedSuppressed = reader.GetInt64(11) != 0; + var hasSymbolCapIssue = reader.GetInt64(12) != 0; + var hasReferenceCapIssue = reader.GetInt64(13) != 0; + var hasFileTooLargeIssue = reader.GetInt64(14) != 0; + var hasStaleIssueMetadata = reader.GetInt64(15) != 0; + var contentReuseEligible = hasLanguage + && hasLines + && hasSymbols + && hasSymbolReferences + && hasCurrentIssueSchema + && symbols <= options.MaxSymbolsPerFile + && references <= options.MaxReferencesPerFile + && !hasSymbolCapIssue + && !hasReferenceCapIssue + && !hasStaleIssueMetadata; + var maxFileSize = options.MaxFileSizeBytes + ?? FileIndexer.DefaultMaxFileSizeBytes; + var statReuseEligible = contentReuseEligible + && size.HasValue + && size.Value <= maxFileSize + && modifiedUtc.HasValue + && !hasFileTooLargeIssue; files[reader.GetString(0)] = new DryRunExistingFileRows( reader.IsDBNull(1) ? null : reader.GetString(1), - reader.GetInt64(2), - reader.GetInt64(3), - reader.GetInt64(4), - reader.GetInt64(5), - reader.GetInt64(6)); + language, + size, + modifiedUtc, + lines, + reader.GetInt64(6), + symbols, + references, + reader.GetInt64(9), + reader.GetInt64(10), + generatedSuppressed, + contentReuseEligible, + statReuseEligible); } return new DryRunDbSnapshot( files, indexedProjectRoot, + metadata, hasChunks, hasSymbols, hasSymbolReferences, @@ -1151,15 +1511,57 @@ private static bool DryRunTableExists(SqliteConnection connection, string tableN return command.ExecuteScalar() != null; } - private static string? DryRunReadMetaString(SqliteConnection connection, string key) + private static bool DryRunColumnExists( + SqliteConnection connection, + string tableName, + string columnName) { - if (!DryRunTableExists(connection, "codeindex_meta")) - return null; + using var command = connection.CreateCommand(); + command.CommandText = $"PRAGMA table_info({tableName})"; + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + if (string.Equals( + reader.GetString(1), + columnName, + StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + private static IReadOnlyDictionary + DryRunReadMetadata(SqliteConnection connection) + { + var metadata = new Dictionary( + StringComparer.Ordinal); + if (!DryRunTableExists(connection, "codeindex_meta") + || !DryRunColumnExists( + connection, + "codeindex_meta", + "key") + || !DryRunColumnExists( + connection, + "codeindex_meta", + "value")) + { + return metadata; + } using var command = connection.CreateCommand(); - command.CommandText = "SELECT value FROM codeindex_meta WHERE key = @key LIMIT 1"; - SqliteCommandPolicy.Add(command, "@key", key); - return command.ExecuteScalar() as string; + command.CommandText = "SELECT key, value FROM codeindex_meta"; + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + metadata[reader.GetString(0)] = reader.IsDBNull(1) + ? null + : reader.GetString(1); + } + + return metadata; } private static int WriteDryRunInterrupted(IndexCommandOptions options, JsonSerializerOptions jsonOptions) => WriteCommandError( @@ -1275,15 +1677,36 @@ private void MarkUnknown(string metric, string reason) private sealed record DryRunDbSnapshot( IReadOnlyDictionary Files, string? IndexedProjectRoot, + IReadOnlyDictionary Metadata, bool ChunksAvailable, bool SymbolsAvailable, bool SymbolReferencesAvailable, bool ReferenceLinesAvailable, bool FileIssuesAvailable) { + internal string? SymbolKindFilterSignature + => GetMeta(SymbolKindFilterMetaKey); + + internal bool SymbolsOnlyGraphOmitted => string.Equals( + GetMeta(DbContext.SymbolsOnlyGraphOmittedMetaKey), + "true", + StringComparison.OrdinalIgnoreCase); + + internal bool? CSharpStaticInterfaceSourceEvidence + => bool.TryParse( + GetMeta( + DbContext.CSharpStaticInterfaceSourceEvidenceMetaKey), + out var value) + ? value + : null; + + internal string? GetMeta(string key) + => Metadata.TryGetValue(key, out var value) ? value : null; + public static DryRunDbSnapshot Empty { get; } = new( new Dictionary(StringComparer.Ordinal), null, + new Dictionary(StringComparer.Ordinal), false, false, false, @@ -1293,11 +1716,18 @@ private sealed record DryRunDbSnapshot( private readonly record struct DryRunExistingFileRows( string? Checksum, + string? Language, + long? Size, + DateTime? ModifiedUtc, + long? Lines, long Chunks, long Symbols, long SymbolReferences, long ReferenceLines, - long FileIssues); + long FileIssues, + bool GeneratedExtractionSuppressed, + bool ContentReuseEligible, + bool StatReuseEligible); private readonly record struct DryRunScanMetadata( bool HadErrors, @@ -1341,14 +1771,60 @@ private readonly record struct DryRunFileProbe( string? DetectionSource, FileIndexer.LanguageDetectionConfidence? DetectionConfidence, LoadedFileRecord? Loaded, - bool PolicySkipped) + bool PolicySkipped, + DryRunPolicySkipKind PolicySkipKind, + long? Size, + DateTime? Modified) { - public static DryRunFileProbe FromError(string message) => new(false, string.Empty, null, message, Unsupported: false, UnknownExtension: false, null, null, null, PolicySkipped: false); - public static DryRunFileProbe FromUnsupported() => new(false, string.Empty, null, null, Unsupported: true, UnknownExtension: false, null, null, null, PolicySkipped: false); - public static DryRunFileProbe FromUnknownExtension() => new(false, string.Empty, null, null, Unsupported: false, UnknownExtension: true, null, null, null, PolicySkipped: false); + public static DryRunFileProbe FromError(string message) => new( + false, + string.Empty, + null, + message, + Unsupported: false, + UnknownExtension: false, + null, + null, + null, + PolicySkipped: false, + DryRunPolicySkipKind.None, + null, + null); + + public static DryRunFileProbe FromUnsupported() => new( + false, + string.Empty, + null, + null, + Unsupported: true, + UnknownExtension: false, + null, + null, + null, + PolicySkipped: false, + DryRunPolicySkipKind.None, + null, + null); + + public static DryRunFileProbe FromUnknownExtension() => new( + false, + string.Empty, + null, + null, + Unsupported: false, + UnknownExtension: true, + null, + null, + null, + PolicySkipped: false, + DryRunPolicySkipKind.None, + null, + null); + public static DryRunFileProbe FromPolicySkip( FileRecord record, - string message) => new( + string message, + DryRunPolicySkipKind policySkipKind) => new( false, record.Lang ?? "unknown", record.Checksum, @@ -1358,6 +1834,16 @@ public static DryRunFileProbe FromPolicySkip( null, null, null, - PolicySkipped: true); + PolicySkipped: true, + policySkipKind, + record.Size, + record.Modified); + } + + private enum DryRunPolicySkipKind + { + None, + Binary, + FileTooLarge, } } diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs index 96edb561b..331c10239 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs @@ -439,6 +439,215 @@ public void Third() { } } } + [Fact] + public void Run_DryRun_ReestimatesUnchangedFileWithPersistedCapIssue_Issue4893() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "app.cs"), + """ + public class App + { + public void First() { } + public void Second() { } + } + """); + var (indexExitCode, _) = RunAndCaptureJson([ + projectRoot, + "--max-symbols-per-file", + "1", + "--json", + ]); + Assert.Equal(CommandExitCodes.Success, indexExitCode); + + var (exitCode, json) = RunAndCaptureJson([ + projectRoot, + "--dry-run", + "--max-symbols-per-file", + "1", + "--json", + ]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(1, json.GetProperty("projected_file_updates").GetInt32()); + Assert.Equal(0, json.GetProperty("projected_file_skips").GetInt32()); + Assert.Equal(1, json.GetProperty("projected_symbol_cap_hits").GetInt32()); + Assert.Equal( + 0, + json.GetProperty("estimated_table_mutations") + .GetProperty("symbols") + .GetInt32()); + } + finally + { + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_DryRun_ReusesUnchangedIndexedBinaryFile_Issue4893() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllBytes( + Path.Combine(projectRoot, "binary.cs"), + [0x70, 0x00, 0x71]); + var (indexExitCode, _) = RunAndCaptureJson([ + projectRoot, + "--json", + ]); + Assert.Equal(CommandExitCodes.Success, indexExitCode); + + var dbPath = Path.Combine( + projectRoot, + ".cdidx", + "codeindex.db"); + SqliteConnection.ClearAllPools(); + var before = File.ReadAllBytes(dbPath); + + var (exitCode, json) = RunAndCaptureJson([ + projectRoot, + "--dry-run", + "--json", + ]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(0, json.GetProperty("projected_file_updates").GetInt32()); + Assert.Equal(1, json.GetProperty("projected_file_skips").GetInt32()); + Assert.Equal(0, json.GetProperty("projected_policy_skips").GetInt32()); + Assert.Equal(0, json.GetProperty("errors_total").GetInt32()); + SqliteConnection.ClearAllPools(); + Assert.Equal(before, File.ReadAllBytes(dbPath)); + + var (refreshExitCode, refreshJson) = RunAndCaptureJson([ + projectRoot, + "--json", + ]); + Assert.Equal(CommandExitCodes.Success, refreshExitCode); + Assert.Equal( + 1, + refreshJson.GetProperty("summary") + .GetProperty("files_skipped") + .GetInt32()); + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_DryRun_DoesNotReuseAcrossFilterOrExtractorContractChange_Issue4893() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "app.cs"), + "public class App { public void Run() { } }\n"); + var (indexExitCode, _) = RunAndCaptureJson([ + projectRoot, + "--json", + ]); + Assert.Equal(CommandExitCodes.Success, indexExitCode); + + var (filterExitCode, filterJson) = RunAndCaptureJson([ + projectRoot, + "--dry-run", + "--include-symbol-kind", + "class", + "--json", + ]); + Assert.Equal(CommandExitCodes.Success, filterExitCode); + Assert.Equal( + 1, + filterJson.GetProperty("projected_file_updates").GetInt32()); + Assert.Equal( + 0, + filterJson.GetProperty("projected_file_skips").GetInt32()); + + var dbPath = Path.Combine( + projectRoot, + ".cdidx", + "codeindex.db"); + using (var connection = new SqliteConnection( + $"Data Source={dbPath}")) + { + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = """ + INSERT INTO codeindex_meta(key, value) + VALUES (@key, 'stale') + ON CONFLICT(key) DO UPDATE SET value = excluded.value + """; + command.Parameters.AddWithValue( + "@key", + DbContext.GetSymbolExtractorVersionMetaKey("csharp")); + command.ExecuteNonQuery(); + } + SqliteConnection.ClearAllPools(); + + var (contractExitCode, contractJson) = RunAndCaptureJson([ + projectRoot, + "--dry-run", + "--json", + ]); + Assert.Equal(CommandExitCodes.Success, contractExitCode); + Assert.Equal( + 1, + contractJson.GetProperty("projected_file_updates").GetInt32()); + Assert.Equal( + 0, + contractJson.GetProperty("projected_file_skips").GetInt32()); + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_DryRun_RejectsScopedSymbolFilterChange_Issue4893() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "app.cs"), + "public class App { }\n"); + var (indexExitCode, _) = RunAndCaptureJson([ + projectRoot, + "--json", + ]); + Assert.Equal(CommandExitCodes.Success, indexExitCode); + + var (exitCode, json) = RunAndCaptureJson([ + projectRoot, + "--files", + "app.cs", + "--dry-run", + "--include-symbol-kind", + "class", + "--json", + ]); + + Assert.Equal(CommandExitCodes.UsageError, exitCode); + Assert.Equal("error", json.GetProperty("status").GetString()); + Assert.Contains( + "symbol-kind filter policy cannot change", + json.GetProperty("message").GetString()); + } + finally + { + DeleteDirectory(projectRoot); + } + } + [Fact] public void Run_DryRun_ParseEstimateFailureReturnsExplicitUnknown_Issue4893() { From 40b28d63e341d4405b1a4e22fc682a2d85f0cecf Mon Sep 17 00:00:00 2001 From: Widthdom Date: Thu, 30 Jul 2026 20:30:25 +0900 Subject: [PATCH 3/3] Honor dry-run invalidation plans (#4893) --- DEVELOPER_GUIDE.md | 4 +- README.md | 14 +- changelog.d/unreleased/4893.changed.md | 4 +- .../Cli/IndexCommandRunner.DryRun.cs | 193 ++++++++++++++-- .../IndexCommandRunnerDryRunTests.cs | 212 ++++++++++++++++++ 5 files changed, 397 insertions(+), 30 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index bf237fb6e..7c9df6e3d 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -33,7 +33,7 @@ Development contracts: |---|---| | Formatting and warnings | CI enforces repository formatting with `.editorconfig` and treats compiler warnings as errors through `Directory.Build.props`, so local changes should pass the format check before opening a PR. Existing trim-analysis warnings are explicitly listed in `WarningsNotAsErrors` until they are fixed without blocking ordinary compiler-warning enforcement, and ILLink keeps reporting trim warnings without failing trimmed publish smoke tests. | | CLI help | `cdidx --help` stays brief, `cdidx --help-all` prints the full command/flag/example reference, `cdidx --help-flags` prints only shared flag tables, and `cdidx --help` prints one command's usage line. Keep new commands visible in the brief summary only when they are a primary user workflow; every command must remain present in the full help and command-specific usage table. | -| `index --dry-run` mutation estimates | Dry-run remains strictly non-mutating: it does not create or lock the database and does not alter sources or index artifacts. It uses read-only stat/checksum data together with the production cap-issue, symbol-filter, extractor-version, and graph-contract reuse gates to project unchanged skips; projects updates, content-policy skips, deletes, purges, and symbol/reference cap hits; and performs the normal chunk/symbol/reference/content-diagnostic extraction for at most 100 planned update files. `estimated_table_mutations` counts projected delete-and-insert row operations, not final table cardinality. Its nullable values must stay synchronized with `estimated_table_mutation_details`: `source` identifies filesystem planning or parse-only plus index-snapshot input, `confidence` is `exact`, `estimate`, or `unknown`, and stable `unknown_reasons` cover candidate truncation, parse-estimate truncation/failure, and partial-index table absence. A numeric zero is reserved for measured no work and remains qualified by its confidence. Parse-only estimates intentionally omit post-extraction hook mutations and cross-file C# workspace augmentation, so child-table values remain estimates. | +| `index --dry-run` mutation estimates | Dry-run remains strictly non-mutating: it does not create or lock the database and does not alter sources or index artifacts. It uses read-only stat/checksum data together with the production cap-issue, symbol-filter, extractor-version, graph-contract, forced extractor/config refresh, hotspot-marker trust, and C# workspace reuse gates to project unchanged skips; projects updates, content-policy skips, deletes, purges, and symbol/reference cap hits; and performs the normal chunk/symbol/reference/content-diagnostic extraction for at most 100 planned update files. `estimated_table_mutations` counts projected delete-and-insert row operations, not final table cardinality. Its nullable values must stay synchronized with `estimated_table_mutation_details`: `source` identifies filesystem planning or parse-only plus index-snapshot input, `confidence` is `exact`, `estimate`, or `unknown`, and stable `unknown_reasons` cover candidate truncation, parse-estimate truncation/failure, C# workspace expansion, unreadable index snapshots, and partial-index table absence. A numeric zero is reserved for measured no work and remains qualified by its confidence. Parse-only estimates intentionally omit post-extraction hook mutations and cross-file C# workspace augmentation, so child-table values remain estimates; when a changed C# source proves that production must expand the refresh across otherwise reusable C# files, those child-table metrics become explicitly unknown instead of retaining incomplete parse totals. | | `cdidx validate` | This is the user-facing integrity scan for indexed content issues such as replacement characters, BOMs, NUL bytes, mixed line endings, UTF-16 BOMs, and likely non-UTF8 content. Keep its CLI usage, README entry, and help summary in sync when adding validation issue kinds or filters. | | `cdidx doctor` | This is the copy-pasteable environment summary for support requests. Keep it redacted by default: secret-like `CDIDX_*` values must not be printed, and new diagnostic fields should be stable enough for issue triage. Full environment inventory filters (`--env-domain`, `--env-category`, and `--env-sensitivity`) use case-insensitive exact values and compose with AND; filtered JSON summaries describe the returned inventory rather than the global catalog. `--max-json-bytes` is valid only with `--json --env-inventory=full`, counts the serialized UTF-8 document plus its newline, and returns a structured usage error rather than an oversized successful document. The `github` block reports `proxy_default_credentials` as `enabled` / `disabled` and the bounded `max_request_timeout_s`; never print proxy credential material or raw secret values. `license --json` returns the versioned `license`, `commercial_use`, `trademark`, and controlling `documents` contract. | | Exception diagnostics | User-facing CLI, JSON, MCP, file-issue, and local diagnostic output must not echo raw `ex.Message` directly. Route exception prose through `CommandErrorWriter.FormatSanitizedExceptionMessage`, `DiagnosticSanitizer.ForMessage`, or an existing bounded `DiagnosticRedactor` helper, and use stable error codes/categories when the message is not needed for recovery. Intentional broad catches should match the `risky-code/broad-exception-catch` taxonomy and normalize to bounded diagnostics, private best-effort suppression, or a documented fallback. | @@ -3389,7 +3389,7 @@ net9 CI lane に合わせる場合は `FRAMEWORK=net9.0 make test` を使いま |---|---| | formatting と warning | CI は `.editorconfig` による repository formatting を強制し、`Directory.Build.props` により compiler warning を error として扱います。ローカル変更は PR 前に formatting check を通してください。既存の trim 解析警告は、通常の警告エラー化を止めずに修正を進められるよう `WarningsNotAsErrors` に明示列挙されています。ILLink は trimmed publish の smoke test を失敗させずに trim warning を報告し続けます。 | | CLI help | `cdidx --help` は短い概要、`cdidx --help-all` は全コマンド・flag・例の一覧、`cdidx --help-flags` は共有 flag table のみ、`cdidx --help` は 1 コマンドの usage line を出します。新しいコマンドは主要な user workflow である場合だけ簡易概要に載せ、full help とコマンド固有の usage table には必ず載せてください。 | -| `index --dry-run` の mutation 推定 | dry-run は厳密に非変更のままにし、database の作成・lock や source / index artifact の変更を行いません。read-only な stat / checksum と production の cap issue、symbol filter、extractor version、graph contract の再利用条件を組み合わせて未変更 skip を判定し、update、内容に対する policy skip、delete、purge、symbol / reference 上限到達を予測し、update 予定 file のうち最大100件に通常の chunk / symbol / reference / content diagnostic 抽出を適用します。`estimated_table_mutations` は最終 table row 数ではなく、delete と insert の予測 row operation 数です。nullable な値は `estimated_table_mutation_details` と常に同期させます。`source` は filesystem plan または parse-only と index snapshot の入力元、`confidence` は `exact` / `estimate` / `unknown`、安定した `unknown_reasons` は candidate の切り詰め、parse 推定の切り詰め・失敗、partial index の table 不足を表します。数値のゼロは、処理がないと計測できた場合だけに使い、その信頼度も併記します。parse-only 推定では extraction 後の hook mutation と C# の cross-file workspace augmentation を意図的に省くため、child table の値は estimate のままです。 | +| `index --dry-run` の mutation 推定 | dry-run は厳密に非変更のままにし、database の作成・lock や source / index artifact の変更を行いません。read-only な stat / checksum と production の cap issue、symbol filter、extractor version、graph contract、extractor / config の強制 refresh、hotspot marker の trust、C# workspace の再利用条件を組み合わせて未変更 skip を判定し、update、内容に対する policy skip、delete、purge、symbol / reference 上限到達を予測し、update 予定 file のうち最大100件に通常の chunk / symbol / reference / content diagnostic 抽出を適用します。`estimated_table_mutations` は最終 table row 数ではなく、delete と insert の予測 row operation 数です。nullable な値は `estimated_table_mutation_details` と常に同期させます。`source` は filesystem plan または parse-only と index snapshot の入力元、`confidence` は `exact` / `estimate` / `unknown`、安定した `unknown_reasons` は candidate の切り詰め、parse 推定の切り詰め・失敗、C# workspace の展開、index snapshot の読み取り不能、partial index の table 不足を表します。数値のゼロは、処理がないと計測できた場合だけに使い、その信頼度も併記します。parse-only 推定では extraction 後の hook mutation と C# の cross-file workspace augmentation を意図的に省くため、child table の値は estimate のままです。変更された C# source によって production が本来再利用可能な C# file にも refresh を展開すると判明した場合は、不完全な parse 合計を残さず、それらの child-table metric を明示的な unknown にします。 | | `cdidx validate` | replacement character、BOM、NUL byte、混在改行、UTF-16 BOM、非 UTF-8 らしい内容など、indexed content の問題を user-facing に検査する integrity scan です。validation issue の種別や filter を追加する場合は、CLI usage、README entry、help summary を同期してください。 | | `cdidx doctor` | support request 向けにコピーしやすい environment summary です。既定では redacted に保ち、secret 風の `CDIDX_*` 値は出力しないでください。新しい diagnostic field は issue triage に使える程度に安定したものだけにします。full environment inventory の filter（`--env-domain`、`--env-category`、`--env-sensitivity`）は大文字小文字を区別しない完全一致で AND 合成し、filtered JSON summary は global catalog ではなく返却 inventory を表します。`--max-json-bytes` は `--json --env-inventory=full` とだけ組み合わせ、serialize した UTF-8 文書と改行を数え、上限を超える成功文書の代わりに structured usage error を返します。`github` block は `proxy_default_credentials` を `enabled` / `disabled` として出力し、bounded な `max_request_timeout_s` も出します。proxy credential material や raw secret value は出力しないでください。`license --json` は version 付きの `license`、`commercial_use`、`trademark`、controlling `documents` contract を返します。 | | 例外診断 | user-facing な CLI / JSON / MCP / file issue / local diagnostic output では raw `ex.Message` を直接 echo しないでください。例外の prose は `CommandErrorWriter.FormatSanitizedExceptionMessage`、`DiagnosticSanitizer.ForMessage`、または既存の bounded な `DiagnosticRedactor` helper を通し、回復に message が不要な場合は安定した error code/category を使ってください。意図的に残す broad catch は `risky-code/broad-exception-catch` taxonomy に沿い、bounded diagnostic、private な best-effort suppression、または documented fallback に正規化してください。 | diff --git a/README.md b/README.md index e618ea64f..5462aa0d3 100644 --- a/README.md +++ b/README.md @@ -238,9 +238,11 @@ estimates for `chunks`, `symbols`, `symbol_references`, `reference_lines`, and `estimated_table_mutations` values are paired with `estimated_table_mutation_details`, whose `source`, `confidence`, and `unknown_reasons` distinguish an estimated or exact zero from unavailable -work. Candidate-path truncation, the parse-estimate cap, parse failures, and -missing tables in a partial index therefore produce `null` plus a stable reason -instead of a misleading zero. See the +work. Candidate-path truncation, the parse-estimate cap, parse failures, +cross-file C# refresh expansion, an unreadable index snapshot, and missing +tables in a partial index therefore produce `null` plus a stable reason instead +of a misleading zero. Reuse projections also honor production extractor/config +refreshes and hotspot-marker trust. See the [Developer Guide](DEVELOPER_GUIDE.md#build--test) for the implementation contract and limitations. @@ -709,8 +711,10 @@ nullable な `estimated_table_mutations` の各値には `estimated_table_mutation_details` が対応し、 `source`、`confidence`、`unknown_reasons` により、推定値または正確なゼロと 計測不能を区別します。そのため candidate path の切り詰め、parse 推定の上限、 -parse failure、partial index の table 不足は、誤解を招くゼロではなく安定した -理由付きの `null` になります。実装上の契約と制限は +parse failure、C# の cross-file refresh 展開、index snapshot の読み取り不能、 +partial index の table 不足は、誤解を招くゼロではなく安定した理由付きの `null` +になります。再利用予測には production の extractor / config refresh と hotspot +marker の trust も反映します。実装上の契約と制限は [開発者ガイド](DEVELOPER_GUIDE.md#ビルドテスト) を参照してください。 ## Status JSON 契約 diff --git a/changelog.d/unreleased/4893.changed.md b/changelog.d/unreleased/4893.changed.md index d9f65c76a..ba4106062 100644 --- a/changelog.d/unreleased/4893.changed.md +++ b/changelog.d/unreleased/4893.changed.md @@ -13,8 +13,8 @@ affected: ## English -- **Index dry-run now reports useful mutation estimates and explicit unknowns (#4893)** — New and changed files receive bounded parse-only chunk, symbol, reference, reference-line, and issue estimates; unchanged, deleted, policy-skipped, symbol-capped, and reference-capped files expose their projected outcomes; and truncation, parse failures, or partial index schemas return nullable metrics with source, confidence, and stable unknown reasons instead of misleading zeros, while preserving dry-run's non-mutation guarantee. +- **Index dry-run now reports useful mutation estimates and explicit unknowns (#4893)** — New and changed files receive bounded parse-only chunk, symbol, reference, reference-line, and issue estimates; unchanged, deleted, policy-skipped, symbol-capped, and reference-capped files expose their projected outcomes; production extractor/config, hotspot-marker, and C# workspace invalidations are reflected in reuse projections; and truncation, parse failures, unreadable snapshots, or partial index schemas return nullable metrics with source, confidence, and stable unknown reasons instead of misleading zeros, while preserving dry-run's non-mutation guarantee. ## 日本語 -- **index dry-run が有用な mutation 推定と明示的な unknown を返すようになりました (#4893)** — 新規・変更 file には上限付きの parse-only による chunk、symbol、reference、reference-line、issue 推定を返し、未変更、削除、policy skip、symbol 上限、reference 上限の予測結果も公開します。切り詰め、parse failure、partial index schema の場合は誤解を招くゼロではなく、入力元、信頼度、安定した unknown reason を持つ nullable metric を返し、dry-run の非変更保証を維持します。 +- **index dry-run が有用な mutation 推定と明示的な unknown を返すようになりました (#4893)** — 新規・変更 file には上限付きの parse-only による chunk、symbol、reference、reference-line、issue 推定を返し、未変更、削除、policy skip、symbol 上限、reference 上限の予測結果も公開します。production の extractor / config、hotspot marker、C# workspace による無効化も再利用予測へ反映し、切り詰め、parse failure、snapshot の読み取り不能、partial index schema の場合は誤解を招くゼロではなく、入力元、信頼度、安定した unknown reason を持つ nullable metric を返して、dry-run の非変更保証を維持します。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs b/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs index b551fbbee..1dc5facab 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.DryRun.cs @@ -79,6 +79,8 @@ private static int RunDryRun( var projectedDeletePaths = new HashSet(StringComparer.Ordinal); var projectedPurgePaths = new HashSet(StringComparer.Ordinal); var mutationEstimates = new DryRunMutationEstimateAccumulator(); + if (dbSnapshot.ReadFailed) + mutationEstimates.MarkAllUnknown("index_snapshot_unavailable"); var estimatedSymbolsDroppedByKindFilter = 0L; var projectedFileUpdates = 0; var projectedFileSkips = 0; @@ -87,6 +89,9 @@ private static int RunDryRun( var projectedReferenceCapHits = 0; var parseEstimateFilesProcessed = 0; var parseEstimateFilesTruncated = false; + var projectedCSharpSkips = new List(); + var csharpWorkspaceContractDetected = false; + var csharpWorkspaceEstimateUnavailable = false; var unsupportedTotal = 0; var unknownExtensionTotal = 0; using var symbolExtractionWorker = new LazyDisposable( @@ -150,6 +155,28 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) } } + var currentHotspotFamilyMarkerFingerprints = + GetHotspotFamilyMarkerFingerprints(dryIndexer, cancellationToken); + var priorHotspotFamilyVersions = FileIndexer + .GetHotspotFamilyMarkerLanguages() + .ToDictionary( + static language => language, + language => dbSnapshot.GetMeta( + DbContext.GetHotspotFamilyVersionMetaKey(language)), + StringComparer.Ordinal); + var priorHotspotFamilyMarkerFingerprints = FileIndexer + .GetHotspotFamilyMarkerLanguages() + .ToDictionary( + static language => language, + language => dbSnapshot.GetMeta( + DbContext.GetHotspotFamilyMarkerFingerprintMetaKey(language)), + StringComparer.Ordinal); + var hotspotFamilyTrustMatchesCurrent = + GetHotspotFamilyTrustMatchesCurrent( + priorHotspotFamilyVersions, + priorHotspotFamilyMarkerFingerprints, + currentHotspotFamilyMarkerFingerprints); + if (!TryResolveDryRunCandidates( options, dryIndexer, @@ -162,6 +189,8 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) out dryDeleteCandidates, out authoritativeFullScan, out dryScanMetadata, + out var forceExtractorRefresh, + out var forceJavaScriptTypeScriptRefresh, out var exitCode)) { return exitCode; @@ -223,10 +252,20 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) probe.Modified, dryIndexer.IsGeneratedCodeExtractionSuppressed(dbRelativePath), authoritativeFullScan, - normalizedProjectRoot); + normalizedProjectRoot, + forceExtractorRefresh, + forceJavaScriptTypeScriptRefresh, + hotspotFamilyTrustMatchesCurrent); if (projectedBinarySkip) { projectedFileSkips++; + if (probe.Language == "csharp") + { + projectedCSharpSkips.Add( + new DryRunProjectedCSharpSkip( + dbRelativePath, + PolicySkipped: true)); + } continue; } @@ -320,10 +359,20 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) probe.Loaded!.Value.Record, dryIndexer.IsGeneratedCodeExtractionSuppressed(dbRelativePath), authoritativeFullScan, - normalizedProjectRoot); + normalizedProjectRoot, + forceExtractorRefresh, + forceJavaScriptTypeScriptRefresh, + hotspotFamilyTrustMatchesCurrent); if (projectedSkip) { projectedFileSkips++; + if (probe.Language == "csharp") + { + projectedCSharpSkips.Add( + new DryRunProjectedCSharpSkip( + dbRelativePath, + PolicySkipped: false)); + } } else { @@ -336,6 +385,8 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) { parseEstimateFilesTruncated = true; mutationEstimates.MarkParseUnknown("parse_estimate_file_limit_reached"); + if (probe.Language == "csharp") + csharpWorkspaceEstimateUnavailable = true; } else { @@ -359,6 +410,8 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) projectedSymbolCapHits++; if (parsedEstimate.ReferenceCapHit) projectedReferenceCapHits++; + csharpWorkspaceContractDetected |= + parsedEstimate.CSharpStaticInterfaceContract; } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -367,6 +420,8 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) catch (Exception ex) { mutationEstimates.MarkParseUnknown("parse_estimation_failed"); + if (probe.Language == "csharp") + csharpWorkspaceEstimateUnavailable = true; RecordDryRunError( displayRelativePath, $"Parse-only mutation estimate unavailable: {CommandErrorWriter.FormatSanitizedExceptionMessage(ex)}"); @@ -378,6 +433,28 @@ void RecordDryRunScanErrors(IEnumerable scanErrors) langCounts[probe.Language] = langCounts.GetValueOrDefault(probe.Language) + 1; } + if (authoritativeFullScan + && projectedCSharpSkips.Count > 0 + && (csharpWorkspaceContractDetected + || csharpWorkspaceEstimateUnavailable)) + { + projectedFileSkips -= projectedCSharpSkips.Count; + projectedFileUpdates += projectedCSharpSkips.Count; + projectedPolicySkips += projectedCSharpSkips.Count( + static skip => skip.PolicySkipped); + foreach (var skip in projectedCSharpSkips) + { + AddEstimatedExistingUpdateMutations( + mutationEstimates, + dbSnapshot, + skip.RelativePath); + } + mutationEstimates.MarkParseUnknown( + csharpWorkspaceContractDetected + ? "csharp_workspace_augmentation_required" + : "csharp_workspace_preflight_unavailable"); + } + foreach (var relativePath in dryDeleteCandidates) { if (candidatePathsProcessed >= dryRunPathLimit) @@ -519,21 +596,34 @@ private static bool TryResolveDryRunCandidates( out IEnumerable dryDeleteCandidates, out bool authoritativeFullScan, out DryRunScanMetadata scanMetadata, + out bool forceExtractorRefresh, + out bool forceJavaScriptTypeScriptRefresh, out int exitCode) { dryCandidates = []; dryDeleteCandidates = []; authoritativeFullScan = false; scanMetadata = DryRunScanMetadata.Empty; + forceExtractorRefresh = false; + forceJavaScriptTypeScriptRefresh = false; exitCode = CommandExitCodes.Success; if (options.UpdateFiles.Count > 0) { // --files: only the specified files / --files: 指定ファイルのみ var relevantIgnoreFileChanged = ContainsRelevantIgnoreFileUpdate(projectPath, options.UpdateFiles); + forceJavaScriptTypeScriptRefresh = + ContainsJavaScriptTypeScriptConfigPath(normalizedUpdatePaths); + forceExtractorRefresh = + ContainsExtractorConfigurationPath( + projectPath, + normalizedUpdatePaths) + || normalizedUpdatePaths.Any( + FileIndexer.IsAmbiguousLanguageProjectMarkerPath); if (relevantIgnoreFileChanged || ContainsIgnoreFilePath(normalizedUpdatePaths) - || ContainsExtractorConfigurationPath(projectPath, normalizedUpdatePaths)) + || forceJavaScriptTypeScriptRefresh + || forceExtractorRefresh) { FileIndexer.ScanFilesResult scanResult; try @@ -621,9 +711,16 @@ private static bool TryResolveDryRunCandidates( } } + forceJavaScriptTypeScriptRefresh = + ContainsJavaScriptTypeScriptConfigPath(changedFiles); + forceExtractorRefresh = + ContainsExtractorConfigurationPath(projectPath, changedFiles) + || changedFiles.Any( + FileIndexer.IsAmbiguousLanguageProjectMarkerPath); if (relevantIgnoreFileChanged || ContainsIgnoreFilePath(changedFiles) - || ContainsExtractorConfigurationPath(projectPath, changedFiles)) + || forceJavaScriptTypeScriptRefresh + || forceExtractorRefresh) { FileIndexer.ScanFilesResult scanResult; try @@ -958,6 +1055,10 @@ private static DryRunParsedMutationEstimate BuildDryRunParsedMutationEstimate( symbolExtractionWorker, cancellationToken); var symbols = symbolExtraction.Symbols; + var csharpStaticInterfaceContract = + record.Lang == "csharp" + && CSharpStaticInterfacePrepass + .HasCSharpStaticInterfaceContractSymbol(symbols); if (symbols.Count > options.MaxSymbolsPerFile) { var issueCount = symbolExtraction.RegexTimeoutIssue == null ? 1 : 2; @@ -969,7 +1070,8 @@ private static DryRunParsedMutationEstimate BuildDryRunParsedMutationEstimate( issueCount, 0, SymbolCapHit: true, - ReferenceCapHit: false); + ReferenceCapHit: false, + csharpStaticInterfaceContract); } SymbolExtractor.ApplyFamilyScope( @@ -987,7 +1089,8 @@ private static DryRunParsedMutationEstimate BuildDryRunParsedMutationEstimate( issueCount, symbolsDroppedByKindFilter, SymbolCapHit: true, - ReferenceCapHit: false); + ReferenceCapHit: false, + csharpStaticInterfaceContract); } FileIndexer.ValidateSymbolLineRanges(record, symbols); @@ -1066,7 +1169,8 @@ private static DryRunParsedMutationEstimate BuildDryRunParsedMutationEstimate( issues.Count, symbolsDroppedByKindFilter, SymbolCapHit: false, - ReferenceCapHit: referenceCapHit); + ReferenceCapHit: referenceCapHit, + csharpStaticInterfaceContract); } private static bool IsDryRunLoadedFileReusable( @@ -1076,15 +1180,23 @@ private static bool IsDryRunLoadedFileReusable( FileRecord record, bool generatedExtractionSuppressed, bool authoritativeFullScan, - string projectRoot) + string projectRoot, + bool forceExtractorRefresh, + bool forceJavaScriptTypeScriptRefresh, + IReadOnlyDictionary + hotspotFamilyTrustMatchesCurrent) { if (!snapshot.Files.TryGetValue(relativePath, out var existing) || !IsDryRunReuseAllowed( options, snapshot, + relativePath, record.Lang, authoritativeFullScan, - projectRoot) + projectRoot, + forceExtractorRefresh, + forceJavaScriptTypeScriptRefresh, + hotspotFamilyTrustMatchesCurrent) || !existing.ContentReuseEligible || existing.GeneratedExtractionSuppressed != generatedExtractionSuppressed) @@ -1117,7 +1229,11 @@ private static bool IsDryRunStatReusable( DateTime? modified, bool generatedExtractionSuppressed, bool authoritativeFullScan, - string projectRoot) + string projectRoot, + bool forceExtractorRefresh, + bool forceJavaScriptTypeScriptRefresh, + IReadOnlyDictionary + hotspotFamilyTrustMatchesCurrent) { if (!size.HasValue || !modified.HasValue @@ -1128,9 +1244,13 @@ private static bool IsDryRunStatReusable( || !IsDryRunReuseAllowed( options, snapshot, + relativePath, language, authoritativeFullScan, - projectRoot)) + projectRoot, + forceExtractorRefresh, + forceJavaScriptTypeScriptRefresh, + hotspotFamilyTrustMatchesCurrent)) { return false; } @@ -1146,11 +1266,17 @@ private static bool IsDryRunStatReusable( private static bool IsDryRunReuseAllowed( IndexCommandOptions options, DryRunDbSnapshot snapshot, + string indexPath, string? language, bool authoritativeFullScan, - string projectRoot) + string projectRoot, + bool forceExtractorRefresh, + bool forceJavaScriptTypeScriptRefresh, + IReadOnlyDictionary + hotspotFamilyTrustMatchesCurrent) { if (options.Rebuild + || forceExtractorRefresh || string.IsNullOrWhiteSpace(language) || !string.Equals( snapshot.SymbolKindFilterSignature, @@ -1161,8 +1287,20 @@ private static bool IsDryRunReuseAllowed( return false; } - if (authoritativeFullScan && snapshot.SymbolsOnlyGraphOmitted) + if (authoritativeFullScan + && (options.SymbolsOnly + || snapshot.SymbolsOnlyGraphOmitted + || !AllowReuseWithCurrentHotspotFamilyTrust( + language, + hotspotFamilyTrustMatchesCurrent))) + return false; + + if (forceJavaScriptTypeScriptRefresh + && (IsJavaScriptTypeScriptLanguage(language) + || IsJavaScriptTypeScriptConfigPath(indexPath))) + { return false; + } if (language == "csharp") { @@ -1182,7 +1320,7 @@ private static bool IsDryRunReuseAllowed( DbContext.CSharpSymbolNameContractVersionMetaKey), currentContract, StringComparison.Ordinal) - || snapshot.CSharpStaticInterfaceSourceEvidence == true) + || snapshot.CSharpStaticInterfaceSourceEvidence is not false) { return false; } @@ -1487,19 +1625,20 @@ FROM files f hasSymbols, hasSymbolReferences, hasReferenceLines, - hasFileIssues); + hasFileIssues, + ReadFailed: false); } catch (SqliteException) { - return DryRunDbSnapshot.Empty; + return DryRunDbSnapshot.ReadFailure; } catch (IOException) { - return DryRunDbSnapshot.Empty; + return DryRunDbSnapshot.ReadFailure; } catch (UnauthorizedAccessException) { - return DryRunDbSnapshot.Empty; + return DryRunDbSnapshot.ReadFailure; } } @@ -1580,7 +1719,12 @@ private readonly record struct DryRunParsedMutationEstimate( long FileIssues, long SymbolsDroppedByKindFilter, bool SymbolCapHit, - bool ReferenceCapHit); + bool ReferenceCapHit, + bool CSharpStaticInterfaceContract = false); + + private readonly record struct DryRunProjectedCSharpSkip( + string RelativePath, + bool PolicySkipped); private sealed class DryRunMutationEstimateAccumulator { @@ -1682,7 +1826,8 @@ private sealed record DryRunDbSnapshot( bool SymbolsAvailable, bool SymbolReferencesAvailable, bool ReferenceLinesAvailable, - bool FileIssuesAvailable) + bool FileIssuesAvailable, + bool ReadFailed) { internal string? SymbolKindFilterSignature => GetMeta(SymbolKindFilterMetaKey); @@ -1711,7 +1856,13 @@ internal bool? CSharpStaticInterfaceSourceEvidence false, false, false, - false); + false, + ReadFailed: false); + + public static DryRunDbSnapshot ReadFailure { get; } = Empty with + { + ReadFailed = true, + }; } private readonly record struct DryRunExistingFileRows( diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs index 331c10239..e6217053b 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerDryRunTests.cs @@ -648,6 +648,218 @@ public void Run_DryRun_RejectsScopedSymbolFilterChange_Issue4893() } } + [Fact] + public void Run_DryRun_ExtractorConfigCommitForcesFullRefresh_Issue4893() + { + var projectRoot = CreateTempProject(); + try + { + RunGit(projectRoot, "init"); + File.WriteAllText( + Path.Combine(projectRoot, "app.cs"), + "public class App { }\n"); + RunGit(projectRoot, "add", "app.cs"); + RunGit(projectRoot, "commit", "-m", "initial"); + var (indexExitCode, _) = RunAndCaptureJson([ + projectRoot, + "--json", + ]); + Assert.Equal(CommandExitCodes.Success, indexExitCode); + + var patternDirectory = Path.Combine( + projectRoot, + ".cdidx", + "patterns"); + Directory.CreateDirectory(patternDirectory); + File.WriteAllText( + Path.Combine(patternDirectory, "custom.json"), + "{}\n"); + RunGit( + projectRoot, + "add", + "-f", + ".cdidx/patterns/custom.json"); + RunGit(projectRoot, "commit", "-m", "change extractor config"); + + var (exitCode, json) = RunAndCaptureJson([ + projectRoot, + "--commits", + "HEAD", + "--dry-run", + "--json", + ]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal( + 1, + json.GetProperty("projected_file_updates").GetInt32()); + Assert.Equal( + 0, + json.GetProperty("projected_file_skips").GetInt32()); + } + finally + { + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_DryRun_CSharpContractRefreshesCrossFileSkips_Issue4893() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "contract.cs"), + "public interface IContract { }\n"); + File.WriteAllText( + Path.Combine(projectRoot, "consumer.cs"), + "public class Consumer { }\n"); + var (indexExitCode, _) = RunAndCaptureJson([ + projectRoot, + "--json", + ]); + Assert.Equal(CommandExitCodes.Success, indexExitCode); + + File.WriteAllText( + Path.Combine(projectRoot, "contract.cs"), + """ + public interface IContract + where T : IContract + { + static abstract T Parse(string value); + } + """); + + var (exitCode, json) = RunAndCaptureJson([ + projectRoot, + "--dry-run", + "--json", + ]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal( + 2, + json.GetProperty("projected_file_updates").GetInt32()); + Assert.Equal( + 0, + json.GetProperty("projected_file_skips").GetInt32()); + var symbolDetails = json + .GetProperty("estimated_table_mutation_details") + .GetProperty("symbols"); + Assert.Equal( + "unknown", + symbolDetails.GetProperty("confidence").GetString()); + Assert.Contains( + "csharp_workspace_augmentation_required", + symbolDetails.GetProperty("unknown_reasons") + .EnumerateArray() + .Select(static value => value.GetString())); + } + finally + { + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_DryRun_HotspotMarkerChangeInvalidatesFamilyReuse_Issue4893() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "one.targets"), + "\n"); + File.WriteAllText( + Path.Combine(projectRoot, "two.targets"), + "\n"); + File.WriteAllText( + Path.Combine(projectRoot, "sample.csproj"), + "\n"); + var (indexExitCode, _) = RunAndCaptureJson([ + projectRoot, + "--json", + ]); + Assert.Equal(CommandExitCodes.Success, indexExitCode); + + File.WriteAllText( + Path.Combine(projectRoot, "second.csproj"), + "\n"); + + var (exitCode, json) = RunAndCaptureJson([ + projectRoot, + "--dry-run", + "--json", + ]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal( + 4, + json.GetProperty("projected_file_updates").GetInt32()); + Assert.Equal( + 0, + json.GetProperty("projected_file_skips").GetInt32()); + } + finally + { + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_DryRun_SnapshotReadFailureReturnsExplicitUnknown_Issue4893() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "app.cs"), + "public class App { }\n"); + var indexDirectory = Path.Combine(projectRoot, ".cdidx"); + Directory.CreateDirectory(indexDirectory); + var dbPath = Path.Combine(indexDirectory, "codeindex.db"); + File.WriteAllText(dbPath, "not a sqlite database\n"); + var before = File.ReadAllBytes(dbPath); + + var (exitCode, json) = RunAndCaptureJson([ + projectRoot, + "--dry-run", + "--json", + ]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + foreach (var metric in new[] + { + "files", + "chunks", + "symbols", + "symbol_references", + "reference_lines", + "file_issues", + }) + { + Assert.Equal( + JsonValueKind.Null, + json.GetProperty("estimated_table_mutations") + .GetProperty(metric) + .ValueKind); + Assert.Contains( + "index_snapshot_unavailable", + json.GetProperty("estimated_table_mutation_details") + .GetProperty(metric) + .GetProperty("unknown_reasons") + .EnumerateArray() + .Select(static value => value.GetString())); + } + Assert.Equal(before, File.ReadAllBytes(dbPath)); + } + finally + { + DeleteDirectory(projectRoot); + } + } + [Fact] public void Run_DryRun_ParseEstimateFailureReturnsExplicitUnknown_Issue4893() {