From 65c20bd883ea67b9d19d7832c43fb9e9038634ae Mon Sep 17 00:00:00 2001 From: Widthdom Date: Wed, 3 Jun 2026 01:44:16 +0900 Subject: [PATCH] Classify validate replacement characters (#2814) --- DEVELOPER_GUIDE.md | 12 +++ README.md | 2 + USER_GUIDE.md | 20 +++-- changelog.d/unreleased/2814.changed.md | 27 ++++++ src/CodeIndex/Cli/ConsoleUi.cs | 4 +- src/CodeIndex/Database/DbContext.cs | 12 ++- src/CodeIndex/Database/DbReader.cs | 10 ++- src/CodeIndex/Database/DbWriter.cs | 34 +++++++- src/CodeIndex/Indexer/Scanning/FileIndexer.cs | 38 ++++++++- src/CodeIndex/Mcp/McpToolDefinitions.cs | 2 +- src/CodeIndex/Models/FileIssue.cs | 7 ++ tests/CodeIndex.Tests/DatabaseTests.cs | 84 +++++++++++++++++++ tests/CodeIndex.Tests/FileIndexerTests.cs | 67 +++++++++++---- .../QueryCommandRunnerTests.cs | 39 +++++++++ 14 files changed, 323 insertions(+), 35 deletions(-) create mode 100644 changelog.d/unreleased/2814.changed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 3788e86aee..2c165f5f84 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -229,6 +229,12 @@ Directory scan / shared path filter (built-in skip lists + `.gitignore` / `.cdid → Populate FTS5 index ``` +`FileIssue` rows may include nullable `origin` and `severity` metadata. +For `replacement_char`, `origin: source_literal` means the file contains a +valid encoded U+FFFD literal, while `origin: decode_replacement` means the +decoder inserted U+FFFD for invalid bytes. `severity: info` is used for source +literals, and `severity: warning` is used for likely encoding damage. + Scoped `--files` / `--commits` refreshes reuse the same path filter as full scans. Before scanning a nested project root, `FileIndexer` loads ignore files from the resolved ignore-rule root through each existing ancestor directory down to the project root's parent, then loads the project directory's own rules during the normal walk. Within each directory, `FileIndexer` loads `.gitignore` before `.cdidxignore`, appends both rule sets in that order, and honors later `!` patterns as re-includes. If an ancestor ignore directory cannot be read, scanning fails closed with a scan error instead of silently skipping those rules; `ScanFilesResult.AncestorIgnoreDirectories` records the resolved ancestor list for troubleshooting. If a commit-scoped refresh includes `.gitignore` or `.cdidxignore` changes, `IndexCommandRunner` falls back to a full scan so newly ignored files are purged safely. Malformed ignore lines are reported as scan errors and skipped instead of aborting the whole run. Directory symlinks default to `--follow-symlinks none`; `internal` follows only targets that resolve under the workspace root, and `all` preserves the broad historical behavior. Dangling symlinks are counted and warned separately. On Windows, files and directories with Hidden or System attributes are rejected before language detection; clear those attributes before indexing project-owned sources because ignore rules cannot re-include them. Incremental refreshes that mutate `fts_chunks` increment `codeindex_meta.fts_incremental_writes_since_optimize`. When the counter reaches `DbWriter.DefaultFtsOptimizeIncrementalWriteThreshold`, the update path runs `INSERT INTO fts_chunks(fts_chunks) VALUES('optimize')`, resets the counter, and stamps `fts_last_optimized_at`. Users can run the same maintenance directly with `cdidx optimize --db ` or `cdidx index --optimize`; this may briefly hold the writer lock on large indexes. @@ -2124,6 +2130,12 @@ CI で `NU1004 The packages lock file is inconsistent with the project dependenc → FTS5インデックス反映 ``` +`FileIssue` rows には nullable な `origin` / `severity` metadata が入ることがある。 +`replacement_char` では `origin: source_literal` が正規にエンコードされた U+FFFD +literal、`origin: decode_replacement` が不正 byte に対して decoder が挿入した U+FFFD +を意味する。source literal は `severity: info`、エンコーディング破損の可能性は +`severity: warning` として返す。 + `--files` / `--commits` の部分更新も、フルスキャンと同じパスフィルタを再利用する。各ディレクトリでは `FileIndexer` が `.gitignore` を `.cdidxignore` より先に読み、この順序でルールを追加し、後続の `!` パターンを再包含として扱う。commit 単位更新に `.gitignore` または `.cdidxignore` の変更が含まれる場合、`IndexCommandRunner` は newly ignored file を安全に purge するため自動でフルスキャンへフォールバックする。malformed な ignore 行は走査エラーとして報告し、その行だけをスキップして index 全体は継続する。Windows では Hidden または System 属性が付いたファイルとディレクトリを言語検出前に拒否する。プロジェクト所有のソースを索引したい場合、ignore ルールでは再包含できないため先にそれらの属性を外す。 ### ignore ファイルの解析 diff --git a/README.md b/README.md index b8c10797a0..bd5ad6e5ed 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,7 @@ downgrading `cdidx`. | Area | What cdidx provides | |---|---| | Search surfaces | CLI-first output for humans and machines; full-text, symbol, reference, caller/callee, dependency, map, inspect, and excerpt commands. `search`, `definition`, `references`, `callers`, `callees`, `find`, and `validate` support `--format count|compact|csv|tsv|lsp|qf|sarif` for token-budgeted agents, scripts, editors, and CI reports. `cdidx lsp --db .cdidx/codeindex.db` starts a read-only stdio Language Server Protocol shim for LSP-native editors. | +| Validation diagnostics | `validate --json` and MCP `validate` annotate `replacement_char` rows with `origin` (`source_literal` or `decode_replacement`) and `severity` so agents can separate intentional U+FFFD literals from likely encoding damage. | | Definition and impact diagnostics | `definition --json` includes C# `disambiguator` hints for overloads, partial types, and extension receivers when indexed metadata can distinguish them. `impact --json` and MCP `impact_analysis` include `impact_failure_chain` and `suggestion_type` for zero-result routing; `impact --strict` exits non-zero when resolution or graph preconditions are unmet. | | Ranking and filters | Public/exported symbol matches rank ahead of protected, internal, and private matches. Use `--no-visibility-rank` for legacy order, and `--visibility` / `--exclude-visibility` with `symbols`, `definition`, `unused`, and `hotspots`. Query defaults can be adjusted with `CDIDX_DEFAULT_LIMIT`, `CDIDX_DEFAULT_SNIPPET_LINES`, and `CDIDX_DEFAULT_MAX_LINE_WIDTH`; explicit CLI flags still win. | | Project scoping | `.sln` / `.csproj`-aware --project <name|path> filters for indexing and queries, plus `--solution ` when a workspace has multiple solution files. | @@ -437,6 +438,7 @@ upgrade / downgrade 後はインストール済み補完 script を再生成し | 分野 | 内容 | |---|---| | 検索面 | CLI-first の人間向け / 機械処理向け出力。全文検索、シンボル、参照、caller/callee、依存関係、map、inspect、excerpt コマンドを提供します。`cdidx lsp --db .cdidx/codeindex.db` は LSP-native editor 向けの read-only stdio Language Server Protocol shim を起動します。 | +| validation 診断 | `validate --json` と MCP `validate` は `replacement_char` 行に `origin` (`source_literal` / `decode_replacement`) と `severity` を付け、意図的な U+FFFD literal とエンコーディング破損の可能性を agent が分離できるようにします。 | | definition / impact 診断 | `definition --json` は C# overload、partial type、extension receiver を区別できる場合に `disambiguator` を返します。`impact --json` と MCP `impact_analysis` は 0 件時の経路判断用に `impact_failure_chain` と `suggestion_type` を返し、`impact --strict` は解決または graph の前提条件が満たされない場合に非 0 で終了します。 | | 順位と filter | public/exported なシンボル一致を protected、internal、private より優先します。従来順は `--no-visibility-rank`、可視性の include / exclude は `symbols`、`definition`、`unused`、`hotspots` の `--visibility` / `--exclude-visibility` で指定できます。query 既定値は `CDIDX_DEFAULT_LIMIT`、`CDIDX_DEFAULT_SNIPPET_LINES`、`CDIDX_DEFAULT_MAX_LINE_WIDTH` で調整でき、明示 CLI flag が常に優先されます。 | | project scope | `.sln` / `.csproj` を使った --project <name|path> filter で index と query を .NET project 配下へ絞り込めます。workspace に solution が複数ある場合は `--solution ` を指定します。 | diff --git a/USER_GUIDE.md b/USER_GUIDE.md index c003da3232..c22e2a3eab 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -280,7 +280,7 @@ sections below show examples and option details for the most common workflows. | Analysis | `impact` | Traverse transitive callers from a resolved symbol | `impact_analysis` | | Analysis | `unused` | Find symbols defined but not referenced, with confidence buckets | `unused_symbols` | | Analysis | `hotspots` | Rank high-impact symbols or statements by reference volume | `symbol_hotspots` | -| Analysis | `validate` | Report encoding and line-ending issues in indexed files | `validate` | +| Analysis | `validate` | Report encoding and line-ending issues in indexed files; U+FFFD rows include origin/severity metadata | `validate` | | Status | `status` | Show DB statistics, freshness, and readiness metadata | `status` | | Status | `languages` | List language extensions and symbol/graph capabilities | `languages` | | Diagnostics | `db --integrity-check` | Run SQLite `PRAGMA integrity_check` against the DB | -- | @@ -400,13 +400,16 @@ conflicting instructions. ```bash cdidx validate -cdidx validate --kind replacement-character --path src/ +cdidx validate --kind replacement_char --path src/ cdidx validate --json --path legacy/ ``` `validate` reports indexed files that are likely to produce misleading snippets or symbol names: U+FFFD replacement characters, UTF-16 BOMs, null bytes, mixed or CR-only line endings, likely non-UTF-8 content, and Git LFS pointer placeholders. +For `replacement_char`, JSON and MCP responses include `origin` (`source_literal` +or `decode_replacement`) and `severity` so agents can distinguish intentional +U+FFFD literals from likely encoding damage. LFS pointers are recorded as `lfs_pointer_skipped` and their placeholder body is not indexed; run `git lfs pull` and then `cdidx index .` to index the real file content. @@ -1945,7 +1948,7 @@ The MCP `tools/list` response includes an `examples` array for every registered | `unused_symbols` | Find symbols defined but never referenced, with confidence buckets for dead-code triage | | `symbol_hotspots` | Find high-impact hotspots. `groupBy` supports `symbol`, `file`, and `statement`; SQL scopes default to statement grouping while non-SQL scopes default to symbol grouping. | | `batch_query` | Execute multiple queries in a single call (MCP only, max 10). The response includes a top-level `metadata` object with `submitted`, `executed`, `errors`, `total_elapsed_ms`, `success_count`, and `failure_count`; every entry in `results` carries `request_index`, `ok`, `elapsed_ms`, and compact `args_summary` fields so callers can correlate partial failures and slow inner queries without relying on positional guesses. | -| `validate` | Report encoding issues (U+FFFD, BOM, null bytes, mixed/CR-only line endings, UTF-16 BOM detection, likely non-UTF8 encodings) | +| `validate` | Report encoding issues (U+FFFD with origin/severity, BOM, null bytes, mixed/CR-only line endings, UTF-16 BOM detection, likely non-UTF8 encodings) | | `languages` | List all supported languages, file extensions, and capabilities | | `ping` | Lightweight connection check | | `index` | Index or re-index a project directory | @@ -2412,7 +2415,7 @@ cdidx index . --quiet | Analysis | `impact` | 解決した symbol から transitive callers を探索 | `impact_analysis` | | Analysis | `unused` | 参照されていない可能性がある symbols を confidence bucket 付きで表示 | `unused_symbols` | | Analysis | `hotspots` | reference volume で high-impact symbols/statements を ranking | `symbol_hotspots` | -| Analysis | `validate` | indexed files の encoding / line-ending 問題を報告 | `validate` | +| Analysis | `validate` | indexed files の encoding / line-ending 問題を報告。U+FFFD 行には origin/severity metadata が付く | `validate` | | Status | `status` | DB stats、freshness、readiness metadata を表示 | `status` | | Status | `languages` | language extensions と symbol/graph capabilities を一覧 | `languages` | | Diagnostics | `db --integrity-check` | DB に対して SQLite `PRAGMA integrity_check` を実行 | -- | @@ -2522,13 +2525,16 @@ render できます。 ```bash cdidx validate -cdidx validate --kind replacement-character --path src/ +cdidx validate --kind replacement_char --path src/ cdidx validate --json --path legacy/ ``` `validate` は、snippet や symbol name を誤らせやすい indexed file を報告します。 対象は U+FFFD replacement character、UTF-16 BOM、null byte、mixed / CR-only line -ending、likely non-UTF-8 content、Git LFS pointer placeholder などです。LFS pointer +ending、likely non-UTF-8 content、Git LFS pointer placeholder などです。 +`replacement_char` の JSON / MCP response には `origin` (`source_literal` / +`decode_replacement`) と `severity` が入り、意図的な U+FFFD literal と +エンコーディング破損の可能性を agent が区別できます。LFS pointer は `lfs_pointer_skipped` として記録され、placeholder 本文は index されません。 実体を index するには `git lfs pull` の後に `cdidx index .` を再実行してください。 @@ -4077,7 +4083,7 @@ OpenAI Codex CLI (`codex.json` または `~/.codex/config.json`): | `unused_symbols` | 定義されているが参照されていないシンボルを bucket 付きで検索（デッドコード検出向け） | | `symbol_hotspots` | 影響の大きい hotspot を検索。`groupBy` は `symbol` / `file` / `statement` を指定でき、SQL scope は statement grouping、非 SQL scope は symbol grouping が既定。 | | `batch_query` | 複数クエリを1回で実行（MCP専用、最大10件）。レスポンスにはトップレベル `metadata`（`submitted` / `executed` / `errors` / `total_elapsed_ms` / `success_count` / `failure_count`）と各 `results` エントリの `request_index` / `ok` / `elapsed_ms` / `args_summary` が含まれ、位置だけに依存せず部分失敗や遅い内部クエリを把握できます。 | -| `validate` | エンコーディング問題（U+FFFD、BOM、null バイト、改行混在 / CR-only 行末、UTF-16 BOM 検出、UTF-8 以外と推定されるエンコーディング）を報告 | +| `validate` | エンコーディング問題（origin/severity 付き U+FFFD、BOM、null バイト、改行混在 / CR-only 行末、UTF-16 BOM 検出、UTF-8 以外と推定されるエンコーディング）を報告 | | `languages` | 対応言語一覧を拡張子・機能付きで表示 | | `ping` | 軽量な接続確認 | | `index` | プロジェクトのインデックス作成・更新 | diff --git a/changelog.d/unreleased/2814.changed.md b/changelog.d/unreleased/2814.changed.md new file mode 100644 index 0000000000..f073cbc87d --- /dev/null +++ b/changelog.d/unreleased/2814.changed.md @@ -0,0 +1,27 @@ +--- +category: changed +issues: + - 2814 +affected: + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - src/CodeIndex/Models/FileIssue.cs + - src/CodeIndex/Database/DbContext.cs + - src/CodeIndex/Database/DbReader.cs + - src/CodeIndex/Database/DbWriter.cs + - src/CodeIndex/Cli/ConsoleUi.cs + - src/CodeIndex/Mcp/McpToolDefinitions.cs + - README.md + - USER_GUIDE.md + - DEVELOPER_GUIDE.md + - tests/CodeIndex.Tests/FileIndexerTests.cs + - tests/CodeIndex.Tests/QueryCommandRunnerTests.cs + - tests/CodeIndex.Tests/DatabaseTests.cs +--- + +## English + +- **`validate` now classifies U+FFFD replacement-character rows (#2814)** — `replacement_char` issues now include `origin` and `severity` metadata so JSON and MCP consumers can distinguish intentional `source_literal` U+FFFD characters from `decode_replacement` encoding damage. + +## 日本語 + +- **`validate` が U+FFFD replacement character 行を分類するようになりました (#2814)** — `replacement_char` issue に `origin` と `severity` metadata を追加し、JSON / MCP consumer が意図的な `source_literal` の U+FFFD とエンコーディング破損を示す `decode_replacement` を区別できるようにしました。 diff --git a/src/CodeIndex/Cli/ConsoleUi.cs b/src/CodeIndex/Cli/ConsoleUi.cs index 5ac2518578..b07f32eb15 100644 --- a/src/CodeIndex/Cli/ConsoleUi.cs +++ b/src/CodeIndex/Cli/ConsoleUi.cs @@ -769,7 +769,7 @@ public static void PrintUsageBrief(bool showBanner = true) Console.WriteLine(" inspect Bundle definition, graph, and nearby symbol context"); Console.WriteLine(" status Show database statistics, freshness, config, and logs"); Console.WriteLine(" doctor Print a redacted environment summary for bug reports"); - Console.WriteLine(" validate Report encoding issues (U+FFFD, BOM, null bytes, mixed line endings, UTF-16 BOM, likely non-UTF8)"); + Console.WriteLine(" validate Report encoding issues (U+FFFD origin/severity, BOM, null bytes, mixed line endings, UTF-16 BOM, likely non-UTF8)"); Console.WriteLine(" impact Show transitive callers; type queries may return heuristic file-level dependency hints"); Console.WriteLine(" deps Show file-level dependency edges from the reference graph"); Console.WriteLine(" unused Find symbols defined but never referenced (dead code)"); @@ -880,7 +880,7 @@ private static void PrintCommandSummary() Console.WriteLine(" db prune --dry-run|--apply Count or delete orphaned DB rows"); Console.WriteLine(" diff Compare two index databases; exit 0 identical, 1 drift, 2 schema mismatch, 3 unreadable"); Console.WriteLine(" report --output Build a redacted crash-repro tarball (.tgz) for bug reports"); - Console.WriteLine(" validate Report encoding issues (U+FFFD, BOM, null bytes, mixed line endings, UTF-16 BOM, likely non-UTF8)"); + Console.WriteLine(" validate Report encoding issues (U+FFFD origin/severity, BOM, null bytes, mixed line endings, UTF-16 BOM, likely non-UTF8)"); Console.WriteLine(" impact Show transitive callers; type queries may return heuristic file-level dependency hints"); Console.WriteLine(" deps Show file-level dependency edges from the reference graph"); Console.WriteLine(" unused Find symbols defined but never referenced (dead code)"); diff --git a/src/CodeIndex/Database/DbContext.cs b/src/CodeIndex/Database/DbContext.cs index f3948409a2..435454957c 100644 --- a/src/CodeIndex/Database/DbContext.cs +++ b/src/CodeIndex/Database/DbContext.cs @@ -1599,7 +1599,9 @@ CREATE TABLE IF NOT EXISTS file_issues ( file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, kind TEXT NOT NULL, line INTEGER NOT NULL DEFAULT 0, - message TEXT NOT NULL + message TEXT NOT NULL, + origin TEXT, + severity TEXT )"); // Key-value metadata: fold algorithm version, future per-subsystem schema markers @@ -1631,6 +1633,8 @@ value TEXT EnsureColumn("symbols", "family_key", "TEXT"); EnsureColumn("symbols", "visibility", "TEXT"); EnsureColumn("symbols", "return_type", "TEXT"); + EnsureColumn("file_issues", "origin", "TEXT"); + EnsureColumn("file_issues", "severity", "TEXT"); EnsureColumn("symbols", "is_metadata_target", "INTEGER"); var rebuildsSymbolReferences = !ColumnIsNotNull("symbol_references", "file_id"); EnsureColumn( @@ -1804,10 +1808,12 @@ CREATE TABLE file_issues ( file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, kind TEXT NOT NULL, line INTEGER NOT NULL DEFAULT 0, - message TEXT NOT NULL + message TEXT NOT NULL, + origin TEXT, + severity TEXT ) """, - "id, file_id, kind, line, message"); + "id, file_id, kind, line, message, origin, severity"); } finally { diff --git a/src/CodeIndex/Database/DbReader.cs b/src/CodeIndex/Database/DbReader.cs index d9575093ab..0f7b839e6e 100644 --- a/src/CodeIndex/Database/DbReader.cs +++ b/src/CodeIndex/Database/DbReader.cs @@ -81,6 +81,7 @@ public partial class DbReader : IDisposable internal readonly bool _hasChunksTable; internal readonly bool _hasReferenceLinesTable; internal readonly bool _canUseReferenceLines; + private readonly HashSet _issueColumns; public bool IncludeGenerated { get; set; } private static readonly AsyncLocal IncludeGeneratedScope = new(); private static readonly AsyncLocal GeneratedColumnAvailableScope = new(); @@ -514,6 +515,9 @@ private DbReader( _hasChunksTable = HasTable("chunks"); _hasReferencesTable = HasTable("symbol_references") && (userVersion & DbContext.GraphReadyFlag) != 0; _hasIssuesPhysicalTable = HasTable("file_issues"); + _issueColumns = _hasIssuesPhysicalTable + ? LoadColumns("file_issues") + : new HashSet(StringComparer.OrdinalIgnoreCase); _hasIssuesTable = _hasIssuesPhysicalTable && (userVersion & DbContext.IssuesReadyFlag) != 0; _hasReferenceLinesTable = HasTable("reference_lines"); _canUseReferenceLines = _hasReferencesTable && _hasReferenceLinesTable && _referenceColumns.Contains("reference_line_id"); @@ -1734,8 +1738,10 @@ string text when DateTime.TryParse(text, CultureInfo.InvariantCulture, DateTimeS { if (!_hasIssuesTable) return new List(); using var cmd = _conn.CreateCommand(); + var originColumn = _issueColumns.Contains("origin") ? "i.origin" : "NULL"; + var severityColumn = _issueColumns.Contains("severity") ? "i.severity" : "NULL"; var sql = @" - SELECT f.path, i.kind, i.line, i.message + SELECT f.path, i.kind, i.line, i.message, " + originColumn + @" AS origin, " + severityColumn + @" AS severity FROM file_issues i JOIN files f ON i.file_id = f.id WHERE 1=1"; @@ -1770,6 +1776,8 @@ FROM file_issues i Kind = reader.GetString(1), Line = reader.GetInt32(2), Message = reader.GetString(3), + Origin = reader.IsDBNull(4) ? null : reader.GetString(4), + Severity = reader.IsDBNull(5) ? null : reader.GetString(5), }); } return results; diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index 1b1c50487d..4ee8316afa 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -41,6 +41,7 @@ public class DbWriter private int _transactionDepth; private int _transactionOwnerThreadId; private Guid _transactionOwnerToken; + private bool? _hasIssueMetadataColumns; // Outermost SqliteTransaction currently held open by this writer (null when no // transaction is active OR after the outermost transaction has been committed / // rolled back). Tracked so cached prepared commands can be re-pointed at the live @@ -486,6 +487,8 @@ private void RunPassiveWalCheckpoint() return null; if (!SymbolExtractorVersionMatchesCurrent(language)) return null; + if (HasStaleIssueMetadata(relativePath)) + return null; // Keep the unchanged check and timestamp touch in one SQLite statement so // concurrent row drift cannot slip between a SELECT and a later UPDATE (#1735). @@ -534,6 +537,31 @@ ELSE generated } } + private bool HasStaleIssueMetadata(string relativePath) + { + if (!HasIssueMetadataColumns()) + { + return false; + } + + using var cmd = _conn.CreateCommand(); + cmd.CommandText = @" + SELECT 1 + FROM file_issues i + JOIN files f ON i.file_id = f.id + WHERE f.path = @path + AND i.kind IN ('replacement_char', 'non_utf8_likely') + AND (i.origin IS NULL OR i.severity IS NULL) + LIMIT 1"; + cmd.Parameters.AddWithValue("@path", relativePath); + return cmd.ExecuteScalar() != null; + } + + private bool HasIssueMetadataColumns() => + _hasIssueMetadataColumns ??= TableExists("file_issues") + && ColumnExists("file_issues", "origin") + && ColumnExists("file_issues", "severity"); + /// /// Check whether the DB currently contains any indexed files for the given language. /// 指定言語の indexed file が DB に存在するか確認する。 @@ -1536,11 +1564,13 @@ public void InsertIssues(long fileId, IReadOnlyList if (issues.Count == 0) return; using var cmd = _conn.CreateCommand(); - cmd.CommandText = "INSERT INTO file_issues (file_id, kind, line, message) VALUES (@fid, @kind, @line, @message)"; + cmd.CommandText = "INSERT INTO file_issues (file_id, kind, line, message, origin, severity) VALUES (@fid, @kind, @line, @message, @origin, @severity)"; var pFid = cmd.Parameters.Add("@fid", SqliteType.Integer); var pKind = cmd.Parameters.Add("@kind", SqliteType.Text); var pLine = cmd.Parameters.Add("@line", SqliteType.Integer); var pMessage = cmd.Parameters.Add("@message", SqliteType.Text); + var pOrigin = cmd.Parameters.Add("@origin", SqliteType.Text); + var pSeverity = cmd.Parameters.Add("@severity", SqliteType.Text); foreach (var issue in issues) { @@ -1548,6 +1578,8 @@ public void InsertIssues(long fileId, IReadOnlyList pKind.Value = issue.Kind; pLine.Value = issue.Line; pMessage.Value = issue.Message; + pOrigin.Value = issue.Origin ?? (object)DBNull.Value; + pSeverity.Value = issue.Severity ?? (object)DBNull.Value; cmd.ExecuteNonQuery(); } } diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs index 8e5da8f1c5..a39baf4833 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.cs @@ -3332,7 +3332,11 @@ public static List ValidateContent(string relativePath, byte[] rawByt const double NonUtf8LikelyRatioThreshold = 0.01; const int NonUtf8LikelyMinCount = 5; var fffdCount = CountReplacementChars(content); - var nonUtf8Likely = fffdCount >= NonUtf8LikelyMinCount + var replacementCharOrigin = fffdCount > 0 + ? DetermineReplacementCharOrigin(rawBytes, isUtf16, utf16BigEndian, hasUtf16Bom) + : null; + var nonUtf8Likely = replacementCharOrigin == FileIssue.OriginDecodeReplacement + && fffdCount >= NonUtf8LikelyMinCount && content.Length > 0 && (double)fffdCount / content.Length >= NonUtf8LikelyRatioThreshold; if (nonUtf8Likely) @@ -3344,6 +3348,8 @@ public static List ValidateContent(string relativePath, byte[] rawByt Kind = "non_utf8_likely", Line = 0, Message = $"Likely non-UTF8 encoding ({fffdCount} U+FFFD over {content.Length} chars, {ratioPercent:F1}%); source may be SHIFT_JIS, GBK, ISO-8859-1, or UTF-16 without BOM", + Origin = FileIssue.OriginDecodeReplacement, + Severity = FileIssue.SeverityWarning, }); } @@ -3359,12 +3365,17 @@ public static List ValidateContent(string relativePath, byte[] rawByt { // Find line number / 行番号を特定 var lineNum = content[..i].Count(c => c == '\n') + 1; + var isSourceLiteral = replacementCharOrigin == FileIssue.OriginSourceLiteral; issues.Add(new FileIssue { Path = relativePath, Kind = "replacement_char", Line = lineNum, - Message = $"U+FFFD replacement character at line {lineNum}", + Message = isSourceLiteral + ? $"U+FFFD source literal at line {lineNum}" + : $"U+FFFD decoder replacement character at line {lineNum}", + Origin = replacementCharOrigin, + Severity = isSourceLiteral ? FileIssue.SeverityInfo : FileIssue.SeverityWarning, }); // Skip to next line to avoid reporting every char on the same line // 同じ行の連続報告を避けるため次の行までスキップ @@ -3729,6 +3740,29 @@ private static int CountReplacementChars(string content) return count; } + private static string DetermineReplacementCharOrigin(byte[] rawBytes, bool isUtf16, bool utf16BigEndian, bool hasUtf16Bom) + { + try + { + if (isUtf16) + { + _ = new UnicodeEncoding(utf16BigEndian, byteOrderMark: hasUtf16Bom, throwOnInvalidBytes: true) + .GetString(rawBytes); + } + else + { + _ = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true) + .GetString(rawBytes); + } + + return FileIssue.OriginSourceLiteral; + } + catch (DecoderFallbackException) + { + return FileIssue.OriginDecodeReplacement; + } + } + /// /// Compute SHA256 checksum from file bytes after collapsing CRLF / CR to LF. /// Matches the line-ending normalization that BuildRecord applies to the decoded diff --git a/src/CodeIndex/Mcp/McpToolDefinitions.cs b/src/CodeIndex/Mcp/McpToolDefinitions.cs index 72af399497..a8c32b3aee 100644 --- a/src/CodeIndex/Mcp/McpToolDefinitions.cs +++ b/src/CodeIndex/Mcp/McpToolDefinitions.cs @@ -366,7 +366,7 @@ private JsonNode HandleToolsList(JsonNode? id) ReadOnlyAnnotations()), CreateToolDefinition( "validate", - "Report encoding issues found during indexing: U+FFFD replacement chars, BOM markers, null bytes, mixed/CR-only line endings, UTF-16 BOM detection, likely non-UTF8 encodings. / インデックス時に検出したエンコーディング問題を報告。", + "Report encoding issues found during indexing: U+FFFD replacement chars, BOM markers, null bytes, mixed/CR-only line endings, UTF-16 BOM detection, likely non-UTF8 encodings. replacement_char rows include origin/severity metadata so agents can separate source literals from decoder replacements. / インデックス時に検出したエンコーディング問題を報告。replacement_char 行は source literal と decoder replacement を分ける origin/severity metadata を含む。", new JsonObject { ["type"] = "object", diff --git a/src/CodeIndex/Models/FileIssue.cs b/src/CodeIndex/Models/FileIssue.cs index fa30e78a92..160d2dc9fe 100644 --- a/src/CodeIndex/Models/FileIssue.cs +++ b/src/CodeIndex/Models/FileIssue.cs @@ -6,8 +6,15 @@ namespace CodeIndex.Models; /// public class FileIssue { + public const string OriginSourceLiteral = "source_literal"; + public const string OriginDecodeReplacement = "decode_replacement"; + public const string SeverityInfo = "info"; + public const string SeverityWarning = "warning"; + public string Path { get; set; } = string.Empty; public string Kind { get; set; } = string.Empty; public int Line { get; set; } public string Message { get; set; } = string.Empty; + public string? Origin { get; set; } + public string? Severity { get; set; } } diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 836fba933e..ec36b7573f 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -1333,6 +1333,90 @@ public void GetUnchangedFileId_ReturnIdIfUnchanged() Assert.Null(id2); } + [Fact] + public void GetUnchangedFileId_ReturnsNullWhenIssueMetadataMissing() + { + var modified = new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var file = new FileRecord + { + Path = "src/literal.py", + Lang = "python", + Size = 50, + Lines = 5, + Modified = modified, + }; + var fileId = _writer.UpsertFile(file); + _writer.InsertIssues(fileId, + [ + new FileIssue + { + Path = file.Path, + Kind = "replacement_char", + Line = 1, + Message = "legacy replacement_char row without metadata", + }, + ]); + + Assert.Null(_writer.GetUnchangedFileId(file.Path, modified)); + + _writer.InsertIssues(fileId, + [ + new FileIssue + { + Path = file.Path, + Kind = "replacement_char", + Line = 1, + Message = "U+FFFD source literal at line 1", + Origin = FileIssue.OriginSourceLiteral, + Severity = FileIssue.SeverityInfo, + }, + ]); + + Assert.NotNull(_writer.GetUnchangedFileId(file.Path, modified)); + } + + [Fact] + public void GetUnchangedFileId_ReturnsNullWhenNonUtf8LikelyMetadataMissing() + { + var modified = new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var file = new FileRecord + { + Path = "src/garbled.py", + Lang = "python", + Size = 50, + Lines = 5, + Modified = modified, + }; + var fileId = _writer.UpsertFile(file); + _writer.InsertIssues(fileId, + [ + new FileIssue + { + Path = file.Path, + Kind = "non_utf8_likely", + Line = 0, + Message = "legacy non_utf8_likely row without metadata", + }, + ]); + + Assert.Null(_writer.GetUnchangedFileId(file.Path, modified)); + + _writer.InsertIssues(fileId, + [ + new FileIssue + { + Path = file.Path, + Kind = "non_utf8_likely", + Line = 0, + Message = "Likely non-UTF8 encoding", + Origin = FileIssue.OriginDecodeReplacement, + Severity = FileIssue.SeverityWarning, + }, + ]); + + Assert.NotNull(_writer.GetUnchangedFileId(file.Path, modified)); + } + [Fact] public void GetUnchangedFileId_WithNullChecksumUsesModifiedAndSize() { diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index 00063c3b06..800494655f 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -4063,20 +4063,22 @@ public void ValidateContent_HighFffdRatio_EmitsAggregateNonUtf8Likely() // SHIFT_JIS / GBK / ISO-8859-1 を UTF-8 で読んで化けた content は per-line // `replacement_char` で埋め尽くすのではなく `non_utf8_likely` 1 件に集約する。 // Closes #1540. - // Build content with > 1% U+FFFD ratio and many lines. - var sb = new System.Text.StringBuilder(); + // Build invalid UTF-8 bytes that decode to > 1% U+FFFD ratio and many lines. + var raw = new List(); for (int i = 0; i < 50; i++) { - sb.Append("alpha � beta\n"); + raw.AddRange(System.Text.Encoding.UTF8.GetBytes("alpha ")); + raw.Add(0xFF); + raw.AddRange(System.Text.Encoding.UTF8.GetBytes(" beta\n")); } - var content = sb.ToString(); - // Raw bytes do not matter here for non_utf8_likely (it reads `content`), so use - // ASCII-safe bytes that won't trip the raw-byte heuristics. - var rawBytes = System.Text.Encoding.UTF8.GetBytes("placeholder\n"); + var rawBytes = raw.ToArray(); + var content = new System.Text.UTF8Encoding(false, throwOnInvalidBytes: false).GetString(rawBytes); var issues = FileIndexer.ValidateContent("garbled.cs", rawBytes, content); - Assert.Contains(issues, i => i.Kind == "non_utf8_likely"); + var issue = Assert.Single(issues.Where(i => i.Kind == "non_utf8_likely")); + Assert.Equal(FileIssue.OriginDecodeReplacement, issue.Origin); + Assert.Equal(FileIssue.SeverityWarning, issue.Severity); // Per-line replacement_char emission must be suppressed when the aggregate fires. // アグリゲートが出た場合は per-line replacement_char を抑止する。 Assert.DoesNotContain(issues, i => i.Kind == "replacement_char"); @@ -4093,20 +4095,49 @@ public void ValidateContent_LowFffdRatio_KeepsPerLineReplacementCharIssues() // Closes #1540. // 4 U+FFFD chars in a long file → far below 1% ratio AND below the minimum-count // floor of 5, so the aggregate must not fire. - var sb = new System.Text.StringBuilder(); - sb.Append("line1 clean\n"); - sb.Append("line2 has � here\n"); - sb.Append("line3 has � here\n"); - for (int i = 0; i < 200; i++) sb.Append("filler ascii ascii ascii\n"); - sb.Append("trailing �\n"); - sb.Append("another �\n"); - var content = sb.ToString(); - var rawBytes = System.Text.Encoding.UTF8.GetBytes("placeholder\n"); + var raw = new List(); + void AddUtf8(string text) => raw.AddRange(System.Text.Encoding.UTF8.GetBytes(text)); + + AddUtf8("line1 clean\n"); + AddUtf8("line2 has "); + raw.Add(0xFF); + AddUtf8(" here\n"); + AddUtf8("line3 has "); + raw.Add(0xFF); + AddUtf8(" here\n"); + for (int i = 0; i < 200; i++) AddUtf8("filler ascii ascii ascii\n"); + AddUtf8("trailing "); + raw.Add(0xFF); + AddUtf8("\n"); + AddUtf8("another "); + raw.Add(0xFF); + AddUtf8("\n"); + var rawBytes = raw.ToArray(); + var content = new System.Text.UTF8Encoding(false, throwOnInvalidBytes: false).GetString(rawBytes); var issues = FileIndexer.ValidateContent("partial.cs", rawBytes, content); Assert.DoesNotContain(issues, i => i.Kind == "non_utf8_likely"); - Assert.Contains(issues, i => i.Kind == "replacement_char"); + Assert.Contains(issues, i => + i.Kind == "replacement_char" + && i.Origin == FileIssue.OriginDecodeReplacement + && i.Severity == FileIssue.SeverityWarning); + } + + [Fact] + public void ValidateContent_SourceLiteralFffd_AnnotatesInfoOrigin() + { + var content = "line1 clean\nline2 has \uFFFD literal\n"; + var rawBytes = System.Text.Encoding.UTF8.GetBytes(content); + + var issues = FileIndexer.ValidateContent("literal.cs", rawBytes, content); + + var issue = Assert.Single(issues.Where(i => i.Kind == "replacement_char")); + Assert.Equal(2, issue.Line); + Assert.Equal(FileIssue.OriginSourceLiteral, issue.Origin); + Assert.Equal(FileIssue.SeverityInfo, issue.Severity); + Assert.Contains("source literal", issue.Message, StringComparison.Ordinal); + Assert.DoesNotContain(issues, i => i.Kind == "non_utf8_likely"); } [Fact] diff --git a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs index 6e6fc5a98c..e0a26cefbc 100644 --- a/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/QueryCommandRunnerTests.cs @@ -4715,6 +4715,45 @@ public void RunValidate_KindFilterNarrowsIssues() } } + [Fact] + public void RunValidate_ReplacementCharJson_IncludesOriginAndSeverity() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_validate_replacement_origin"); + try + { + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + Directory.CreateDirectory(Path.Combine(projectRoot, "src")); + File.WriteAllText( + Path.Combine(projectRoot, "src", "literal.cs"), + "class Literal { const char Value = '\uFFFD'; }\n"); + + var (indexExitCode, _, indexStderr) = CaptureConsole(() => IndexCommandRunner.Run( + [projectRoot, "--db", dbPath, "--json", "--quiet"], + _jsonOptions)); + Assert.Equal(CommandExitCodes.Success, indexExitCode); + Assert.Equal(string.Empty, indexStderr); + + var (exitCode, stdout, stderr) = CaptureConsole(() => QueryCommandRunner.RunValidate( + ["--db", dbPath, "--json", "--kind", "replacement_char"], + _jsonOptions)); + + using var document = ParseJsonOutput(stdout); + var json = document.RootElement; + var issue = json.GetProperty("issues")[0]; + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, stderr); + Assert.Equal(1, json.GetProperty("count").GetInt32()); + Assert.Equal("replacement_char", issue.GetProperty("kind").GetString()); + Assert.Equal(FileIssue.OriginSourceLiteral, issue.GetProperty("origin").GetString()); + Assert.Equal(FileIssue.SeverityInfo, issue.GetProperty("severity").GetString()); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + // `validate --kind replacement_chra` previously filtered the file_issues table by an // unknown kind, returned zero rows, and printed the same "No encoding issues found." // message a genuinely-clean repo would print — silently masking the typo. Round-2 adds