Surface oversize file diagnostics

USER_GUIDE.md

@@ -1857,9 +1857,9 @@ When SQLite returns permission-style errors such as `SQLITE_AUTH`, `SQLITE_PERM`
     - Recovery: fix file/directory permissions, or exclude the path via `.cdidxignore`. The index keeps running across the rest of the tree; no rebuild is required after permissions are fixed — a normal `cdidx index .` will pick up the now-readable files.
 
 12. **File rejected: too large**
-    - Symptom: `--verbose` shows `[ERR ] <path>: File too large (N MiB > M MiB limit). Override with --max-file-bytes <bytes> or CDIDX_MAX_FILE_BYTES=<bytes> when this source file is intentionally indexable.` (the file becomes part of the run's `errors` count), and the file does not appear in search.
+    - Symptom: `validate --kind file_too_large` reports `File too large (N MiB > M MiB limit). Override with --max-file-bytes <bytes> or CDIDX_MAX_FILE_BYTES=<bytes> when this source file is intentionally indexable.` The file is listed in `files`, but no chunks, symbols, or references are indexed for it, so it does not appear in search.
     - Cause: the file exceeds the configured per-file size limit. Indexing huge generated files would waste tokens and bloat the DB.
-    - Recovery: shrink or split the file, add it to `.cdidxignore`, or raise the limit with `cdidx index . --max-file-bytes 50M` / `CDIDX_MAX_FILE_BYTES=50M` when the file is legitimate source. Generated artifacts should generally be gitignored too. Note that any `[ERR ]` files in a run leave readiness flags unstamped, so resolve oversize entries before depending on `graph_table_available` / `issues_table_available` results.
+    - Recovery: shrink or split the file, add it to `.cdidxignore`, or raise the limit with `cdidx index . --max-file-bytes 50M` / `CDIDX_MAX_FILE_BYTES=50M` when the file is legitimate source. Generated artifacts should generally be gitignored too.
 
 13. **Feature unavailable on trimmed / AOT build** (`E009_FEATURE_UNAVAILABLE`)
     - Symptom: `Error [E009_FEATURE_UNAVAILABLE]: ...` when invoking flags such as `--json` on a build that lacks the required code paths.

changelog.d/unreleased/1603.fixed.md

@@ -0,0 +1,19 @@
+---
+category: fixed
+issues:
+  - 1603
+affected:
+  - src/CodeIndex/Cli/IndexCommandRunner.cs
+  - src/CodeIndex/Cli/QueryCommandRunner.cs
+  - src/CodeIndex/Indexer/Scanning/FileIndexer.cs
+  - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs
+  - USER_GUIDE.md
+---
+
+## English
+
+- **Oversize files now surface through `validate` (#1603)** — files above the configured indexing size limit are persisted with a `file_too_large` issue instead of disappearing as per-file errors, so `validate --kind file_too_large` explains why their chunks, symbols, and references are absent.
+
+## 日本語
+
+- **サイズ上限を超えたファイルが `validate` で見えるようになりました (#1603)** — 設定された indexing size limit を超えたファイルは per-file error として消えるのではなく `file_too_large` issue として保存されるため、`validate --kind file_too_large` で chunks / symbols / references が存在しない理由を確認できます。

src/CodeIndex/Cli/IndexCommandRunner.cs

@@ -2143,6 +2143,41 @@ void ThrowIfUpdateCancelled()
                     continue;
                 }
 
+                if (ex is FileIndexer.FileTooLargeSkippedException fileTooLarge)
+                {
+                    if (fileBatchMarked)
+                        writer.ClearBatchInProgress();
+
+                    DemoteReadinessOnce();
+                    writer.MarkBatchInProgress();
+                    using var txn = writer.BeginTransaction();
+                    var skippedRecord = indexer.BuildSkippedFileRecord(absPath);
+                    writer.PurgeStaleFilesSharingChecksum(projectRoot, skippedRecord.Path, skippedRecord.Checksum);
+                    if (projectRootWritten)
+                        writer.PurgeStaleFilesSharingDirectoryAndStem(projectRoot, skippedRecord.Path);
+                    WriteProjectRootOnce();
+                    var fileId = writer.UpsertFile(skippedRecord);
+                    writer.InsertChunks([]);
+                    writer.InsertSymbols([]);
+                    writer.InsertReferences([]);
+                    writer.InsertIssues(fileId,
+                    [
+                        new FileIssue
+                        {
+                            Path = fileTooLarge.RelativePath,
+                            Kind = "file_too_large",
+                            Line = 0,
+                            Message = fileTooLarge.Message,
+                        },
+                    ]);
+                    writer.ClearBatchInProgress();
+                    txn.Commit();
+
+                    updated++;
+                    ftsMutated = true;
+                    continue;
+                }
+
                 if (ex is FileNotFoundException or DirectoryNotFoundException)
                 {
                     if (fileBatchMarked)
@@ -3472,6 +3507,20 @@ void StopJsonHeartbeat()
                         {
                             extractionResults.Add(FullScanFileWorkItem.Skipped(filePath, ex.Message), cancellationToken);
                         }
+                        catch (FileIndexer.FileTooLargeSkippedException ex)
+                        {
+                            var record = indexer.BuildSkippedFileRecord(filePath);
+                            var issue = new FileIssue
+                            {
+                                Path = ex.RelativePath,
+                                Kind = "file_too_large",
+                                Line = 0,
+                                Message = ex.Message,
+                            };
+                            extractionResults.Add(
+                                FullScanFileWorkItem.Success(filePath, record, string.Empty, [], ex.Message, [], [], [], [issue]),
+                                cancellationToken);
+                        }
                         catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException)
                         {
                             var relativePath = FileIndexer.NormalizePathSeparators(Path.GetRelativePath(projectRoot, filePath));

src/CodeIndex/Cli/QueryCommandRunner.cs

@@ -3350,7 +3350,7 @@ private static JsonObject BuildUnusedResultsByBucketJson(IEnumerable<UnusedSymbo
     // `--kind replacement_chra` のようなタイプミスを did-you-mean で救うため、
     // FileIndexer.cs 内の `Kind = "..."` 代入と同期させる (#1582)。
     private static readonly string[] AllValidValidateKinds =
-        ["bom", "cr_only_line_endings", "line_too_long", "mixed_line_endings", "mixed_line_endings_three_way", "non_utf8_likely", "null_byte", "replacement_char", "utf16_bom"];
+        ["bom", "cr_only_line_endings", "file_too_large", "line_too_long", "mixed_line_endings", "mixed_line_endings_three_way", "non_utf8_likely", "null_byte", "replacement_char", "utf16_bom"];
 
     public static int RunValidate(string[] cmdArgs, JsonSerializerOptions jsonOptions)
     {

src/CodeIndex/Indexer/Scanning/FileIndexer.cs

@@ -2455,7 +2455,11 @@ public static string NormalizePathSeparators(string path)
         {
             var initialLength = stream.Length;
             if (initialLength > _maxFileSizeBytes)
-                throw new InvalidOperationException(BuildFileTooLargeMessage(initialLength, grewDuringRead: false));
+                throw new FileTooLargeSkippedException(
+                    NormalizePathSeparators(relativePath),
+                    initialLength,
+                    _maxFileSizeBytes,
+                    BuildFileTooLargeMessage(initialLength, grewDuringRead: false));
 
             // Pre-size the accumulator to the observed length but cap initial capacity
             // at the configured limit so a tampered Length cannot force a huge up-front allocation.
@@ -2470,7 +2474,11 @@ public static string NormalizePathSeparators(string path)
             {
                 total += read;
                 if (total > _maxFileSizeBytes)
-                    throw new InvalidOperationException(BuildFileTooLargeMessage(total, grewDuringRead: true));
+                    throw new FileTooLargeSkippedException(
+                        NormalizePathSeparators(relativePath),
+                        total,
+                        _maxFileSizeBytes,
+                        BuildFileTooLargeMessage(total, grewDuringRead: true));
                 accumulator.Write(buffer, 0, read);
             }
             bytes = accumulator.ToArray();
@@ -2577,6 +2585,27 @@ public static string NormalizePathSeparators(string path)
         return (record, content, bytes, warning);
     }
 
+    public FileRecord BuildSkippedFileRecord(string absolutePath)
+    {
+        if (!IsFilePathSyntaxIndexable(absolutePath))
+            throw new InvalidOperationException("Cannot index a file path that contains NUL or control characters.");
+
+        var relativePath = Path.GetRelativePath(_projectRoot, absolutePath);
+        var normalizedRelativePath = NormalizePathSeparators(relativePath);
+        var ioPath = LongPath.EnsureWindowsPrefix(absolutePath);
+        var info = new FileInfo(ioPath);
+        return new FileRecord
+        {
+            Path = normalizedRelativePath,
+            Lang = TryDetectLanguage(absolutePath).Language,
+            Size = info.Exists ? info.Length : 0,
+            Lines = 0,
+            Checksum = null,
+            Modified = info.Exists ? info.LastWriteTimeUtc : DateTime.MinValue,
+            Generated = HasGeneratedCodeFileName(normalizedRelativePath),
+        };
+    }
+
     internal static bool IsFilePathSyntaxIndexable(string path)
     {
         foreach (var c in path)
@@ -3078,6 +3107,17 @@ internal static bool ContainsIndexBlockingNullByte(byte[] rawBytes)
 
     internal sealed class BinaryFileSkippedException(string message) : InvalidOperationException(message);
 
+    internal sealed class FileTooLargeSkippedException(
+        string relativePath,
+        long actualBytes,
+        long limitBytes,
+        string message) : InvalidOperationException(message)
+    {
+        public string RelativePath { get; } = relativePath;
+        public long ActualBytes { get; } = actualBytes;
+        public long LimitBytes { get; } = limitBytes;
+    }
+
     public static bool HasConflictMarkers(string content) =>
         TryGetConflictMarkerLine(content, out _);
 

tests/CodeIndex.Tests/FileIndexerTests.cs

@@ -343,7 +343,7 @@ public void BuildRecordWithRawBytes_OverExplicitMaxFileBytes_ThrowsActionableOve
 
             var indexer = new FileIndexer(tempDir, ignoreCase: false, ignoreRuleRoot: null, maxFileSizeBytes: 4);
 
-            var ex = Assert.Throws<InvalidOperationException>(() => indexer.BuildRecordWithRawBytes(path));
+            var ex = Assert.Throws<FileIndexer.FileTooLargeSkippedException>(() => indexer.BuildRecordWithRawBytes(path));
             Assert.Contains("File too large", ex.Message);
             Assert.Contains("--max-file-bytes", ex.Message);
             Assert.Contains(FileIndexer.MaxFileSizeEnvironmentVariable, ex.Message);
@@ -3470,10 +3470,10 @@ public void StripLineLeadingInvisibles_ConsecutiveLineLeadingInvisibles_AllStrip
     }
 
     [Fact]
-    public void BuildRecord_ThrowsForOversizedFile()
+    public void BuildRecord_ThrowsFileTooLargeSkippedExceptionForOversizedFile()
     {
-        // Files exceeding the default cap should throw InvalidOperationException
-        // 既定上限を超えるファイルはInvalidOperationExceptionを投げる
+        // Files exceeding the default cap should carry structured skip metadata.
+        // 既定上限を超えるファイルは structured skip metadata を持つ例外を投げる。
         var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}");
         try
         {
@@ -3485,7 +3485,10 @@ public void BuildRecord_ThrowsForOversizedFile()
                 stream.SetLength(FileIndexer.DefaultMaxFileSizeBytes + 1);
 
             var indexer = new FileIndexer(tempDir);
-            Assert.Throws<InvalidOperationException>(() => indexer.BuildRecord(filePath));
+            var ex = Assert.Throws<FileIndexer.FileTooLargeSkippedException>(() => indexer.BuildRecord(filePath));
+            Assert.Equal("large.py", ex.RelativePath);
+            Assert.Equal(FileIndexer.DefaultMaxFileSizeBytes + 1, ex.ActualBytes);
+            Assert.Equal(FileIndexer.DefaultMaxFileSizeBytes, ex.LimitBytes);
         }
         finally
         {
@@ -3512,7 +3515,7 @@ public void BuildRecord_DefaultRejectsTenMiBFileBeforeReadingPayload()
             var indexer = new FileIndexer(tempDir);
             var before = GC.GetAllocatedBytesForCurrentThread();
 
-            var ex = Assert.Throws<InvalidOperationException>(() => indexer.BuildRecord(filePath));
+            var ex = Assert.Throws<FileIndexer.FileTooLargeSkippedException>(() => indexer.BuildRecord(filePath));
 
             var allocated = GC.GetAllocatedBytesForCurrentThread() - before;
             Assert.Contains("File too large", ex.Message);