diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 48e7febd9b..4738521887 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -1857,9 +1857,9 @@ When SQLite returns permission-style errors such as `SQLITE_AUTH`, `SQLITE_PERM` - Recovery: fix file/directory permissions, or exclude the path via `.cdidxignore`. The index keeps running across the rest of the tree; no rebuild is required after permissions are fixed — a normal `cdidx index .` will pick up the now-readable files. 12. **File rejected: too large** - - Symptom: `--verbose` shows `[ERR ] : File too large (N MiB > M MiB limit). Override with --max-file-bytes or CDIDX_MAX_FILE_BYTES= when this source file is intentionally indexable.` (the file becomes part of the run's `errors` count), and the file does not appear in search. + - Symptom: `validate --kind file_too_large` reports `File too large (N MiB > M MiB limit). Override with --max-file-bytes or CDIDX_MAX_FILE_BYTES= when this source file is intentionally indexable.` The file is listed in `files`, but no chunks, symbols, or references are indexed for it, so it does not appear in search. - Cause: the file exceeds the configured per-file size limit. Indexing huge generated files would waste tokens and bloat the DB. - - Recovery: shrink or split the file, add it to `.cdidxignore`, or raise the limit with `cdidx index . --max-file-bytes 50M` / `CDIDX_MAX_FILE_BYTES=50M` when the file is legitimate source. Generated artifacts should generally be gitignored too. Note that any `[ERR ]` files in a run leave readiness flags unstamped, so resolve oversize entries before depending on `graph_table_available` / `issues_table_available` results. + - Recovery: shrink or split the file, add it to `.cdidxignore`, or raise the limit with `cdidx index . --max-file-bytes 50M` / `CDIDX_MAX_FILE_BYTES=50M` when the file is legitimate source. Generated artifacts should generally be gitignored too. 13. **Feature unavailable on trimmed / AOT build** (`E009_FEATURE_UNAVAILABLE`) - Symptom: `Error [E009_FEATURE_UNAVAILABLE]: ...` when invoking flags such as `--json` on a build that lacks the required code paths. diff --git a/changelog.d/unreleased/1603.fixed.md b/changelog.d/unreleased/1603.fixed.md new file mode 100644 index 0000000000..bf7bbb37e0 --- /dev/null +++ b/changelog.d/unreleased/1603.fixed.md @@ -0,0 +1,19 @@ +--- +category: fixed +issues: + - 1603 +affected: + - src/CodeIndex/Cli/IndexCommandRunner.cs + - src/CodeIndex/Cli/QueryCommandRunner.cs + - src/CodeIndex/Indexer/Scanning/FileIndexer.cs + - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs + - USER_GUIDE.md +--- + +## English + +- **Oversize files now surface through `validate` (#1603)** — files above the configured indexing size limit are persisted with a `file_too_large` issue instead of disappearing as per-file errors, so `validate --kind file_too_large` explains why their chunks, symbols, and references are absent. + +## 日本語 + +- **サイズ上限を超えたファイルが `validate` で見えるようになりました (#1603)** — 設定された indexing size limit を超えたファイルは per-file error として消えるのではなく `file_too_large` issue として保存されるため、`validate --kind file_too_large` で chunks / symbols / references が存在しない理由を確認できます。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.cs b/src/CodeIndex/Cli/IndexCommandRunner.cs index cc712b7497..50ca078fcf 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.cs @@ -2143,6 +2143,41 @@ void ThrowIfUpdateCancelled() continue; } + if (ex is FileIndexer.FileTooLargeSkippedException fileTooLarge) + { + if (fileBatchMarked) + writer.ClearBatchInProgress(); + + DemoteReadinessOnce(); + writer.MarkBatchInProgress(); + using var txn = writer.BeginTransaction(); + var skippedRecord = indexer.BuildSkippedFileRecord(absPath); + writer.PurgeStaleFilesSharingChecksum(projectRoot, skippedRecord.Path, skippedRecord.Checksum); + if (projectRootWritten) + writer.PurgeStaleFilesSharingDirectoryAndStem(projectRoot, skippedRecord.Path); + WriteProjectRootOnce(); + var fileId = writer.UpsertFile(skippedRecord); + writer.InsertChunks([]); + writer.InsertSymbols([]); + writer.InsertReferences([]); + writer.InsertIssues(fileId, + [ + new FileIssue + { + Path = fileTooLarge.RelativePath, + Kind = "file_too_large", + Line = 0, + Message = fileTooLarge.Message, + }, + ]); + writer.ClearBatchInProgress(); + txn.Commit(); + + updated++; + ftsMutated = true; + continue; + } + if (ex is FileNotFoundException or DirectoryNotFoundException) { if (fileBatchMarked) @@ -3472,6 +3507,20 @@ void StopJsonHeartbeat() { extractionResults.Add(FullScanFileWorkItem.Skipped(filePath, ex.Message), cancellationToken); } + catch (FileIndexer.FileTooLargeSkippedException ex) + { + var record = indexer.BuildSkippedFileRecord(filePath); + var issue = new FileIssue + { + Path = ex.RelativePath, + Kind = "file_too_large", + Line = 0, + Message = ex.Message, + }; + extractionResults.Add( + FullScanFileWorkItem.Success(filePath, record, string.Empty, [], ex.Message, [], [], [], [issue]), + cancellationToken); + } catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException) { var relativePath = FileIndexer.NormalizePathSeparators(Path.GetRelativePath(projectRoot, filePath)); diff --git a/src/CodeIndex/Cli/QueryCommandRunner.cs b/src/CodeIndex/Cli/QueryCommandRunner.cs index 54c80a1239..86dd7cbdaa 100644 --- a/src/CodeIndex/Cli/QueryCommandRunner.cs +++ b/src/CodeIndex/Cli/QueryCommandRunner.cs @@ -3350,7 +3350,7 @@ private static JsonObject BuildUnusedResultsByBucketJson(IEnumerable _maxFileSizeBytes) - throw new InvalidOperationException(BuildFileTooLargeMessage(initialLength, grewDuringRead: false)); + throw new FileTooLargeSkippedException( + NormalizePathSeparators(relativePath), + initialLength, + _maxFileSizeBytes, + BuildFileTooLargeMessage(initialLength, grewDuringRead: false)); // Pre-size the accumulator to the observed length but cap initial capacity // at the configured limit so a tampered Length cannot force a huge up-front allocation. @@ -2470,7 +2474,11 @@ public static string NormalizePathSeparators(string path) { total += read; if (total > _maxFileSizeBytes) - throw new InvalidOperationException(BuildFileTooLargeMessage(total, grewDuringRead: true)); + throw new FileTooLargeSkippedException( + NormalizePathSeparators(relativePath), + total, + _maxFileSizeBytes, + BuildFileTooLargeMessage(total, grewDuringRead: true)); accumulator.Write(buffer, 0, read); } bytes = accumulator.ToArray(); @@ -2577,6 +2585,27 @@ public static string NormalizePathSeparators(string path) return (record, content, bytes, warning); } + public FileRecord BuildSkippedFileRecord(string absolutePath) + { + if (!IsFilePathSyntaxIndexable(absolutePath)) + throw new InvalidOperationException("Cannot index a file path that contains NUL or control characters."); + + var relativePath = Path.GetRelativePath(_projectRoot, absolutePath); + var normalizedRelativePath = NormalizePathSeparators(relativePath); + var ioPath = LongPath.EnsureWindowsPrefix(absolutePath); + var info = new FileInfo(ioPath); + return new FileRecord + { + Path = normalizedRelativePath, + Lang = TryDetectLanguage(absolutePath).Language, + Size = info.Exists ? info.Length : 0, + Lines = 0, + Checksum = null, + Modified = info.Exists ? info.LastWriteTimeUtc : DateTime.MinValue, + Generated = HasGeneratedCodeFileName(normalizedRelativePath), + }; + } + internal static bool IsFilePathSyntaxIndexable(string path) { foreach (var c in path) @@ -3078,6 +3107,17 @@ internal static bool ContainsIndexBlockingNullByte(byte[] rawBytes) internal sealed class BinaryFileSkippedException(string message) : InvalidOperationException(message); + internal sealed class FileTooLargeSkippedException( + string relativePath, + long actualBytes, + long limitBytes, + string message) : InvalidOperationException(message) + { + public string RelativePath { get; } = relativePath; + public long ActualBytes { get; } = actualBytes; + public long LimitBytes { get; } = limitBytes; + } + public static bool HasConflictMarkers(string content) => TryGetConflictMarkerLine(content, out _); diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index b9bb2b0c34..8d345584ef 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -343,7 +343,7 @@ public void BuildRecordWithRawBytes_OverExplicitMaxFileBytes_ThrowsActionableOve var indexer = new FileIndexer(tempDir, ignoreCase: false, ignoreRuleRoot: null, maxFileSizeBytes: 4); - var ex = Assert.Throws(() => indexer.BuildRecordWithRawBytes(path)); + var ex = Assert.Throws(() => indexer.BuildRecordWithRawBytes(path)); Assert.Contains("File too large", ex.Message); Assert.Contains("--max-file-bytes", ex.Message); Assert.Contains(FileIndexer.MaxFileSizeEnvironmentVariable, ex.Message); @@ -3470,10 +3470,10 @@ public void StripLineLeadingInvisibles_ConsecutiveLineLeadingInvisibles_AllStrip } [Fact] - public void BuildRecord_ThrowsForOversizedFile() + public void BuildRecord_ThrowsFileTooLargeSkippedExceptionForOversizedFile() { - // Files exceeding the default cap should throw InvalidOperationException - // 既定上限を超えるファイルはInvalidOperationExceptionを投げる + // Files exceeding the default cap should carry structured skip metadata. + // 既定上限を超えるファイルは structured skip metadata を持つ例外を投げる。 var tempDir = Path.Combine(Path.GetTempPath(), $"codeindex_test_{Guid.NewGuid():N}"); try { @@ -3485,7 +3485,10 @@ public void BuildRecord_ThrowsForOversizedFile() stream.SetLength(FileIndexer.DefaultMaxFileSizeBytes + 1); var indexer = new FileIndexer(tempDir); - Assert.Throws(() => indexer.BuildRecord(filePath)); + var ex = Assert.Throws(() => indexer.BuildRecord(filePath)); + Assert.Equal("large.py", ex.RelativePath); + Assert.Equal(FileIndexer.DefaultMaxFileSizeBytes + 1, ex.ActualBytes); + Assert.Equal(FileIndexer.DefaultMaxFileSizeBytes, ex.LimitBytes); } finally { @@ -3512,7 +3515,7 @@ public void BuildRecord_DefaultRejectsTenMiBFileBeforeReadingPayload() var indexer = new FileIndexer(tempDir); var before = GC.GetAllocatedBytesForCurrentThread(); - var ex = Assert.Throws(() => indexer.BuildRecord(filePath)); + var ex = Assert.Throws(() => indexer.BuildRecord(filePath)); var allocated = GC.GetAllocatedBytesForCurrentThread() - before; Assert.Contains("File too large", ex.Message); diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index e6492476f5..5afb2cb748 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -80,6 +80,76 @@ public void Run_NullByteFile_SkipsWithoutPersistingPartialRows() } } + [Fact] + public void Run_FileAboveMaxFileBytes_PersistsFileTooLargeIssue() + { + var projectRoot = CreateTempProject(); + try + { + var filePath = Path.Combine(projectRoot, "large.py"); + File.WriteAllText(filePath, "print('start')\n" + new string('a', 256)); + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--max-file-bytes", "128", "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + Assert.Equal(0, json.GetProperty("summary").GetProperty("errors").GetInt32()); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + Assert.Equal(1, CountRows(dbPath, "files")); + Assert.Equal(0, CountRows(dbPath, "chunks")); + Assert.Equal(0, CountRows(dbPath, "symbols")); + Assert.Equal(0, CountRows(dbPath, "symbol_references")); + + using var db = new DbContext(dbPath); + db.TryMigrateForRead(); + var reader = new DbReader(db.Connection, db.IsReadOnly); + var issue = Assert.Single(reader.GetIssues("file_too_large")); + Assert.Equal("large.py", issue.Path); + Assert.Equal(0, issue.Line); + Assert.Contains("File too large", issue.Message); + Assert.Contains("--max-file-bytes", issue.Message); + Assert.Contains(FileIndexer.MaxFileSizeEnvironmentVariable, issue.Message); + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void RunFiles_FileAboveMaxFileBytes_PersistsFileTooLargeIssue() + { + var projectRoot = CreateTempProject(); + try + { + var filePath = Path.Combine(projectRoot, "large.py"); + File.WriteAllText(filePath, "print('start')\n" + new string('a', 256)); + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--files", "large.py", "--max-file-bytes", "128", "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + Assert.Equal(0, json.GetProperty("summary").GetProperty("errors").GetInt32()); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + Assert.Equal(1, CountRows(dbPath, "files")); + Assert.Equal(0, CountRows(dbPath, "chunks")); + Assert.Equal(0, CountRows(dbPath, "symbols")); + Assert.Equal(0, CountRows(dbPath, "symbol_references")); + + var issue = Assert.Single(ReadFileIssues(dbPath, "file_too_large")); + Assert.Equal("large.py", issue.Path); + Assert.Contains("File too large", issue.Message); + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + [Fact] public void Run_NewIndexDatabase_RunsAnalyzeAfterSuccessfulIndex() { @@ -3442,7 +3512,7 @@ public interface IParseable } [Fact] - public void Run_FullScan_WithIndexingErrors_PrintsRecoveryWarning() + public void Run_FullScan_WithOversizedFile_PrintsSkipWarningWithoutRecoveryWarning() { var projectRoot = CreateTempProject(); try @@ -3453,8 +3523,9 @@ public void Run_FullScan_WithIndexingErrors_PrintsRecoveryWarning() var (exitCode, _, stderr) = RunCliInSubprocess([projectRoot], projectRoot); Assert.Equal(CommandExitCodes.Success, exitCode); - Assert.Contains("Some files failed to index", stderr); - Assert.Contains("rerun `cdidx index", stderr); + Assert.Contains("[WARN] File too large", stderr); + Assert.DoesNotContain("Some files failed to index", stderr); + Assert.DoesNotContain("rerun `cdidx index", stderr); } finally { @@ -3547,7 +3618,7 @@ public void Run_FullScan_CancelledAfterReadinessDemotion_RollsBackExistingIndex( } [Fact] - public void Run_UpdateMode_WithIndexingErrors_PrintsRecoveryWarning() + public void Run_UpdateMode_WithOversizedFile_PrintsSkipWarningWithoutRecoveryWarning() { var projectRoot = CreateTempProject(); try @@ -3562,8 +3633,9 @@ public void Run_UpdateMode_WithIndexingErrors_PrintsRecoveryWarning() var (exitCode, _, stderr) = RunCliInSubprocess([projectRoot, "--files", "huge.py"], projectRoot); Assert.Equal(CommandExitCodes.Success, exitCode); - Assert.Contains("Some files failed to update", stderr); - Assert.Contains("rerun `cdidx index", stderr); + Assert.Equal(string.Empty, stderr); + Assert.DoesNotContain("Some files failed to update", stderr); + Assert.DoesNotContain("rerun `cdidx index", stderr); } finally { @@ -6652,7 +6724,7 @@ public void Run_UpdateMode_DoesNotRestampFoldReadyWhenSymbolExtractorVersionMism } [Fact] - public void Run_UpdateMode_ClearsHotspotFamilyTrustOnPartialFailure() + public void Run_UpdateMode_RestampsHotspotFamilyTrustOnOversizedFileSkip() { var projectRoot = CreateTempProject(); try @@ -6671,11 +6743,11 @@ public void Run_UpdateMode_ClearsHotspotFamilyTrustOnPartialFailure() var (exitCode2, json2) = RunAndCaptureJson([projectRoot, "--files", "app.cs", "--json"]); Assert.Equal(CommandExitCodes.Success, exitCode2); - Assert.Equal("partial", json2.GetProperty("status").GetString()); - Assert.Equal(1, json2.GetProperty("summary").GetProperty("errors").GetInt32()); + Assert.Equal("success", json2.GetProperty("status").GetString()); + Assert.Equal(0, json2.GetProperty("summary").GetProperty("errors").GetInt32()); using var verifyDb = new DbContext(dbPath); - Assert.Null(verifyDb.GetMetaString(DbContext.GetHotspotFamilyVersionMetaKey("csharp"))); + Assert.Equal(DbContext.HotspotFamilyVersion.ToString(System.Globalization.CultureInfo.InvariantCulture), verifyDb.GetMetaString(DbContext.GetHotspotFamilyVersionMetaKey("csharp"))); } finally { @@ -7842,6 +7914,35 @@ private static int CountRows(string dbPath, string tableName) return Convert.ToInt32(command.ExecuteScalar()); } + private static List ReadFileIssues(string dbPath, string kind) + { + using var connection = new SqliteConnection($"Data Source={dbPath}"); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = """ + SELECT f.path, i.kind, i.line, i.message + FROM file_issues i + JOIN files f ON f.id = i.file_id + WHERE i.kind = @kind + ORDER BY f.path + """; + command.Parameters.AddWithValue("@kind", kind); + using var reader = command.ExecuteReader(); + var issues = new List(); + while (reader.Read()) + { + issues.Add(new FileIssue + { + Path = reader.GetString(0), + Kind = reader.GetString(1), + Line = reader.GetInt32(2), + Message = reader.GetString(3), + }); + } + + return issues; + } + private (int ExitCode, JsonElement Json, string Stderr) RunAndCaptureJsonWithStderr(string[] args) { lock (TestConsoleLock.Gate)