From e81c439913fe37382efe7362058f2f99185c5b4f Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 6 Jun 2026 20:09:10 +0900 Subject: [PATCH 1/3] Fix bounded report log tail selection (#3026) --- USER_GUIDE.md | 8 ++-- changelog.d/unreleased/3026.fixed.md | 17 +++++++++ src/CodeIndex/Cli/ReportCommandRunner.cs | 33 ++++++++++++++--- .../ReportCommandRunnerTests.cs | 37 +++++++++++++++++++ 4 files changed, 86 insertions(+), 9 deletions(-) create mode 100644 changelog.d/unreleased/3026.fixed.md diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 623ade2668..4426a98540 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -403,7 +403,7 @@ be audited whenever the matching help text changes. | Status stale-after hint | `24h`, overridden by `--stale-after`, `CDIDX_STALE_AFTER`, or `.cdidxrc.json`; max `30d` | status runner | | Color mode | `auto`, overridden by `--color`, `CLICOLOR_FORCE`, `NO_COLOR`, or `CLICOLOR=0` | `ConsoleUi` | | ANSI palette | `basic` fallback, auto-upgraded from terminal hints unless overridden | `ConsoleUi` | -| Report log tail | `200` lines (`--log-lines`), clamped to `2000` | report runner help | +| Report log tail | `200` lines (`--log-lines`), clamped to `2000`, from at most the 32 newest lifecycle log files | report runner help | | Report per-log tail read | `1,048,576` bytes | `ReportCommandRunner` | | JSON envelope capture | `10,485,760` characters | `JsonEnvelopeWrapper` | | CLI batch line | `1,048,576` characters | `QueryCommandRunner` | @@ -1166,7 +1166,7 @@ cdidx report --output report.tgz --json |---|---|---| | `--output ` / `-o ` | (required) | Destination `.tar.gz`. The directory is created if missing; on POSIX, the archive and tar entries are owner-readable/writable only. | | `--db ` | `.cdidx/codeindex.db` | Override the database whose schema is summarized. If absent, `schema.txt` records that no DB was found. Schema summaries cap table entries at 64, displayed table names at 96 characters, and row-count scans at 1000 rows per table. | -| `--log-lines ` | `200` | How many trailing lifecycle-log lines to include (`0` disables the tail; values above `2000` are clamped). Each log file contributes from a bounded 1,048,576-byte tail window instead of being loaded fully. | +| `--log-lines ` | `200` | How many trailing lifecycle-log lines to include (`0` disables the tail; values above `2000` are clamped). Report collection considers at most the 32 newest lifecycle log files; each file contributes from a bounded 1,048,576-byte tail window instead of being loaded fully. | | `--no-log` | | Skip the lifecycle log entirely. | | `--include-args` | | Keep literal `cwd=` and `args=` values in the log tail (opt-in; share only with trusted recipients). | | `--json` | | Print a stable summary envelope (`output_path`, `version`, `files`, `schema_tables`, `log_lines_included`, `log_included`, `db_included`, `db_path`) instead of the human-friendly output. | @@ -2687,7 +2687,7 @@ render できます。 | Status stale-after hint | `24h`。`--stale-after` / `CDIDX_STALE_AFTER` / `.cdidxrc.json` で上書き、最大 `30d` | status runner | | Color mode | `auto`。`--color` / `CLICOLOR_FORCE` / `NO_COLOR` / `CLICOLOR=0` で上書き | `ConsoleUi` | | ANSI palette | `basic` fallback。terminal hints で自動昇格、または明示上書き | `ConsoleUi` | -| Report log tail | `200` lines（`--log-lines`）、最大 `2000` に clamp | report runner help | +| Report log tail | `200` lines（`--log-lines`）、最大 `2000` に clamp、最新 32 件までの lifecycle log file から収集 | report runner help | | Report per-log tail read | `1,048,576` bytes | `ReportCommandRunner` | | JSON envelope capture | `10,485,760` 文字 | `JsonEnvelopeWrapper` | | CLI batch line | `1,048,576` 文字 | `QueryCommandRunner` | @@ -3459,7 +3459,7 @@ cdidx report --output report.tgz --json |---|---|---| | `--output ` / `-o ` | （必須） | 出力先 `.tar.gz`。親ディレクトリが無ければ作成します。POSIX では archive と tar entry は owner の読み書きのみになります。 | | `--db ` | `.cdidx/codeindex.db` | スキーマ要約対象の DB を上書きします。存在しなければ `schema.txt` に「DB が見つからなかった」旨が記録されます。スキーマ要約は table entry を 64 件、表示 table 名を 96 文字、行数 scan を table ごとに 1000 行までに制限します。 | -| `--log-lines ` | `200` | ライフサイクルログ末尾を何行含めるか（`0` で末尾を含めません。`2000` を超える値は clamp されます）。各ログファイルは全体を読み込まず、末尾 1,048,576 byte の範囲から収集します。 | +| `--log-lines ` | `200` | ライフサイクルログ末尾を何行含めるか（`0` で末尾を含めません。`2000` を超える値は clamp されます）。report 収集は最新 32 件までの lifecycle log file を対象にし、各ログファイルは全体を読み込まず、末尾 1,048,576 byte の範囲から収集します。 | | `--no-log` | | ライフサイクルログを完全に省略します。 | | `--include-args` | | ログ末尾の `cwd=` / `args=` 値を伏字化せずそのまま含めます（信頼できる相手にだけ使用してください）。 | | `--json` | | 人間向け出力の代わりに、安定したサマリ JSON（`output_path` / `version` / `files` / `schema_tables` / `log_lines_included` / `log_included` / `db_included` / `db_path`）を出力します。 | diff --git a/changelog.d/unreleased/3026.fixed.md b/changelog.d/unreleased/3026.fixed.md new file mode 100644 index 0000000000..38dc00c80d --- /dev/null +++ b/changelog.d/unreleased/3026.fixed.md @@ -0,0 +1,17 @@ +--- +category: fixed +issues: + - 3026 +affected: + - USER_GUIDE.md + - src/CodeIndex/Cli/ReportCommandRunner.cs + - tests/CodeIndex.Tests/ReportCommandRunnerTests.cs +--- + +## English + +- **Report log tails now keep a bounded recent-file candidate set (#3026)** — `cdidx report` no longer sorts and materializes every `stderr-*.log` file before reading the latest lifecycle log lines. + +## 日本語 + +- **report のログ末尾収集が最新ファイル候補を bounded に保持するようになりました (#3026)** — `cdidx report` は最新の lifecycle log 行を読む前にすべての `stderr-*.log` をソートして materialize しなくなりました。 diff --git a/src/CodeIndex/Cli/ReportCommandRunner.cs b/src/CodeIndex/Cli/ReportCommandRunner.cs index c890133d78..900aaef452 100644 --- a/src/CodeIndex/Cli/ReportCommandRunner.cs +++ b/src/CodeIndex/Cli/ReportCommandRunner.cs @@ -23,6 +23,7 @@ public static class ReportCommandRunner internal const int DefaultLogLines = 200; internal const int MaxLogLines = 2000; internal const int MaxLogFileTailBytes = 1024 * 1024; + internal const int MaxRecentLogFiles = 32; internal const int MaxSchemaTables = 64; internal const int MaxSchemaTableNameDisplayChars = 96; internal const int MaxSchemaRowCountScanRows = 1000; @@ -175,7 +176,9 @@ internal static string BuildReadme(string version, bool includeLog, bool include sb.AppendLine("- `schema.txt` — capped SQLite table list and bounded row counts (no table row contents)."); if (includeLog) { - sb.AppendLine("- `log/stderr-recent.log` — last N lines of the cdidx lifecycle log"); + sb.AppendLine( + "- `log/stderr-recent.log` — up to the requested last N lifecycle-log lines, " + + $"selected from the {MaxRecentLogFiles} newest log files"); sb.AppendLine(includeArgs ? " (includes literal `args=` lines; path-bearing lifecycle fields stay redacted)." : " (`args=` and path-bearing lifecycle fields are redacted; rerun with `--include-args` to keep arguments literal)."); @@ -282,10 +285,8 @@ internal static string BuildRecentLogTail(int maxLines, bool includeArgs, out in if (string.IsNullOrWhiteSpace(logDir) || !Directory.Exists(logDir)) return $"no cdidx lifecycle log directory found (looked at: {RedactedPlaceholder}).\n"; - var logFiles = new DirectoryInfo(logDir) - .EnumerateFiles("stderr-*.log", SearchOption.TopDirectoryOnly) - .OrderByDescending(f => f.Name, StringComparer.Ordinal) - .ToList(); + var logFiles = SelectRecentLogFiles( + new DirectoryInfo(logDir).EnumerateFiles("stderr-*.log", SearchOption.TopDirectoryOnly)); if (logFiles.Count == 0) return $"no cdidx lifecycle log files found in: {RedactedPlaceholder}\n"; @@ -319,6 +320,28 @@ internal static string BuildRecentLogTail(int maxLines, bool includeArgs, out in return sb.ToString(); } + private static IReadOnlyList SelectRecentLogFiles(IEnumerable files) + { + var recent = new List(MaxRecentLogFiles); + foreach (var file in files) + { + var insertAt = recent.FindIndex( + existing => string.Compare(file.Name, existing.Name, StringComparison.Ordinal) > 0); + if (insertAt < 0) + { + if (recent.Count < MaxRecentLogFiles) + recent.Add(file); + continue; + } + + recent.Insert(insertAt, file); + if (recent.Count > MaxRecentLogFiles) + recent.RemoveAt(recent.Count - 1); + } + + return recent; + } + internal static IReadOnlyList ReadLogFileTailLines(string path, int maxLines) { if (maxLines <= 0) diff --git a/tests/CodeIndex.Tests/ReportCommandRunnerTests.cs b/tests/CodeIndex.Tests/ReportCommandRunnerTests.cs index 90c1f321ee..128acc3bb7 100644 --- a/tests/CodeIndex.Tests/ReportCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/ReportCommandRunnerTests.cs @@ -497,6 +497,43 @@ public void BuildRecentLogTail_LargeLogReadsBoundedTail_Issue2837() } } + [Fact] + public void BuildRecentLogTail_ManyLogFilesKeepsNewestBoundedCandidates_Issue3026() + { + var workDir = CreateWorkDir(); + var logDir = Path.Combine(workDir, "logs"); + Directory.CreateDirectory(logDir); + var fileCount = ReportCommandRunner.MaxRecentLogFiles + 3; + for (var i = 0; i < fileCount; i++) + { + File.WriteAllText( + Path.Combine(logDir, $"stderr-20260518-{i:D4}.log"), + $"line-{i:D4}\n"); + } + + var previousLogDir = Environment.GetEnvironmentVariable("CDIDX_GLOBAL_TOOL_LOG_DIR"); + Environment.SetEnvironmentVariable("CDIDX_GLOBAL_TOOL_LOG_DIR", logDir); + try + { + var logText = ReportCommandRunner.BuildRecentLogTail(2, includeArgs: false, out var linesIncluded); + + var secondNewest = $"line-{fileCount - 2:D4}"; + var newest = $"line-{fileCount - 1:D4}"; + Assert.Equal(2, linesIncluded); + Assert.Contains(secondNewest, logText); + Assert.Contains(newest, logText); + Assert.True( + logText.IndexOf(secondNewest, StringComparison.Ordinal) < + logText.IndexOf(newest, StringComparison.Ordinal)); + Assert.DoesNotContain("line-0000", logText); + } + finally + { + Environment.SetEnvironmentVariable("CDIDX_GLOBAL_TOOL_LOG_DIR", previousLogDir); + TryDeleteDirectory(workDir); + } + } + [Fact] public void Run_IncludeArgs_PreservesLiteralArgsButRedactsPaths() { From ca8f07d1030c43bb52a193b530fadf98a528bb86 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 6 Jun 2026 20:11:26 +0900 Subject: [PATCH 2/3] Cap private log hardening work (#3027) --- changelog.d/unreleased/3027.fixed.md | 16 +++++++ src/CodeIndex/Cli/PrivateLogFile.cs | 7 +++ tests/CodeIndex.Tests/GlobalToolLogTests.cs | 48 +++++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 changelog.d/unreleased/3027.fixed.md diff --git a/changelog.d/unreleased/3027.fixed.md b/changelog.d/unreleased/3027.fixed.md new file mode 100644 index 0000000000..d85439e700 --- /dev/null +++ b/changelog.d/unreleased/3027.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 3027 +affected: + - src/CodeIndex/Cli/PrivateLogFile.cs + - tests/CodeIndex.Tests/GlobalToolLogTests.cs +--- + +## English + +- **Private log hardening now caps best-effort startup work (#3027)** — lifecycle log permission hardening stops after a bounded number of existing files so startup does not scan an unbounded log directory. + +## 日本語 + +- **private log hardening の起動時 best-effort 作業を cap しました (#3027)** — lifecycle log の permission hardening は既存ファイルを bounded 件数で止めるため、巨大なログディレクトリを無制限に走査しなくなりました。 diff --git a/src/CodeIndex/Cli/PrivateLogFile.cs b/src/CodeIndex/Cli/PrivateLogFile.cs index 9dd6d06feb..8c0edaeca5 100644 --- a/src/CodeIndex/Cli/PrivateLogFile.cs +++ b/src/CodeIndex/Cli/PrivateLogFile.cs @@ -6,6 +6,7 @@ namespace CodeIndex.Cli; internal static class PrivateLogFile { internal const UnixFileMode PrivateFileMode = UnixFileMode.UserRead | UnixFileMode.UserWrite; + internal const int MaxExistingFilesToHarden = 128; internal static FileStream OpenAppend(string path, FileShare share = FileShare.ReadWrite) { @@ -49,8 +50,14 @@ internal static void HardenExisting(string directory, string pattern) try { + var hardened = 0; foreach (var file in new DirectoryInfo(directory).EnumerateFiles(pattern, SearchOption.TopDirectoryOnly)) + { TrySetPrivatePermissions(file.FullName); + hardened++; + if (hardened >= MaxExistingFilesToHarden) + break; + } } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { diff --git a/tests/CodeIndex.Tests/GlobalToolLogTests.cs b/tests/CodeIndex.Tests/GlobalToolLogTests.cs index b37f04ee4a..c4f7342536 100644 --- a/tests/CodeIndex.Tests/GlobalToolLogTests.cs +++ b/tests/CodeIndex.Tests/GlobalToolLogTests.cs @@ -8,6 +8,17 @@ namespace CodeIndex.Tests; [Collection("SQLite pool sensitive")] public class GlobalToolLogTests { + private const UnixFileMode PermissionBits = + UnixFileMode.UserRead | + UnixFileMode.UserWrite | + UnixFileMode.UserExecute | + UnixFileMode.GroupRead | + UnixFileMode.GroupWrite | + UnixFileMode.GroupExecute | + UnixFileMode.OtherRead | + UnixFileMode.OtherWrite | + UnixFileMode.OtherExecute; + [Fact] public void PrivateLogFile_OpenAppend_OnUnixCreatesPrivateFile() { @@ -31,6 +42,43 @@ public void PrivateLogFile_OpenAppend_OnUnixCreatesPrivateFile() } } + [Fact] + public void PrivateLogFile_HardenExisting_CapsBestEffortWork_Issue3027() + { + if (OperatingSystem.IsWindows()) + return; + + var directory = Path.Combine(Path.GetTempPath(), $"cdidx_private_log_harden_{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + var fileCount = PrivateLogFile.MaxExistingFilesToHarden + 2; + try + { + for (var i = 0; i < fileCount; i++) + { + var path = Path.Combine(directory, $"stderr-{i:D4}.log"); + File.WriteAllText(path, "x"); + File.SetUnixFileMode(path, PermissionBits); + } + + PrivateLogFile.HardenExisting(directory, "stderr-*.log"); + + var privateCount = 0; + for (var i = 0; i < fileCount; i++) + { + var path = Path.Combine(directory, $"stderr-{i:D4}.log"); + if ((File.GetUnixFileMode(path) & PermissionBits) == PrivateLogFile.PrivateFileMode) + privateCount++; + } + + Assert.Equal(PrivateLogFile.MaxExistingFilesToHarden, privateCount); + } + finally + { + if (Directory.Exists(directory)) + Directory.Delete(directory, recursive: true); + } + } + [Fact] public void FormatArgs_RedactsSensitiveArgumentsByDefault() { From e5dca899198e875fbdc755765939b8fb35d3165f Mon Sep 17 00:00:00 2001 From: Widthdom Date: Sat, 6 Jun 2026 20:15:34 +0900 Subject: [PATCH 3/3] Bound private log pruning retention (#3028) --- changelog.d/unreleased/3028.fixed.md | 16 +++++ src/CodeIndex/Cli/PrivateLogFile.cs | 74 ++++++++++++++++++--- tests/CodeIndex.Tests/GlobalToolLogTests.cs | 36 ++++++++++ 3 files changed, 117 insertions(+), 9 deletions(-) create mode 100644 changelog.d/unreleased/3028.fixed.md diff --git a/changelog.d/unreleased/3028.fixed.md b/changelog.d/unreleased/3028.fixed.md new file mode 100644 index 0000000000..a9aac9c589 --- /dev/null +++ b/changelog.d/unreleased/3028.fixed.md @@ -0,0 +1,16 @@ +--- +category: fixed +issues: + - 3028 +affected: + - src/CodeIndex/Cli/PrivateLogFile.cs + - tests/CodeIndex.Tests/GlobalToolLogTests.cs +--- + +## English + +- **Private log pruning now keeps only a bounded retained-file set in memory (#3028)** — lifecycle log cleanup no longer sorts and materializes every matching log file before deleting older entries. + +## 日本語 + +- **private log pruning が保持対象だけを bounded にメモリへ載せるようになりました (#3028)** — lifecycle log cleanup は古いエントリを削除する前に一致する全ログファイルをソートして materialize しなくなりました。 diff --git a/src/CodeIndex/Cli/PrivateLogFile.cs b/src/CodeIndex/Cli/PrivateLogFile.cs index 8c0edaeca5..57f589fec5 100644 --- a/src/CodeIndex/Cli/PrivateLogFile.cs +++ b/src/CodeIndex/Cli/PrivateLogFile.cs @@ -69,15 +69,19 @@ internal static void PruneOldFiles(string directory, string pattern, int retaine { try { - var oldFiles = new DirectoryInfo(directory) - .EnumerateFiles(pattern, SearchOption.TopDirectoryOnly) - .OrderByDescending(file => file.LastWriteTimeUtc) - .ThenByDescending(file => file.Name, StringComparer.Ordinal) - .Skip(retainedFileCount) - .ToList(); - - foreach (var file in oldFiles) - file.Delete(); + var directoryInfo = new DirectoryInfo(directory); + var retainedFiles = SelectRetainedFiles( + directoryInfo.EnumerateFiles(pattern, SearchOption.TopDirectoryOnly), + retainedFileCount); + var retainedPaths = new HashSet( + retainedFiles.Select(file => file.FullName), + OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal); + + foreach (var file in directoryInfo.EnumerateFiles(pattern, SearchOption.TopDirectoryOnly)) + { + if (ShouldPruneFile(file, retainedPaths, retainedFiles, retainedFileCount)) + file.Delete(); + } } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { @@ -85,6 +89,58 @@ internal static void PruneOldFiles(string directory, string pattern, int retaine } } + private static IReadOnlyList SelectRetainedFiles(IEnumerable files, int retainedFileCount) + { + if (retainedFileCount <= 0) + return []; + + var retained = new List(retainedFileCount); + foreach (var file in files) + AddRetainedFile(retained, file, retainedFileCount); + + return retained; + } + + private static void AddRetainedFile(List retained, FileInfo file, int retainedFileCount) + { + var insertAt = retained.FindIndex(existing => CompareRetentionOrder(file, existing) > 0); + if (insertAt < 0) + { + if (retained.Count < retainedFileCount) + retained.Add(file); + return; + } + + retained.Insert(insertAt, file); + if (retained.Count > retainedFileCount) + retained.RemoveAt(retained.Count - 1); + } + + private static bool ShouldPruneFile( + FileInfo file, + HashSet retainedPaths, + IReadOnlyList retainedFiles, + int retainedFileCount) + { + if (retainedPaths.Contains(file.FullName)) + return false; + if (retainedFileCount <= 0) + return true; + if (retainedFiles.Count < retainedFileCount) + return false; + + return CompareRetentionOrder(file, retainedFiles[^1]) < 0; + } + + private static int CompareRetentionOrder(FileInfo left, FileInfo right) + { + var modified = left.LastWriteTimeUtc.CompareTo(right.LastWriteTimeUtc); + if (modified != 0) + return modified; + + return string.Compare(left.Name, right.Name, StringComparison.Ordinal); + } + internal static bool TryRotateSlots(string path, int retainedFileCount) { try diff --git a/tests/CodeIndex.Tests/GlobalToolLogTests.cs b/tests/CodeIndex.Tests/GlobalToolLogTests.cs index c4f7342536..9e12a2170f 100644 --- a/tests/CodeIndex.Tests/GlobalToolLogTests.cs +++ b/tests/CodeIndex.Tests/GlobalToolLogTests.cs @@ -79,6 +79,42 @@ public void PrivateLogFile_HardenExisting_CapsBestEffortWork_Issue3027() } } + [Fact] + public void PrivateLogFile_PruneOldFiles_KeepsNewestFilesWithoutMaterializingAll_Issue3028() + { + var directory = Path.Combine(Path.GetTempPath(), $"cdidx_private_log_prune_{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + const int retainedFileCount = 5; + const int fileCount = retainedFileCount + 17; + var timestamp = DateTime.UtcNow.AddHours(-1); + try + { + for (var i = 0; i < fileCount; i++) + { + var path = Path.Combine(directory, $"stderr-{i:D4}.log"); + File.WriteAllText(path, "x"); + File.SetLastWriteTimeUtc(path, timestamp); + } + + PrivateLogFile.PruneOldFiles(directory, "stderr-*.log", retainedFileCount); + + var remaining = Directory.GetFiles(directory, "stderr-*.log") + .Select(Path.GetFileName) + .OrderBy(name => name, StringComparer.Ordinal) + .ToArray(); + var expected = Enumerable.Range(fileCount - retainedFileCount, retainedFileCount) + .Select(i => $"stderr-{i:D4}.log") + .OrderBy(name => name, StringComparer.Ordinal) + .ToArray(); + Assert.Equal(expected, remaining); + } + finally + { + if (Directory.Exists(directory)) + Directory.Delete(directory, recursive: true); + } + } + [Fact] public void FormatArgs_RedactsSensitiveArgumentsByDefault() {