diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 7aec5917fd..6412d50b79 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -1359,8 +1359,10 @@ Each record is a single JSON object on its own line with these fields: | `wal_checkpoint_ms` | number (optional) | Reserved for future WAL checkpoint timing | | `files_indexed` | number (optional) | Reserved for future per-index file counts | | `error` | string (optional) | Short error category, when the command failed in a way worth tagging | +| `_length` | number (optional) | Original character count when `tool`, `source`, `language`, or `error` was truncated | +| `_truncated` | boolean (optional) | Present and `true` when `tool`, `source`, `language`, or `error` was truncated | -Optional fields are omitted from the JSON when null so future consumers can grow new columns without breaking older parsers. The file is local-only and uses the relaxed JSON encoder so timestamps stay human-readable in `tail` / `grep` workflows. +Optional fields are omitted from the JSON when null so future consumers can grow new columns without breaking older parsers. Metrics string fields are bounded before serialization. If `tool`, `source`, `language`, or `error` is too long, the emitted value is clipped and the record includes the matching `_length` / `_truncated` metadata so consumers can detect the truncation. Each serialized JSON object is also kept within an 8 KiB event budget, so pathological escaping can reduce string fields below the normal per-field cap. The file is local-only and uses the relaxed JSON encoder so timestamps stay human-readable in `tail` / `grep` workflows. Example output: @@ -3565,8 +3567,10 @@ MCP ツールで catch-all まで突き抜けた例外（想定外の SQLite 例 | `wal_checkpoint_ms` | number（任意） | 将来の WAL チェックポイント時間計測用に予約 | | `files_indexed` | number（任意） | 将来の index 当たり処理ファイル数用に予約 | | `error` | string（任意） | タグ付けに値する失敗時の短いエラーカテゴリ | +| `_length` | number（任意） | `tool` / `source` / `language` / `error` が切り詰められた場合の元の文字数 | +| `_truncated` | boolean（任意） | `tool` / `source` / `language` / `error` が切り詰められた場合に `true` で付与 | -任意フィールドは値が null のとき JSON から省略されるため、後でフィールドを追加しても古いパーサを壊しません。ファイルはローカル専用で、`tail` / `grep` ワークフローでも timestamp が人間可読のまま残るよう relaxed エンコーダを使用します。 +任意フィールドは値が null のとき JSON から省略されるため、後でフィールドを追加しても古いパーサを壊しません。metrics の文字列フィールドは serialization 前に制限されます。`tool` / `source` / `language` / `error` が長すぎる場合、出力値は切り詰められ、対応する `_length` / `_truncated` metadata により consumer が truncation を検出できます。各 serialized JSON object は 8 KiB の event budget 内にも収められるため、escape による膨張が激しい入力では通常の field 単位上限よりさらに短くなる場合があります。ファイルはローカル専用で、`tail` / `grep` ワークフローでも timestamp が人間可読のまま残るよう relaxed エンコーダを使用します。 出力例: diff --git a/changelog.d/unreleased/3224.fixed.md b/changelog.d/unreleased/3224.fixed.md new file mode 100644 index 0000000000..d334ce9145 --- /dev/null +++ b/changelog.d/unreleased/3224.fixed.md @@ -0,0 +1,17 @@ +--- +category: fixed +issues: + - 3224 +affected: + - USER_GUIDE.md + - src/CodeIndex/Cli/MetricsSink.cs + - tests/CodeIndex.Tests/MetricsSinkTests.cs +--- + +## English + +- **Metrics JSONL events now bound string fields (#3224)** — `--metrics` and `CDIDX_METRICS` records now clamp oversized string fields, emit `*_length` / `*_truncated` metadata, and keep each serialized event within a fixed byte budget. + +## 日本語 + +- **Metrics JSONL event の文字列フィールドを制限するようになりました (#3224)** — `--metrics` と `CDIDX_METRICS` のレコードは巨大な文字列フィールドを切り詰め、`*_length` / `*_truncated` metadata を出力し、各 serialized event を固定 byte budget 内に収めます。 diff --git a/src/CodeIndex/Cli/MetricsSink.cs b/src/CodeIndex/Cli/MetricsSink.cs index 4ca2c9b6b9..bad7363760 100644 --- a/src/CodeIndex/Cli/MetricsSink.cs +++ b/src/CodeIndex/Cli/MetricsSink.cs @@ -22,6 +22,8 @@ internal static class MetricsSink internal const string EnvVarName = "CDIDX_METRICS"; internal const long DefaultMaxBytes = 50L * 1024 * 1024; internal const int RotationKeep = 3; + internal const int MaxStringFieldChars = 1024; + internal const int MaxSerializedEventBytes = 8 * 1024; internal static IDisposable? TryStart(string? explicitPath) => TryStart(explicitPath, DefaultMaxBytes); @@ -145,6 +147,35 @@ private void RotateIfNeededLocked() } internal static string SerializeEvent(MetricsEvent evt) + { + var encoded = SerializeEventBytes(evt, MaxStringFieldChars); + if (encoded.Length <= MaxSerializedEventBytes) + return Encoding.UTF8.GetString(encoded); + + // JSON escaping can expand already-clamped strings, especially control chars. + // Re-clamp all string fields until the serialized JSON object fits the event cap. + var low = 0; + var high = MaxStringFieldChars - 1; + var best = SerializeEventBytes(evt, 0); + while (low <= high) + { + var mid = low + ((high - low) / 2); + var candidate = SerializeEventBytes(evt, mid); + if (candidate.Length <= MaxSerializedEventBytes) + { + best = candidate; + low = mid + 1; + } + else + { + high = mid - 1; + } + } + + return Encoding.UTF8.GetString(best); + } + + private static byte[] SerializeEventBytes(MetricsEvent evt, int maxStringChars) { using var buffer = new MemoryStream(); using (var jw = new Utf8JsonWriter(buffer, new JsonWriterOptions @@ -162,12 +193,12 @@ internal static string SerializeEvent(MetricsEvent evt) { jw.WriteStartObject(); jw.WriteString("timestamp", evt.Timestamp.ToString("O", CultureInfo.InvariantCulture)); - jw.WriteString("tool", evt.Tool); - jw.WriteString("source", evt.Source); + WriteBoundedString(jw, "tool", evt.Tool, maxStringChars); + WriteBoundedString(jw, "source", evt.Source, maxStringChars); jw.WriteNumber("elapsed_ms", Math.Round(evt.ElapsedMs, 3)); jw.WriteNumber("exit_code", evt.ExitCode); if (evt.Language is { } lang) - jw.WriteString("language", lang); + WriteBoundedString(jw, "language", lang, maxStringChars); if (evt.BytesRead is { } br) jw.WriteNumber("bytes_read", br); if (evt.BytesWritten is { } bw) @@ -177,13 +208,39 @@ internal static string SerializeEvent(MetricsEvent evt) if (evt.FilesIndexed is { } fi) jw.WriteNumber("files_indexed", fi); if (evt.Error is { } err) - jw.WriteString("error", err); + WriteBoundedString(jw, "error", err, maxStringChars); jw.WriteEndObject(); } - return Encoding.UTF8.GetString(buffer.ToArray()); + return buffer.ToArray(); + } + + private static void WriteBoundedString(Utf8JsonWriter jw, string name, string value, int maxChars) + { + var bounded = BoundString(value, maxChars); + jw.WriteString(name, bounded.Text); + if (!bounded.Truncated) + return; + + jw.WriteNumber(name + "_length", bounded.OriginalLength); + jw.WriteBoolean(name + "_truncated", true); + } + + private static BoundedMetricsString BoundString(string value, int maxChars) + { + var safeMax = Math.Max(0, maxChars); + if (value.Length <= safeMax) + return new BoundedMetricsString(value, value.Length, Truncated: false); + + var end = safeMax; + if (end > 0 && end < value.Length && char.IsHighSurrogate(value[end - 1]) && char.IsLowSurrogate(value[end])) + end--; + + return new BoundedMetricsString(value.Substring(0, end), value.Length, Truncated: true); } } +internal readonly record struct BoundedMetricsString(string Text, int OriginalLength, bool Truncated); + /// /// Structured metrics record emitted to the JSONL sink. Optional fields are omitted from /// the payload when null so consumers can grow new fields without breaking older parsers. diff --git a/tests/CodeIndex.Tests/MetricsSinkTests.cs b/tests/CodeIndex.Tests/MetricsSinkTests.cs index 0b41474dfd..9adbc9d974 100644 --- a/tests/CodeIndex.Tests/MetricsSinkTests.cs +++ b/tests/CodeIndex.Tests/MetricsSinkTests.cs @@ -1,3 +1,4 @@ +using System.Text; using System.Text.Json; using CodeIndex.Cli; @@ -299,6 +300,78 @@ public void SerializeEvent_IncludesAllOptionalFieldsWhenSet() Assert.False(root.TryGetProperty("error", out _)); } + [Fact] + public void SerializeEvent_TruncatesOversizedStringFieldsAndWritesMetadata() + { + var oversizedTool = new string('t', MetricsSink.MaxStringFieldChars + 11); + var oversizedSource = new string('s', MetricsSink.MaxStringFieldChars + 12); + var oversizedLanguage = new string('l', MetricsSink.MaxStringFieldChars + 13); + var oversizedError = new string('e', MetricsSink.MaxStringFieldChars + 14); + var evt = new MetricsEvent( + Timestamp: new DateTimeOffset(2026, 5, 16, 0, 0, 0, TimeSpan.Zero), + Tool: oversizedTool, + Source: oversizedSource, + ElapsedMs: 1.25, + ExitCode: 1, + Language: oversizedLanguage, + Error: oversizedError); + + var json = MetricsSink.SerializeEvent(evt); + using var doc = JsonDocument.Parse(json); + var root = doc.RootElement; + + Assert.Equal(MetricsSink.MaxStringFieldChars, root.GetProperty("tool").GetString()!.Length); + Assert.Equal(oversizedTool.Length, root.GetProperty("tool_length").GetInt32()); + Assert.True(root.GetProperty("tool_truncated").GetBoolean()); + Assert.Equal(MetricsSink.MaxStringFieldChars, root.GetProperty("source").GetString()!.Length); + Assert.Equal(oversizedSource.Length, root.GetProperty("source_length").GetInt32()); + Assert.True(root.GetProperty("source_truncated").GetBoolean()); + Assert.Equal(MetricsSink.MaxStringFieldChars, root.GetProperty("language").GetString()!.Length); + Assert.Equal(oversizedLanguage.Length, root.GetProperty("language_length").GetInt32()); + Assert.True(root.GetProperty("language_truncated").GetBoolean()); + Assert.Equal(MetricsSink.MaxStringFieldChars, root.GetProperty("error").GetString()!.Length); + Assert.Equal(oversizedError.Length, root.GetProperty("error_length").GetInt32()); + Assert.True(root.GetProperty("error_truncated").GetBoolean()); + } + + [Fact] + public void Record_OversizedEscapedEventWritesSingleBoundedJsonlLine() + { + var metricsPath = Path.Combine(Path.GetTempPath(), $"cdidx_metrics_bounded_{Guid.NewGuid():N}.jsonl"); + try + { + using var session = MetricsSink.TryStartForTesting(metricsPath, maxBytes: 1024 * 1024); + Assert.NotNull(session); + var oversizedEscaped = new string('\u0001', 50_000); + + MetricsSink.Record(new MetricsEvent( + Timestamp: new DateTimeOffset(2026, 5, 16, 0, 0, 0, TimeSpan.Zero), + Tool: oversizedEscaped, + Source: oversizedEscaped, + ElapsedMs: 1.0, + ExitCode: 1, + Language: oversizedEscaped, + Error: oversizedEscaped)); + + var line = Assert.Single(File.ReadAllLines(metricsPath)); + Assert.True( + Encoding.UTF8.GetByteCount(line) <= MetricsSink.MaxSerializedEventBytes, + $"Metrics event was {Encoding.UTF8.GetByteCount(line)} bytes."); + using var doc = JsonDocument.Parse(line); + var root = doc.RootElement; + Assert.True(root.GetProperty("tool_truncated").GetBoolean()); + Assert.True(root.GetProperty("source_truncated").GetBoolean()); + Assert.True(root.GetProperty("language_truncated").GetBoolean()); + Assert.True(root.GetProperty("error_truncated").GetBoolean()); + Assert.Equal(oversizedEscaped.Length, root.GetProperty("error_length").GetInt32()); + } + finally + { + if (File.Exists(metricsPath)) + File.Delete(metricsPath); + } + } + private static (int ExitCode, string Stdout, string Stderr) CaptureConsole(Func action) => ConsoleCapture.Capture(action); }