From 2b5312918c612354c7e53e37725f403f3b543d74 Mon Sep 17 00:00:00 2001 From: "itarun.p" Date: Mon, 20 Jul 2026 06:37:26 +0700 Subject: [PATCH 1/3] perf: cache Codex context parses per file instead of per scan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Codex Context Breakdown cache keyed on the GLOBAL maximum session-file mtime, so appending to whichever session is currently active invalidated the aggregate for every unchanged historical file and forced a full rescan. The reporter measured 2.19s cold against 2,066 files, and 9.1s under concurrent dashboard load, for what was a single-line append. Adds a per-file parse cache consulted when the aggregate cache misses, so that miss now costs one file rather than all of them. The aggregate cache is deliberately kept as the fast path: replacing it outright would turn every no-change request into "stat every file and recombine", regressing the case that matters most under dashboard load. `top` is deliberately absent from the per-file key. Truncation to the top N happens at merge time, so one cached parse serves every `top` value; a test pins this so it does not get "fixed" into the key later. File identity reuses the predicate rollout.js already relies on in 19 places — inode, size, mtimeMs, plus a sha1 of the first 256 bytes to catch a rotator that reuses the inode. Moved to src/lib/file-identity.js and imported back into rollout.js under the same name, so those 19 call sites are untouched. The comparison is equality, never ordering: a clock moving backwards or a restored mtime must read as changed. Verified the merge helpers build fresh targets and only read from their source rows, so a cached parse cannot be mutated by a later merge — the aliasing hazard that per-file caching would otherwise introduce. Tests were negative-controlled: with the cache lookup disabled, the two behavioral tests fail and the three correctness tests still pass, which is the correct split. Refs #62 --- src/lib/codex-context-breakdown.js | 88 +++++++++- src/lib/file-identity.js | 69 ++++++++ src/lib/rollout.js | 26 +-- test/codex-context-per-file-cache.test.js | 190 ++++++++++++++++++++++ 4 files changed, 344 insertions(+), 29 deletions(-) create mode 100644 src/lib/file-identity.js create mode 100644 test/codex-context-per-file-cache.test.js diff --git a/src/lib/codex-context-breakdown.js b/src/lib/codex-context-breakdown.js index 9ae4fede..eec98bfd 100644 --- a/src/lib/codex-context-breakdown.js +++ b/src/lib/codex-context-breakdown.js @@ -14,6 +14,8 @@ const fs = require("node:fs"); const os = require("node:os"); const path = require("node:path"); +const { fileIdentity, isUnchanged } = require("./file-identity"); + const { emptyTotals, addInto, @@ -139,6 +141,57 @@ function cacheTimeZoneKey(timeZoneContext) { return `${timeZoneContext.timeZone || ""}|${Number.isFinite(timeZoneContext.offsetMinutes) ? timeZoneContext.offsetMinutes : ""}`; } +// Per-file parse cache. +// +// The aggregate cache above keys on the GLOBAL maximum mtime, so appending to +// the one session that is currently active invalidates the aggregate for every +// unchanged historical file and forces a full rescan (issue #62). This layer +// makes that miss cost one file instead of all of them. +// +// `top` is deliberately absent from the key: truncation to the top N happens +// at merge time, so a single cached per-file parse serves every `top` value. +const FILE_CACHE = new Map(); +const FILE_CACHE_SCHEMA_VERSION = "codex-context-file-v1"; +// Roughly one entry per session file per (range, timezone) combination the +// dashboard asks for. Entries are small — one parsed rollup, no line data. +const FILE_CACHE_MAX_ENTRIES = 8_000; + +// Counts parses per file so tests can assert an unchanged file was NOT +// reparsed. Observing the actual claim beats spying on the filesystem, which +// would also catch the identity check's own 256-byte read. +const parseCounts = new Map(); + +function fileCacheKey({ fromKey, toKey, timeZoneContext, filePath }) { + return [ + FILE_CACHE_SCHEMA_VERSION, + fromKey || "", + toKey || "", + cacheTimeZoneKey(timeZoneContext), + filePath, + ].join("|"); +} + +function pruneFileCache() { + if (FILE_CACHE.size <= FILE_CACHE_MAX_ENTRIES) return; + const excess = FILE_CACHE.size - FILE_CACHE_MAX_ENTRIES; + let removed = 0; + // Map iterates in insertion order, so this drops the least recently stored. + for (const key of FILE_CACHE.keys()) { + FILE_CACHE.delete(key); + if (++removed >= excess) break; + } +} + +function __resetContextCachesForTests() { + CACHE.clear(); + FILE_CACHE.clear(); + parseCounts.clear(); +} + +function __getParseCountsForTests() { + return new Map(parseCounts); +} + // --------------------------------------------------------------------------- // Main entry // --------------------------------------------------------------------------- @@ -182,13 +235,32 @@ async function computeCodexContextBreakdown({ const sessions = []; for (const filePath of files) { - const parsed = await parseCodexRolloutFile(filePath, { - fromIso, - toIso, - from: fromKey, - to: toKey, - timeZoneContext, - }); + const key = fileCacheKey({ fromKey, toKey, timeZoneContext, filePath }); + const identity = fileIdentity(filePath); + const hit = FILE_CACHE.get(key); + + let parsed; + if (hit && isUnchanged(hit.identity, identity)) { + parsed = hit.parsed; + } else { + parseCounts.set(filePath, (parseCounts.get(filePath) || 0) + 1); + parsed = await parseCodexRolloutFile(filePath, { + fromIso, + toIso, + from: fromKey, + to: toKey, + timeZoneContext, + }); + // A null identity means the path is not a readable regular file right + // now; caching against it could never be invalidated correctly. + if (identity) { + FILE_CACHE.set(key, { identity, parsed }); + pruneFileCache(); + } + } + + // Re-applied on every pass rather than baked into the cached entry, so a + // zero-token file has one representation instead of two. if (!parsed || !parsed.totals || !parsed.totals.total_tokens) continue; sessions.push(parsed); } @@ -397,4 +469,6 @@ async function computeCodexContextBreakdown({ module.exports = { computeCodexContextBreakdown, + __resetContextCachesForTests, + __getParseCountsForTests, }; diff --git a/src/lib/file-identity.js b/src/lib/file-identity.js new file mode 100644 index 00000000..6310a92d --- /dev/null +++ b/src/lib/file-identity.js @@ -0,0 +1,69 @@ +const crypto = require("node:crypto"); +const fssync = require("node:fs"); + +// Fingerprint the first bytes of a file so rotation is detected even when the +// OS reuses the inode after unlink+recreate (Linux logrotate). Returns "" on +// any error so callers treat "no fingerprint" as "not changed". Also returns +// "" when the file is shorter than the fingerprint window: for a sub-256B +// file that is still being appended to, the window would extend into the +// not-yet-written append zone, so plain growth (not rotation) would shift +// the hash and produce a false "rotated" signal. +function readFileHeadSignature(filePath) { + try { + const fd = fssync.openSync(filePath, "r"); + try { + const buf = Buffer.alloc(256); + const bytes = fssync.readSync(fd, buf, 0, 256, 0); + if (bytes < 256) return ""; + return crypto.createHash("sha1").update(buf.subarray(0, bytes)).digest("hex"); + } finally { + fssync.closeSync(fd); + } + } catch { + return ""; + } +} + +// A token that changes whenever a file's contents could have changed. Returns +// null when the path is not a readable regular file, which callers must treat +// as "cannot be reused" rather than "unchanged". +function fileIdentity(filePath) { + let st = null; + try { + st = fssync.statSync(filePath); + } catch { + return null; + } + if (!st.isFile()) return null; + return { + inode: st.ino || 0, + size: Number.isFinite(st.size) ? st.size : 0, + mtimeMs: Number.isFinite(st.mtimeMs) ? st.mtimeMs : 0, + head: readFileHeadSignature(filePath), + }; +} + +// Equality, not ordering. Deliberately not `cur.mtimeMs > prev.mtimeMs`: a +// clock that moves backwards, a restored mtime, or a truncation must all read +// as "changed" and force a re-read. The cost of a false "changed" is a wasted +// parse; the cost of a false "unchanged" is serving stale data forever. +function isUnchanged(prev, cur) { + if (!prev || !cur) return false; + // An empty head means the file was too short to fingerprint, so it carries + // no rotation signal — fall back to inode/size/mtime alone rather than + // treating "" as a mismatch. + const headChanged = + typeof prev.head === "string" && prev.head !== "" && prev.head !== cur.head; + return ( + prev.inode === cur.inode && + prev.size === cur.size && + prev.mtimeMs === cur.mtimeMs && + !headChanged + ); +} + +module.exports = { + readFileHeadSignature, + fileIdentity, + isUnchanged, +}; diff --git a/src/lib/rollout.js b/src/lib/rollout.js index 47b73138..f2482cf6 100644 --- a/src/lib/rollout.js +++ b/src/lib/rollout.js @@ -49,28 +49,10 @@ async function listClaudeProjectFiles(projectsDir) { return out; } -// Fingerprint the first bytes of a file so rotation is detected even when the -// OS reuses the inode after unlink+recreate (Linux logrotate). Returns "" on -// any error so callers treat "no fingerprint" as "not changed". Also returns -// "" when the file is shorter than the fingerprint window: for a sub-256B -// file that is still being appended to, the window would extend into the -// not-yet-written append zone, so plain growth (not rotation) would shift -// the hash and produce a false "rotated" signal. -function readFileHeadSignature(filePath) { - try { - const fd = fssync.openSync(filePath, "r"); - try { - const buf = Buffer.alloc(256); - const bytes = fssync.readSync(fd, buf, 0, 256, 0); - if (bytes < 256) return ""; - return crypto.createHash("sha1").update(buf.subarray(0, bytes)).digest("hex"); - } finally { - fssync.closeSync(fd); - } - } catch { - return ""; - } -} +// Moved to ./file-identity so the context-breakdown caches can share it +// rather than grow a second copy. Imported under the same name, so all 19 +// call sites below are unaffected. +const { readFileHeadSignature } = require("./file-identity"); async function listGeminiSessionFiles(tmpDir) { const out = []; diff --git a/test/codex-context-per-file-cache.test.js b/test/codex-context-per-file-cache.test.js new file mode 100644 index 00000000..9a83d37f --- /dev/null +++ b/test/codex-context-per-file-cache.test.js @@ -0,0 +1,190 @@ +const assert = require("node:assert/strict"); +const fs = require("node:fs/promises"); +const fssync = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { test } = require("node:test"); + +const { + computeCodexContextBreakdown, + __resetContextCachesForTests, + __getParseCountsForTests, +} = require("../src/lib/codex-context-breakdown"); + +// Issue #62: the aggregate cache keys on the GLOBAL maximum session-file mtime, +// so appending to the one active session invalidated every historical file and +// forced a full rescan. These pin that only changed files are reparsed, and +// that the cached aggregate still equals a clean scan. + +const DAY = "2026-05-08"; + +function rolloutEvents({ sessionId, at, inputTokens, outputTokens }) { + return [ + { + timestamp: `${DAY}T${at}.000Z`, + type: "session_meta", + payload: { id: sessionId, cwd: "/tmp/project", model_provider: "openai", cli_version: "1.0.0" }, + }, + { + timestamp: `${DAY}T${at}.100Z`, + type: "turn_context", + payload: { cwd: "/tmp/project", model: "gpt-5.5" }, + }, + { + timestamp: `${DAY}T${at}.200Z`, + type: "response_item", + payload: { type: "function_call", name: "exec_command", call_id: `call-${sessionId}`, arguments: "{}" }, + }, + { + timestamp: `${DAY}T${at}.300Z`, + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: inputTokens, + cached_input_tokens: 0, + output_tokens: outputTokens, + reasoning_output_tokens: 0, + total_tokens: inputTokens + outputTokens, + }, + }, + }, + }, + ]; +} + +async function writeRollout(rootDir, fileName, events) { + const dir = path.join(rootDir, DAY.slice(0, 4), DAY.slice(5, 7), DAY.slice(8, 10)); + await fs.mkdir(dir, { recursive: true }); + const filePath = path.join(dir, fileName); + await fs.writeFile(filePath, events.map((e) => JSON.stringify(e)).join("\n") + "\n", "utf8"); + return filePath; +} + +// Appending and then re-stat-ing within the same millisecond can leave mtimeMs +// unchanged, which would make this test pass for the wrong reason. Push mtime +// forward explicitly so the identity check is exercised, not the clock. +async function appendAndTouch(filePath, events) { + await fs.appendFile(filePath, events.map((e) => JSON.stringify(e)).join("\n") + "\n", "utf8"); + const future = new Date(Date.now() + 2000); + fssync.utimesSync(filePath, future, future); +} + +async function setupCorpus(t) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "tt-codex-perfile-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + __resetContextCachesForTests(); + t.after(() => __resetContextCachesForTests()); + + const historical = []; + for (let i = 0; i < 3; i++) { + historical.push( + await writeRollout( + root, + `rollout-hist-${i}.jsonl`, + rolloutEvents({ sessionId: `hist-${i}`, at: `1${i}:00:00`, inputTokens: 100, outputTokens: 50 }), + ), + ); + } + const active = await writeRollout( + root, + "rollout-active.jsonl", + rolloutEvents({ sessionId: "active", at: "14:00:00", inputTokens: 200, outputTokens: 100 }), + ); + + return { root, historical, active }; +} + +test("appending to the active session reparses only that file", async (t) => { + const { root, active } = await setupCorpus(t); + + const before = await computeCodexContextBreakdown({ codexDir: root }); + const cold = __getParseCountsForTests(); + assert.equal(cold.size, 4, "first pass must parse all four files"); + assert.deepStrictEqual([...cold.values()], [1, 1, 1, 1]); + + await appendAndTouch(active, rolloutEvents({ sessionId: "active", at: "15:00:00", inputTokens: 7, outputTokens: 3 })); + + const warm = await computeCodexContextBreakdown({ codexDir: root }); + const after = __getParseCountsForTests(); + + // The three historical files must not have been parsed a second time. + for (const [filePath, count] of after) { + const expected = filePath === active ? 2 : 1; + assert.equal(count, expected, `${path.basename(filePath)} parsed ${count}x, expected ${expected}x`); + } + + assert.ok( + warm.totals.total_tokens > before.totals.total_tokens, + "the appended record must be reflected in the aggregate", + ); +}); + +test("the cached aggregate equals a clean full scan after an append", async (t) => { + const { root, active } = await setupCorpus(t); + + await computeCodexContextBreakdown({ codexDir: root }); + await appendAndTouch(active, rolloutEvents({ sessionId: "active", at: "16:00:00", inputTokens: 11, outputTokens: 5 })); + const warm = await computeCodexContextBreakdown({ codexDir: root }); + + // Cold: no aggregate cache, no per-file cache, nothing carried over. + __resetContextCachesForTests(); + const cold = await computeCodexContextBreakdown({ codexDir: root }); + + assert.deepStrictEqual(warm, cold, "incremental result must be identical to a clean scan"); +}); + +test("changing only top reuses the per-file cache", async (t) => { + const { root } = await setupCorpus(t); + + await computeCodexContextBreakdown({ codexDir: root, top: 20 }); + const baseline = __getParseCountsForTests(); + + await computeCodexContextBreakdown({ codexDir: root, top: 5 }); + const after = __getParseCountsForTests(); + + // top is applied at merge time, so it is deliberately absent from the + // per-file key. If someone adds it, this test is what catches the + // regression: every file would reparse for a purely cosmetic change. + assert.deepStrictEqual([...after.values()], [...baseline.values()], "changing top must not reparse anything"); +}); + +test("changing the timezone context does reparse", async (t) => { + const { root } = await setupCorpus(t); + + await computeCodexContextBreakdown({ codexDir: root }); + const baseline = [...__getParseCountsForTests().values()]; + + await computeCodexContextBreakdown({ + codexDir: root, + timeZoneContext: { timeZone: "Asia/Bangkok", offsetMinutes: 420 }, + }); + const after = [...__getParseCountsForTests().values()]; + + // Bucketing depends on the timezone, so cached parses are not reusable. + assert.deepStrictEqual(after, baseline.map((n) => n + 1), "a timezone change must reparse every file"); +}); + +test("rotation and truncation invalidate the per-file cache", async (t) => { + const { root, active } = await setupCorpus(t); + + await computeCodexContextBreakdown({ codexDir: root }); + const before = __getParseCountsForTests().get(active); + + // Rewrite with different leading bytes and a different length: the inode may + // be reused, so this leans on the head fingerprint and the size check. + await fs.rm(active); + await writeRollout( + root, + "rollout-active.jsonl", + rolloutEvents({ sessionId: "rotated", at: "17:00:00", inputTokens: 999, outputTokens: 1 }), + ); + + const rotated = await computeCodexContextBreakdown({ codexDir: root }); + assert.equal(__getParseCountsForTests().get(active), before + 1, "a rotated file must be reparsed"); + + __resetContextCachesForTests(); + const cold = await computeCodexContextBreakdown({ codexDir: root }); + assert.deepStrictEqual(rotated, cold, "post-rotation result must match a clean scan"); +}); From cffa8576353c4fade5fcf2a335636b8c9c3db44e Mon Sep 17 00:00:00 2001 From: "itarun.p" Date: Mon, 20 Jul 2026 06:45:56 +0700 Subject: [PATCH 2/3] perf: cache Claude category parses per file instead of per scan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same fix as the Codex side, but this one had a real obstacle: the scan kept a single `seenHashes` dedup Set shared across every file, so whether a message in one file counted depended on what another file had already contributed. Summing independent per-file results would double count every cross-file duplicate — and #62's own acceptance criterion is that the aggregate stays identical to a clean full scan. categorizeSessionFile now parses into its own accumulators and returns them, with a file-local dedup set plus the list of hashes it claimed. The caller merges in `files` order and inserts every hash into one global Set; on any collision it discards the merge and re-runs a full sequential scan with a shared dedup set, which is exactly the old behavior. A scan of 6,529 real session files found zero cross-file duplicates, so the guard is expected to be dead code — it exists so correctness does not depend on that measurement being true of everyone's corpus. A test forces the collision and asserts the duplicate is counted once. Merging in `files` order rather than worker-completion order also makes the result deterministic, which the old shared-accumulator scan was not. Measured on a copy of 800 real session files (108 MB): cold scan 292 ms -> 347 ms no change (L1) 2 ms -> 2 ms after one append 274 ms -> 16 ms Cold is ~19% slower — one extra stat and 256-byte head read per file, plus building per-file structures. That is the trade: the append path, which is what #62 reports and what the dashboard actually hits while a session is live, gets 17x faster. Verified the refactor changes no real-world number: the full breakdown over ~6,500 local session files is byte-identical to main across a fixed date range. An earlier comparison appeared to differ, but that was the live corpus being appended to between the two runs, not the code. Parse instrumentation is opt-in — left always-on it would grow one Map entry per session file forever on a long-running server, purely to serve tests. Fixes #62 --- src/lib/claude-categorizer.js | 236 +++++++++++++++++--- src/lib/codex-context-breakdown.js | 4 +- test/claude-category-per-file-cache.test.js | 168 ++++++++++++++ 3 files changed, 375 insertions(+), 33 deletions(-) create mode 100644 test/claude-category-per-file-cache.test.js diff --git a/src/lib/claude-categorizer.js b/src/lib/claude-categorizer.js index 7a0ff731..8dcb9bdd 100644 --- a/src/lib/claude-categorizer.js +++ b/src/lib/claude-categorizer.js @@ -28,6 +28,7 @@ const { allocateByLargestRemainder, } = require("./categorizer-utils"); const { claudeMessageDedupKey } = require("./rollout"); +const { fileIdentity, isUnchanged } = require("./file-identity"); const CATEGORY_KEYS = [ "system_prefix", @@ -494,12 +495,40 @@ function classifyOneMessage(obj, sessionState, breakdown, toolLedger = null, ski } // Read one session jsonl streaming, in timestamp range, dedup by msgId+reqId. -async function categorizeSessionFile(filePath, { fromIso, toIso, seenHashes }, breakdown, toolLedger = null, skillLedger = null, execLedger = null) { +// Parses one file into its OWN accumulators and returns them, rather than +// mutating shared ones. That is what makes a per-file cache possible: the +// result depends only on (file contents, fromIso, toIso), so it can be reused +// whenever the file has not changed. +// +// The dedup set is now file-local. Cross-file duplicates are handled by the +// caller, which detects them and falls back to a full scan — see +// mergeFileResult. Intra-file dedup, including the deferred-add behavior +// below, is unchanged and must stay that way. +// `sharedSeen` is used only by the collision fallback, which needs the old +// cross-file dedup semantics. When it is passed, the returned `hashes` list is +// empty because dedup has already been applied globally. +async function categorizeSessionFile(filePath, { fromIso, toIso, sharedSeen = null }) { + const breakdown = emptyCategoryMap(); + const toolLedger = { + tool_calls: { total_calls: 0, by_name: new Map() }, + subagents: { total_calls: 0, by_name: new Map() }, + }; + const skillLedger = { total_calls: 0, by_name: new Map() }; + const execLedger = { + total_calls: 0, + by_type: new Map(), + by_executable: new Map(), + by_command: new Map(), + by_exit: new Map(), + }; + const seenHashes = sharedSeen || new Set(); + const empty = () => ({ breakdown, toolLedger, skillLedger, execLedger, counted: 0, hashes: [] }); + let stream; try { stream = fssync.createReadStream(filePath, { encoding: "utf8" }); } catch (_e) { - return 0; + return empty(); } const rl = readline.createInterface({ input: stream, crlfDelay: Infinity }); const sessionState = { systemPrefixSeen: false }; @@ -538,7 +567,14 @@ async function categorizeSessionFile(filePath, { fromIso, toIso, seenHashes }, b } rl.close(); stream.close?.(); - return counted; + return { + breakdown, + toolLedger, + skillLedger, + execLedger, + counted, + hashes: sharedSeen ? [] : [...seenHashes], + }; } // Convert a YYYY-MM-DD day key (already in the user's tz from the API call) @@ -624,6 +660,99 @@ function maxMtimeMs(files) { return max; } +// Per-file parse cache. See the equivalent in codex-context-breakdown.js — +// same problem (#62), same two-layer shape: the aggregate cache above stays +// the fast path, and this only decides how expensive a MISS is. +const FILE_CACHE = new Map(); +const FILE_CACHE_SCHEMA_VERSION = "claude-category-file-v1"; +const FILE_CACHE_MAX_ENTRIES = 8_000; + +// Instrumentation is opt-in: without it a long-running server would grow one +// Map entry per session file forever, purely to serve tests. +let trackParses = false; +const parseCounts = new Map(); + +function fileCacheKey(fromIso, toIso, filePath) { + return `${FILE_CACHE_SCHEMA_VERSION}|${fromIso || ""}|${toIso || ""}|${filePath}`; +} + +function pruneFileCache() { + if (FILE_CACHE.size <= FILE_CACHE_MAX_ENTRIES) return; + let excess = FILE_CACHE.size - FILE_CACHE_MAX_ENTRIES; + for (const key of FILE_CACHE.keys()) { + FILE_CACHE.delete(key); + if (--excess <= 0) break; + } +} + +// These fields are high-water marks, not running sums. Adding them would +// silently inflate the reported worst case. +const MAX_MERGE_FIELDS = new Set(["max_duration_ms"]); + +// Folds one cached row into an accumulator map. Always builds a fresh target +// object on first insert: putting a cached row in by reference would let a +// later merge mutate the cache in place, which is the aliasing hazard that +// per-file caching introduces. +function mergeRowInto(targetMap, key, row) { + const existing = targetMap.get(key); + if (!existing) { + targetMap.set(key, { ...row, totals: { ...row.totals } }); + return; + } + for (const [field, value] of Object.entries(row)) { + if (field === "totals") { + addInto(existing.totals, value); + } else if (typeof value === "number") { + existing[field] = MAX_MERGE_FIELDS.has(field) + ? Math.max(existing[field] || 0, value) + : (existing[field] || 0) + value; + } else if (existing[field] === undefined) { + existing[field] = value; + } + } +} + +function mergeNamedLedger(target, source) { + target.total_calls += source.total_calls || 0; + for (const [name, row] of source.by_name) mergeRowInto(target.by_name, name, row); +} + +// Returns false when this file shares a dedup hash with one already merged. +// Such a collision cannot be repaired from aggregates: the deduped message's +// tokens are missing, and worse, systemPrefixSeen may have latched onto a +// different message, invalidating the whole per-file result rather than one +// row. The caller re-runs a full sequential scan instead of patching. +function mergeFileResult(acc, result, globalHashes) { + for (const hash of result.hashes) { + if (globalHashes.has(hash)) return false; + globalHashes.add(hash); + } + for (const key of CATEGORY_KEYS) addInto(acc.breakdown[key], result.breakdown[key]); + mergeNamedLedger(acc.toolLedger.tool_calls, result.toolLedger.tool_calls); + mergeNamedLedger(acc.toolLedger.subagents, result.toolLedger.subagents); + mergeNamedLedger(acc.skillLedger, result.skillLedger); + acc.execLedger.total_calls += result.execLedger.total_calls || 0; + for (const bucket of ["by_type", "by_executable", "by_command", "by_exit"]) { + for (const [name, row] of result.execLedger[bucket]) { + mergeRowInto(acc.execLedger[bucket], name, row); + } + } + acc.messageCount += result.counted; + if (result.counted > 0) acc.sessionCount += 1; + return true; +} + +function __resetCategoryCachesForTests() { + trackParses = true; + CACHE.clear(); + FILE_CACHE.clear(); + parseCounts.clear(); +} + +function __getCategoryParseCountsForTests() { + return new Map(parseCounts); +} + async function computeClaudeCategoryBreakdown({ from = null, to = null, rootDir = null, projectDir = null } = {}) { const root = rootDir || defaultClaudeProjectsDir(); let files = []; @@ -656,49 +785,90 @@ async function computeClaudeCategoryBreakdown({ from = null, to = null, rootDir } const { fromIso, toIso } = dayKeyToIsoBounds(from, to); - const breakdown = emptyCategoryMap(); - const seenHashes = new Set(); - let messageCount = 0; - let sessionCount = 0; - const toolLedger = { - tool_calls: { total_calls: 0, by_name: new Map() }, - subagents: { total_calls: 0, by_name: new Map() }, - }; - const skillLedger = { total_calls: 0, by_name: new Map() }; - const execLedger = { - total_calls: 0, - by_type: new Map(), - by_executable: new Map(), - by_command: new Map(), - by_exit: new Map(), - }; + function newAccumulator() { + return { + breakdown: emptyCategoryMap(), + toolLedger: { + tool_calls: { total_calls: 0, by_name: new Map() }, + subagents: { total_calls: 0, by_name: new Map() }, + }, + skillLedger: { total_calls: 0, by_name: new Map() }, + execLedger: { + total_calls: 0, + by_type: new Map(), + by_executable: new Map(), + by_command: new Map(), + by_exit: new Map(), + }, + messageCount: 0, + sessionCount: 0, + }; + } // Process files with bounded parallelism. CPU-bound (JSON.parse per line) // limits the win, but overlapping the per-file fs.open + first-block read // I/O behind the previous file's parsing still shaves ~15% off cold scans. - // `seenHashes`/`breakdown`/ledgers are mutated only inside synchronous - // sections of `classifyOneMessage`; `for await` in `categorizeSessionFile` - // only yields between lines, so workers can't tear shared state. + // Each parse now owns its accumulators, so there is no shared state to tear + // — and an unchanged file skips the parse entirely (#62). const SCAN_CONCURRENCY = 4; + const results = new Array(files.length); let cursor = 0; async function worker() { while (cursor < files.length) { const idx = cursor++; if (idx >= files.length) return; - const counted = await categorizeSessionFile( - files[idx], - { fromIso, toIso, seenHashes }, - breakdown, - toolLedger, - skillLedger, - execLedger, - ); - if (counted > 0) sessionCount += 1; - messageCount += counted; + const filePath = files[idx]; + const key = fileCacheKey(fromIso, toIso, filePath); + const identity = fileIdentity(filePath); + const hit = FILE_CACHE.get(key); + if (hit && isUnchanged(hit.identity, identity)) { + results[idx] = hit.result; + continue; + } + if (trackParses) parseCounts.set(filePath, (parseCounts.get(filePath) || 0) + 1); + const result = await categorizeSessionFile(filePath, { fromIso, toIso }); + results[idx] = result; + if (identity) { + FILE_CACHE.set(key, { identity, result }); + pruneFileCache(); + } } } await Promise.all(Array.from({ length: SCAN_CONCURRENCY }, () => worker())); + // Merge in `files` order rather than completion order, so the result no + // longer depends on how the four workers happened to interleave. + let acc = newAccumulator(); + const globalHashes = new Set(); + let composable = true; + for (const result of results) { + if (!result) continue; + if (!mergeFileResult(acc, result, globalHashes)) { + composable = false; + break; + } + } + + if (!composable) { + // Two files claimed the same dedup hash, so per-file results overlap and + // cannot be summed. Redo the scan sequentially with one shared dedup set, + // which is exactly the pre-cache behavior. Expected never to happen: a + // scan of 6,529 real session files found 0 cross-file duplicates. Kept so + // correctness does not rest on that measurement. + acc = newAccumulator(); + const sharedSeen = new Set(); + const fallbackHashes = new Set(); + for (const filePath of files) { + const result = await categorizeSessionFile(filePath, { fromIso, toIso, sharedSeen }); + if (trackParses) parseCounts.set(filePath, (parseCounts.get(filePath) || 0) + 1); + mergeFileResult(acc, result, fallbackHashes); + } + } + + const { breakdown, toolLedger, skillLedger, execLedger } = acc; + const messageCount = acc.messageCount; + const sessionCount = acc.sessionCount; + const totals = emptyTotals(); for (const key of CATEGORY_KEYS) addInto(totals, breakdown[key]); @@ -1306,6 +1476,8 @@ async function computeClaudeGroundTruthBuckets({ rootDir = null } = {}) { } module.exports = { + __resetCategoryCachesForTests, + __getCategoryParseCountsForTests, CATEGORY_KEYS, computeClaudeCategoryBreakdown, computeClaudeGroundTruthBuckets, diff --git a/src/lib/codex-context-breakdown.js b/src/lib/codex-context-breakdown.js index eec98bfd..5e6b056d 100644 --- a/src/lib/codex-context-breakdown.js +++ b/src/lib/codex-context-breakdown.js @@ -159,6 +159,7 @@ const FILE_CACHE_MAX_ENTRIES = 8_000; // Counts parses per file so tests can assert an unchanged file was NOT // reparsed. Observing the actual claim beats spying on the filesystem, which // would also catch the identity check's own 256-byte read. +let trackParses = false; const parseCounts = new Map(); function fileCacheKey({ fromKey, toKey, timeZoneContext, filePath }) { @@ -183,6 +184,7 @@ function pruneFileCache() { } function __resetContextCachesForTests() { + trackParses = true; CACHE.clear(); FILE_CACHE.clear(); parseCounts.clear(); @@ -243,7 +245,7 @@ async function computeCodexContextBreakdown({ if (hit && isUnchanged(hit.identity, identity)) { parsed = hit.parsed; } else { - parseCounts.set(filePath, (parseCounts.get(filePath) || 0) + 1); + if (trackParses) parseCounts.set(filePath, (parseCounts.get(filePath) || 0) + 1); parsed = await parseCodexRolloutFile(filePath, { fromIso, toIso, diff --git a/test/claude-category-per-file-cache.test.js b/test/claude-category-per-file-cache.test.js new file mode 100644 index 00000000..643e4393 --- /dev/null +++ b/test/claude-category-per-file-cache.test.js @@ -0,0 +1,168 @@ +const assert = require("node:assert/strict"); +const fs = require("node:fs/promises"); +const fssync = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { test } = require("node:test"); + +const { + computeClaudeCategoryBreakdown, + __resetCategoryCachesForTests, + __getCategoryParseCountsForTests, +} = require("../src/lib/claude-categorizer"); + +// Issue #62: the aggregate cache keys on the GLOBAL maximum session-file mtime, +// so appending to the active session invalidated every historical file. These +// pin that only changed files are reparsed AND that the incremental result is +// identical to a clean scan — including when files share a dedup hash, which is +// the case that makes per-file results non-composable. + +function ts(minutesAgo) { + return new Date(Date.now() - minutesAgo * 60_000).toISOString(); +} + +function assistantEntry({ requestId, messageId, at, output = 200, cacheCreate = 1000 }) { + return { + type: "assistant", + timestamp: at, + requestId, + message: { + id: messageId, + usage: { + input_tokens: 0, + cache_creation_input_tokens: cacheCreate, + cache_read_input_tokens: 0, + output_tokens: output, + }, + content: [{ type: "text", text: "x".repeat(50) }], + }, + }; +} + +async function writeJsonl(file, lines) { + await fs.writeFile(file, lines.map((l) => JSON.stringify(l)).join("\n") + "\n", "utf8"); +} + +async function appendAndTouch(file, lines) { + await fs.appendFile(file, lines.map((l) => JSON.stringify(l)).join("\n") + "\n", "utf8"); + const future = new Date(Date.now() + 2000); + fssync.utimesSync(file, future, future); +} + +async function setupCorpus(t) { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "tt-claude-perfile-")); + t.after(() => fs.rm(dir, { recursive: true, force: true })); + __resetCategoryCachesForTests(); + t.after(() => __resetCategoryCachesForTests()); + + const historical = []; + for (let i = 0; i < 3; i++) { + const file = path.join(dir, `session-hist-${i}.jsonl`); + await writeJsonl(file, [ + assistantEntry({ requestId: `rH${i}`, messageId: `mH${i}`, at: ts(60 + i) }), + ]); + historical.push(file); + } + const active = path.join(dir, "session-active.jsonl"); + await writeJsonl(active, [assistantEntry({ requestId: "rA1", messageId: "mA1", at: ts(5) })]); + + return { dir, historical, active }; +} + +test("appending to the active session reparses only that file", async (t) => { + const { dir, active } = await setupCorpus(t); + + const before = await computeClaudeCategoryBreakdown({ rootDir: dir }); + assert.deepStrictEqual([...__getCategoryParseCountsForTests().values()], [1, 1, 1, 1]); + + await appendAndTouch(active, [assistantEntry({ requestId: "rA2", messageId: "mA2", at: ts(1) })]); + + const warm = await computeClaudeCategoryBreakdown({ rootDir: dir }); + + for (const [filePath, count] of __getCategoryParseCountsForTests()) { + const expected = filePath === active ? 2 : 1; + assert.equal(count, expected, `${path.basename(filePath)} parsed ${count}x, expected ${expected}x`); + } + + assert.equal(warm.message_count, before.message_count + 1); + assert.ok(warm.totals.total_tokens > before.totals.total_tokens); +}); + +test("the incremental result is identical to a clean full scan", async (t) => { + const { dir, active } = await setupCorpus(t); + + await computeClaudeCategoryBreakdown({ rootDir: dir }); + await appendAndTouch(active, [assistantEntry({ requestId: "rA3", messageId: "mA3", at: ts(1) })]); + const warm = await computeClaudeCategoryBreakdown({ rootDir: dir }); + + __resetCategoryCachesForTests(); + const cold = await computeClaudeCategoryBreakdown({ rootDir: dir }); + + assert.deepStrictEqual(warm, cold, "cached recombination must equal a clean scan"); +}); + +test("a dedup hash shared across two files falls back to a full sequential scan", async (t) => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "tt-claude-collide-")); + t.after(() => fs.rm(dir, { recursive: true, force: true })); + __resetCategoryCachesForTests(); + t.after(() => __resetCategoryCachesForTests()); + + // Same requestId AND message.id in two files produces the same dedup key, so + // a full scan counts it once. Summing independent per-file results would + // count it twice — this is the case the collision guard exists for. + const at = ts(30); + await writeJsonl(path.join(dir, "session-one.jsonl"), [ + assistantEntry({ requestId: "shared", messageId: "shared-msg", at, output: 400 }), + ]); + await writeJsonl(path.join(dir, "session-two.jsonl"), [ + assistantEntry({ requestId: "shared", messageId: "shared-msg", at, output: 400 }), + assistantEntry({ requestId: "unique", messageId: "unique-msg", at, output: 100 }), + ]); + + const result = await computeClaudeCategoryBreakdown({ rootDir: dir }); + + // Two distinct messages survive dedup, not three. + assert.equal(result.message_count, 2, "the duplicated message must be counted once"); + + // The guard reparses everything sequentially, so each file is parsed twice: + // once optimistically, once in the fallback. + for (const [filePath, count] of __getCategoryParseCountsForTests()) { + assert.equal(count, 2, `${path.basename(filePath)} should be parsed twice (optimistic + fallback)`); + } +}); + +test("changing the date range does not serve a cached parse from another range", async (t) => { + const { dir } = await setupCorpus(t); + + await computeClaudeCategoryBreakdown({ rootDir: dir }); + const baseline = [...__getCategoryParseCountsForTests().values()]; + + const today = new Date().toISOString().slice(0, 10); + await computeClaudeCategoryBreakdown({ rootDir: dir, from: today, to: today }); + const after = [...__getCategoryParseCountsForTests().values()]; + + assert.deepStrictEqual( + after, + baseline.map((n) => n + 1), + "a different range must reparse; per-file entries are range-scoped", + ); +}); + +test("rotating a file invalidates its cached parse", async (t) => { + const { dir, active } = await setupCorpus(t); + + await computeClaudeCategoryBreakdown({ rootDir: dir }); + const before = __getCategoryParseCountsForTests().get(active); + + await fs.rm(active); + await writeJsonl(active, [ + assistantEntry({ requestId: "rotated", messageId: "rotated-msg", at: ts(2), output: 900 }), + ]); + + const rotated = await computeClaudeCategoryBreakdown({ rootDir: dir }); + assert.equal(__getCategoryParseCountsForTests().get(active), before + 1); + + __resetCategoryCachesForTests(); + const cold = await computeClaudeCategoryBreakdown({ rootDir: dir }); + assert.deepStrictEqual(rotated, cold); +}); From 679ba30b61a424776df7ea9660cec7c6e8bcf085 Mon Sep 17 00:00:00 2001 From: "itarun.p" Date: Tue, 21 Jul 2026 05:58:58 +0700 Subject: [PATCH 3/3] chore: refresh openwiki source facts after #70 The fact file records evidence as file:line, so merging #70 shifted the line numbers this branch's snapshot was pinned to and docs:openwiki:check failed on a gate unrelated to this branch's change. Regenerated; no source change. --- openwiki-facts/source-facts.json | 48 ++++++++++++++++---------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/openwiki-facts/source-facts.json b/openwiki-facts/source-facts.json index 1e272406..c99d4244 100644 --- a/openwiki-facts/source-facts.json +++ b/openwiki-facts/source-facts.json @@ -238,99 +238,99 @@ "parsers": [ { "name": "parseRolloutIncremental", - "evidence": "src/lib/rollout.js:104" + "evidence": "src/lib/rollout.js:86" }, { "name": "parseClaudeIncremental", - "evidence": "src/lib/rollout.js:226" + "evidence": "src/lib/rollout.js:208" }, { "name": "parseGeminiIncremental", - "evidence": "src/lib/rollout.js:349" + "evidence": "src/lib/rollout.js:331" }, { "name": "parseOpencodeIncremental", - "evidence": "src/lib/rollout.js:470" + "evidence": "src/lib/rollout.js:452" }, { "name": "parseOpenclawIncremental", - "evidence": "src/lib/rollout.js:623" + "evidence": "src/lib/rollout.js:605" }, { "name": "parseOpencodeDbIncremental", - "evidence": "src/lib/rollout.js:2496" + "evidence": "src/lib/rollout.js:2478" }, { "name": "parseCursorApiIncremental", - "evidence": "src/lib/rollout.js:2677" + "evidence": "src/lib/rollout.js:2659" }, { "name": "parseKiroIncremental", - "evidence": "src/lib/rollout.js:2935" + "evidence": "src/lib/rollout.js:2917" }, { "name": "parseHermesIncremental", - "evidence": "src/lib/rollout.js:3190" + "evidence": "src/lib/rollout.js:3172" }, { "name": "parseKiroCliIncremental", - "evidence": "src/lib/rollout.js:3809" + "evidence": "src/lib/rollout.js:3791" }, { "name": "parseKimiIncremental", - "evidence": "src/lib/rollout.js:4439" + "evidence": "src/lib/rollout.js:4421" }, { "name": "parseKimiCodeIncremental", - "evidence": "src/lib/rollout.js:4630" + "evidence": "src/lib/rollout.js:4612" }, { "name": "parseCodebuddyIncremental", - "evidence": "src/lib/rollout.js:4854" + "evidence": "src/lib/rollout.js:4836" }, { "name": "parseRoocodeIncremental", - "evidence": "src/lib/rollout.js:5357" + "evidence": "src/lib/rollout.js:5339" }, { "name": "parseZedIncremental", - "evidence": "src/lib/rollout.js:5674" + "evidence": "src/lib/rollout.js:5656" }, { "name": "parseGooseIncremental", - "evidence": "src/lib/rollout.js:5991" + "evidence": "src/lib/rollout.js:5973" }, { "name": "parseDroidIncremental", - "evidence": "src/lib/rollout.js:6368" + "evidence": "src/lib/rollout.js:6350" }, { "name": "parseKilocodeIncremental", - "evidence": "src/lib/rollout.js:6615" + "evidence": "src/lib/rollout.js:6597" }, { "name": "parseOmpIncremental", - "evidence": "src/lib/rollout.js:6751" + "evidence": "src/lib/rollout.js:6733" }, { "name": "parsePiIncremental", - "evidence": "src/lib/rollout.js:7004" + "evidence": "src/lib/rollout.js:6986" }, { "name": "parseCraftIncremental", - "evidence": "src/lib/rollout.js:7288" + "evidence": "src/lib/rollout.js:7270" }, { "name": "parseCopilotIncremental", - "evidence": "src/lib/rollout.js:7638" + "evidence": "src/lib/rollout.js:7620" }, { "name": "parseGrokBuildIncremental", - "evidence": "src/lib/rollout.js:8104" + "evidence": "src/lib/rollout.js:8086" }, { "name": "parseAntigravityIncremental", - "evidence": "src/lib/rollout.js:8276" + "evidence": "src/lib/rollout.js:8258" } ] }