diff --git a/openwiki-facts/source-facts.json b/openwiki-facts/source-facts.json index 1e272406..c99d4244 100644 --- a/openwiki-facts/source-facts.json +++ b/openwiki-facts/source-facts.json @@ -238,99 +238,99 @@ "parsers": [ { "name": "parseRolloutIncremental", - "evidence": "src/lib/rollout.js:104" + "evidence": "src/lib/rollout.js:86" }, { "name": "parseClaudeIncremental", - "evidence": "src/lib/rollout.js:226" + "evidence": "src/lib/rollout.js:208" }, { "name": "parseGeminiIncremental", - "evidence": "src/lib/rollout.js:349" + "evidence": "src/lib/rollout.js:331" }, { "name": "parseOpencodeIncremental", - "evidence": "src/lib/rollout.js:470" + "evidence": "src/lib/rollout.js:452" }, { "name": "parseOpenclawIncremental", - "evidence": "src/lib/rollout.js:623" + "evidence": "src/lib/rollout.js:605" }, { "name": "parseOpencodeDbIncremental", - "evidence": "src/lib/rollout.js:2496" + "evidence": "src/lib/rollout.js:2478" }, { "name": "parseCursorApiIncremental", - "evidence": "src/lib/rollout.js:2677" + "evidence": "src/lib/rollout.js:2659" }, { "name": "parseKiroIncremental", - "evidence": "src/lib/rollout.js:2935" + "evidence": "src/lib/rollout.js:2917" }, { "name": "parseHermesIncremental", - "evidence": "src/lib/rollout.js:3190" + "evidence": "src/lib/rollout.js:3172" }, { "name": "parseKiroCliIncremental", - "evidence": "src/lib/rollout.js:3809" + "evidence": "src/lib/rollout.js:3791" }, { "name": "parseKimiIncremental", - "evidence": "src/lib/rollout.js:4439" + "evidence": "src/lib/rollout.js:4421" }, { "name": "parseKimiCodeIncremental", - "evidence": "src/lib/rollout.js:4630" + "evidence": "src/lib/rollout.js:4612" }, { "name": "parseCodebuddyIncremental", - "evidence": "src/lib/rollout.js:4854" + "evidence": "src/lib/rollout.js:4836" }, { "name": "parseRoocodeIncremental", - "evidence": "src/lib/rollout.js:5357" + "evidence": "src/lib/rollout.js:5339" }, { "name": "parseZedIncremental", - "evidence": "src/lib/rollout.js:5674" + "evidence": "src/lib/rollout.js:5656" }, { "name": "parseGooseIncremental", - "evidence": "src/lib/rollout.js:5991" + "evidence": "src/lib/rollout.js:5973" }, { "name": "parseDroidIncremental", - "evidence": "src/lib/rollout.js:6368" + "evidence": "src/lib/rollout.js:6350" }, { "name": "parseKilocodeIncremental", - "evidence": "src/lib/rollout.js:6615" + "evidence": "src/lib/rollout.js:6597" }, { "name": "parseOmpIncremental", - "evidence": "src/lib/rollout.js:6751" + "evidence": "src/lib/rollout.js:6733" }, { "name": "parsePiIncremental", - "evidence": "src/lib/rollout.js:7004" + "evidence": "src/lib/rollout.js:6986" }, { "name": "parseCraftIncremental", - "evidence": "src/lib/rollout.js:7288" + "evidence": "src/lib/rollout.js:7270" }, { "name": "parseCopilotIncremental", - "evidence": "src/lib/rollout.js:7638" + "evidence": "src/lib/rollout.js:7620" }, { "name": "parseGrokBuildIncremental", - "evidence": "src/lib/rollout.js:8104" + "evidence": "src/lib/rollout.js:8086" }, { "name": "parseAntigravityIncremental", - "evidence": "src/lib/rollout.js:8276" + "evidence": "src/lib/rollout.js:8258" } ] } diff --git a/src/lib/claude-categorizer.js b/src/lib/claude-categorizer.js index 7a0ff731..8dcb9bdd 100644 --- a/src/lib/claude-categorizer.js +++ b/src/lib/claude-categorizer.js @@ -28,6 +28,7 @@ const { allocateByLargestRemainder, } = require("./categorizer-utils"); const { claudeMessageDedupKey } = require("./rollout"); +const { fileIdentity, isUnchanged } = require("./file-identity"); const CATEGORY_KEYS = [ "system_prefix", @@ -494,12 +495,40 @@ function classifyOneMessage(obj, sessionState, breakdown, toolLedger = null, ski } // Read one session jsonl streaming, in timestamp range, dedup by msgId+reqId. -async function categorizeSessionFile(filePath, { fromIso, toIso, seenHashes }, breakdown, toolLedger = null, skillLedger = null, execLedger = null) { +// Parses one file into its OWN accumulators and returns them, rather than +// mutating shared ones. That is what makes a per-file cache possible: the +// result depends only on (file contents, fromIso, toIso), so it can be reused +// whenever the file has not changed. +// +// The dedup set is now file-local. Cross-file duplicates are handled by the +// caller, which detects them and falls back to a full scan — see +// mergeFileResult. Intra-file dedup, including the deferred-add behavior +// below, is unchanged and must stay that way. +// `sharedSeen` is used only by the collision fallback, which needs the old +// cross-file dedup semantics. When it is passed, the returned `hashes` list is +// empty because dedup has already been applied globally. +async function categorizeSessionFile(filePath, { fromIso, toIso, sharedSeen = null }) { + const breakdown = emptyCategoryMap(); + const toolLedger = { + tool_calls: { total_calls: 0, by_name: new Map() }, + subagents: { total_calls: 0, by_name: new Map() }, + }; + const skillLedger = { total_calls: 0, by_name: new Map() }; + const execLedger = { + total_calls: 0, + by_type: new Map(), + by_executable: new Map(), + by_command: new Map(), + by_exit: new Map(), + }; + const seenHashes = sharedSeen || new Set(); + const empty = () => ({ breakdown, toolLedger, skillLedger, execLedger, counted: 0, hashes: [] }); + let stream; try { stream = fssync.createReadStream(filePath, { encoding: "utf8" }); } catch (_e) { - return 0; + return empty(); } const rl = readline.createInterface({ input: stream, crlfDelay: Infinity }); const sessionState = { systemPrefixSeen: false }; @@ -538,7 +567,14 @@ async function categorizeSessionFile(filePath, { fromIso, toIso, seenHashes }, b } rl.close(); stream.close?.(); - return counted; + return { + breakdown, + toolLedger, + skillLedger, + execLedger, + counted, + hashes: sharedSeen ? [] : [...seenHashes], + }; } // Convert a YYYY-MM-DD day key (already in the user's tz from the API call) @@ -624,6 +660,99 @@ function maxMtimeMs(files) { return max; } +// Per-file parse cache. See the equivalent in codex-context-breakdown.js — +// same problem (#62), same two-layer shape: the aggregate cache above stays +// the fast path, and this only decides how expensive a MISS is. +const FILE_CACHE = new Map(); +const FILE_CACHE_SCHEMA_VERSION = "claude-category-file-v1"; +const FILE_CACHE_MAX_ENTRIES = 8_000; + +// Instrumentation is opt-in: without it a long-running server would grow one +// Map entry per session file forever, purely to serve tests. +let trackParses = false; +const parseCounts = new Map(); + +function fileCacheKey(fromIso, toIso, filePath) { + return `${FILE_CACHE_SCHEMA_VERSION}|${fromIso || ""}|${toIso || ""}|${filePath}`; +} + +function pruneFileCache() { + if (FILE_CACHE.size <= FILE_CACHE_MAX_ENTRIES) return; + let excess = FILE_CACHE.size - FILE_CACHE_MAX_ENTRIES; + for (const key of FILE_CACHE.keys()) { + FILE_CACHE.delete(key); + if (--excess <= 0) break; + } +} + +// These fields are high-water marks, not running sums. Adding them would +// silently inflate the reported worst case. +const MAX_MERGE_FIELDS = new Set(["max_duration_ms"]); + +// Folds one cached row into an accumulator map. Always builds a fresh target +// object on first insert: putting a cached row in by reference would let a +// later merge mutate the cache in place, which is the aliasing hazard that +// per-file caching introduces. +function mergeRowInto(targetMap, key, row) { + const existing = targetMap.get(key); + if (!existing) { + targetMap.set(key, { ...row, totals: { ...row.totals } }); + return; + } + for (const [field, value] of Object.entries(row)) { + if (field === "totals") { + addInto(existing.totals, value); + } else if (typeof value === "number") { + existing[field] = MAX_MERGE_FIELDS.has(field) + ? Math.max(existing[field] || 0, value) + : (existing[field] || 0) + value; + } else if (existing[field] === undefined) { + existing[field] = value; + } + } +} + +function mergeNamedLedger(target, source) { + target.total_calls += source.total_calls || 0; + for (const [name, row] of source.by_name) mergeRowInto(target.by_name, name, row); +} + +// Returns false when this file shares a dedup hash with one already merged. +// Such a collision cannot be repaired from aggregates: the deduped message's +// tokens are missing, and worse, systemPrefixSeen may have latched onto a +// different message, invalidating the whole per-file result rather than one +// row. The caller re-runs a full sequential scan instead of patching. +function mergeFileResult(acc, result, globalHashes) { + for (const hash of result.hashes) { + if (globalHashes.has(hash)) return false; + globalHashes.add(hash); + } + for (const key of CATEGORY_KEYS) addInto(acc.breakdown[key], result.breakdown[key]); + mergeNamedLedger(acc.toolLedger.tool_calls, result.toolLedger.tool_calls); + mergeNamedLedger(acc.toolLedger.subagents, result.toolLedger.subagents); + mergeNamedLedger(acc.skillLedger, result.skillLedger); + acc.execLedger.total_calls += result.execLedger.total_calls || 0; + for (const bucket of ["by_type", "by_executable", "by_command", "by_exit"]) { + for (const [name, row] of result.execLedger[bucket]) { + mergeRowInto(acc.execLedger[bucket], name, row); + } + } + acc.messageCount += result.counted; + if (result.counted > 0) acc.sessionCount += 1; + return true; +} + +function __resetCategoryCachesForTests() { + trackParses = true; + CACHE.clear(); + FILE_CACHE.clear(); + parseCounts.clear(); +} + +function __getCategoryParseCountsForTests() { + return new Map(parseCounts); +} + async function computeClaudeCategoryBreakdown({ from = null, to = null, rootDir = null, projectDir = null } = {}) { const root = rootDir || defaultClaudeProjectsDir(); let files = []; @@ -656,49 +785,90 @@ async function computeClaudeCategoryBreakdown({ from = null, to = null, rootDir } const { fromIso, toIso } = dayKeyToIsoBounds(from, to); - const breakdown = emptyCategoryMap(); - const seenHashes = new Set(); - let messageCount = 0; - let sessionCount = 0; - const toolLedger = { - tool_calls: { total_calls: 0, by_name: new Map() }, - subagents: { total_calls: 0, by_name: new Map() }, - }; - const skillLedger = { total_calls: 0, by_name: new Map() }; - const execLedger = { - total_calls: 0, - by_type: new Map(), - by_executable: new Map(), - by_command: new Map(), - by_exit: new Map(), - }; + function newAccumulator() { + return { + breakdown: emptyCategoryMap(), + toolLedger: { + tool_calls: { total_calls: 0, by_name: new Map() }, + subagents: { total_calls: 0, by_name: new Map() }, + }, + skillLedger: { total_calls: 0, by_name: new Map() }, + execLedger: { + total_calls: 0, + by_type: new Map(), + by_executable: new Map(), + by_command: new Map(), + by_exit: new Map(), + }, + messageCount: 0, + sessionCount: 0, + }; + } // Process files with bounded parallelism. CPU-bound (JSON.parse per line) // limits the win, but overlapping the per-file fs.open + first-block read // I/O behind the previous file's parsing still shaves ~15% off cold scans. - // `seenHashes`/`breakdown`/ledgers are mutated only inside synchronous - // sections of `classifyOneMessage`; `for await` in `categorizeSessionFile` - // only yields between lines, so workers can't tear shared state. + // Each parse now owns its accumulators, so there is no shared state to tear + // — and an unchanged file skips the parse entirely (#62). const SCAN_CONCURRENCY = 4; + const results = new Array(files.length); let cursor = 0; async function worker() { while (cursor < files.length) { const idx = cursor++; if (idx >= files.length) return; - const counted = await categorizeSessionFile( - files[idx], - { fromIso, toIso, seenHashes }, - breakdown, - toolLedger, - skillLedger, - execLedger, - ); - if (counted > 0) sessionCount += 1; - messageCount += counted; + const filePath = files[idx]; + const key = fileCacheKey(fromIso, toIso, filePath); + const identity = fileIdentity(filePath); + const hit = FILE_CACHE.get(key); + if (hit && isUnchanged(hit.identity, identity)) { + results[idx] = hit.result; + continue; + } + if (trackParses) parseCounts.set(filePath, (parseCounts.get(filePath) || 0) + 1); + const result = await categorizeSessionFile(filePath, { fromIso, toIso }); + results[idx] = result; + if (identity) { + FILE_CACHE.set(key, { identity, result }); + pruneFileCache(); + } } } await Promise.all(Array.from({ length: SCAN_CONCURRENCY }, () => worker())); + // Merge in `files` order rather than completion order, so the result no + // longer depends on how the four workers happened to interleave. + let acc = newAccumulator(); + const globalHashes = new Set(); + let composable = true; + for (const result of results) { + if (!result) continue; + if (!mergeFileResult(acc, result, globalHashes)) { + composable = false; + break; + } + } + + if (!composable) { + // Two files claimed the same dedup hash, so per-file results overlap and + // cannot be summed. Redo the scan sequentially with one shared dedup set, + // which is exactly the pre-cache behavior. Expected never to happen: a + // scan of 6,529 real session files found 0 cross-file duplicates. Kept so + // correctness does not rest on that measurement. + acc = newAccumulator(); + const sharedSeen = new Set(); + const fallbackHashes = new Set(); + for (const filePath of files) { + const result = await categorizeSessionFile(filePath, { fromIso, toIso, sharedSeen }); + if (trackParses) parseCounts.set(filePath, (parseCounts.get(filePath) || 0) + 1); + mergeFileResult(acc, result, fallbackHashes); + } + } + + const { breakdown, toolLedger, skillLedger, execLedger } = acc; + const messageCount = acc.messageCount; + const sessionCount = acc.sessionCount; + const totals = emptyTotals(); for (const key of CATEGORY_KEYS) addInto(totals, breakdown[key]); @@ -1306,6 +1476,8 @@ async function computeClaudeGroundTruthBuckets({ rootDir = null } = {}) { } module.exports = { + __resetCategoryCachesForTests, + __getCategoryParseCountsForTests, CATEGORY_KEYS, computeClaudeCategoryBreakdown, computeClaudeGroundTruthBuckets, diff --git a/src/lib/codex-context-breakdown.js b/src/lib/codex-context-breakdown.js index 9ae4fede..5e6b056d 100644 --- a/src/lib/codex-context-breakdown.js +++ b/src/lib/codex-context-breakdown.js @@ -14,6 +14,8 @@ const fs = require("node:fs"); const os = require("node:os"); const path = require("node:path"); +const { fileIdentity, isUnchanged } = require("./file-identity"); + const { emptyTotals, addInto, @@ -139,6 +141,59 @@ function cacheTimeZoneKey(timeZoneContext) { return `${timeZoneContext.timeZone || ""}|${Number.isFinite(timeZoneContext.offsetMinutes) ? timeZoneContext.offsetMinutes : ""}`; } +// Per-file parse cache. +// +// The aggregate cache above keys on the GLOBAL maximum mtime, so appending to +// the one session that is currently active invalidates the aggregate for every +// unchanged historical file and forces a full rescan (issue #62). This layer +// makes that miss cost one file instead of all of them. +// +// `top` is deliberately absent from the key: truncation to the top N happens +// at merge time, so a single cached per-file parse serves every `top` value. +const FILE_CACHE = new Map(); +const FILE_CACHE_SCHEMA_VERSION = "codex-context-file-v1"; +// Roughly one entry per session file per (range, timezone) combination the +// dashboard asks for. Entries are small — one parsed rollup, no line data. +const FILE_CACHE_MAX_ENTRIES = 8_000; + +// Counts parses per file so tests can assert an unchanged file was NOT +// reparsed. Observing the actual claim beats spying on the filesystem, which +// would also catch the identity check's own 256-byte read. +let trackParses = false; +const parseCounts = new Map(); + +function fileCacheKey({ fromKey, toKey, timeZoneContext, filePath }) { + return [ + FILE_CACHE_SCHEMA_VERSION, + fromKey || "", + toKey || "", + cacheTimeZoneKey(timeZoneContext), + filePath, + ].join("|"); +} + +function pruneFileCache() { + if (FILE_CACHE.size <= FILE_CACHE_MAX_ENTRIES) return; + const excess = FILE_CACHE.size - FILE_CACHE_MAX_ENTRIES; + let removed = 0; + // Map iterates in insertion order, so this drops the least recently stored. + for (const key of FILE_CACHE.keys()) { + FILE_CACHE.delete(key); + if (++removed >= excess) break; + } +} + +function __resetContextCachesForTests() { + trackParses = true; + CACHE.clear(); + FILE_CACHE.clear(); + parseCounts.clear(); +} + +function __getParseCountsForTests() { + return new Map(parseCounts); +} + // --------------------------------------------------------------------------- // Main entry // --------------------------------------------------------------------------- @@ -182,13 +237,32 @@ async function computeCodexContextBreakdown({ const sessions = []; for (const filePath of files) { - const parsed = await parseCodexRolloutFile(filePath, { - fromIso, - toIso, - from: fromKey, - to: toKey, - timeZoneContext, - }); + const key = fileCacheKey({ fromKey, toKey, timeZoneContext, filePath }); + const identity = fileIdentity(filePath); + const hit = FILE_CACHE.get(key); + + let parsed; + if (hit && isUnchanged(hit.identity, identity)) { + parsed = hit.parsed; + } else { + if (trackParses) parseCounts.set(filePath, (parseCounts.get(filePath) || 0) + 1); + parsed = await parseCodexRolloutFile(filePath, { + fromIso, + toIso, + from: fromKey, + to: toKey, + timeZoneContext, + }); + // A null identity means the path is not a readable regular file right + // now; caching against it could never be invalidated correctly. + if (identity) { + FILE_CACHE.set(key, { identity, parsed }); + pruneFileCache(); + } + } + + // Re-applied on every pass rather than baked into the cached entry, so a + // zero-token file has one representation instead of two. if (!parsed || !parsed.totals || !parsed.totals.total_tokens) continue; sessions.push(parsed); } @@ -397,4 +471,6 @@ async function computeCodexContextBreakdown({ module.exports = { computeCodexContextBreakdown, + __resetContextCachesForTests, + __getParseCountsForTests, }; diff --git a/src/lib/file-identity.js b/src/lib/file-identity.js new file mode 100644 index 00000000..6310a92d --- /dev/null +++ b/src/lib/file-identity.js @@ -0,0 +1,69 @@ +const crypto = require("node:crypto"); +const fssync = require("node:fs"); + +// Fingerprint the first bytes of a file so rotation is detected even when the +// OS reuses the inode after unlink+recreate (Linux logrotate). Returns "" on +// any error so callers treat "no fingerprint" as "not changed". Also returns +// "" when the file is shorter than the fingerprint window: for a sub-256B +// file that is still being appended to, the window would extend into the +// not-yet-written append zone, so plain growth (not rotation) would shift +// the hash and produce a false "rotated" signal. +function readFileHeadSignature(filePath) { + try { + const fd = fssync.openSync(filePath, "r"); + try { + const buf = Buffer.alloc(256); + const bytes = fssync.readSync(fd, buf, 0, 256, 0); + if (bytes < 256) return ""; + return crypto.createHash("sha1").update(buf.subarray(0, bytes)).digest("hex"); + } finally { + fssync.closeSync(fd); + } + } catch { + return ""; + } +} + +// A token that changes whenever a file's contents could have changed. Returns +// null when the path is not a readable regular file, which callers must treat +// as "cannot be reused" rather than "unchanged". +function fileIdentity(filePath) { + let st = null; + try { + st = fssync.statSync(filePath); + } catch { + return null; + } + if (!st.isFile()) return null; + return { + inode: st.ino || 0, + size: Number.isFinite(st.size) ? st.size : 0, + mtimeMs: Number.isFinite(st.mtimeMs) ? st.mtimeMs : 0, + head: readFileHeadSignature(filePath), + }; +} + +// Equality, not ordering. Deliberately not `cur.mtimeMs > prev.mtimeMs`: a +// clock that moves backwards, a restored mtime, or a truncation must all read +// as "changed" and force a re-read. The cost of a false "changed" is a wasted +// parse; the cost of a false "unchanged" is serving stale data forever. +function isUnchanged(prev, cur) { + if (!prev || !cur) return false; + // An empty head means the file was too short to fingerprint, so it carries + // no rotation signal — fall back to inode/size/mtime alone rather than + // treating "" as a mismatch. + const headChanged = + typeof prev.head === "string" && prev.head !== "" && prev.head !== cur.head; + return ( + prev.inode === cur.inode && + prev.size === cur.size && + prev.mtimeMs === cur.mtimeMs && + !headChanged + ); +} + +module.exports = { + readFileHeadSignature, + fileIdentity, + isUnchanged, +}; diff --git a/src/lib/rollout.js b/src/lib/rollout.js index 76b705ae..7d81bbea 100644 --- a/src/lib/rollout.js +++ b/src/lib/rollout.js @@ -53,28 +53,10 @@ async function listClaudeProjectFiles(projectsDir) { return out; } -// Fingerprint the first bytes of a file so rotation is detected even when the -// OS reuses the inode after unlink+recreate (Linux logrotate). Returns "" on -// any error so callers treat "no fingerprint" as "not changed". Also returns -// "" when the file is shorter than the fingerprint window: for a sub-256B -// file that is still being appended to, the window would extend into the -// not-yet-written append zone, so plain growth (not rotation) would shift -// the hash and produce a false "rotated" signal. -function readFileHeadSignature(filePath) { - try { - const fd = fssync.openSync(filePath, "r"); - try { - const buf = Buffer.alloc(256); - const bytes = fssync.readSync(fd, buf, 0, 256, 0); - if (bytes < 256) return ""; - return crypto.createHash("sha1").update(buf.subarray(0, bytes)).digest("hex"); - } finally { - fssync.closeSync(fd); - } - } catch { - return ""; - } -} +// Moved to ./file-identity so the context-breakdown caches can share it +// rather than grow a second copy. Imported under the same name, so all 19 +// call sites below are unaffected. +const { readFileHeadSignature } = require("./file-identity"); async function listGeminiSessionFiles(tmpDir) { const out = []; diff --git a/test/claude-category-per-file-cache.test.js b/test/claude-category-per-file-cache.test.js new file mode 100644 index 00000000..643e4393 --- /dev/null +++ b/test/claude-category-per-file-cache.test.js @@ -0,0 +1,168 @@ +const assert = require("node:assert/strict"); +const fs = require("node:fs/promises"); +const fssync = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { test } = require("node:test"); + +const { + computeClaudeCategoryBreakdown, + __resetCategoryCachesForTests, + __getCategoryParseCountsForTests, +} = require("../src/lib/claude-categorizer"); + +// Issue #62: the aggregate cache keys on the GLOBAL maximum session-file mtime, +// so appending to the active session invalidated every historical file. These +// pin that only changed files are reparsed AND that the incremental result is +// identical to a clean scan — including when files share a dedup hash, which is +// the case that makes per-file results non-composable. + +function ts(minutesAgo) { + return new Date(Date.now() - minutesAgo * 60_000).toISOString(); +} + +function assistantEntry({ requestId, messageId, at, output = 200, cacheCreate = 1000 }) { + return { + type: "assistant", + timestamp: at, + requestId, + message: { + id: messageId, + usage: { + input_tokens: 0, + cache_creation_input_tokens: cacheCreate, + cache_read_input_tokens: 0, + output_tokens: output, + }, + content: [{ type: "text", text: "x".repeat(50) }], + }, + }; +} + +async function writeJsonl(file, lines) { + await fs.writeFile(file, lines.map((l) => JSON.stringify(l)).join("\n") + "\n", "utf8"); +} + +async function appendAndTouch(file, lines) { + await fs.appendFile(file, lines.map((l) => JSON.stringify(l)).join("\n") + "\n", "utf8"); + const future = new Date(Date.now() + 2000); + fssync.utimesSync(file, future, future); +} + +async function setupCorpus(t) { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "tt-claude-perfile-")); + t.after(() => fs.rm(dir, { recursive: true, force: true })); + __resetCategoryCachesForTests(); + t.after(() => __resetCategoryCachesForTests()); + + const historical = []; + for (let i = 0; i < 3; i++) { + const file = path.join(dir, `session-hist-${i}.jsonl`); + await writeJsonl(file, [ + assistantEntry({ requestId: `rH${i}`, messageId: `mH${i}`, at: ts(60 + i) }), + ]); + historical.push(file); + } + const active = path.join(dir, "session-active.jsonl"); + await writeJsonl(active, [assistantEntry({ requestId: "rA1", messageId: "mA1", at: ts(5) })]); + + return { dir, historical, active }; +} + +test("appending to the active session reparses only that file", async (t) => { + const { dir, active } = await setupCorpus(t); + + const before = await computeClaudeCategoryBreakdown({ rootDir: dir }); + assert.deepStrictEqual([...__getCategoryParseCountsForTests().values()], [1, 1, 1, 1]); + + await appendAndTouch(active, [assistantEntry({ requestId: "rA2", messageId: "mA2", at: ts(1) })]); + + const warm = await computeClaudeCategoryBreakdown({ rootDir: dir }); + + for (const [filePath, count] of __getCategoryParseCountsForTests()) { + const expected = filePath === active ? 2 : 1; + assert.equal(count, expected, `${path.basename(filePath)} parsed ${count}x, expected ${expected}x`); + } + + assert.equal(warm.message_count, before.message_count + 1); + assert.ok(warm.totals.total_tokens > before.totals.total_tokens); +}); + +test("the incremental result is identical to a clean full scan", async (t) => { + const { dir, active } = await setupCorpus(t); + + await computeClaudeCategoryBreakdown({ rootDir: dir }); + await appendAndTouch(active, [assistantEntry({ requestId: "rA3", messageId: "mA3", at: ts(1) })]); + const warm = await computeClaudeCategoryBreakdown({ rootDir: dir }); + + __resetCategoryCachesForTests(); + const cold = await computeClaudeCategoryBreakdown({ rootDir: dir }); + + assert.deepStrictEqual(warm, cold, "cached recombination must equal a clean scan"); +}); + +test("a dedup hash shared across two files falls back to a full sequential scan", async (t) => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "tt-claude-collide-")); + t.after(() => fs.rm(dir, { recursive: true, force: true })); + __resetCategoryCachesForTests(); + t.after(() => __resetCategoryCachesForTests()); + + // Same requestId AND message.id in two files produces the same dedup key, so + // a full scan counts it once. Summing independent per-file results would + // count it twice — this is the case the collision guard exists for. + const at = ts(30); + await writeJsonl(path.join(dir, "session-one.jsonl"), [ + assistantEntry({ requestId: "shared", messageId: "shared-msg", at, output: 400 }), + ]); + await writeJsonl(path.join(dir, "session-two.jsonl"), [ + assistantEntry({ requestId: "shared", messageId: "shared-msg", at, output: 400 }), + assistantEntry({ requestId: "unique", messageId: "unique-msg", at, output: 100 }), + ]); + + const result = await computeClaudeCategoryBreakdown({ rootDir: dir }); + + // Two distinct messages survive dedup, not three. + assert.equal(result.message_count, 2, "the duplicated message must be counted once"); + + // The guard reparses everything sequentially, so each file is parsed twice: + // once optimistically, once in the fallback. + for (const [filePath, count] of __getCategoryParseCountsForTests()) { + assert.equal(count, 2, `${path.basename(filePath)} should be parsed twice (optimistic + fallback)`); + } +}); + +test("changing the date range does not serve a cached parse from another range", async (t) => { + const { dir } = await setupCorpus(t); + + await computeClaudeCategoryBreakdown({ rootDir: dir }); + const baseline = [...__getCategoryParseCountsForTests().values()]; + + const today = new Date().toISOString().slice(0, 10); + await computeClaudeCategoryBreakdown({ rootDir: dir, from: today, to: today }); + const after = [...__getCategoryParseCountsForTests().values()]; + + assert.deepStrictEqual( + after, + baseline.map((n) => n + 1), + "a different range must reparse; per-file entries are range-scoped", + ); +}); + +test("rotating a file invalidates its cached parse", async (t) => { + const { dir, active } = await setupCorpus(t); + + await computeClaudeCategoryBreakdown({ rootDir: dir }); + const before = __getCategoryParseCountsForTests().get(active); + + await fs.rm(active); + await writeJsonl(active, [ + assistantEntry({ requestId: "rotated", messageId: "rotated-msg", at: ts(2), output: 900 }), + ]); + + const rotated = await computeClaudeCategoryBreakdown({ rootDir: dir }); + assert.equal(__getCategoryParseCountsForTests().get(active), before + 1); + + __resetCategoryCachesForTests(); + const cold = await computeClaudeCategoryBreakdown({ rootDir: dir }); + assert.deepStrictEqual(rotated, cold); +}); diff --git a/test/codex-context-per-file-cache.test.js b/test/codex-context-per-file-cache.test.js new file mode 100644 index 00000000..9a83d37f --- /dev/null +++ b/test/codex-context-per-file-cache.test.js @@ -0,0 +1,190 @@ +const assert = require("node:assert/strict"); +const fs = require("node:fs/promises"); +const fssync = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { test } = require("node:test"); + +const { + computeCodexContextBreakdown, + __resetContextCachesForTests, + __getParseCountsForTests, +} = require("../src/lib/codex-context-breakdown"); + +// Issue #62: the aggregate cache keys on the GLOBAL maximum session-file mtime, +// so appending to the one active session invalidated every historical file and +// forced a full rescan. These pin that only changed files are reparsed, and +// that the cached aggregate still equals a clean scan. + +const DAY = "2026-05-08"; + +function rolloutEvents({ sessionId, at, inputTokens, outputTokens }) { + return [ + { + timestamp: `${DAY}T${at}.000Z`, + type: "session_meta", + payload: { id: sessionId, cwd: "/tmp/project", model_provider: "openai", cli_version: "1.0.0" }, + }, + { + timestamp: `${DAY}T${at}.100Z`, + type: "turn_context", + payload: { cwd: "/tmp/project", model: "gpt-5.5" }, + }, + { + timestamp: `${DAY}T${at}.200Z`, + type: "response_item", + payload: { type: "function_call", name: "exec_command", call_id: `call-${sessionId}`, arguments: "{}" }, + }, + { + timestamp: `${DAY}T${at}.300Z`, + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: inputTokens, + cached_input_tokens: 0, + output_tokens: outputTokens, + reasoning_output_tokens: 0, + total_tokens: inputTokens + outputTokens, + }, + }, + }, + }, + ]; +} + +async function writeRollout(rootDir, fileName, events) { + const dir = path.join(rootDir, DAY.slice(0, 4), DAY.slice(5, 7), DAY.slice(8, 10)); + await fs.mkdir(dir, { recursive: true }); + const filePath = path.join(dir, fileName); + await fs.writeFile(filePath, events.map((e) => JSON.stringify(e)).join("\n") + "\n", "utf8"); + return filePath; +} + +// Appending and then re-stat-ing within the same millisecond can leave mtimeMs +// unchanged, which would make this test pass for the wrong reason. Push mtime +// forward explicitly so the identity check is exercised, not the clock. +async function appendAndTouch(filePath, events) { + await fs.appendFile(filePath, events.map((e) => JSON.stringify(e)).join("\n") + "\n", "utf8"); + const future = new Date(Date.now() + 2000); + fssync.utimesSync(filePath, future, future); +} + +async function setupCorpus(t) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "tt-codex-perfile-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + __resetContextCachesForTests(); + t.after(() => __resetContextCachesForTests()); + + const historical = []; + for (let i = 0; i < 3; i++) { + historical.push( + await writeRollout( + root, + `rollout-hist-${i}.jsonl`, + rolloutEvents({ sessionId: `hist-${i}`, at: `1${i}:00:00`, inputTokens: 100, outputTokens: 50 }), + ), + ); + } + const active = await writeRollout( + root, + "rollout-active.jsonl", + rolloutEvents({ sessionId: "active", at: "14:00:00", inputTokens: 200, outputTokens: 100 }), + ); + + return { root, historical, active }; +} + +test("appending to the active session reparses only that file", async (t) => { + const { root, active } = await setupCorpus(t); + + const before = await computeCodexContextBreakdown({ codexDir: root }); + const cold = __getParseCountsForTests(); + assert.equal(cold.size, 4, "first pass must parse all four files"); + assert.deepStrictEqual([...cold.values()], [1, 1, 1, 1]); + + await appendAndTouch(active, rolloutEvents({ sessionId: "active", at: "15:00:00", inputTokens: 7, outputTokens: 3 })); + + const warm = await computeCodexContextBreakdown({ codexDir: root }); + const after = __getParseCountsForTests(); + + // The three historical files must not have been parsed a second time. + for (const [filePath, count] of after) { + const expected = filePath === active ? 2 : 1; + assert.equal(count, expected, `${path.basename(filePath)} parsed ${count}x, expected ${expected}x`); + } + + assert.ok( + warm.totals.total_tokens > before.totals.total_tokens, + "the appended record must be reflected in the aggregate", + ); +}); + +test("the cached aggregate equals a clean full scan after an append", async (t) => { + const { root, active } = await setupCorpus(t); + + await computeCodexContextBreakdown({ codexDir: root }); + await appendAndTouch(active, rolloutEvents({ sessionId: "active", at: "16:00:00", inputTokens: 11, outputTokens: 5 })); + const warm = await computeCodexContextBreakdown({ codexDir: root }); + + // Cold: no aggregate cache, no per-file cache, nothing carried over. + __resetContextCachesForTests(); + const cold = await computeCodexContextBreakdown({ codexDir: root }); + + assert.deepStrictEqual(warm, cold, "incremental result must be identical to a clean scan"); +}); + +test("changing only top reuses the per-file cache", async (t) => { + const { root } = await setupCorpus(t); + + await computeCodexContextBreakdown({ codexDir: root, top: 20 }); + const baseline = __getParseCountsForTests(); + + await computeCodexContextBreakdown({ codexDir: root, top: 5 }); + const after = __getParseCountsForTests(); + + // top is applied at merge time, so it is deliberately absent from the + // per-file key. If someone adds it, this test is what catches the + // regression: every file would reparse for a purely cosmetic change. + assert.deepStrictEqual([...after.values()], [...baseline.values()], "changing top must not reparse anything"); +}); + +test("changing the timezone context does reparse", async (t) => { + const { root } = await setupCorpus(t); + + await computeCodexContextBreakdown({ codexDir: root }); + const baseline = [...__getParseCountsForTests().values()]; + + await computeCodexContextBreakdown({ + codexDir: root, + timeZoneContext: { timeZone: "Asia/Bangkok", offsetMinutes: 420 }, + }); + const after = [...__getParseCountsForTests().values()]; + + // Bucketing depends on the timezone, so cached parses are not reusable. + assert.deepStrictEqual(after, baseline.map((n) => n + 1), "a timezone change must reparse every file"); +}); + +test("rotation and truncation invalidate the per-file cache", async (t) => { + const { root, active } = await setupCorpus(t); + + await computeCodexContextBreakdown({ codexDir: root }); + const before = __getParseCountsForTests().get(active); + + // Rewrite with different leading bytes and a different length: the inode may + // be reused, so this leans on the head fingerprint and the size check. + await fs.rm(active); + await writeRollout( + root, + "rollout-active.jsonl", + rolloutEvents({ sessionId: "rotated", at: "17:00:00", inputTokens: 999, outputTokens: 1 }), + ); + + const rotated = await computeCodexContextBreakdown({ codexDir: root }); + assert.equal(__getParseCountsForTests().get(active), before + 1, "a rotated file must be reparsed"); + + __resetContextCachesForTests(); + const cold = await computeCodexContextBreakdown({ codexDir: root }); + assert.deepStrictEqual(rotated, cold, "post-rotation result must match a clean scan"); +});