diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2db7e708..a559b7ba 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -62,6 +62,10 @@ This is the most common kind of contribution. The pattern: 2. **Add a hook installer in `src/commands/init.js`** — most tools support a config file or hook script you can patch. Make it idempotent (safe to re-run). 3. **Add a status check in `src/commands/status.js`** — show whether the hook is installed and whether data has been collected. 4. **Add a parser test in its own `test/-parser.test.js`** — use a real (anonymized) sample log fixture. Recent tools each got their own file (`droid-parser.test.js`, `zed-parser.test.js`, `goose-parser.test.js`); only the older ones share `rollout-parser.test.js`. + + **Also add a conformance fixture: `test/fixtures/parser-conformance/.cjs`.** `test/parser-conformance.test.js` enumerates every `parse*Incremental` from `src/lib/rollout.js` **source**, so a new parser without a fixture fails the build — it cannot be silently skipped. The alternative is an entry in `allowlist.json` explaining why a fixture is not possible (a SQLite schema, a live API shape), and that list is a ratchet: it may only shrink. + + The fixture checks the parser's *output* — the column invariant `total = input + output + cache_creation + cache_read + reasoning`, non-negative counts, a model, a 30-minute UTC bucket, and that re-parsing the same input does not double-count. It deliberately does **not** prove you read the provider's format correctly; a parser that double-counts cache reads and inflates the total to match still passes. That is what step 4's real sample log is for. Copy `craft.cjs` for the shape. 5. **Add the tool to the Supported tools list in `README.md`** — the blockquote under "🔌 Supported tools". Leave the "20+" count alone; a hard number in prose has no validator and goes stale silently. Look at how Claude Code, Codex, or Gemini are wired in for reference — they're the simplest examples. diff --git a/test/fixtures/parser-conformance/allowlist.json b/test/fixtures/parser-conformance/allowlist.json new file mode 100644 index 00000000..42c0042a --- /dev/null +++ b/test/fixtures/parser-conformance/allowlist.json @@ -0,0 +1,28 @@ +{ + "_why": "Parsers with no conformance fixture yet. This list is a RATCHET: parser-conformance.test.js fails if it is longer than max_size, and also fails if it is SHORTER, telling you to lower max_size in the same commit. An allowlist that can grow is a TODO list, and this repo has evidence about what happens to those.", + "_how_to_remove_an_entry": "Add test/fixtures/parser-conformance/.cjs (see craft.cjs for the shape), delete the entry here, and lower max_size by one.", + "max_size": 21, + "parsers": { + "parseRolloutIncremental": "Codex rollout JSONL. Covered behaviourally by test/rollout-parser.test.js; no conformance fixture yet.", + "parseClaudeIncremental": "Covered behaviourally by test/rollout-parser.test.js; no conformance fixture yet.", + "parseGeminiIncremental": "Covered behaviourally by test/rollout-parser.test.js; no conformance fixture yet.", + "parseOpencodeIncremental": "Covered behaviourally elsewhere; no conformance fixture yet.", + "parseOpenclawIncremental": "No conformance fixture yet.", + "parseOpencodeDbIncremental": "Reads a SQLite database. A fixture needs a real schema, not a JSONL line.", + "parseCursorApiIncremental": "Talks to Cursor's API shape rather than a local log; covered by test/cursor-parser.test.js.", + "parseKiroIncremental": "No conformance fixture yet; test/fixtures/kiro-cli exists for the CLI variant.", + "parseHermesIncremental": "Reads a SQLite database across profile directories. A fixture needs a real schema, not a JSONL line.", + "parseKiroCliIncremental": "No conformance fixture yet.", + "parseKimiCodeIncremental": "No conformance fixture yet.", + "parseRoocodeIncremental": "Covered by test/roocode-parser.test.js; no conformance fixture yet.", + "parseZedIncremental": "Covered by test/zed-parser.test.js; no conformance fixture yet.", + "parseGooseIncremental": "Covered by test/goose-parser.test.js; no conformance fixture yet.", + "parseDroidIncremental": "Covered by test/droid-parser.test.js; no conformance fixture yet.", + "parseKilocodeIncremental": "Covered by test/kilocode-parser.test.js; no conformance fixture yet.", + "parseOmpIncremental": "No conformance fixture yet.", + "parsePiIncremental": "No conformance fixture yet.", + "parseCopilotIncremental": "Reads the Copilot quota API shape; counts are unverified against a live account (#105).", + "parseGrokBuildIncremental": "No conformance fixture yet.", + "parseAntigravityIncremental": "Finds the app by lsof-ing listening ports; not reducible to a file fixture today (#105)." + } +} diff --git a/test/fixtures/parser-conformance/codebuddy.cjs b/test/fixtures/parser-conformance/codebuddy.cjs new file mode 100644 index 00000000..82869543 --- /dev/null +++ b/test/fixtures/parser-conformance/codebuddy.cjs @@ -0,0 +1,58 @@ +"use strict"; + +// Conformance fixture for parseCodebuddyIncremental. +// +// CodeBuddy writes JSONL where assistant messages carry `providerData.rawUsage` +// in OpenAI's shape. The load-bearing detail, and the reason this parser is +// worth a fixture at all: **`prompt_tokens` INCLUDES the cached tokens**, so +// `input_tokens` is `prompt_tokens - cache_read`. Getting that wrong +// double-counts cache reads — the cached-input-semantics failure class +// CLAUDE.md records at 1.6-7x magnitude. +// +// The numbers below are chosen so a parser that forgot the subtraction would +// break the column-sum invariant rather than merely be a bit off. + +const fs = require("node:fs"); +const path = require("node:path"); + +const assistantMessage = (uuid, tsMs, rawUsage) => + JSON.stringify({ + type: "message", + role: "assistant", + sessionId: "codebuddy-session-1", + uuid, + timestamp: tsMs, + providerData: { rawUsage }, + }) + "\n"; + +// Fixed instant, so the 30-minute bucket is deterministic rather than +// whatever the clock says when CI happens to run. +const T = Date.parse("2026-05-14T09:12:30.000Z"); + +module.exports = { + source: "codebuddy", + parser: "parseCodebuddyIncremental", + build(dir) { + const log = path.join(dir, "codebuddy-session-1.jsonl"); + fs.writeFileSync( + log, + [ + assistantMessage("cb-msg-1", T, { + // 3000 prompt tokens of which 2400 were cache reads -> 600 real input. + prompt_tokens: 3000, + completion_tokens: 250, + prompt_tokens_details: { cached_tokens: 2400 }, + cache_creation_input_tokens: 100, + }), + // A user turn, which carries no usage and must be skipped. + JSON.stringify({ type: "message", role: "user", uuid: "cb-user-1" }) + "\n", + assistantMessage("cb-msg-2", T + 60_000, { + prompt_tokens: 40, + completion_tokens: 8, + prompt_tokens_details: {}, + }), + ].join(""), + ); + return { projectFiles: [log], defaultModel: "codebuddy-default" }; + }, +}; diff --git a/test/fixtures/parser-conformance/craft.cjs b/test/fixtures/parser-conformance/craft.cjs new file mode 100644 index 00000000..4c3fc3fe --- /dev/null +++ b/test/fixtures/parser-conformance/craft.cjs @@ -0,0 +1,51 @@ +"use strict"; + +// Conformance fixture for parseCraftIncremental. +// +// Craft writes one JSONL file per session whose FIRST line is a header carrying +// running totals: `{ id, tokenUsage: { inputTokens, outputTokens, +// cacheReadTokens, cacheCreationTokens, totalTokens } }`. The header is +// rewritten in place as the session grows, so the parser contributes deltas. +// That snapshot-vs-cumulative shape is one of the failure classes CLAUDE.md +// records at 1.6-7x magnitude, which is why it is worth a fixture. + +const fs = require("node:fs"); +const path = require("node:path"); + +const session = (id, usage) => + JSON.stringify({ + id, + model: "claude-sonnet-5", + tokenUsage: usage, + }) + "\n"; + +module.exports = { + source: "craft", + parser: "parseCraftIncremental", + build(dir) { + const one = path.join(dir, "session-one.jsonl"); + const two = path.join(dir, "session-two.jsonl"); + fs.writeFileSync( + one, + session("craft-session-one", { + inputTokens: 1200, + outputTokens: 340, + cacheReadTokens: 800, + cacheCreationTokens: 60, + totalTokens: 2400, + }), + ); + // Zero cache columns: a row that omits them must still satisfy the column + // sum rather than being excused from it. + fs.writeFileSync( + two, + session("craft-session-two", { + inputTokens: 90, + outputTokens: 10, + cacheReadTokens: 0, + cacheCreationTokens: 0, + }), + ); + return { sessionFiles: [one, two] }; + }, +}; diff --git a/test/fixtures/parser-conformance/kimi.cjs b/test/fixtures/parser-conformance/kimi.cjs new file mode 100644 index 00000000..70399f58 --- /dev/null +++ b/test/fixtures/parser-conformance/kimi.cjs @@ -0,0 +1,52 @@ +"use strict"; + +// Conformance fixture for parseKimiIncremental. +// +// Kimi writes a wire log of JSONL envelopes; only `message.type === "StatusUpdate"` +// carries usage, under `payload.token_usage`, and `payload.message_id` is the +// dedup key. The four token fields arrive already separated — `input_other` is +// non-cached input — so this fixture's job is to hold that mapping still and to +// prove the message_id dedup survives a second parse. + +const fs = require("node:fs"); +const path = require("node:path"); + +// Fixed instant so the 30-minute bucket is deterministic rather than whatever +// the clock says when CI runs. Kimi's timestamp is epoch SECONDS, not ms. +const T_SECONDS = Date.parse("2026-05-14T09:12:30.000Z") / 1000; + +const statusUpdate = (messageId, usage, atSeconds) => + JSON.stringify({ + timestamp: atSeconds, + message: { + type: "StatusUpdate", + payload: { message_id: messageId, token_usage: usage }, + }, + }) + "\n"; + +module.exports = { + source: "kimi", + parser: "parseKimiIncremental", + build(dir) { + const wire = path.join(dir, "wire.jsonl"); + fs.writeFileSync( + wire, + [ + statusUpdate( + "kimi-msg-1", + { input_other: 500, output: 120, input_cache_read: 2000, input_cache_creation: 40 }, + T_SECONDS, + ), + // A second message in the same file, and one line the parser must skip + // rather than choke on. + JSON.stringify({ message: { type: "Heartbeat" } }) + "\n", + statusUpdate( + "kimi-msg-2", + { input_other: 30, output: 7, input_cache_read: 0, input_cache_creation: 0 }, + T_SECONDS + 60, + ), + ].join(""), + ); + return { wireFiles: [wire], model: "kimi-k2" }; + }, +}; diff --git a/test/parser-conformance.test.js b/test/parser-conformance.test.js new file mode 100644 index 00000000..4da4781a --- /dev/null +++ b/test/parser-conformance.test.js @@ -0,0 +1,229 @@ +"use strict"; + +// Parser conformance ratchet. +// +// Test coverage in this repo is inverted relative to risk. CLAUDE.md:169-172 +// records the historical failure modes — dedup failures, cached-input +// semantics, snapshot-vs-cumulative confusion — as 1.6x to 7x magnitude errors +// in the numbers the whole product exists to report. That is the class with no +// gate: README:116 tells contributors "a new provider is usually one parser file +// away", and today that is true and unguarded. +// +// So this harness ENUMERATES THE PARSERS FROM SOURCE rather than from a +// hand-written list. A parser added without a fixture and without an allowlist +// entry fails here; it cannot be silently skipped. The allowlist is a ratchet +// that may only shrink, because an allowlist that can grow is a TODO list. +// +// What a fixture proves: the parser's OUTPUT satisfies the queue's column +// invariant, bucket alignment and non-negativity, and that parsing the same +// input twice does not double-count. +// +// What it does NOT prove, and this matters more than the list above: that the +// parser reads the provider's real format correctly. A parser that double-counts +// cache reads into input_tokens and inflates total_tokens to match is internally +// consistent — the column sum passes it while the number the user reads is +// several times too big. There is a test below that pins exactly that blind +// spot, so nobody reads this harness as covering more than it does. Only a real +// sample log with known-correct expected values catches it, which is what the +// per-tool tests are for. This is the floor under them, not a replacement. + +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { test } = require("node:test"); + +const rollout = require("../src/lib/rollout"); + +const FIXTURE_DIR = path.join(__dirname, "fixtures", "parser-conformance"); +const ALLOWLIST = require("./fixtures/parser-conformance/allowlist.json"); + +// From source, not from Object.keys(rollout) — an export could be renamed or a +// parser could be defined and never exported, and either way the list has to +// reflect what is actually in the file. +function enumerateParsers() { + const source = fs.readFileSync(path.join(__dirname, "..", "src", "lib", "rollout.js"), "utf8"); + return [...source.matchAll(/^(?:async\s+)?function\s+(parse\w*Incremental)\s*\(/gm)].map( + (m) => m[1], + ); +} + +function fixtureFor(parser) { + for (const entry of fs.readdirSync(FIXTURE_DIR)) { + if (!entry.endsWith(".cjs")) continue; + const mod = require(path.join(FIXTURE_DIR, entry)); + if (mod.parser === parser) return mod; + } + return null; +} + +function readQueue(queuePath) { + if (!fs.existsSync(queuePath)) return []; + return fs + .readFileSync(queuePath, "utf8") + .split("\n") + .filter((l) => l.trim()) + .map((l) => JSON.parse(l)); +} + +const COLUMNS = [ + "input_tokens", + "output_tokens", + "cache_creation_input_tokens", + "cached_input_tokens", + "reasoning_output_tokens", +]; + +const THIRTY_MINUTES_MS = 30 * 60 * 1000; + +function assertRowConforms(row, label) { + for (const column of COLUMNS) { + const value = Number(row[column] || 0); + assert.ok( + Number.isFinite(value) && value >= 0, + `${label}: ${column} must be a non-negative number, got ${JSON.stringify(row[column])}`, + ); + } + // CLAUDE.md's "Token normalization" block, which lived in prose and nothing + // enforced. A miswritten row was aggregated and rendered, not flagged. + const sum = COLUMNS.reduce((acc, column) => acc + Number(row[column] || 0), 0); + assert.equal( + Number(row.total_tokens || 0), + sum, + `${label}: total_tokens must equal the sum of the columns`, + ); + + assert.ok( + typeof row.model === "string" && row.model.trim().length > 0, + `${label}: model must be present — "unknown" is the placeholder for "we did not record one" (#94)`, + ); + + const bucket = Date.parse(row.hour_start); + assert.ok(Number.isFinite(bucket), `${label}: hour_start must be a timestamp`); + assert.equal( + bucket % THIRTY_MINUTES_MS, + 0, + `${label}: hour_start must land on a 30-minute UTC boundary, got ${row.hour_start}`, + ); +} + +const GOOD_ROW = { + input_tokens: 600, + output_tokens: 250, + cache_creation_input_tokens: 100, + cached_input_tokens: 2400, + reasoning_output_tokens: 0, + total_tokens: 3350, + model: "claude-sonnet-5", + hour_start: "2026-05-14T09:00:00.000Z", +}; + +test("the conformance check rejects the rows it exists to reject", () => { + // A checker nothing has ever failed is a checker nobody has tested. Each case + // below is one of the shapes CLAUDE.md records as having really happened. + assert.doesNotThrow(() => assertRowConforms(GOOD_ROW, "control")); + + const rejects = { + "column sum off — the invariant that lived only in prose": { total_tokens: 3351 }, + "a negative count": { input_tokens: -1, total_tokens: 2749 }, + "no model at all": { model: "" }, + "a bucket off the 30-minute boundary": { hour_start: "2026-05-14T09:07:00.000Z" }, + }; + for (const [label, override] of Object.entries(rejects)) { + assert.throws( + () => assertRowConforms({ ...GOOD_ROW, ...override }, label), + assert.AssertionError, + `the check must reject: ${label}`, + ); + } +}); + +test("what the column invariant does NOT catch, stated rather than assumed", () => { + // Worth pinning, because it is easy to read the invariant as covering more + // than it does. A parser that double-counts cache reads into input_tokens AND + // inflates total_tokens to match is internally consistent, so the column sum + // passes it — while the number the user reads is 4x too big. That is the + // cached-input-semantics class CLAUDE.md records at 1.6-7x magnitude. + // + // Nothing structural catches that. Only a real sample log with known-correct + // expected values does, which is what the per-tool tests are for. This + // harness is the floor under them, not a replacement. + const doubleCounted = { + ...GOOD_ROW, + input_tokens: 3000, // should be 600: prompt_tokens minus the 2400 cache reads + total_tokens: 5750, // inflated to match, so the sum still balances + }; + assert.doesNotThrow( + () => assertRowConforms(doubleCounted, "double-counted"), + "if this ever throws, the harness got stronger and this note should be rewritten", + ); +}); + +const parsers = enumerateParsers(); + +test("the harness finds the parsers, so a rename cannot empty it", () => { + assert.ok(parsers.length >= 20, `expected the full parser set, found ${parsers.length}`); + assert.ok(parsers.includes("parseClaudeIncremental")); +}); + +test("every parser has a fixture or an allowlist entry — a new one cannot opt out", () => { + const uncovered = parsers.filter( + (parser) => !fixtureFor(parser) && !ALLOWLIST.parsers[parser], + ); + assert.deepEqual( + uncovered, + [], + `these parsers have neither a conformance fixture nor a recorded reason: ${uncovered.join(", ")}.` + + ` Add test/fixtures/parser-conformance/.cjs, or add an entry to allowlist.json saying why not.`, + ); +}); + +test("the allowlist may only shrink", () => { + const size = Object.keys(ALLOWLIST.parsers).length; + assert.ok( + size <= ALLOWLIST.max_size, + `the allowlist grew to ${size}, above its committed ceiling of ${ALLOWLIST.max_size}.` + + ` Write a fixture instead of raising the ceiling.`, + ); + assert.equal( + size, + ALLOWLIST.max_size, + `the allowlist is down to ${size} — lower max_size to ${size} in the same commit so the` + + ` ratchet records the ground gained.`, + ); +}); + +test("no allowlist entry names a parser that no longer exists", () => { + const stale = Object.keys(ALLOWLIST.parsers).filter((p) => !parsers.includes(p)); + assert.deepEqual(stale, [], `stale allowlist entries: ${stale.join(", ")}`); +}); + +for (const parser of parsers) { + const fixture = fixtureFor(parser); + if (!fixture) continue; + + test(`${parser}: output conforms, and parsing twice does not double-count`, async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "tt-conformance-")); + const queuePath = path.join(root, "queue.jsonl"); + const input = path.join(root, "input"); + fs.mkdirSync(input, { recursive: true }); + + const options = fixture.build(input); + const cursors = {}; + + await rollout[parser]({ ...options, cursors, queuePath }); + const first = readQueue(queuePath).filter((r) => r.source === fixture.source); + assert.ok(first.length > 0, "the fixture must actually produce rows, or it proves nothing"); + first.forEach((row, i) => assertRowConforms(row, `${parser} row ${i}`)); + + // The "run sync twice" lesson from CLAUDE.md:172 — the one that catches + // dedup-key instability. Same input, carried cursors: nothing new. + await rollout[parser]({ ...options, cursors, queuePath }); + const second = readQueue(queuePath).filter((r) => r.source === fixture.source); + assert.deepEqual( + second, + first, + "re-parsing unchanged input must not add or alter rows", + ); + }); +}