diff --git a/CHANGELOG.md b/CHANGELOG.md index 97d51957..b6a82e7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ - `codeburn sync push --attribution` (opt-in): sends git attribution spans — the session→commit correlation from `codeburn yield` (`codeburn.session.attribution` and `codeburn.commit` span types with normalized repo remote, commit SHAs, merged/reverted state, and PR links). Nothing new is sent without the flag; local-only repos and Windows filesystem paths are never emitted as repo identities, and sessions whose project path no longer resolves never inherit the push-time working directory's repo. See docs/sync/README.md "Git attribution". ### Fixed (CLI) +- **Copilot CLI sessions report their input and cache tokens.** The Copilot CLI writes the same `producer: 'copilot-agent'` in its `session.start` events that VS Code transcripts carry, so content-based detection classified every CLI session as a transcript and skipped its `session.shutdown` rollup — the only place the CLI records input, cache-read and cache-write tokens — leaving cache hit rate at 0.0% and dramatically underreporting cost. Whether a file is a transcript is now decided by where discovery found it, never by its contents. Resumed sessions, whose legs each append a cumulative rollup, are billed as per-leg deltas so a growing session never double-counts or goes stale; the GitHub Copilot desktop app writes the same session store, so its usage is covered by the same fix. The copilot session cache takes a parse-version bump and the daily cache bumps from v16 to v17 for the one-time re-parse that heals already-recorded days whose logs still exist. (#944) +- **Copilot CLI subagent runs are attributed to their agent.** Newer CLIs announce delegation with `subagent.started`/`subagent.completed` rather than `subagent.selected`, so delegated turns lost their agent label; the label now also clears when the subagent completes instead of bleeding onto the parent's later turns. Rides the #944 re-parse, so already-cached sessions gain the attribution. (#944) - **`--project` / `--exclude` now apply to the headline totals, not just the detail panels.** The durable headline unions the carry-forward daily cache with today's live parse, and the cached days were sliced to the requested provider but never to the requested project — so the Overview panel counted excluded projects while By Project / By Activity / By Model (built from the name-filtered parse) left them out, and the two could not be reconciled. Cost, calls, sessions and savings are now sliced out of the per-project day stats the cache has carried since v15. Tokens, models and categories have no per-project split in the cache, so under a project filter they come from the (project-filtered) live parse instead; cached days — or provider slices — carried from before v15 have no project split at all, so they cannot be attributed to a filtered project, and the terminal overview now states how much was set aside rather than folding it into the total. (#864) - **Codex parser corrections**: fork-replay no longer double-counts `patch_apply_end` and `mcp_tool_call_end`; `exec` is normalized to Bash; `custom_tool_call` events are handled; token_count lines larger than 32 KiB now parse exact token counts instead of estimating. Codex session cache bumps from v7 to v8 for a one-time re-parse. Only tool attribution changes for ordinary sessions, leaving their cost identical; sessions that logged an oversized token_count line are repriced from exact counts instead of an estimate. (#805) diff --git a/src/daily-cache.ts b/src/daily-cache.ts index 1c6bf57b..f3cbfd02 100644 --- a/src/daily-cache.ts +++ b/src/daily-cache.ts @@ -5,7 +5,13 @@ import { homedir } from 'os' import { join } from 'path' import type { DateRange, ProjectSummary } from './types.js' -// Bumped to 16: Codex discovery is structural instead of originator-gated +// Bumped to 17: copilot CLI sessions were misclassified as VS Code transcripts +// (#944), so days finalized at v16 or earlier carry output-only copilot costs — +// the session.shutdown rollup's input/cache tokens were dropped. Raising +// MIN_SUPPORTED_VERSION forces the one-time re-derivation under the +// provenance-based classification; sourceless days carry forward as-is. +// +// v16: Codex discovery is structural instead of originator-gated // (#873/#626), so rollouts written by third-party frontends driving // `codex app-server` ("t3code_desktop", "JetBrains.IntelliJ IDEA", ...) now // contribute usage that v15 rollups never contained. Those files were rejected @@ -67,8 +73,8 @@ import type { DateRange, ProjectSummary } from './types.js' // that older binaries skipped. v8 added local-model savings to the daily // rollup; the `savingsConfigHash` field is invalidated separately when the // user changes their `localModelSavings` mapping. -export const DAILY_CACHE_VERSION = 16 -const MIN_SUPPORTED_VERSION = 16 +export const DAILY_CACHE_VERSION = 17 +const MIN_SUPPORTED_VERSION = 17 // Version-suffixed so different binaries each own a distinct file and never // clobber an incompatible schema. Bumping the version mints a fresh filename; // adoptOlderDailyCaches then unions days out of every previous file (including diff --git a/src/providers/copilot.ts b/src/providers/copilot.ts index c3b7ff06..e472fdb7 100644 --- a/src/providers/copilot.ts +++ b/src/providers/copilot.ts @@ -194,6 +194,9 @@ type SubagentSelectedData = { agentName: string agentDisplayName?: string tools?: string[] + // Present on subagent.started/completed (CLI ≥ ~1.0.7x): the delegation + // tool call that launched the run, used to pair completed with started. + toolCallId?: string } // Per-model usage rollup the CLI writes into session.shutdown. inputTokens is @@ -217,6 +220,8 @@ type CopilotEvent = | { type: 'user.message'; data: UserMessageData; timestamp?: string } | { type: 'assistant.message'; data: AssistantMessageData; timestamp?: string } | { type: 'subagent.selected'; data: SubagentSelectedData; timestamp?: string } + | { type: 'subagent.started'; data: SubagentSelectedData; timestamp?: string } + | { type: 'subagent.completed'; data: SubagentSelectedData; timestamp?: string } | { type: 'session.shutdown'; data: SessionShutdownData; timestamp?: string } type ChatJournalPathSegment = string | number @@ -693,50 +698,56 @@ function inferTranscriptModel(lines: string[]): string { } // --------------------------------------------------------------------------- -// JSONL parser (handles both regular session-state events and VS Code -// transcript format via session.start { producer: 'copilot-agent' }) +// JSONL parser (handles both regular CLI session-state events and the VS Code +// transcript format — the same event vocabulary, but transcripts carry no +// token counts and no session.shutdown rollup) // --------------------------------------------------------------------------- +/** + * `isTranscript` comes from discovery (where the file lives), never from + * content: the Copilot CLI writes the same session.start producer + * ('copilot-agent') that VS Code transcripts carry, so producer sniffing + * misread every CLI session as a transcript and dropped its session.shutdown + * input/cache rollup (#944). + */ function createJsonlParser( source: SessionSource, - seenKeys: Set + seenKeys: Set, + isTranscript: boolean ): SessionParser { return { async *parse(): AsyncGenerator { const content = await readSessionFile(source.path) if (!content) return - const sessionId = basename(dirname(source.path)) + // CLI session-state files live at /events.jsonl; transcripts + // at transcripts/.jsonl — keying the latter on the parent dir + // would collapse every transcript into one "transcripts" session (and + // one shared dedup namespace). + const sessionId = isTranscript + ? basename(source.path, '.jsonl') + : basename(dirname(source.path)) const lines = content.split('\n').filter((l) => l.trim()) - // Detect VS Code transcript format: the first session.start event has - // { producer: 'copilot-agent' } and no outputTokens in messages. - let isTranscript = false let currentModel = '' let pendingUserMessage = '' - // Track the active subagent for this session (from subagent.selected events). - // Resets when a new subagent is selected. - let currentSubagentType: string | undefined - - // First pass: detect format and infer transcript model if needed. - for (const line of lines) { - try { - const ev = JSON.parse(line) as CopilotEvent - if (ev.type === 'session.start') { - const data = ev.data as SessionStartData & { producer?: string } - if (data.producer === 'copilot-agent') { - isTranscript = true - } - break - } - if (ev.type === 'session.model_change') break // regular format - } catch { - continue - } - } + // Subagent attribution. Older CLIs write subagent.selected — sticky + // until replaced, never cleared. CLI ≥ ~1.0.7x brackets each run with + // started/completed instead; runs can nest or overlap, so completed + // removes ONLY its own toolCallId's entry and the label falls back to + // the still-active run (or the sticky selected value) rather than + // wiping attribution for everything in flight. + let selectedSubagentType: string | undefined + const activeSubagents: Array<{ toolCallId: string; name: string }> = [] + const currentSubagentType = (): string | undefined => + activeSubagents[activeSubagents.length - 1]?.name ?? selectedSubagentType if (isTranscript) { + // Tool-call-id prefix inference seeds the model; it must not gate the + // whole file, or a transcript carrying explicit model info + // (session.model_change / per-message model) but no tool calls would + // yield nothing. Messages that still end up modelless are skipped + // individually below. currentModel = inferTranscriptModel(lines) - if (!currentModel) return // no toolCallIds to infer model from } // Shutdown rollups may lack their own timestamp; remember the last @@ -744,6 +755,15 @@ function createJsonlParser( // timestamp, which the date-range filters silently drop. let lastEventTimestamp = '' + // A resumed session appends one session.shutdown PER LEG, each carrying + // CUMULATIVE per-model totals. Emitting each rollup whole would need the + // cache to update a prior call in place — the durable merge is + // append-only by dedup key — so we emit per-leg DELTAS keyed by + // occurrence instead: re-parses of a growing file append only the new + // leg, and each leg lands on its own timestamp. + const prevShutdownUsage = new Map() + const shutdownCountByModel = new Map() + for (const line of lines) { let event: CopilotEvent try { @@ -766,7 +786,34 @@ function createJsonlParser( } if (event.type === 'subagent.selected') { - currentSubagentType = (event.data as SubagentSelectedData).agentName + selectedSubagentType = (event.data as SubagentSelectedData).agentName + continue + } + + if (event.type === 'subagent.started') { + const data = event.data as SubagentSelectedData + activeSubagents.push({ toolCallId: data.toolCallId ?? '', name: data.agentName }) + continue + } + + if (event.type === 'subagent.completed') { + const id = (event.data as SubagentSelectedData).toolCallId ?? '' + if (!id) { + // ID-less completion (transitional CLIs that key nothing, like + // subagent.selected): end the most recently started run; explicit + // no-op on an empty stack. + activeSubagents.pop() + continue + } + for (let i = activeSubagents.length - 1; i >= 0; i--) { + if (activeSubagents[i]!.toolCallId === id) { + activeSubagents.splice(i, 1) + break + } + } + // A non-empty id that matches nothing refers to a run we never saw + // start — leave the active runs alone rather than evicting an + // unrelated one. continue } @@ -783,11 +830,12 @@ function createJsonlParser( // is gated to the CLI (non-transcript) format, leaving VS Code, // JetBrains and OTel sources untouched. // - // We emit one supplementary call per model carrying ONLY the - // input/cache tokens the per-turn events lack; output is excluded so - // the assistant.message output (and its cost) is not double-counted. - // Combined with the per-turn output cost, this yields the full, - // CLI-measured session cost. + // We emit one supplementary call per model PER SHUTDOWN LEG (resumed + // sessions write one cumulative rollup per leg; see the delta + // tracking above) carrying ONLY the input/cache tokens the per-turn + // events lack; output is excluded so the assistant.message output + // (and its cost) is not double-counted. Combined with the per-turn + // output cost, this yields the full, CLI-measured session cost. if (isTranscript) continue const shutdownData = event.data as SessionShutdownData const modelMetrics = shutdownData.modelMetrics @@ -801,23 +849,49 @@ function createJsonlParser( const usage = metrics['usage'] if (!isRecord(usage)) continue - const cacheReadTokens = numberOrZero(usage['cacheReadTokens']) - const cacheWriteTokens = numberOrZero(usage['cacheWriteTokens']) - const reasoningTokens = numberOrZero(usage['reasoningTokens']) + const cumulative: Required = { + inputTokens: numberOrZero(usage['inputTokens']), + outputTokens: numberOrZero(usage['outputTokens']), + cacheReadTokens: numberOrZero(usage['cacheReadTokens']), + cacheWriteTokens: numberOrZero(usage['cacheWriteTokens']), + reasoningTokens: numberOrZero(usage['reasoningTokens']), + } + const prevRaw = prevShutdownUsage.get(model) + prevShutdownUsage.set(model, cumulative) + const n = (shutdownCountByModel.get(model) ?? 0) + 1 + shutdownCountByModel.set(model, n) + + // A cumulative total BELOW the previous rollup means the CLI reset + // its counters (a fresh accounting epoch): delta from zero, else + // this leg's post-reset usage would be clamped away entirely. + // inputTokens is the monotonic sentinel — it is cache-inclusive, + // so any usage at all grows it. + const prev = + prevRaw && cumulative.inputTokens < numberOrZero(prevRaw.inputTokens) + ? undefined + : prevRaw + + // This leg's contribution: cumulative minus the previous rollup. + // The clamp guards any remaining non-monotonic field. + const delta = (k: keyof ShutdownModelUsage): number => + Math.max(0, cumulative[k] - numberOrZero(prev?.[k])) + const cacheReadTokens = delta('cacheReadTokens') + const cacheWriteTokens = delta('cacheWriteTokens') + const reasoningTokens = delta('reasoningTokens') // usage.inputTokens is cache-INCLUSIVE (input + cache_read + // cache_write). calculateCost expects the uncached input alone with // cache tokens billed separately, so subtract the cache components. // Clamp at 0 in case a future schema reports input non-inclusively. const inputTokens = Math.max( 0, - numberOrZero(usage['inputTokens']) - cacheReadTokens - cacheWriteTokens + delta('inputTokens') - cacheReadTokens - cacheWriteTokens ) // Nothing this call would add over the per-turn events, so skip it // to avoid an empty $0 row (output is intentionally excluded). - if (inputTokens === 0 && cacheReadTokens === 0 && cacheWriteTokens === 0) continue + if (inputTokens === 0 && cacheReadTokens === 0 && cacheWriteTokens === 0 && reasoningTokens === 0) continue - const dedupKey = `copilot:${sessionId}:shutdown:${model}` + const dedupKey = `copilot:${sessionId}:shutdown:${model}:${n}` if (seenKeys.has(dedupKey)) continue seenKeys.add(dedupKey) @@ -898,6 +972,7 @@ function createJsonlParser( // Cost will be lower than actual API cost. This is the original // behaviour — OTel data (below) replaces it when available. const costUSD = calculateCost(currentModel, 0, outputTokens, 0, 0, 0) + const subagentType = currentSubagentType() yield { provider: 'copilot', @@ -914,7 +989,7 @@ function createJsonlParser( tools, bashCommands, skills: skills.length > 0 ? skills : undefined, - subagentTypes: currentSubagentType ? [currentSubagentType] : undefined, + subagentTypes: subagentType ? [subagentType] : undefined, timestamp: event.timestamp ?? '', speed: 'standard' as const, deduplicationKey: dedupKey, @@ -1837,6 +1912,12 @@ interface JsonlSessionSource extends SessionSource { sourceType: 'jsonl' } +// A VS Code workspaceStorage transcript. Distinct from 'jsonl' (CLI +// session-state) so classification rides provenance, not file contents (#944). +interface TranscriptSessionSource extends SessionSource { + sourceType: 'transcript' +} + interface ChatSessionSource extends SessionSource { sourceType: 'chatsession' } @@ -1874,6 +1955,10 @@ function isJetBrainsSource(source: SessionSource): source is JetBrainsSessionSou return (source as JetBrainsSessionSource).sourceType === 'jetbrains' } +function isTranscriptSource(source: SessionSource): source is TranscriptSessionSource { + return (source as TranscriptSessionSource).sourceType === 'transcript' +} + // --------------------------------------------------------------------------- // Session discovery: JSONL (original) // --------------------------------------------------------------------------- @@ -2242,8 +2327,8 @@ async function discoverEmptyWindowChatSessions( */ async function discoverTranscriptSessions( workspaceStorageDirs: string[] -): Promise { - const sources: JsonlSessionSource[] = [] +): Promise { + const sources: TranscriptSessionSource[] = [] for (const wsDir of workspaceStorageDirs) { let hashDirs: string[] @@ -2275,7 +2360,7 @@ async function discoverTranscriptSessions( path: join(transcriptsDir, file), project, provider: 'copilot', - sourceType: 'jsonl', + sourceType: 'transcript', }) } } @@ -2418,7 +2503,7 @@ export function createCopilotProvider( if (isJetBrainsSource(source)) { return createJetBrainsParser(source, seenKeys) } - return createJsonlParser(source, seenKeys) + return createJsonlParser(source, seenKeys, isTranscriptSource(source)) }, } } diff --git a/src/session-cache.ts b/src/session-cache.ts index 4d0e31b7..a86aea89 100644 --- a/src/session-cache.ts +++ b/src/session-cache.ts @@ -224,7 +224,10 @@ export const PROVIDER_PARSE_VERSIONS: Record = { codex: 'mcp-attribution-v5-est-cost-active-timing-mcp-wait-rich-capture-v1-cross-provider-pr-v1', cursor: 'composer-anchored-crediting-v1-est-cost', 'cursor-agent': 'workspaceless-transcript-v1', - copilot: 'cli-shutdown-cost-v1-skills', + // source-provenance-v1 (#944): CLI sessions were misread as VS Code + // transcripts (both carry producer 'copilot-agent'), skipping the shutdown + // input/cache rollup; this bump re-parses them so the missing tokens land. + copilot: 'cli-shutdown-cost-v1-skills-source-provenance-v1', grok: 'estimated-cost-v1', hermes: 'reasoning-output-accounting-v1-est-cost', 'lingtai-tui': 'token-ledger-registry-activity-v3', diff --git a/tests/parser.test.ts b/tests/parser.test.ts index b4dd0631..6d54b78f 100644 --- a/tests/parser.test.ts +++ b/tests/parser.test.ts @@ -605,3 +605,77 @@ describe('(h) provider filter excludes claude from the orphan pass', () => { expect(totalCost(after)).toBeCloseTo(costBefore, 10) }) }) + +// ═══════════════════════════════════════════════════════════════════════════ +// (f) Growing resumed CLI session: durable merge appends only the new leg +// ═══════════════════════════════════════════════════════════════════════════ +// Resumed Copilot CLI sessions append one CUMULATIVE session.shutdown per leg +// (#944). The parser emits per-leg deltas keyed by occurrence; this exercises +// the PRODUCTION merge path — the durable union-by-dedup-key merge against the +// on-disk cache when the file grows between parses — which the unit tests +// (which pre-seed seenKeys) cannot reach. +describe('(f) growing resumed CLI session durable merge', () => { + it('totals equal the final cumulative rollup after the file grows a leg', async () => { + const sessionStateDir = join(tmpHome, 'session-state') + await mkdir(sessionStateDir, { recursive: true }) + vi.stubEnv('CODEBURN_COPILOT_SESSION_STATE_DIR', sessionStateDir) + vi.stubEnv('CODEBURN_COPILOT_DISABLE_OTEL', '1') + vi.stubEnv('CODEBURN_COPILOT_WS_STORAGE_DIR', join(tmpHome, 'no-ws')) + vi.stubEnv('CODEBURN_COPILOT_GLOBAL_STORAGE_DIR', join(tmpHome, 'no-global')) + vi.stubEnv('CODEBURN_COPILOT_JETBRAINS_DIR', join(tmpHome, 'no-jb')) + + const base = Date.now() - 5 * 24 * 60 * 60 * 1000 + const at = (offsetSec: number): string => new Date(base + offsetSec * 1000).toISOString() + const dir = join(sessionStateDir, 'sess-grow') + await mkdir(dir, { recursive: true }) + await writeFile(join(dir, 'workspace.yaml'), 'id: sess-grow\ncwd: /home/user/testproj\n') + const eventsPath = join(dir, 'events.jsonl') + + // Cumulative rollups from a real resumed CLI 1.0.78 session. + const shutdown = (ts: string, inputTokens: number, cacheReadTokens: number, cacheWriteTokens: number, outputTokens: number) => + JSON.stringify({ + type: 'session.shutdown', + timestamp: ts, + data: { + shutdownType: 'routine', + modelMetrics: { + 'claude-sonnet-4-5': { + requests: { count: 1, cost: 1 }, + usage: { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, reasoningTokens: 0 }, + }, + }, + }, + }) + const leg1 = [ + JSON.stringify({ type: 'session.model_change', timestamp: at(0), data: { newModel: 'claude-sonnet-4-5' } }), + JSON.stringify({ type: 'assistant.message', timestamp: at(10), data: { messageId: 'msg-1', outputTokens: 17, toolRequests: [] } }), + shutdown(at(20), 24672, 0, 24670, 17), + ] + await writeFile(eventsPath, leg1.join('\n') + '\n') + + const sumUsage = (projects: Awaited>) => { + const calls = projects.flatMap(p => p.sessions).flatMap(s => s.turns).flatMap(t => t.assistantCalls) + return { + input: calls.reduce((s, c) => s + c.usage.inputTokens, 0), + cacheRead: calls.reduce((s, c) => s + c.usage.cacheReadInputTokens, 0), + cacheWrite: calls.reduce((s, c) => s + c.usage.cacheCreationInputTokens, 0), + } + } + + const first = sumUsage(await parseAllSessions(undefined, 'copilot')) + expect(first).toEqual({ input: 2, cacheRead: 0, cacheWrite: 24670 }) + + // The session resumes: leg 2 appends per-turn events plus a CUMULATIVE + // rollup. The cached leg-1 delta must be kept once and only the leg-2 + // delta appended — totals equal the final cumulative rollup exactly. + clearSessionCache() + await writeFile(eventsPath, [ + ...leg1, + JSON.stringify({ type: 'assistant.message', timestamp: at(100), data: { messageId: 'msg-2', outputTokens: 132, toolRequests: [] } }), + shutdown(at(120), 74463, 49489, 24968, 149), + ].join('\n') + '\n') + + const second = sumUsage(await parseAllSessions(undefined, 'copilot')) + expect(second).toEqual({ input: 74463 - 49489 - 24968, cacheRead: 49489, cacheWrite: 24968 }) + }) +}) diff --git a/tests/providers/copilot.test.ts b/tests/providers/copilot.test.ts index a5d95517..ae128be9 100644 --- a/tests/providers/copilot.test.ts +++ b/tests/providers/copilot.test.ts @@ -124,6 +124,18 @@ async function collectCalls(source: { path: string; project: string; provider: s return calls } +// Write a transcript inside the test's tmpDir sandbox, but at the production +// directory shape — {ws}/{hash}/GitHub.copilot-chat/transcripts/.jsonl — +// because sessionId derivation reads the path structure (file basename for +// transcripts). Never touches the real VS Code storage. +async function createTranscriptFile(sessionId: string, lines: string[]) { + const transcriptsDir = join(tmpDir, 'ws', 'hash1', 'GitHub.copilot-chat', 'transcripts') + await mkdir(transcriptsDir, { recursive: true }) + const path = join(transcriptsDir, `${sessionId}.jsonl`) + await writeFile(path, lines.join('\n') + '\n') + return path +} + describe('copilot provider - JSONL parsing', () => { beforeEach(async () => { tmpDir = await mkdtemp(join(tmpdir(), 'copilot-test-')) @@ -335,8 +347,122 @@ describe('copilot provider - JSONL parsing', () => { expect(calls[0]!.model).toBe('gpt-4.1') }) + it('attributes turns between subagent.started and subagent.completed to the subagent', async () => { + // CLI ≥ ~1.0.7x writes subagent.started/completed (not subagent.selected); + // event shapes from a real delegating 1.0.78 session. The label must cover + // the subagent's turns and clear afterwards, not bleed onto the parent's. + const eventsPath = await createSessionDir('sess-subagent-cli', [ + modelChange('claude-sonnet-5'), + userMessage('delegate a search'), + JSON.stringify({ + type: 'subagent.started', + timestamp: '2026-08-07T10:00:11Z', + data: { toolCallId: 'toolu_01SZnHjC', agentName: 'explore', agentDisplayName: 'Explore Agent' }, + }), + JSON.stringify({ + type: 'assistant.message', + timestamp: '2026-08-07T10:00:14Z', + data: { messageId: 'msg-sub', model: 'claude-haiku-4.5', outputTokens: 197, toolRequests: [] }, + }), + JSON.stringify({ + type: 'subagent.completed', + timestamp: '2026-08-07T10:00:19Z', + data: { toolCallId: 'toolu_01SZnHjC', agentName: 'explore', model: 'claude-haiku-4.5', totalTokens: 26435 }, + }), + JSON.stringify({ + type: 'assistant.message', + timestamp: '2026-08-07T10:00:22Z', + data: { messageId: 'msg-parent', model: 'claude-sonnet-5', outputTokens: 51, toolRequests: [] }, + }), + ]) + + const calls = await collectCalls({ path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'jsonl' }) + const sub = calls.find(c => c.deduplicationKey.endsWith(':msg-sub'))! + expect(sub.subagentTypes).toEqual(['explore']) + expect(sub.model).toBe('claude-haiku-4.5') + const parent = calls.find(c => c.deduplicationKey.endsWith(':msg-parent'))! + expect(parent.subagentTypes).toBeUndefined() + }) + + it('completing a nested subagent restores the outer label, matched by toolCallId', async () => { + const started = (id: string, name: string) => + JSON.stringify({ type: 'subagent.started', timestamp: '2026-08-07T10:00:11Z', data: { toolCallId: id, agentName: name } }) + const completed = (id: string) => + JSON.stringify({ type: 'subagent.completed', timestamp: '2026-08-07T10:00:19Z', data: { toolCallId: id, agentName: 'x' } }) + const msg = (messageId: string, outputTokens = 10) => + JSON.stringify({ type: 'assistant.message', timestamp: '2026-08-07T10:00:14Z', data: { messageId, model: 'claude-sonnet-5', outputTokens, toolRequests: [] } }) + + const eventsPath = await createSessionDir('sess-subagent-nested', [ + modelChange('claude-sonnet-5'), + started('call-A', 'explore'), + started('call-B', 'plan'), + msg('msg-inner'), // while B runs → 'plan' + completed('call-B'), + msg('msg-outer'), // B done, A still active → 'explore', NOT unlabeled + completed('call-A'), + msg('msg-after'), // all done → no label + ]) + + const calls = await collectCalls({ path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'jsonl' }) + const byId = (id: string) => calls.find(c => c.deduplicationKey.endsWith(`:${id}`))! + expect(byId('msg-inner').subagentTypes).toEqual(['plan']) + expect(byId('msg-outer').subagentTypes).toEqual(['explore']) + expect(byId('msg-after').subagentTypes).toBeUndefined() + }) + + it('ignores a completed event whose non-empty toolCallId matches no active run', async () => { + // A completion for a run we never saw start must not evict an unrelated + // active run; only a genuinely ID-less completion may pop the stack. + const eventsPath = await createSessionDir('sess-subagent-unmatched', [ + modelChange('claude-sonnet-5'), + JSON.stringify({ + type: 'subagent.started', + timestamp: '2026-08-07T10:00:11Z', + data: { toolCallId: 'call-A', agentName: 'explore' }, + }), + JSON.stringify({ + type: 'subagent.completed', + timestamp: '2026-08-07T10:00:12Z', + data: { toolCallId: 'call-unknown', agentName: 'phantom' }, + }), + JSON.stringify({ + type: 'assistant.message', + timestamp: '2026-08-07T10:00:14Z', + data: { messageId: 'msg-1', model: 'claude-sonnet-5', outputTokens: 10, toolRequests: [] }, + }), + JSON.stringify({ + type: 'subagent.completed', + timestamp: '2026-08-07T10:00:15Z', + data: { agentName: 'legacy-no-id' }, + }), + JSON.stringify({ + type: 'assistant.message', + timestamp: '2026-08-07T10:00:16Z', + data: { messageId: 'msg-2', model: 'claude-sonnet-5', outputTokens: 12, toolRequests: [] }, + }), + ]) + + const calls = await collectCalls({ path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'jsonl' }) + // The unmatched completion left 'explore' active… + expect(calls.find(c => c.deduplicationKey.endsWith(':msg-1'))!.subagentTypes).toEqual(['explore']) + // …and the ID-less completion (legacy shape) ended it. + expect(calls.find(c => c.deduplicationKey.endsWith(':msg-2'))!.subagentTypes).toBeUndefined() + }) + + it('keeps subagent.selected sticky when no completed event ever arrives', async () => { + // Older CLIs only write subagent.selected; nothing clears it. + const eventsPath = await createSessionDir('sess-subagent-selected', [ + modelChange('claude-sonnet-5'), + JSON.stringify({ type: 'subagent.selected', data: { agentName: 'refactor' } }), + assistantMessage({ messageId: 'msg-1', outputTokens: 25 }), + assistantMessage({ messageId: 'msg-2', outputTokens: 30, timestamp: '2026-04-15T10:01:00Z' }), + ]) + const calls = await collectCalls({ path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'jsonl' }) + expect(calls.map(c => c.subagentTypes)).toEqual([['refactor'], ['refactor']]) + }) + it('infers OpenAI auto bucket for transcript toolCallId prefix call_', async () => { - const eventsPath = await createSessionDir('sess-tr-call', [ + const eventsPath = await createTranscriptFile('sess-tr-call', [ transcriptSessionStart('sess-tr-call'), transcriptUserMessage('check model inference'), transcriptAssistantMessage({ @@ -346,16 +472,20 @@ describe('copilot provider - JSONL parsing', () => { }), ]) - const source = { path: eventsPath, project: 'test', provider: 'copilot' } + const source = { path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'transcript' } const calls: ParsedProviderCall[] = [] for await (const call of copilot.createSessionParser(source, new Set()).parse()) calls.push(call) expect(calls).toHaveLength(1) expect(calls[0]!.model).toBe('copilot-openai-auto') + // Each transcript is its own session, keyed by file basename — NOT the + // shared parent dir name 'transcripts', which would collapse every + // transcript into one session and one dedup namespace. + expect(calls[0]!.sessionId).toBe('sess-tr-call') }) it('infers Anthropic auto bucket for transcript toolCallId prefixes tooluse_/toolu_vrtx_', async () => { - const eventsPath = await createSessionDir('sess-tr-claude', [ + const eventsPath = await createTranscriptFile('sess-tr-claude', [ transcriptSessionStart('sess-tr-claude'), transcriptUserMessage('check model inference'), transcriptAssistantMessage({ @@ -365,7 +495,7 @@ describe('copilot provider - JSONL parsing', () => { }), ]) - const source = { path: eventsPath, project: 'test', provider: 'copilot' } + const source = { path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'transcript' } const calls: ParsedProviderCall[] = [] for await (const call of copilot.createSessionParser(source, new Set()).parse()) calls.push(call) @@ -374,7 +504,7 @@ describe('copilot provider - JSONL parsing', () => { }) it('chooses the dominant inferred transcript model when prefixes are mixed', async () => { - const eventsPath = await createSessionDir('sess-tr-mixed', [ + const eventsPath = await createTranscriptFile('sess-tr-mixed', [ transcriptSessionStart('sess-tr-mixed'), transcriptUserMessage('mixed'), transcriptAssistantMessage({ @@ -394,7 +524,7 @@ describe('copilot provider - JSONL parsing', () => { }), ]) - const source = { path: eventsPath, project: 'test', provider: 'copilot' } + const source = { path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'transcript' } const calls: ParsedProviderCall[] = [] for await (const call of copilot.createSessionParser(source, new Set()).parse()) calls.push(call) @@ -402,8 +532,35 @@ describe('copilot provider - JSONL parsing', () => { expect(calls.every(c => c.model === 'copilot-openai-auto')).toBe(true) }) + it('parses a producerless transcript with explicit model info and no tool calls', async () => { + // Prefix inference has nothing to work with here; the explicit + // session.model_change must still establish the model, and the shutdown + // rollup must stay ignored — provenance, not the producer field, gates it. + const eventsPath = await createTranscriptFile('sess-tr-explicit', [ + JSON.stringify({ type: 'session.start', data: { sessionId: 'sess-tr-explicit' } }), + modelChange('gpt-4.1'), + transcriptUserMessage('hi'), + JSON.stringify({ + type: 'assistant.message', + timestamp: '2026-04-15T10:00:15Z', + data: { messageId: 'msg-1', outputTokens: 80, toolRequests: [] }, + }), + shutdownEvent({ + modelMetrics: { + 'gpt-4.1': { inputTokens: 1000, outputTokens: 80, cacheReadTokens: 500, cacheWriteTokens: 200 }, + }, + }), + ]) + + const calls = await collectCalls({ path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'transcript' }) + expect(calls).toHaveLength(1) + expect(calls[0]!.model).toBe('gpt-4.1') + expect(calls[0]!.outputTokens).toBe(80) + expect(calls.every(c => !c.deduplicationKey.includes(':shutdown:'))).toBe(true) + }) + it('normalizes Copilot MCP tool names from VS Code transcripts', async () => { - const eventsPath = await createSessionDir('sess-tr-mcp-tools', [ + const eventsPath = await createTranscriptFile('sess-tr-mcp-tools', [ transcriptSessionStart('sess-tr-mcp-tools'), transcriptUserMessage('use GitHub MCP'), transcriptAssistantMessage({ @@ -414,7 +571,7 @@ describe('copilot provider - JSONL parsing', () => { }), ]) - const source = { path: eventsPath, project: 'test', provider: 'copilot' } + const source = { path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'transcript' } const calls: ParsedProviderCall[] = [] for await (const call of copilot.createSessionParser(source, new Set()).parse()) calls.push(call) @@ -458,7 +615,7 @@ describe('copilot provider - session.shutdown token/cost rollup', () => { // One per-turn assistant.message call + one supplementary shutdown call. expect(calls).toHaveLength(2) - const shutdown = calls.find(c => c.deduplicationKey === 'copilot:sess-shutdown:shutdown:claude-sonnet-4-5') + const shutdown = calls.find(c => c.deduplicationKey === 'copilot:sess-shutdown:shutdown:claude-sonnet-4-5:1') expect(shutdown).toBeDefined() expect(shutdown!.model).toBe('claude-sonnet-4-5') expect(shutdown!.inputTokens).toBe(4) // 71282 - 35495 - 35783 @@ -546,6 +703,86 @@ describe('copilot provider - session.shutdown token/cost rollup', () => { expect(gpt.costUSD).toBeCloseTo(calculateCost('gpt-5', 50, 0, 0, 5000, 0), 12) }) + it('emits per-leg deltas for a resumed session with cumulative shutdown rollups', async () => { + // Numbers from a real resumed CLI 1.0.78 session (3 legs via --resume): + // each leg appends a session.shutdown whose modelMetrics are CUMULATIVE. + // Emitting deltas keyed by occurrence keeps a growing file append-only + // under the durable union-by-key cache merge — re-parsing after each + // resume adds only the new leg, never double-counting earlier ones. + const legs = [ + { inputTokens: 24672, outputTokens: 17, cacheReadTokens: 0, cacheWriteTokens: 24670 }, + { inputTokens: 74463, outputTokens: 149, cacheReadTokens: 49489, cacheWriteTokens: 24968 }, + { inputTokens: 124783, outputTokens: 243, cacheReadTokens: 99569, cacheWriteTokens: 25204 }, + ] + const lines = [modelChange('claude-sonnet-5'), assistantMessage({ messageId: 'msg-1', outputTokens: 17 })] + for (const [i, leg] of legs.entries()) { + lines.push(shutdownEvent({ modelMetrics: { 'claude-sonnet-5': leg }, timestamp: `2026-08-0${i + 1}T10:00:00Z` })) + } + const eventsPath = await createSessionDir('sess-resumed', lines) + const calls = await collectCalls({ path: eventsPath, project: 'myproject', provider: 'copilot', sourceType: 'jsonl' }) + + const shutdowns = calls.filter(c => c.deduplicationKey.includes(':shutdown:')) + expect(shutdowns.map(c => c.deduplicationKey)).toEqual([ + 'copilot:sess-resumed:shutdown:claude-sonnet-5:1', + 'copilot:sess-resumed:shutdown:claude-sonnet-5:2', + 'copilot:sess-resumed:shutdown:claude-sonnet-5:3', + ]) + // Each leg lands on its own shutdown timestamp (a resumed session can + // span days; whole-rollup emission would collapse them onto one). + expect(shutdowns.map(c => c.timestamp)).toEqual([ + '2026-08-01T10:00:00Z', '2026-08-02T10:00:00Z', '2026-08-03T10:00:00Z', + ]) + // Per-leg deltas sum exactly to the final cumulative rollup. + const sum = (k: 'inputTokens' | 'cacheReadInputTokens' | 'cacheCreationInputTokens') => + shutdowns.reduce((a, c) => a + c[k], 0) + expect(sum('cacheReadInputTokens')).toBe(99569) + expect(sum('cacheCreationInputTokens')).toBe(25204) + expect(sum('inputTokens')).toBe(124783 - 99569 - 25204) + + // A later re-parse of the grown file (prior legs already cached) emits + // only what the seen-key set lacks. + const seen = new Set(calls.map(c => c.deduplicationKey)) + const again = await collectCalls({ path: eventsPath, project: 'myproject', provider: 'copilot', sourceType: 'jsonl' }, seen) + expect(again).toHaveLength(0) + }) + + it('starts a fresh delta baseline when a cumulative rollup goes backwards (counter reset)', async () => { + // Hypothetical but cheap to guard: if the CLI ever resets its counters + // mid-session, the post-reset epoch must be billed from zero — a stale + // high-water baseline would clamp it away (and the reset leg's real usage + // with it). + const eventsPath = await createSessionDir('sess-reset', [ + modelChange('claude-sonnet-5'), + assistantMessage({ messageId: 'msg-1', outputTokens: 10 }), + shutdownEvent({ + modelMetrics: { 'claude-sonnet-5': { inputTokens: 10000, outputTokens: 10, cacheReadTokens: 0, cacheWriteTokens: 5000 } }, + timestamp: '2026-08-01T10:00:00Z', + }), + // Reset: cumulative drops below the previous rollup → new epoch. + shutdownEvent({ + modelMetrics: { 'claude-sonnet-5': { inputTokens: 2000, outputTokens: 5, cacheReadTokens: 0, cacheWriteTokens: 1000 } }, + timestamp: '2026-08-02T10:00:00Z', + }), + shutdownEvent({ + modelMetrics: { 'claude-sonnet-5': { inputTokens: 5000, outputTokens: 8, cacheReadTokens: 2000, cacheWriteTokens: 1500 } }, + timestamp: '2026-08-03T10:00:00Z', + }), + ]) + const calls = await collectCalls({ path: eventsPath, project: 'myproject', provider: 'copilot', sourceType: 'jsonl' }) + const shutdowns = calls.filter(c => c.deduplicationKey.includes(':shutdown:')) + expect(shutdowns).toHaveLength(3) + // Leg 1: epoch-1 usage in full. + expect(shutdowns[0]!.inputTokens).toBe(5000) // 10000 − 0 − 5000 + expect(shutdowns[0]!.cacheCreationInputTokens).toBe(5000) + // Leg 2 (reset): billed from zero, not clamped away against the old baseline. + expect(shutdowns[1]!.inputTokens).toBe(1000) // 2000 − 0 − 1000 + expect(shutdowns[1]!.cacheCreationInputTokens).toBe(1000) + // Leg 3: normal delta within the new epoch. + expect(shutdowns[2]!.inputTokens).toBe(500) // (5000−2000) − 2000 − 500 + expect(shutdowns[2]!.cacheReadInputTokens).toBe(2000) + expect(shutdowns[2]!.cacheCreationInputTokens).toBe(500) + }) + it('keeps shutdown dedup keys stable across re-parses', async () => { const eventsPath = await createSessionDir('sess-reparse', [ modelChange('claude-sonnet-4-5'), @@ -594,7 +831,7 @@ describe('copilot provider - session.shutdown token/cost rollup', () => { }) it('ignores session.shutdown for VS Code transcript sessions', async () => { - const eventsPath = await createSessionDir('sess-tr-shutdown', [ + const eventsPath = await createTranscriptFile('sess-tr-shutdown', [ transcriptSessionStart('sess-tr-shutdown'), transcriptUserMessage('hi'), transcriptAssistantMessage({ messageId: 'msg-1', content: 'done', toolCallIds: ['call_abc'] }), @@ -604,7 +841,7 @@ describe('copilot provider - session.shutdown token/cost rollup', () => { }, }), ]) - const source = { path: eventsPath, project: 'test', provider: 'copilot' } + const source = { path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'transcript' } const calls = await collectCalls(source) // Only the transcript assistant call; the shutdown rollup is CLI-only. @@ -612,6 +849,165 @@ describe('copilot provider - session.shutdown token/cost rollup', () => { expect(calls.every(c => !c.deduplicationKey.includes(':shutdown:'))).toBe(true) expect(calls[0]!.model).toBe('copilot-openai-auto') }) + + // Regression test for #944: events are redacted copies of a real Copilot CLI + // 1.0.78 session. The CLI writes the same producer ('copilot-agent') as VS + // Code transcripts, so content sniffing skipped this session's shutdown + // rollup — reporting 100 of its 49,573 tokens and zero input/cache. + it('parses a CLI session whose session.start carries producer copilot-agent (issue #944)', async () => { + const eventsPath = await createSessionDir('sess-cli-producer', [ + JSON.stringify({ + type: 'session.start', + timestamp: '2026-08-07T17:56:35.573Z', + data: { + sessionId: 'sess-cli-producer', + version: 1, + producer: 'copilot-agent', + copilotVersion: '1.0.78', + startTime: '2026-08-07T17:56:35.554Z', + context: { cwd: '/home/user/myproject' }, + }, + }), + JSON.stringify({ + type: 'session.model_change', + timestamp: '2026-08-07T17:56:36.725Z', + data: { newModel: 'claude-sonnet-5', reasoningEffort: null }, + }), + JSON.stringify({ + type: 'user.message', + timestamp: '2026-08-07T17:56:36.732Z', + data: { content: 'Run echo and summarize the output.' }, + }), + JSON.stringify({ + type: 'assistant.message', + timestamp: '2026-08-07T17:56:38.763Z', + data: { + messageId: 'a982a391-9ee3-4fbd-89a9-26d5af78c890', + model: 'claude-sonnet-5', + content: '', + toolRequests: [{ + toolCallId: 'toolu_017eL3f5aeGiLoALignYMZEN', + name: 'bash', + arguments: { command: 'echo codeburn-repro-944', description: 'Echo test string' }, + type: 'function', + }], + turnId: '0', + outputTokens: 81, + }, + }), + JSON.stringify({ + type: 'assistant.message', + timestamp: '2026-08-07T17:56:40.417Z', + data: { + messageId: '8758ea51-797f-4285-972c-495911e2839f', + model: 'claude-sonnet-5', + content: 'The command printed the string "codeburn-repro-944".', + toolRequests: [], + turnId: '1', + outputTokens: 19, + }, + }), + JSON.stringify({ + type: 'session.shutdown', + timestamp: '2026-08-07T17:56:40.591Z', + data: { + shutdownType: 'routine', + sessionStartTime: 1786125395554, + modelMetrics: { + 'claude-sonnet-5': { + requests: { count: 2, cost: 1 }, + usage: { inputTokens: 49473, outputTokens: 100, cacheReadTokens: 24678, cacheWriteTokens: 24791, reasoningTokens: 0 }, + }, + }, + }, + }), + ]) + + // Discovery tags session-state files 'jsonl'; provenance, not the shared + // producer value, must classify this as a CLI session. + const calls = await collectCalls({ path: eventsPath, project: 'myproject', provider: 'copilot', sourceType: 'jsonl' }) + + // Two per-turn output calls with the REAL model — not the + // 'copilot-anthropic-auto' bucket transcript inference would pick from the + // toolu_ toolCallId prefix. + const perTurn = calls.filter(c => !c.deduplicationKey.includes(':shutdown:')) + expect(perTurn.map(c => c.outputTokens)).toEqual([81, 19]) + expect(perTurn.every(c => c.model === 'claude-sonnet-5')).toBe(true) + + // The shutdown rollup lands: the tokens the misclassification dropped. + const shutdown = calls.find(c => c.deduplicationKey === 'copilot:sess-cli-producer:shutdown:claude-sonnet-5:1') + expect(shutdown).toBeDefined() + expect(shutdown!.inputTokens).toBe(4) // 49473 − 24678 − 24791 (cache-inclusive) + expect(shutdown!.cacheReadInputTokens).toBe(24678) + expect(shutdown!.cacheCreationInputTokens).toBe(24791) + expect(shutdown!.outputTokens).toBe(0) // owned by the per-turn events + expect(shutdown!.costIsEstimated).toBe(false) + expect(shutdown!.costUSD).toBeCloseTo(calculateCost('claude-sonnet-5', 4, 0, 24791, 24678, 0), 12) + expect(shutdown!.costUSD).toBeGreaterThan(0) + }) + + it('treats a bare (untagged) source as CLI format, not transcript', async () => { + // Producer sniffing must not resurface for sources without a sourceType tag + // (the pre-tagging shape): same events, same result as the tagged parse. + const eventsPath = await createSessionDir('sess-cli-untagged', [ + JSON.stringify({ + type: 'session.start', + timestamp: '2026-08-07T17:56:35.573Z', + data: { sessionId: 'sess-cli-untagged', producer: 'copilot-agent', copilotVersion: '1.0.78' }, + }), + modelChange('claude-sonnet-5'), + userMessage('hello'), + assistantMessage({ messageId: 'msg-1', outputTokens: 42 }), + shutdownEvent({ + modelMetrics: { + 'claude-sonnet-5': { inputTokens: 1000, outputTokens: 42, cacheReadTokens: 600, cacheWriteTokens: 300 }, + }, + }), + ]) + + const calls = await collectCalls({ path: eventsPath, project: 'myproject', provider: 'copilot' }) + expect(calls.some(c => c.deduplicationKey.includes(':shutdown:'))).toBe(true) + expect(calls.find(c => c.deduplicationKey.includes(':shutdown:'))!.cacheReadInputTokens).toBe(600) + }) + + it('wires discovery through parsing: a discovered CLI session keeps its shutdown rollup', async () => { + // The full #944 pipeline: discoverSessions must tag the session-state file + // so that the parser it hands off to keeps the shutdown tokens. + await createSessionDir('sess-wire', [ + JSON.stringify({ + type: 'session.start', + timestamp: '2026-08-07T17:56:35.573Z', + data: { sessionId: 'sess-wire', producer: 'copilot-agent', copilotVersion: '1.0.78' }, + }), + modelChange('claude-sonnet-5'), + userMessage('hello'), + assistantMessage({ messageId: 'msg-1', outputTokens: 42 }), + shutdownEvent({ + modelMetrics: { + 'claude-sonnet-5': { inputTokens: 5000, outputTokens: 42, cacheReadTokens: 3000, cacheWriteTokens: 1500 }, + }, + }), + ]) + + // Keep discovery hermetic: a real agent-traces.db on the host must not leak in. + vi.stubEnv('CODEBURN_COPILOT_DISABLE_OTEL', '1') + try { + const provider = createCopilotProvider(tmpDir, '/nonexistent/vscode', '/nonexistent/global', '/nonexistent/jetbrains') + const sessions = await provider.discoverSessions() + expect(sessions).toHaveLength(1) + + const calls: ParsedProviderCall[] = [] + for await (const call of provider.createSessionParser(sessions[0]!, new Set()).parse()) calls.push(call) + + const shutdown = calls.find(c => c.deduplicationKey === 'copilot:sess-wire:shutdown:claude-sonnet-5:1') + expect(shutdown).toBeDefined() + expect(shutdown!.inputTokens).toBe(500) // 5000 − 3000 − 1500 + expect(shutdown!.cacheReadInputTokens).toBe(3000) + expect(shutdown!.cacheCreationInputTokens).toBe(1500) + } finally { + vi.unstubAllEnvs() + } + }) }) describe('copilot provider - chatSessions parsing', () => { @@ -811,6 +1207,9 @@ describe('copilot provider - discoverSessions', () => { expect(sessions).toHaveLength(2) expect(sessions.every(s => s.provider === 'copilot')).toBe(true) expect(sessions.every(s => s.path.endsWith('events.jsonl'))).toBe(true) + // Session-state files are tagged as CLI sources — the tag (not the file's + // producer value) decides transcript vs CLI parsing (#944). + expect(sessions.every(s => (s as { sourceType?: string }).sourceType === 'jsonl')).toBe(true) }) it('reads project name from workspace.yaml cwd', async () => { @@ -864,6 +1263,7 @@ describe('copilot provider - discoverSessions', () => { expect(sessions).toHaveLength(1) expect(sessions[0]!.project).toBe('myapp') expect(sessions[0]!.path).toContain('session-1.jsonl') + expect((sessions[0] as { sourceType?: string }).sourceType).toBe('transcript') }) it('includes VSCodium workspaceStorage paths on all supported platforms', () => {