From 137f443516b76617c54d6e0330f70bc7ca71d6f0 Mon Sep 17 00:00:00 2001 From: Yang Liu Date: Sun, 31 May 2026 13:43:21 +1200 Subject: [PATCH] fix(parser): handle nested cache_creation.input_tokens format (v2.1.152+) The Anthropic API can report cache-write tokens in two formats: - Flat (older): usage.cache_creation_input_tokens: N - Nested (v2.1.152+): usage.cache_creation: { input_tokens: N } Previously the parser only read the flat field, causing cache_creation_input_tokens to show 0 for sessions recorded after the Claude Code v2.1.152 fix. Add CacheCreationUsage sub-struct and optional cache_creation field to EntryUsage so serde handles nested deserialization. Add cache_creation_from_value() helper for raw JSON Value access sites (session.rs scan_session_metadata, IncrementalTokenScanner, subagent.rs scan_subagent_tokens_into). Both the struct method and helper take max(flat, nested) to remain backward-compatible with pre-fix sessions that recorded 0 in the flat field. Fixes #116 --- specs/01-parser-pipeline.md | 15 +++--- src-tauri/src/parser/classify.rs | 2 +- src-tauri/src/parser/entry.rs | 87 ++++++++++++++++++++++++++++++++ src-tauri/src/parser/session.rs | 33 ++++++++---- src-tauri/src/parser/subagent.rs | 7 +-- 5 files changed, 122 insertions(+), 22 deletions(-) diff --git a/specs/01-parser-pipeline.md b/specs/01-parser-pipeline.md index e337e81..7130d79 100644 --- a/specs/01-parser-pipeline.md +++ b/specs/01-parser-pipeline.md @@ -129,13 +129,14 @@ flowchart TD ### Version-Compatibility Normalisations -| Issue | Version | Fix | -| ------------------------------------ | ------------ | -------------------------------------------------------------- | -| Tool inputs JSON-encoded as strings | pre-v2.1.92 | Deserialise inner string → object | -| Fork reference in `forkedFrom` field | pre-v2.1.118 | Map to synthetic `fork-context-ref` | -| Hook payload in content text | all | Regex extraction of teammate ID, color, protocol | -| Large outputs written to disk | v2.1.89+ | `RE_PERSISTED_OUTPUT_PATH` → file read | -| Dynamic Workflow lifecycle types | v2.1.154+ | Add to `NOISE_ENTRY_TYPES`; capture workflow fields on `Entry` | +| Issue | Version | Fix | +| ----------------------------------------------------------------------------------------- | ------------ | ------------------------------------------------------------------------- | +| Tool inputs JSON-encoded as strings | pre-v2.1.92 | Deserialise inner string → object | +| Fork reference in `forkedFrom` field | pre-v2.1.118 | Map to synthetic `fork-context-ref` | +| Hook payload in content text | all | Regex extraction of teammate ID, color, protocol | +| Large outputs written to disk | v2.1.89+ | `RE_PERSISTED_OUTPUT_PATH` → file read | +| Dynamic Workflow lifecycle types | v2.1.154+ | Add to `NOISE_ENTRY_TYPES`; capture workflow fields on `Entry` | +| `cache_creation_input_tokens` always 0 when API uses nested `cache_creation.input_tokens` | v2.1.152+ | `cache_creation_from_value()` reads both flat and nested forms; takes max | --- diff --git a/src-tauri/src/parser/classify.rs b/src-tauri/src/parser/classify.rs index 76cd4a5..c5322cd 100644 --- a/src-tauri/src/parser/classify.rs +++ b/src-tauri/src/parser/classify.rs @@ -459,7 +459,7 @@ pub fn classify(e: Entry) -> Option { input_tokens: e.message.usage.input_tokens, output_tokens: e.message.usage.output_tokens, cache_read_tokens: e.message.usage.cache_read_input_tokens, - cache_creation_tokens: e.message.usage.cache_creation_input_tokens, + cache_creation_tokens: e.message.usage.effective_cache_creation_input_tokens(), }, stop_reason, is_meta: false, diff --git a/src-tauri/src/parser/entry.rs b/src-tauri/src/parser/entry.rs index c997699..1d4721e 100644 --- a/src-tauri/src/parser/entry.rs +++ b/src-tauri/src/parser/entry.rs @@ -145,6 +145,15 @@ pub struct EntryMessage { pub usage: EntryUsage, } +/// Nested cache-creation breakdown returned by the Anthropic API since Claude Code v2.1.152. +/// The API may report cache writes as `usage.cache_creation.input_tokens` instead of (or in +/// addition to) the flat `usage.cache_creation_input_tokens` field. +#[derive(Debug, Deserialize, Default)] +pub struct CacheCreationUsage { + #[serde(default)] + pub input_tokens: i64, +} + #[derive(Debug, Deserialize, Default)] pub struct EntryUsage { #[serde(default)] @@ -153,8 +162,42 @@ pub struct EntryUsage { pub output_tokens: i64, #[serde(default)] pub cache_read_input_tokens: i64, + /// Flat format (pre-v2.1.152). May be 0 when the API uses the nested format. #[serde(default)] pub cache_creation_input_tokens: i64, + /// Nested format (v2.1.152+). Takes precedence when non-zero. + #[serde(default)] + pub cache_creation: Option, +} + +impl EntryUsage { + /// Returns the effective cache-creation token count, handling both the flat and nested + /// API formats. Takes the max so old sessions (flat only) and new sessions (nested only) + /// both read correctly. + pub fn effective_cache_creation_input_tokens(&self) -> i64 { + let nested = self + .cache_creation + .as_ref() + .map(|c| c.input_tokens) + .unwrap_or(0); + self.cache_creation_input_tokens.max(nested) + } +} + +/// Extracts the effective cache-creation token count from a raw `usage` JSON value, +/// handling both the flat `cache_creation_input_tokens` field and the nested +/// `cache_creation.input_tokens` form introduced in Claude Code v2.1.152. +pub(crate) fn cache_creation_from_value(usage: &Value) -> i64 { + let flat = usage + .get("cache_creation_input_tokens") + .and_then(|v| v.as_i64()) + .unwrap_or(0); + let nested = usage + .get("cache_creation") + .and_then(|v| v.get("input_tokens")) + .and_then(|v| v.as_i64()) + .unwrap_or(0); + flat.max(nested) } impl Entry { @@ -264,6 +307,50 @@ mod tests { use super::*; use serde_json::json; + // --- cache_creation_from_value / effective_cache_creation_input_tokens tests --- + + #[test] + fn cache_creation_flat_format() { + let usage = json!({"cache_creation_input_tokens": 200}); + assert_eq!(cache_creation_from_value(&usage), 200); + } + + #[test] + fn cache_creation_nested_format() { + let usage = json!({"cache_creation": {"input_tokens": 300}}); + assert_eq!(cache_creation_from_value(&usage), 300); + } + + #[test] + fn cache_creation_both_formats_returns_max() { + // Both fields present: take the larger value. + let usage = + json!({"cache_creation_input_tokens": 100, "cache_creation": {"input_tokens": 300}}); + assert_eq!(cache_creation_from_value(&usage), 300); + } + + #[test] + fn cache_creation_neither_format_returns_zero() { + let usage = json!({"input_tokens": 50}); + assert_eq!(cache_creation_from_value(&usage), 0); + } + + #[test] + fn entry_usage_deserializes_nested_format() { + let json_str = r#"{"input_tokens":10,"output_tokens":5,"cache_read_input_tokens":0,"cache_creation":{"input_tokens":400}}"#; + let usage: EntryUsage = serde_json::from_str(json_str).unwrap(); + assert_eq!(usage.cache_creation_input_tokens, 0); + assert_eq!(usage.effective_cache_creation_input_tokens(), 400); + } + + #[test] + fn entry_usage_deserializes_flat_format() { + let json_str = r#"{"input_tokens":10,"output_tokens":5,"cache_read_input_tokens":0,"cache_creation_input_tokens":200}"#; + let usage: EntryUsage = serde_json::from_str(json_str).unwrap(); + assert_eq!(usage.cache_creation_input_tokens, 200); + assert_eq!(usage.effective_cache_creation_input_tokens(), 200); + } + // --- parse_entry tests --- #[test] diff --git a/src-tauri/src/parser/session.rs b/src-tauri/src/parser/session.rs index 73ae11d..5893189 100644 --- a/src-tauri/src/parser/session.rs +++ b/src-tauri/src/parser/session.rs @@ -9,7 +9,7 @@ use std::path::PathBuf; use super::chunk::{build_chunks, Chunk}; use super::classify::{classify, ClassifiedMsg}; use super::debuglog::extract_hook_msgs; -use super::entry::{parse_entry, Entry}; +use super::entry::{cache_creation_from_value, parse_entry, Entry}; /// SessionInfo holds metadata about a discovered session file for the picker. #[derive(Debug, Clone, Serialize)] @@ -665,10 +665,7 @@ pub(crate) fn scan_session_metadata(path: &str) -> SessionMetadata { .get("cache_read_input_tokens") .and_then(|v| v.as_i64()) .unwrap_or(0), - cache_create: usage - .get("cache_creation_input_tokens") - .and_then(|v| v.as_i64()) - .unwrap_or(0), + cache_create: cache_creation_from_value(usage), model: model_str.to_string(), has_stop_reason: has_stop, }; @@ -1013,10 +1010,7 @@ impl IncrementalTokenScanner { .get("cache_read_input_tokens") .and_then(|v| v.as_i64()) .unwrap_or(0); - let cc = usage - .get("cache_creation_input_tokens") - .and_then(|v| v.as_i64()) - .unwrap_or(0); + let cc = cache_creation_from_value(usage); if inp + output + cr + cc == 0 { return; @@ -1449,6 +1443,27 @@ mod tests { std::fs::remove_dir_all(&tmp).ok(); } + #[test] + fn incremental_scanner_nested_cache_creation_format() { + // Verifies that the nested `cache_creation.input_tokens` format introduced in + // Claude Code v2.1.152 is counted correctly. + let tmp = env::temp_dir().join("tail-test-scanner-nested-cache"); + std::fs::create_dir_all(&tmp).unwrap(); + let path = tmp.join("session.jsonl"); + + let entry = r#"{"type":"assistant","uuid":"a1","requestId":"r1","message":{"model":"claude-sonnet-4-20250514","role":"assistant","content":[],"usage":{"input_tokens":100,"output_tokens":50,"cache_read_input_tokens":0,"cache_creation":{"input_tokens":300}},"stop_reason":"end_turn"}}"#; + std::fs::write(&path, format!("{entry}\n")).unwrap(); + + let mut scanner = IncrementalTokenScanner::new(); + let totals = scanner.scan_new_bytes(path.to_str().unwrap()); + assert_eq!(totals.input_tokens, 100); + assert_eq!(totals.output_tokens, 50); + assert_eq!(totals.cache_creation_tokens, 300); + assert_eq!(totals.total_tokens, 450); + + std::fs::remove_dir_all(&tmp).ok(); + } + // --- resolve_live_chain_uuids tests --- fn make_entry(uuid: &str, parent_uuid: &str, leaf_uuid: &str, is_sidechain: bool) -> Entry { diff --git a/src-tauri/src/parser/subagent.rs b/src-tauri/src/parser/subagent.rs index 47a2c77..20f8e18 100644 --- a/src-tauri/src/parser/subagent.rs +++ b/src-tauri/src/parser/subagent.rs @@ -8,7 +8,7 @@ use std::path::Path; use super::chunk::*; use super::classify::*; -use super::entry::parse_entry; +use super::entry::{cache_creation_from_value, parse_entry}; /// Per-requestId token snapshot used for deduplication across JSONL files. #[derive(Clone, Default)] @@ -1165,10 +1165,7 @@ pub fn scan_subagent_tokens_into( .get("cache_read_input_tokens") .and_then(|v| v.as_i64()) .unwrap_or(0); - let cc = usage - .get("cache_creation_input_tokens") - .and_then(|v| v.as_i64()) - .unwrap_or(0); + let cc = cache_creation_from_value(usage); if inp + out + cr + cc == 0 { continue; }