fix: parse Grok Build usage logs

docs/superpowers/specs/2026-07-13-grok-parser-design.md

@@ -0,0 +1,49 @@
+# Grok Build 日志解析修复设计
+
+## 背景与目标
+
+AIUsage 已能发现 Grok Build 的默认会话目录，但 `Aggregator` 没有注册 Grok 解析器，导致发现到的日志全部返回空结果。Grok Build 当前的 `updates.jsonl` 使用 JSON-RPC 更新格式，并通过累计的 `totalTokens` 暴露 token 使用量，不能直接套用通用 JSONL 解析器。
+
+本次修复应让自动解析和 `aiusage parse --tool grok` 都能导入既有及新增 Grok Build 会话，同时避免累计 token 重复计数，并自动恢复此前被错误水位跳过的历史日志。
+
+## 解析架构
+
+在 `@aiusage/core` 中新增专用的状态化 `GrokParser`，由 `Aggregator` 为 `grok` 注册。解析器只接受 Grok Build `updates.jsonl` 中的 JSON-RPC 更新，跟踪当前文件内的模型、最近累计 token、当前 turn 的基线与最大累计值。
+
+每遇到新的 `user_message_chunk`，解析器结束上一个 turn；文件结束时通过 `finalize()` 输出最后一个 turn。单个 turn 的 token 使用量为其最大累计值减去 turn 开始前的累计基线。重复或回退的累计值不产生负数或重复记录。由于当前日志没有稳定的 input/output/cache 拆分，正向增量统一记录到 `inputTokens`，其他 token 分类记为 0。
+
+解析器在 `finalize()` 后清空文件级状态，避免多个会话互相污染。若日志没有明确 turn 边界但存在有效累计 token，则生成一个会话级回退记录，保证已有数据不会继续静默丢失。
+
+## 字段提取
+
+- session ID：优先使用 `params.sessionId`，否则使用 `updates.jsonl` 的父目录名。
+- 模型：优先使用 `params.update._meta.modelId`，并兼容 `params._meta.modelId`；缺失时使用 `grok-unknown`。
+- token：兼容 `params._meta.totalTokens` 和 `params.update._meta.totalTokens`，只接受非负有限整数。
+- 时间戳：优先读取 `agentTimestampMs`，兼容常见 timestamp 字段，最后回退到解析上下文时间。
+- 工作目录：从 `sessions/<encoded-project-path>/<session-id>/updates.jsonl` 的项目目录解码；无法可靠解码时不写入 `cwd`。
+- provider 与 cost：provider 由现有模型推断逻辑得到 `xai`；价格存在时复用现有计价逻辑，否则 cost 为 0、来源为 `unknown`。
+
+## 文件发现与水位迁移
+
+Grok 发现逻辑收窄为只返回名为 `updates.jsonl` 的文件，排除 `events.jsonl`、`chat_history.jsonl` 等非用量来源。
+
+水位状态增加 Grok 解析器版本。读取旧水位文件时，如果版本低于当前版本，则仅清空 `files.grok` 并写入当前版本；其他工具水位保持不变。这样修复后的首次解析会自动重扫历史 Grok 日志，之后继续使用正常的字节偏移增量解析。
+
+增量解析时，水位之前的行仍会送入状态化解析器以重建累计基线，但不会重复写入记录；只有水位之后产生的 turn 结果会被导入。
+
+## 错误处理
+
+无效 JSON、缺少用量字段、零增量、重复累计值和回退累计值均被安全忽略，不中断同文件后续行。文件级读取错误沿用现有 `parseLogs` 错误收集机制。迁移只操作 Grok 水位，不删除数据库中的现有记录。
+
+## 测试策略
+
+- Core 单元测试：新版 JSON-RPC 格式、模型/会话/时间戳提取、累计增量、重复与回退值、无 turn 边界回退、多文件状态隔离。
+- Aggregator 测试：确认 `grok` 已注册且可解析记录。
+- Discovery 测试：Windows 默认目录与 `GROK_HOME`，并确认只发现 `updates.jsonl`。
+- Watermark 测试：旧状态仅重置 Grok 水位、保留其他工具水位，并持久化解析器版本。
+- CLI 集成测试：已有 Grok 水位会自动重扫，首次导入历史记录；再次运行不重复导入；不同会话保持独立。
+- 最终验证：运行 core/cli 相关测试、完整测试、lint 与 build。
+
+## 非目标
+
+本次不从 `signals.json` 或 `summary.json` 补齐压缩前 token，不估算 input/output 比例，也不新增 Grok 专属价格。若后续日志提供稳定的分类 token 或汇总字段，可在专用解析器中独立扩展。

packages/cli/src/discovery.ts

@@ -773,6 +773,8 @@ export function discoverTools(env: NodeJS.ProcessEnv = process.env): DetectedToo
                 + findZipFiles(detectedPath).length
             } else if (entry.sourceKey === 'kiro') {
               fileCount += unique([...findJsonlFiles(detectedPath), ...findJsonFiles(detectedPath)]).length
+            } else if (entry.sourceKey === 'grok') {
+              fileCount += findJsonlFiles(detectedPath).filter((p) => basename(p) === 'updates.jsonl').length
             } else {
               fileCount += findJsonlFiles(detectedPath).length
             }
@@ -900,7 +902,7 @@ export function discoverLogFiles(env: NodeJS.ProcessEnv = process.env): { tool:
     { tool: 'kiro', path: kiroDevDataDir(ctx), filter: (p) => extname(p) === '.jsonl' || extname(p) === '.json' },
     { tool: 'kiro', path: probeKiroOverridePath(ctx), filter: (p) => extname(p) === '.jsonl' || extname(p) === '.json' },
     { tool: 'kiro', path: kiroWorkspaceSessionsDir(ctx), filter: (p) => extname(p) === '.json' && basename(p) !== 'sessions.json' },
-    { tool: 'grok', path: probeGrok(ctx), filter: (p) => extname(p) === '.jsonl' },
+    { tool: 'grok', path: probeGrok(ctx), filter: (p) => basename(p) === 'updates.jsonl' },
     { tool: 'antigravity', path: probeAntigravity(ctx) },
     { tool: 'omp', path: probeOmp(ctx) },
     { tool: 'pi', path: probePi(ctx) },

packages/cli/src/watermark.ts

@@ -1,6 +1,8 @@
 import { readFileSync, writeFileSync, existsSync } from 'node:fs'
 import type { Tool } from '@aiusage/core'
 
+const CURRENT_GROK_PARSER_VERSION = 1
+
 export interface WatermarkEntry {
   offset: number
   size: number
@@ -45,6 +47,7 @@ export type FileWatermarkData = Record<Tool, Record<string, WatermarkEntry>>
 
 export interface WatermarkState {
   files: FileWatermarkData
+  grokParserVersion?: number
   opencode?: OpenCodeCursor | null
   hermes?: HermesCursor | null
   qoder?: QoderCursor | null
@@ -103,18 +106,40 @@ export class WatermarkManager {
 
   private load(): WatermarkState {
     if (!existsSync(this.path)) {
-      return { files: defaultFileData() }
+      return { files: defaultFileData(), grokParserVersion: CURRENT_GROK_PARSER_VERSION }
     }
     try {
       const content = readFileSync(this.path, 'utf-8')
       const parsed = JSON.parse(content)
+      let state: WatermarkState
       // Handle legacy format (flat Record<Tool, ...> without 'files' key)
       if (parsed && typeof parsed === 'object' && !('files' in parsed)) {
-        return { files: { ...defaultFileData(), ...parsed } }
+        state = { files: { ...defaultFileData(), ...parsed } }
+      } else {
+        state = {
+          files: { ...defaultFileData(), ...(parsed.files ?? {}) },
+          grokParserVersion: parsed.grokParserVersion,
+          opencode: parsed.opencode ?? null,
+          hermes: parsed.hermes ?? null,
+          qoder: parsed.qoder ?? null,
+          cursor: parsed.cursor ?? null,
+          goose: parsed.goose ?? null,
+          zed: parsed.zed ?? null,
+          kiro: parsed.kiro ?? null,
+          zcode: parsed.zcode ?? null,
+          zcodeTools: parsed.zcodeTools ?? null,
+          trae: parsed.trae ?? null,
+          codebuddyIde: parsed.codebuddyIde ?? null,
+        }
+      }
+
+      if ((state.grokParserVersion ?? 0) < CURRENT_GROK_PARSER_VERSION) {
+        state.files.grok = {}
+        state.grokParserVersion = CURRENT_GROK_PARSER_VERSION
       }
-      return { files: { ...defaultFileData(), ...(parsed.files ?? {}) }, opencode: parsed.opencode ?? null, hermes: parsed.hermes ?? null, qoder: parsed.qoder ?? null, cursor: parsed.cursor ?? null, goose: parsed.goose ?? null, zed: parsed.zed ?? null, kiro: parsed.kiro ?? null, zcode: parsed.zcode ?? null, zcodeTools: parsed.zcodeTools ?? null, trae: parsed.trae ?? null, codebuddyIde: parsed.codebuddyIde ?? null }
+      return state
     } catch {
-      return { files: defaultFileData() }
+      return { files: defaultFileData(), grokParserVersion: CURRENT_GROK_PARSER_VERSION }
     }
   }
 

packages/cli/tests/commands/parse-grok.test.ts

@@ -0,0 +1,126 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import Database from 'better-sqlite3'
+import { mkdirSync, rmSync, statSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { initializeDatabase } from '../../src/db/index.js'
+
+const { testDir } = vi.hoisted(() => ({
+  testDir: '/tmp/aiusage-parse-grok-test',
+}))
+
+vi.mock('node:os', async () => {
+  const actual = await vi.importActual<typeof import('node:os')>('node:os')
+  return {
+    ...actual,
+    homedir: () => testDir,
+  }
+})
+
+const { runParse } = await import('../../src/commands/parse.js')
+
+function update(options: {
+  sessionId: string
+  totalTokens?: number
+  sessionUpdate?: string
+  modelId?: string
+  timestamp: number
+}): Record<string, unknown> {
+  return {
+    method: 'session/update',
+    params: {
+      sessionId: options.sessionId,
+      update: {
+        sessionUpdate: options.sessionUpdate ?? 'agent_message_chunk',
+        ...(options.modelId ? { _meta: { modelId: options.modelId } } : {}),
+      },
+      _meta: {
+        ...(options.totalTokens == null ? {} : { totalTokens: options.totalTokens }),
+        agentTimestampMs: options.timestamp,
+      },
+    },
+  }
+}
+
+function writeJsonl(filePath: string, rows: Record<string, unknown>[]): void {
+  writeFileSync(filePath, `${rows.map(row => JSON.stringify(row)).join('\n')}\n`)
+}
+
+describe('runParse with Grok Build data', () => {
+  let cacheDb: Database.Database
+
+  beforeEach(() => {
+    rmSync(testDir, { recursive: true, force: true })
+    mkdirSync(join(testDir, '.aiusage'), { recursive: true })
+    cacheDb = new Database(':memory:')
+    initializeDatabase(cacheDb)
+  })
+
+  afterEach(() => {
+    cacheDb.close()
+    rmSync(testDir, { recursive: true, force: true })
+  })
+
+  it('replays stale watermarks once and keeps Grok sessions isolated', async () => {
+    const sessionsRoot = join(testDir, '.grok', 'sessions')
+    const sessionADir = join(sessionsRoot, '%43%3A%5Cworkspace%5Capp', 'session-a')
+    const sessionBDir = join(sessionsRoot, '%2Fworkspace%2Fother', 'session-b')
+    mkdirSync(sessionADir, { recursive: true })
+    mkdirSync(sessionBDir, { recursive: true })
+    const sessionAPath = join(sessionADir, 'updates.jsonl')
+    const sessionBPath = join(sessionBDir, 'updates.jsonl')
+
+    writeJsonl(sessionAPath, [
+      update({ sessionId: 'session-a', totalTokens: 100, timestamp: 1_700_000_000_000 }),
+      update({ sessionId: 'session-a', sessionUpdate: 'user_message_chunk', modelId: 'grok-composer-2.5-fast', timestamp: 1_700_000_001_000 }),
+      update({ sessionId: 'session-a', totalTokens: 300, timestamp: 1_700_000_002_000 }),
+      update({ sessionId: 'session-a', sessionUpdate: 'user_message_chunk', modelId: 'grok-composer-2.5-fast', timestamp: 1_700_000_003_000 }),
+      update({ sessionId: 'session-a', totalTokens: 450, timestamp: 1_700_000_004_000 }),
+    ])
+    writeJsonl(sessionBPath, [
+      update({ sessionId: 'session-b', totalTokens: 20, timestamp: 1_700_000_010_000 }),
+      update({ sessionId: 'session-b', sessionUpdate: 'user_message_chunk', modelId: 'grok-3', timestamp: 1_700_000_011_000 }),
+      update({ sessionId: 'session-b', totalTokens: 70, timestamp: 1_700_000_012_000 }),
+    ])
+
+    writeFileSync(join(testDir, '.aiusage', 'config.json'), JSON.stringify({
+      sources: { grok: sessionsRoot },
+    }))
+    writeFileSync(join(testDir, '.aiusage', 'watermark.json'), JSON.stringify({
+      files: {
+        grok: {
+          [sessionAPath]: {
+            offset: statSync(sessionAPath).size,
+            size: statSync(sessionAPath).size,
+            mtime: statSync(sessionAPath).mtimeMs,
+          },
+          [sessionBPath]: {
+            offset: statSync(sessionBPath).size,
+            size: statSync(sessionBPath).size,
+            mtime: statSync(sessionBPath).mtimeMs,
+          },
+        },
+      },
+    }))
+
+    const first = await runParse(cacheDb, 'grok')
+    expect(first.errors).toEqual([])
+    expect(first.parsedCount).toBe(3)
+
+    const rows = cacheDb.prepare(`
+      SELECT session_id, model, input_tokens, cwd
+      FROM records
+      WHERE tool = 'grok'
+      ORDER BY session_id, line_offset
+    `).all()
+    expect(rows).toEqual([
+      { session_id: 'session-a', model: 'grok-composer-2.5-fast', input_tokens: 200, cwd: 'C:\\workspace\\app' },
+      { session_id: 'session-a', model: 'grok-composer-2.5-fast', input_tokens: 150, cwd: 'C:\\workspace\\app' },
+      { session_id: 'session-b', model: 'grok-3', input_tokens: 50, cwd: '/workspace/other' },
+    ])
+
+    const second = await runParse(cacheDb, 'grok')
+    expect(second.errors).toEqual([])
+    expect(second.parsedCount).toBe(0)
+    expect(cacheDb.prepare("SELECT COUNT(*) AS count FROM records WHERE tool = 'grok'").get()).toEqual({ count: 3 })
+  })
+})

packages/cli/tests/discovery.test.ts

@@ -37,6 +37,7 @@ describe('discovery path resolution', () => {
     delete process.env.AIUSAGE_KELIVO_PATH
     delete process.env.AIUSAGE_CODEFUSE_PATH
     delete process.env.CODEFUSE_HOME
+    delete process.env.GROK_HOME
     for (const dir of tempDirs.splice(0)) {
       rmSync(dir, { recursive: true, force: true })
     }
@@ -132,6 +133,37 @@ describe('discovery path resolution', () => {
     expect(discoverLogFiles().find((result) => result.tool === 'codex')?.paths).toEqual([archivedFile])
   })
 
+  it('discovers only Grok Build updates.jsonl from the Windows default path', async () => {
+    const home = makeHome()
+    const sessionDir = join(home, '.grok', 'sessions', '%43%3A%5Cworkspace', 'session-1')
+    mkdirSync(sessionDir, { recursive: true })
+    const updatesPath = join(sessionDir, 'updates.jsonl')
+    writeFileSync(updatesPath, '{}\n')
+    writeFileSync(join(sessionDir, 'events.jsonl'), '{}\n')
+    writeFileSync(join(sessionDir, 'chat_history.jsonl'), '{}\n')
+
+    const { discoverLogFiles, discoverTools } = await loadDiscovery({ home, platform: 'win32' })
+
+    const detected = discoverTools().find((tool) => tool.sourceKey === 'grok')
+    expect(detected?.status).toBe('found')
+    expect(detected?.fileCount).toBe(1)
+    expect(discoverLogFiles().find((result) => result.tool === 'grok')?.paths).toEqual([updatesPath])
+  })
+
+  it('honors GROK_HOME when discovering Grok Build updates', async () => {
+    const home = makeHome()
+    const grokHome = join(home, 'custom-grok')
+    const sessionDir = join(grokHome, 'sessions', '%2Fworkspace', 'session-2')
+    mkdirSync(sessionDir, { recursive: true })
+    const updatesPath = join(sessionDir, 'updates.jsonl')
+    writeFileSync(updatesPath, '{}\n')
+    process.env.GROK_HOME = grokHome
+
+    const { discoverLogFiles } = await loadDiscovery({ home, platform: 'linux' })
+
+    expect(discoverLogFiles().find((result) => result.tool === 'grok')?.paths).toEqual([updatesPath])
+  })
+
   it('counts Codex archived sessions when regular sessions are absent', async () => {
     const home = makeHome()
     const archivedDir = join(home, '.codex', 'archived_sessions', '2026', '06', '03')

packages/cli/tests/watermark.test.ts

@@ -1,7 +1,7 @@
 import { describe, it, expect, beforeEach, afterEach } from 'vitest'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
-import { mkdirSync, writeFileSync, rmSync, existsSync } from 'node:fs'
+import { mkdirSync, writeFileSync, readFileSync, rmSync, existsSync } from 'node:fs'
 import { WatermarkManager } from '../src/watermark.js'
 
 describe('WatermarkManager', () => {
@@ -113,6 +113,35 @@ describe('WatermarkManager', () => {
     expect(wm2.getEntry('claude-code', '/path/file.jsonl')!.offset).toBe(500)
   })
 
+  it('clears only stale Grok entries when upgrading the Grok parser', () => {
+    writeFileSync(watermarkPath, JSON.stringify({
+      files: {
+        grok: { '/old/updates.jsonl': { offset: 100, size: 100, mtime: 1 } },
+        'claude-code': { '/claude/session.jsonl': { offset: 200, size: 200, mtime: 2 } },
+      },
+    }), 'utf-8')
+
+    const wm = new WatermarkManager(watermarkPath)
+    expect(wm.getEntry('grok', '/old/updates.jsonl')).toBeNull()
+    expect(wm.getEntry('claude-code', '/claude/session.jsonl')?.offset).toBe(200)
+
+    wm.save()
+    const saved = JSON.parse(readFileSync(watermarkPath, 'utf-8'))
+    expect(saved.grokParserVersion).toBe(1)
+  })
+
+  it('preserves Grok entries written by the current parser version', () => {
+    writeFileSync(watermarkPath, JSON.stringify({
+      files: {
+        grok: { '/current/updates.jsonl': { offset: 300, size: 300, mtime: 3 } },
+      },
+      grokParserVersion: 1,
+    }), 'utf-8')
+
+    const wm = new WatermarkManager(watermarkPath)
+    expect(wm.getEntry('grok', '/current/updates.jsonl')?.offset).toBe(300)
+  })
+
   it('returns null when no opencode cursor has been set', () => {
     const wm = new WatermarkManager(watermarkPath)
     expect(wm.getOpenCodeCursor()).toBeNull()

packages/core/src/aggregator.ts

@@ -7,6 +7,7 @@ import { OpenClawParser } from './parsers/openclaw.js'
 import { QoderParser } from './parsers/qoder.js'
 import { CopilotParser } from './parsers/copilot.js'
 import { GenericJsonlParser } from './parsers/generic-jsonl.js'
+import { GrokParser } from './parsers/grok.js'
 
 export interface CreateContextOptions {
   tool: Tool
@@ -34,6 +35,7 @@ export class Aggregator {
       ['gemini', new GenericJsonlParser('gemini', 'gemini-unknown')],
       ['kimi', new GenericJsonlParser('kimi', 'kimi-for-coding')],
       ['codebuddy', new GenericJsonlParser('codebuddy', 'codebuddy-unknown')],
+      ['grok', new GrokParser()],
       ['antigravity', new GenericJsonlParser('antigravity', 'antigravity-unknown')],
       ['omp', new GenericJsonlParser('omp', 'omp-unknown')],
       ['pi', new GenericJsonlParser('pi', 'pi-unknown')],