feat(queue): reclaim the 84% of the queue nothing ever reclaimed

CLAUDE.md

@@ -57,7 +57,14 @@ input_tokens                  = non-cached input only (no cache reads/writes)
 cached_input_tokens           = cache reads
 cache_creation_input_tokens   = cache writes
 reasoning_output_tokens       = reasoning tokens (Codex/every-code fold them into output_tokens for cost)
-total_tokens                  = input + output + cache_creation + cache_read + reasoning_output (sum of all columns; Gemini-style rows that omit reasoning still pass invariants if you set the column to 0)
+total_tokens                  = input + output + cache_creation + cache_read + reasoning_output
+                                EXCEPT codex / every-code, where reasoning_output is ALREADY
+                                counted inside output_tokens, so their total_tokens correctly
+                                omits it. `computeRowCost` (pricing/index.js:309) makes the same
+                                distinction and charges their reasoning at zero. Gemini-style rows
+                                that omit reasoning pass with the column set to 0.
+                                Enforced by `doctor` (queue.row_invariant) via
+                                expectedTotal() in src/lib/queue-compact.js.
 ```
 
 **Cost is computed from `input_tokens + output_tokens + cached_input_tokens + cache_creation_input_tokens + reasoning_output_tokens` only — never `total_tokens`** (`computeRowCost` in `src/lib/pricing/index.js`). If a new provider only fills `total_tokens` with input=0/output=0, the dashboard renders **$0 cost** regardless of pricing entries. Distribute the total across columns or extend `computeRowCost`.

src/commands/doctor.js

@@ -12,6 +12,7 @@ async function cmdDoctor(argv = []) {
   const home = os.homedir();
   const { trackerDir } = await resolveTrackerPaths({ home });
   const configPath = path.join(trackerDir, "config.json");
+  const queuePath = path.join(trackerDir, "queue.jsonl");
 
   const configStatus = await readJsonStrict(configPath);
   const config =
@@ -36,6 +37,7 @@ async function cmdDoctor(argv = []) {
       trackerDir,
       configPath,
       cliPath,
+      queuePath,
     },
   });
 

src/commands/sync.js

@@ -6,6 +6,7 @@ const cp = require("node:child_process");
 const readline = require("node:readline");
 
 const { ensureDir, readJson, writeJson, openLock } = require("../lib/fs");
+const { analyzeQueue, compactQueue } = require("../lib/queue-compact");
 const {
   listRolloutFiles,
   listClaudeProjectFiles,
@@ -143,6 +144,13 @@ async function cmdSync(argv) {
     const cursorsPath = path.join(trackerDir, "cursors.json");
     const queuePath = path.join(trackerDir, "queue.jsonl");
     const queueStatePath = path.join(trackerDir, "queue.state.json");
+
+    // Maintenance path: compact and stop. Deliberately inside the lock, since
+    // the whole point is that a concurrent append must not land between the
+    // read and the rename.
+    if (opts.compact) {
+      return runCompaction(queuePath, { auto: opts.auto });
+    }
     const projectQueuePath = path.join(trackerDir, "project.queue.jsonl");
     const projectQueueStatePath = path.join(trackerDir, "project.queue.state.json");
     const grokSignalPath = path.join(trackerDir, "grok-last-session.json");
@@ -982,6 +990,29 @@ async function cmdSync(argv) {
       totalBuckets,
     };
 
+    // Nothing ever reclaimed superseded rows, so the file grows monotonically
+    // and every endpoint call re-parses all of it. #103 proposed compacting
+    // automatically past a threshold. This RECOMMENDS instead, deliberately.
+    //
+    // The reason is not caution in the abstract. Wiring the automatic version up
+    // first, it immediately rewrote an 11 MB production queue — 34,924 lines
+    // down to 5,636 — during `npm test`, because test/init-uninstall.test.js
+    // spawned the notify handler in a child process without an isolated HOME.
+    // That leak is fixed, but the lesson stands: rewriting the user's data file
+    // without being asked turns any path that reaches sync into a data-mutating
+    // path. `--compact` is one command and the user chooses to run it.
+    const queueHealth = analyzeQueue(queuePath);
+    if (
+      !opts.auto &&
+      queueHealth.parseable >= COMPACT_MIN_LINES &&
+      queueHealth.ratio >= COMPACT_RATIO
+    ) {
+      process.stdout.write(
+        `\nQueue: ${Math.round(queueHealth.ratio * 100)}% of ${queueHealth.parseable} rows are superseded` +
+          ` (${Math.round(queueHealth.bytes / 1024)} KiB). Run \`tracker sync --compact\` to reclaim them.\n`,
+      );
+    }
+
     if (!opts.auto) {
       process.stdout.write(
         [
@@ -1002,6 +1033,40 @@ async function cmdSync(argv) {
   }
 }
 
+// When a queue is mostly dead weight and big enough to be worth rewriting, sync
+// says so. Below either threshold the rewrite costs more than it saves.
+const COMPACT_RATIO = 0.5;
+const COMPACT_MIN_LINES = 5000;
+
+// Reporting is not optional. The README tells users they can `cat` this file;
+// silently rewriting it underneath them is the wrong default.
+function runCompaction(queuePath, { auto = false } = {}) {
+  const result = compactQueue(queuePath);
+  if (auto) return result;
+  if (!result.changed) {
+    process.stdout.write(`Queue compaction: nothing to reclaim (${result.reason}).\n`);
+    return result;
+  }
+  const savedKiB = Math.round((result.bytesBefore - result.bytesAfter) / 1024);
+  process.stdout.write(
+    [
+      "Queue compacted:",
+      `- Lines: ${result.totalLines} -> ${result.keptLines}`,
+      `- Superseded rows reclaimed: ${result.superseded}`,
+      `- Size: ${savedKiB} KiB smaller`,
+      result.malformed > 0
+        ? `- Kept ${result.malformed} unparseable line(s) untouched — they are invisible to every reader, but they are still your bytes`
+        : null,
+      "",
+    ]
+      // Not filter(Boolean): that drops the trailing "" too, and the report ran
+      // straight into the next line of output.
+      .filter((line) => line !== null)
+      .join("\n"),
+  );
+  return result;
+}
+
 function parseArgs(argv) {
   const out = {
     auto: false,
@@ -1010,6 +1075,7 @@ function parseArgs(argv) {
     fromOpenclaw: false,
     drain: false,
     repairGrok: false,
+    compact: false,
   };
   for (let i = 0; i < argv.length; i++) {
     const a = argv[i];
@@ -1019,6 +1085,7 @@ function parseArgs(argv) {
     else if (a === "--from-openclaw") out.fromOpenclaw = true;
     else if (a === "--drain") out.drain = true;
     else if (a === "--repair-grok") out.repairGrok = true;
+    else if (a === "--compact") out.compact = true;
     else throw new Error(`Unknown option: ${a}`);
   }
   return out;

src/lib/doctor.js

@@ -1,4 +1,5 @@
 const fs = require("node:fs/promises");
+const { findRowViolations } = require("./queue-compact");
 const { constants } = require("node:fs");
 const path = require("node:path");
 
@@ -28,6 +29,9 @@ async function buildDoctorReport({
   if (paths.cliPath) {
     checks.push(await checkCliEntrypoint(paths.cliPath));
   }
+  if (paths.queuePath) {
+    checks.push(await checkQueueRows(paths.queuePath));
+  }
 
   // No cloud reachability check: TokenTracker is local-only, so there is no
   // remote endpoint whose availability could affect anything here.
@@ -358,8 +362,76 @@ function summarizeChecks(checks = []) {
   return summary;
 }
 
+// CLAUDE.md states the column invariant in prose:
+//
+//   total = input + output + cache_creation + cache_read + reasoning
+//
+// Nothing enforced it at runtime, so a miswritten or corrupt row was aggregated
+// and rendered rather than flagged — and a parser bug of exactly that shape is
+// the class CLAUDE.md records at 1.6-7x magnitude. Same conversion as the
+// curated-expiry and version-lockstep checks: a rule that lived in a document
+// starts running.
+//
+// A warn rather than a fail: the rows are already on disk and already being
+// rendered, so failing the whole health check would be reporting a crisis the
+// user cannot act on in the moment. What they can act on is knowing which rows,
+// and how many.
+const QUEUE_VIOLATIONS_SHOWN = 5;
+
+function queueCheck(status, detail, meta) {
+  return { id: "queue.row_invariant", status, detail, critical: false, meta };
+}
+
+async function readQueueRowsForDoctor(queuePath) {
+  const raw = await fs.readFile(queuePath, "utf8");
+  const rows = [];
+  let malformed = 0;
+  for (const line of raw.split("\n")) {
+    if (!line.trim()) continue;
+    try {
+      rows.push(JSON.parse(line));
+    } catch {
+      malformed += 1;
+    }
+  }
+  return { rows, malformed };
+}
+
+async function checkQueueRows(queuePath) {
+  let rows;
+  let malformed;
+  try {
+    ({ rows, malformed } = await readQueueRowsForDoctor(queuePath));
+  } catch (err) {
+    if (err && err.code === "ENOENT") {
+      return queueCheck("ok", "no queue yet", { path: queuePath });
+    }
+    return queueCheck("warn", `queue unreadable: ${err?.message || err}`, { path: queuePath });
+  }
+
+  const violations = findRowViolations(rows);
+  if (violations.length === 0 && malformed === 0) {
+    return queueCheck("ok", `${rows.length} rows satisfy the column invariant`, {
+      path: queuePath,
+      rows: rows.length,
+    });
+  }
+
+  const parts = [];
+  if (violations.length > 0) parts.push(`${violations.length} row problem(s)`);
+  if (malformed > 0) parts.push(`${malformed} unparseable line(s)`);
+  return queueCheck("warn", `${parts.join(", ")} in ${rows.length + malformed} line(s)`, {
+    path: queuePath,
+    rows: rows.length,
+    malformed,
+    violations: violations.length,
+    examples: violations.slice(0, QUEUE_VIOLATIONS_SHOWN),
+  });
+}
+
 module.exports = {
   buildDoctorReport,
+  checkQueueRows,
   buildBrowserOpenerCheck,
   buildNodeVersionCheck,
   isHeadlessEnvironment,