From 9ff85367823a6079aa110ad5903047593bd25758 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 17 Jul 2026 10:48:11 +0300 Subject: [PATCH 01/30] feat: add nextChunk checkpoint decision function --- src/memory/checkpoint.ts | 26 ++++++++++++++ tests/unit/checkpoint-nextchunk.test.ts | 48 +++++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 src/memory/checkpoint.ts create mode 100644 tests/unit/checkpoint-nextchunk.test.ts diff --git a/src/memory/checkpoint.ts b/src/memory/checkpoint.ts new file mode 100644 index 0000000..2cd5311 --- /dev/null +++ b/src/memory/checkpoint.ts @@ -0,0 +1,26 @@ +export interface OffsetTurn { + role: "user" | "assistant" | "tool"; + text: string; + endByte: number; +} + +export const CHECKPOINT_THRESHOLD = 8000; + +export function nextChunk( + turns: OffsetTurn[], + threshold: number, +): { chunk: string; endByte: number; consumedTurns: number } | null { + let length = 0; + for (let i = 0; i < turns.length; i++) { + length += (i > 0 ? 1 : 0) + turns[i]!.text.length; + if (length >= threshold) { + const consumed = turns.slice(0, i + 1); + return { + chunk: consumed.map((t) => t.text).join("\n"), + endByte: turns[i]!.endByte, + consumedTurns: i + 1, + }; + } + } + return null; +} diff --git a/tests/unit/checkpoint-nextchunk.test.ts b/tests/unit/checkpoint-nextchunk.test.ts new file mode 100644 index 0000000..b60151d --- /dev/null +++ b/tests/unit/checkpoint-nextchunk.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, test } from "bun:test"; +import { nextChunk, type OffsetTurn } from "../../src/memory/checkpoint.js"; + +function turn(text: string, endByte: number): OffsetTurn { + return { role: "user", text, endByte }; +} + +describe("nextChunk", () => { + test("returns null when accumulated turns are below threshold", () => { + const turns = [turn("a".repeat(10), 11), turn("b".repeat(10), 22)]; + expect(nextChunk(turns, 8000)).toBeNull(); + }); + + test("emits the turn that crosses the threshold, whole", () => { + const turns = [ + turn("a".repeat(5000), 5001), + turn("b".repeat(4000), 9002), + turn("c".repeat(10), 9013), + ]; + const result = nextChunk(turns, 8000); + expect(result).not.toBeNull(); + expect(result!.consumedTurns).toBe(2); + expect(result!.endByte).toBe(9002); + expect(result!.chunk).toBe("a".repeat(5000) + "\n" + "b".repeat(4000)); + }); + + test("emits at exactly the threshold", () => { + const turns = [turn("a".repeat(8000), 8001)]; + const result = nextChunk(turns, 8000); + expect(result).not.toBeNull(); + expect(result!.consumedTurns).toBe(1); + }); + + test("emits a single over-threshold turn alone, never split", () => { + const turns = [turn("a".repeat(20000), 20001), turn("b", 20003)]; + const result = nextChunk(turns, 8000); + expect(result!.consumedTurns).toBe(1); + expect(result!.chunk).toBe("a".repeat(20000)); + expect(result!.endByte).toBe(20001); + }); + + test("charges one newline between consumed turns", () => { + const turns = [turn("a".repeat(4000), 4001), turn("b".repeat(3999), 8001)]; + const result = nextChunk(turns, 8000); + expect(result).not.toBeNull(); + expect(result!.consumedTurns).toBe(2); + }); +}); From 353653392de34e34a378db4cf06d2afbe150d36d Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 17 Jul 2026 10:53:45 +0300 Subject: [PATCH 02/30] feat: add checkpoint file read/write with atomic rename --- src/memory/checkpoint.ts | 46 ++++++++++++++++++++++++++++++ tests/unit/checkpoint-file.test.ts | 42 +++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 tests/unit/checkpoint-file.test.ts diff --git a/src/memory/checkpoint.ts b/src/memory/checkpoint.ts index 2cd5311..b917996 100644 --- a/src/memory/checkpoint.ts +++ b/src/memory/checkpoint.ts @@ -1,3 +1,5 @@ +import { rename, unlink } from "node:fs/promises"; + export interface OffsetTurn { role: "user" | "assistant" | "tool"; text: string; @@ -24,3 +26,47 @@ export function nextChunk( } return null; } + +export interface Checkpoint { + byteOffset: number; + segment: number; +} + +const ZERO_CHECKPOINT: Checkpoint = { byteOffset: 0, segment: 0 }; + +export function checkpointPath(sessionId: string): string { + return `/tmp/betterdb-${sessionId}.checkpoint.json`; +} + +export async function readCheckpoint(sessionId: string): Promise { + const file = Bun.file(checkpointPath(sessionId)); + if (!(await file.exists())) { + return { ...ZERO_CHECKPOINT }; + } + try { + const parsed = (await file.json()) as Partial; + if ( + typeof parsed.byteOffset !== "number" || + typeof parsed.segment !== "number" + ) { + return { ...ZERO_CHECKPOINT }; + } + return { byteOffset: parsed.byteOffset, segment: parsed.segment }; + } catch { + return { ...ZERO_CHECKPOINT }; + } +} + +export async function writeCheckpoint( + sessionId: string, + cp: Checkpoint, +): Promise { + const target = checkpointPath(sessionId); + const tmp = `${target}.tmp`; + await Bun.write(tmp, JSON.stringify(cp)); + await rename(tmp, target); +} + +export async function removeCheckpoint(sessionId: string): Promise { + await unlink(checkpointPath(sessionId)).catch(() => {}); +} diff --git a/tests/unit/checkpoint-file.test.ts b/tests/unit/checkpoint-file.test.ts new file mode 100644 index 0000000..4bf1ccf --- /dev/null +++ b/tests/unit/checkpoint-file.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, test, afterEach } from "bun:test"; +import { unlink } from "node:fs/promises"; +import { + checkpointPath, + readCheckpoint, + writeCheckpoint, + removeCheckpoint, +} from "../../src/memory/checkpoint.js"; + +const SID = "checkpoint-file-test"; + +afterEach(async () => { + await unlink(checkpointPath(SID)).catch(() => {}); +}); + +describe("checkpoint file", () => { + test("missing file reads as the zero checkpoint", async () => { + expect(await readCheckpoint(SID)).toEqual({ byteOffset: 0, segment: 0 }); + }); + + test("round-trips a written checkpoint", async () => { + await writeCheckpoint(SID, { byteOffset: 8002, segment: 1 }); + expect(await readCheckpoint(SID)).toEqual({ byteOffset: 8002, segment: 1 }); + }); + + test("overwrites an existing checkpoint", async () => { + await writeCheckpoint(SID, { byteOffset: 100, segment: 1 }); + await writeCheckpoint(SID, { byteOffset: 200, segment: 2 }); + expect(await readCheckpoint(SID)).toEqual({ byteOffset: 200, segment: 2 }); + }); + + test("a corrupt file reads as the zero checkpoint", async () => { + await Bun.write(checkpointPath(SID), "{not json"); + expect(await readCheckpoint(SID)).toEqual({ byteOffset: 0, segment: 0 }); + }); + + test("removeCheckpoint deletes the file", async () => { + await writeCheckpoint(SID, { byteOffset: 5, segment: 1 }); + await removeCheckpoint(SID); + expect(await readCheckpoint(SID)).toEqual({ byteOffset: 0, segment: 0 }); + }); +}); From 651e18d5003afc89601732c911ca8f9ae8fb0a33 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 17 Jul 2026 10:57:52 +0300 Subject: [PATCH 03/30] feat: add shared transcript line parser and byte-offset reader --- src/memory/checkpoint.ts | 38 +++++++++++++++++ src/memory/transcript.ts | 65 +++++++++++++++++++++++++++++ tests/unit/checkpoint-parse.test.ts | 47 +++++++++++++++++++++ tests/unit/transcript-line.test.ts | 48 +++++++++++++++++++++ 4 files changed, 198 insertions(+) create mode 100644 tests/unit/checkpoint-parse.test.ts create mode 100644 tests/unit/transcript-line.test.ts diff --git a/src/memory/checkpoint.ts b/src/memory/checkpoint.ts index b917996..465e0c3 100644 --- a/src/memory/checkpoint.ts +++ b/src/memory/checkpoint.ts @@ -1,4 +1,5 @@ import { rename, unlink } from "node:fs/promises"; +import { parseTranscriptLine } from "./transcript.js"; export interface OffsetTurn { role: "user" | "assistant" | "tool"; @@ -70,3 +71,40 @@ export async function writeCheckpoint( export async function removeCheckpoint(sessionId: string): Promise { await unlink(checkpointPath(sessionId)).catch(() => {}); } + +export async function parseTurnsFrom( + path: string, + fromByte: number, +): Promise { + const file = Bun.file(path); + if (!(await file.exists())) { + return []; + } + + const bytes = new Uint8Array(await file.slice(fromByte).arrayBuffer()); + const decoder = new TextDecoder(); + const turns: OffsetTurn[] = []; + let lineStart = 0; + + for (let i = 0; i <= bytes.length; i++) { + const atEnd = i === bytes.length; + if (!atEnd && bytes[i] !== 0x0a) { + continue; + } + const lineBytes = bytes.subarray(lineStart, i); + const endByte = fromByte + (atEnd ? i : i + 1); + lineStart = i + 1; + if (lineBytes.length === 0) { + continue; + } + const line = decoder.decode(lineBytes).trim(); + if (!line) { + continue; + } + for (const turn of parseTranscriptLine(line)) { + turns.push({ ...turn, endByte }); + } + } + + return turns; +} diff --git a/src/memory/transcript.ts b/src/memory/transcript.ts index 26721bc..a8856da 100644 --- a/src/memory/transcript.ts +++ b/src/memory/transcript.ts @@ -93,3 +93,68 @@ export function selectTranscript( if (prev < turns.length - 1) out.push(GAP_MARKER); return out.join("\n"); } + +interface RawTranscriptEntry { + type?: string; + message?: { content?: unknown }; + tool_name?: string; + name?: string; +} + +function extractContent(content: unknown): string { + if (typeof content === "string") { + return content; + } + if (Array.isArray(content)) { + return content + .filter((b): b is { type: string; text: string } => { + return ( + typeof b === "object" && + b !== null && + (b as { type?: unknown }).type === "text" && + typeof (b as { text?: unknown }).text === "string" + ); + }) + .map((b) => b.text) + .join("\n"); + } + return ""; +} + +export function parseTranscriptLine(line: string): TranscriptTurn[] { + let entry: RawTranscriptEntry; + try { + entry = JSON.parse(line) as RawTranscriptEntry; + } catch { + return []; + } + + if (entry.type === "user" && entry.message?.content !== undefined) { + const content = extractContent(entry.message.content); + if ( + content && + !content.includes("") + ) { + return [{ role: "user", text: `User: ${content}` }]; + } + return []; + } + + if (entry.type === "assistant" && entry.message?.content !== undefined) { + const content = extractContent(entry.message.content); + if (content) { + return [{ role: "assistant", text: `Assistant: ${content.slice(0, 2000)}` }]; + } + return []; + } + + if (entry.type === "tool_use" || entry.type === "tool_result") { + const toolName = entry.tool_name ?? entry.name ?? ""; + if (toolName) { + return [{ role: "tool", text: `Tool: ${toolName}` }]; + } + } + + return []; +} diff --git a/tests/unit/checkpoint-parse.test.ts b/tests/unit/checkpoint-parse.test.ts new file mode 100644 index 0000000..50fe210 --- /dev/null +++ b/tests/unit/checkpoint-parse.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, test, afterEach } from "bun:test"; +import { unlink } from "node:fs/promises"; +import { parseTurnsFrom } from "../../src/memory/checkpoint.js"; + +const FIXTURE = "/tmp/betterdb-parse-test.jsonl"; + +afterEach(async () => { + await unlink(FIXTURE).catch(() => {}); +}); + +async function writeLines(lines: object[]): Promise { + await Bun.write(FIXTURE, lines.map((l) => JSON.stringify(l)).join("\n") + "\n"); +} + +describe("parseTurnsFrom", () => { + test("returns [] for a missing file", async () => { + expect(await parseTurnsFrom("/tmp/does-not-exist.jsonl", 0)).toEqual([]); + }); + + test("parses all turns from offset 0 with end-byte offsets", async () => { + await writeLines([ + { type: "user", message: { content: "hello" } }, + { type: "assistant", message: { content: "hi" } }, + ]); + const turns = await parseTurnsFrom(FIXTURE, 0); + expect(turns.map((t) => t.text)).toEqual(["User: hello", "Assistant: hi"]); + expect(turns[0]!.endByte).toBeGreaterThan(0); + expect(turns[1]!.endByte).toBeGreaterThan(turns[0]!.endByte); + }); + + test("resuming from a prior endByte yields only later turns", async () => { + await writeLines([ + { type: "user", message: { content: "first" } }, + { type: "user", message: { content: "second" } }, + ]); + const all = await parseTurnsFrom(FIXTURE, 0); + const resumed = await parseTurnsFrom(FIXTURE, all[0]!.endByte); + expect(resumed.map((t) => t.text)).toEqual(["User: second"]); + }); + + test("the final endByte equals the file size", async () => { + await writeLines([{ type: "user", message: { content: "only" } }]); + const size = Bun.file(FIXTURE).size; + const turns = await parseTurnsFrom(FIXTURE, 0); + expect(turns[turns.length - 1]!.endByte).toBe(size); + }); +}); diff --git a/tests/unit/transcript-line.test.ts b/tests/unit/transcript-line.test.ts new file mode 100644 index 0000000..268ebe4 --- /dev/null +++ b/tests/unit/transcript-line.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, test } from "bun:test"; +import { parseTranscriptLine } from "../../src/memory/transcript.js"; + +describe("parseTranscriptLine", () => { + test("extracts a user turn from string content", () => { + const line = JSON.stringify({ type: "user", message: { content: "hello" } }); + expect(parseTranscriptLine(line)).toEqual([{ role: "user", text: "User: hello" }]); + }); + + test("extracts a user turn from block content", () => { + const line = JSON.stringify({ + type: "user", + message: { content: [{ type: "text", text: "hi there" }] }, + }); + expect(parseTranscriptLine(line)).toEqual([{ role: "user", text: "User: hi there" }]); + }); + + test("drops system-generated command messages", () => { + const line = JSON.stringify({ + type: "user", + message: { content: "/foo" }, + }); + expect(parseTranscriptLine(line)).toEqual([]); + }); + + test("caps an assistant turn at 2000 chars", () => { + const line = JSON.stringify({ + type: "assistant", + message: { content: "x".repeat(5000) }, + }); + const turns = parseTranscriptLine(line); + expect(turns).toHaveLength(1); + expect(turns[0]!.text).toBe("Assistant: " + "x".repeat(2000)); + }); + + test("extracts a tool turn by name", () => { + const line = JSON.stringify({ type: "tool_use", tool_name: "Edit" }); + expect(parseTranscriptLine(line)).toEqual([{ role: "tool", text: "Tool: Edit" }]); + }); + + test("returns [] for malformed JSON", () => { + expect(parseTranscriptLine("{not json")).toEqual([]); + }); + + test("returns [] for an unrecognized entry type", () => { + expect(parseTranscriptLine(JSON.stringify({ type: "summary" }))).toEqual([]); + }); +}); From cffd1698d0847b5e97f37f4da6857281648972a4 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 17 Jul 2026 11:03:10 +0300 Subject: [PATCH 04/30] test: cover final-line endByte with no trailing newline --- tests/unit/checkpoint-parse.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/checkpoint-parse.test.ts b/tests/unit/checkpoint-parse.test.ts index 50fe210..fda79c6 100644 --- a/tests/unit/checkpoint-parse.test.ts +++ b/tests/unit/checkpoint-parse.test.ts @@ -44,4 +44,15 @@ describe("parseTurnsFrom", () => { const turns = await parseTurnsFrom(FIXTURE, 0); expect(turns[turns.length - 1]!.endByte).toBe(size); }); + + test("final endByte equals file size when no trailing newline", async () => { + await Bun.write( + FIXTURE, + JSON.stringify({ type: "user", message: { content: "no newline" } }), + ); + const size = Bun.file(FIXTURE).size; + const turns = await parseTurnsFrom(FIXTURE, 0); + expect(turns).toHaveLength(1); + expect(turns[turns.length - 1]!.endByte).toBe(size); + }); }); From beadcab237568f6de146211dc56a80201b64712d Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 17 Jul 2026 11:04:30 +0300 Subject: [PATCH 05/30] feat: add Stop checkpoint hook that queues without summarizing --- src/hooks/stop-checkpoint.ts | 68 ++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 src/hooks/stop-checkpoint.ts diff --git a/src/hooks/stop-checkpoint.ts b/src/hooks/stop-checkpoint.ts new file mode 100644 index 0000000..bca5d42 --- /dev/null +++ b/src/hooks/stop-checkpoint.ts @@ -0,0 +1,68 @@ +import { readRawPayload, runHook } from "./_utils.js"; +import { getValkeyClient } from "../client/valkey.js"; +import { getCwdProject, getGitBranch } from "../memory/capture.js"; +import { isConfigured } from "../config.js"; +import { + CHECKPOINT_THRESHOLD, + nextChunk, + parseTurnsFrom, + readCheckpoint, + writeCheckpoint, +} from "../memory/checkpoint.js"; + +runHook(async () => { + if (!isConfigured()) { + return; + } + + const payload = await readRawPayload(); + const sessionId = payload["session_id"] as string; + const cwd = payload["cwd"] as string | undefined; + const transcriptPath = payload["transcript_path"] as string | undefined; + + if (!sessionId || !transcriptPath) { + return; + } + if (cwd) { + process.chdir(cwd); + } + + const transcript = Bun.file(transcriptPath); + if (!(await transcript.exists())) { + return; + } + + const checkpoint = await readCheckpoint(sessionId); + + if (transcript.size <= checkpoint.byteOffset) { + return; + } + + const turns = await parseTurnsFrom(transcriptPath, checkpoint.byteOffset); + const result = nextChunk(turns, CHECKPOINT_THRESHOLD); + if (result === null) { + return; + } + + let valkeyClient; + try { + valkeyClient = await getValkeyClient(); + } catch { + return; + } + + await valkeyClient.pushIngestQueue(result.chunk, { + project: getCwdProject(), + branch: getGitBranch(), + timestamp: new Date().toISOString(), + sessionId, + segment: checkpoint.segment, + }); + + await writeCheckpoint(sessionId, { + byteOffset: result.endByte, + segment: checkpoint.segment + 1, + }); + + await valkeyClient.quit(); +}); From 8d34dbf0907c601cee73d0d2252e1356e30ca30a Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 17 Jul 2026 11:09:11 +0300 Subject: [PATCH 06/30] fix: flush only the post-checkpoint tail at session end --- src/hooks/session-end.ts | 93 +++++++++------------------------------- 1 file changed, 20 insertions(+), 73 deletions(-) diff --git a/src/hooks/session-end.ts b/src/hooks/session-end.ts index ccf21cf..7158bd4 100644 --- a/src/hooks/session-end.ts +++ b/src/hooks/session-end.ts @@ -6,7 +6,13 @@ import { getCwdProject, } from "../memory/capture.js"; import { SessionEventSchema } from "../memory/schema.js"; -import { selectTranscript, type TranscriptTurn } from "../memory/transcript.js"; +import { selectTranscript } from "../memory/transcript.js"; +import { + parseTurnsFrom, + readCheckpoint, + removeCheckpoint, + type OffsetTurn, +} from "../memory/checkpoint.js"; import { config, isConfigured } from "../config.js"; import { unlink } from "node:fs/promises"; import { join } from "node:path"; @@ -39,11 +45,11 @@ runHook(async () => { } const eventFilePath = `/tmp/betterdb-${sessionId}.jsonl`; - let turns: TranscriptTurn[] = []; + const checkpoint = await readCheckpoint(sessionId); + let turns: OffsetTurn[] = []; - // Prefer transcript_path — contains the full conversation including user messages if (transcriptPath) { - turns = await parseTranscriptTurns(transcriptPath); + turns = await parseTurnsFrom(transcriptPath, checkpoint.byteOffset); } // Fall back to JSONL event file (tool calls captured by PostToolUse hook). @@ -66,7 +72,7 @@ runHook(async () => { .buildTranscript() .split("\n") .filter(Boolean) - .map((text) => ({ role: "tool" as const, text })); + .map((text) => ({ role: "tool" as const, text, endByte: 0 })); } } @@ -78,7 +84,7 @@ runHook(async () => { // Nothing to store if (!transcript || transcript.length < 20) { - await cleanup(eventFilePath); + await cleanup(eventFilePath, sessionId); return; } @@ -86,7 +92,7 @@ runHook(async () => { try { valkeyClient = await getValkeyClient(); } catch { - await cleanup(eventFilePath); + await cleanup(eventFilePath, sessionId); return; // Valkey unreachable — skip silently } @@ -98,12 +104,13 @@ runHook(async () => { branch, timestamp: new Date().toISOString(), sessionId, + segment: checkpoint.segment, }); await spawnDrain(); await valkeyClient.quit(); - await cleanup(eventFilePath); + await cleanup(eventFilePath, sessionId); }); /** @@ -144,71 +151,11 @@ async function spawnDrain(): Promise { ); } -/** - * Parse Claude Code's transcript JSONL into role-tagged turns. - * The JSONL contains objects with type: "user" | "assistant" and message content. - * We extract user/assistant/tool turns so selectTranscript can rank them. - */ -async function parseTranscriptTurns(path: string): Promise { - const file = Bun.file(path); - if (!(await file.exists())) return []; - - const raw = await file.text(); - const turns: TranscriptTurn[] = []; - - for (const line of raw.split("\n").filter(Boolean)) { - try { - const entry = JSON.parse(line); - if (entry.type === "user" && entry.message?.content) { - const content = - typeof entry.message.content === "string" - ? entry.message.content - : Array.isArray(entry.message.content) - ? entry.message.content - .filter((b: { type: string }) => b.type === "text") - .map((b: { text: string }) => b.text) - .join("\n") - : ""; - // Skip system-generated messages (commands, caveats) - if ( - content && - !content.includes("") - ) { - turns.push({ role: "user", text: `User: ${content}` }); - } - } else if (entry.type === "assistant" && entry.message?.content) { - const content = - typeof entry.message.content === "string" - ? entry.message.content - : Array.isArray(entry.message.content) - ? entry.message.content - .filter((b: { type: string }) => b.type === "text") - .map((b: { text: string }) => b.text) - .join("\n") - : ""; - if (content) { - turns.push({ - role: "assistant", - text: `Assistant: ${content.slice(0, 2000)}`, - }); - } - } else if (entry.type === "tool_use" || entry.type === "tool_result") { - // Include tool names for context but keep it brief - const toolName = entry.tool_name ?? entry.name ?? ""; - if (toolName) { - turns.push({ role: "tool", text: `Tool: ${toolName}` }); - } - } - } catch { - // Skip malformed lines - } - } - - return turns; -} - -async function cleanup(eventFilePath: string): Promise { +async function cleanup( + eventFilePath: string, + sessionId: string, +): Promise { + await removeCheckpoint(sessionId); try { await unlink(eventFilePath); } catch { From 3c228709c11c79fafb0eb28298039562ee1fdf50 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 17 Jul 2026 11:14:23 +0300 Subject: [PATCH 07/30] feat: register the Stop checkpoint hook in all install paths --- scripts/install-hooks.sh | 2 ++ scripts/register-hooks.ts | 5 +++++ src/index.ts | 8 ++++++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/install-hooks.sh b/scripts/install-hooks.sh index 64eec2b..de90677 100755 --- a/scripts/install-hooks.sh +++ b/scripts/install-hooks.sh @@ -64,6 +64,7 @@ const hooks = { PreToolUse: [{ matcher: '', hooks: [{ type: 'command', command: wrap('pre-tool') }] }], PostToolUse: [{ matcher: '', hooks: [{ type: 'command', command: wrap('post-tool') }] }], SessionEnd: [{ hooks: [{ type: 'command', command: wrap('session-end') }] }], + Stop: [{ hooks: [{ type: 'command', command: wrap('stop-checkpoint') }] }], }; fs.writeFileSync(settingsPath, JSON.stringify({ ...existing, hooks }, null, 2)); console.log('Hook configuration written to: ' + settingsPath); @@ -92,6 +93,7 @@ echo "" echo "Hooks written to: ~/.claude/settings.json" echo " SessionStart → $DIST_DIR/session-start" echo " SessionEnd → $DIST_DIR/session-end" +echo " Stop → $DIST_DIR/stop-checkpoint" echo " PreToolUse → $DIST_DIR/pre-tool" echo " PostToolUse → $DIST_DIR/post-tool" echo "" diff --git a/scripts/register-hooks.ts b/scripts/register-hooks.ts index eb01dfd..839ff18 100644 --- a/scripts/register-hooks.ts +++ b/scripts/register-hooks.ts @@ -29,6 +29,7 @@ const hookFiles = [ "pre-tool.ts", "post-tool.ts", "session-end.ts", + "stop-checkpoint.ts", ]; for (const file of hookFiles) { if (!existsSync(join(hooksDir, file))) { @@ -84,6 +85,9 @@ const betterdbHooks: Record = { SessionEnd: [ { hooks: [{ type: "command", command: cmd("session-end.ts") }] }, ], + Stop: [ + { hooks: [{ type: "command", command: cmd("stop-checkpoint.ts") }] }, + ], }; for (const [event, entries] of Object.entries(betterdbHooks)) { @@ -103,5 +107,6 @@ console.log(" SessionStart → session-start.ts"); console.log(" PreToolUse → pre-tool.ts"); console.log(" PostToolUse → post-tool.ts"); console.log(" SessionEnd → session-end.ts"); +console.log(" Stop → stop-checkpoint.ts"); console.log(`\n Plugin root: ${resolvedRoot}`); console.log("\n Restart Claude Code for hooks to take effect."); diff --git a/src/index.ts b/src/index.ts index 4ffbc16..cd78d63 100644 --- a/src/index.ts +++ b/src/index.ts @@ -32,6 +32,7 @@ const PKG_ROOT = resolve(import.meta.dir, ".."); const BINARIES = [ { src: "src/hooks/session-start.ts", out: "session-start" }, { src: "src/hooks/session-end.ts", out: "session-end" }, + { src: "src/hooks/stop-checkpoint.ts", out: "stop-checkpoint" }, { src: "src/hooks/pre-tool.ts", out: "pre-tool" }, { src: "src/hooks/post-tool.ts", out: "post-tool" }, { src: "src/hooks/drain.ts", out: "drain" }, @@ -259,11 +260,14 @@ async function runInstall() { SessionEnd: [ { hooks: [{ type: "command", command: join(BIN_DIR, "session-end") }] }, ], + Stop: [ + { hooks: [{ type: "command", command: join(BIN_DIR, "stop-checkpoint") }] }, + ], }; settings["hooks"] = mergeHooks(existingHooks, betterdbHooks); writeFileSync(settingsPath, JSON.stringify(settings, null, 2) + "\n"); - console.log(" Registered 4 hooks in ~/.claude/settings.json"); + console.log(" Registered 5 hooks in ~/.claude/settings.json"); // Register MCP server globally (-s user) so it's available in all projects const mcpBin = join(BIN_DIR, "mcp-server"); @@ -359,7 +363,7 @@ async function runInstall() { // 7. PRINT SUMMARY console.log("\n=== Installation Complete ===\n"); console.log(` ✅ Compiled ${BINARIES.length} binaries to ${BIN_DIR}/`); - console.log(" ✅ Registered 4 hooks with Claude Code"); + console.log(" ✅ Registered 5 hooks with Claude Code"); console.log(" ✅ Registered MCP server: betterdb-memory"); console.log(" ✅ Valkey index ready"); console.log(` ✅ Config saved to ${CONFIG_PATH}`); From d3de5996062604a0133fdf0adec94a4aa26d85d3 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 17 Jul 2026 11:18:08 +0300 Subject: [PATCH 08/30] test: assert checkpoint loop queues distinct segments --- tests/integration/checkpoint-capture.test.ts | 83 ++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 tests/integration/checkpoint-capture.test.ts diff --git a/tests/integration/checkpoint-capture.test.ts b/tests/integration/checkpoint-capture.test.ts new file mode 100644 index 0000000..48ac0ee --- /dev/null +++ b/tests/integration/checkpoint-capture.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, test, afterAll } from "bun:test"; +import { unlink } from "node:fs/promises"; +import { getValkeyClient } from "../../src/client/valkey.js"; +import { + CHECKPOINT_THRESHOLD, + nextChunk, + parseTurnsFrom, + readCheckpoint, + writeCheckpoint, + removeCheckpoint, + checkpointPath, +} from "../../src/memory/checkpoint.js"; + +const SKIP = Bun.env.BETTERDB_SKIP_INTEGRATION === "true"; +const SID = "checkpoint-capture-it"; +const FIXTURE = `/tmp/betterdb-${SID}.transcript.jsonl`; + +const META = { + project: "it", + branch: "it", + timestamp: "2026-07-17T00:00:00.000Z", + sessionId: SID, +}; + +async function seedTranscript(): Promise { + const lines: string[] = []; + for (let i = 0; i < 12; i++) { + lines.push( + JSON.stringify({ + type: "user", + message: { content: `turn ${i} ` + "x".repeat(2500) }, + }), + ); + } + await Bun.write(FIXTURE, lines.join("\n") + "\n"); +} + +describe.skipIf(SKIP)("checkpoint capture integration", () => { + afterAll(async () => { + await unlink(FIXTURE).catch(() => {}); + await unlink(checkpointPath(SID)).catch(() => {}); + }); + + test("produces sequential segments plus a tail", async () => { + await seedTranscript(); + await removeCheckpoint(SID); + + const vk = await getValkeyClient(); + // Start from a clean queue without assuming a raw() accessor. + await vk.popIngestQueue(10000); + + for (let guard = 0; guard < 20; guard++) { + const cp = await readCheckpoint(SID); + const turns = await parseTurnsFrom(FIXTURE, cp.byteOffset); + const result = nextChunk(turns, CHECKPOINT_THRESHOLD); + if (result === null) { + break; + } + await vk.pushIngestQueue(result.chunk, { ...META, segment: cp.segment }); + await writeCheckpoint(SID, { + byteOffset: result.endByte, + segment: cp.segment + 1, + }); + } + + const cp = await readCheckpoint(SID); + const tail = await parseTurnsFrom(FIXTURE, cp.byteOffset); + const tailText = tail.map((t) => t.text).join("\n"); + if (tailText.length >= 20) { + await vk.pushIngestQueue(tailText, { ...META, segment: cp.segment }); + } + + const items = await vk.popIngestQueue(50); + await vk.quit(); + + expect(items.length).toBeGreaterThanOrEqual(3); + const segments = items.map((i) => (i.meta as { segment: number }).segment); + expect(segments).toEqual([...Array(items.length).keys()]); + for (const item of items) { + expect(item.transcript.length).toBeGreaterThan(0); + } + }); +}); From 3d2ac6b7df4d2986bd61c97c48ffd4c9121a044f Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 17 Jul 2026 11:24:50 +0300 Subject: [PATCH 09/30] test: exercise the tail-flush path in checkpoint capture --- tests/integration/checkpoint-capture.test.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/integration/checkpoint-capture.test.ts b/tests/integration/checkpoint-capture.test.ts index 48ac0ee..1b1bc01 100644 --- a/tests/integration/checkpoint-capture.test.ts +++ b/tests/integration/checkpoint-capture.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test, afterAll } from "bun:test"; import { unlink } from "node:fs/promises"; -import { getValkeyClient } from "../../src/client/valkey.js"; +import { getValkeyClient, resetValkeyClient } from "../../src/client/valkey.js"; import { CHECKPOINT_THRESHOLD, nextChunk, @@ -24,7 +24,7 @@ const META = { async function seedTranscript(): Promise { const lines: string[] = []; - for (let i = 0; i < 12; i++) { + for (let i = 0; i < 14; i++) { lines.push( JSON.stringify({ type: "user", @@ -39,6 +39,7 @@ describe.skipIf(SKIP)("checkpoint capture integration", () => { afterAll(async () => { await unlink(FIXTURE).catch(() => {}); await unlink(checkpointPath(SID)).catch(() => {}); + await resetValkeyClient(); }); test("produces sequential segments plus a tail", async () => { @@ -49,11 +50,14 @@ describe.skipIf(SKIP)("checkpoint capture integration", () => { // Start from a clean queue without assuming a raw() accessor. await vk.popIngestQueue(10000); + let loopSegments = 0; + let exitedViaNull = false; for (let guard = 0; guard < 20; guard++) { const cp = await readCheckpoint(SID); const turns = await parseTurnsFrom(FIXTURE, cp.byteOffset); const result = nextChunk(turns, CHECKPOINT_THRESHOLD); if (result === null) { + exitedViaNull = true; break; } await vk.pushIngestQueue(result.chunk, { ...META, segment: cp.segment }); @@ -61,19 +65,25 @@ describe.skipIf(SKIP)("checkpoint capture integration", () => { byteOffset: result.endByte, segment: cp.segment + 1, }); + loopSegments++; } const cp = await readCheckpoint(SID); const tail = await parseTurnsFrom(FIXTURE, cp.byteOffset); const tailText = tail.map((t) => t.text).join("\n"); + let tailPushed = false; if (tailText.length >= 20) { await vk.pushIngestQueue(tailText, { ...META, segment: cp.segment }); + tailPushed = true; } const items = await vk.popIngestQueue(50); await vk.quit(); - expect(items.length).toBeGreaterThanOrEqual(3); + expect(exitedViaNull).toBe(true); + expect(loopSegments).toBeGreaterThanOrEqual(2); + expect(tailPushed).toBe(true); + expect(items.length).toBe(loopSegments + 1); const segments = items.map((i) => (i.meta as { segment: number }).segment); expect(segments).toEqual([...Array(items.length).keys()]); for (const item of items) { From 6912f5ecd47bf0e5789660ab4b0c86172523d04c Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 17 Jul 2026 12:49:00 +0300 Subject: [PATCH 10/30] refactor: derive hook registration from a single typed spec - Add src/hook-spec.ts as the sole source of truth for the five lifecycle hooks (event, source file, binary, matcher) - Derive hook counts, the settings map, the summary listing, and the hook entries of BINARIES from HOOK_SPECS - Replace Record hook maps with typed HookEntry - Drop stripLegacyBetterdbHooks: Stop is registered again, so both install paths already replace the legacy session-end entry on merge --- scripts/register-hooks.ts | 59 +++++++------------- src/hook-migration.ts | 38 ------------- src/hook-spec.ts | 58 ++++++++++++++++++++ src/index.ts | 49 ++++------------- tests/unit/hook-migration.test.ts | 89 ------------------------------- 5 files changed, 87 insertions(+), 206 deletions(-) delete mode 100644 src/hook-migration.ts create mode 100644 src/hook-spec.ts delete mode 100644 tests/unit/hook-migration.test.ts diff --git a/scripts/register-hooks.ts b/scripts/register-hooks.ts index 839ff18..9a264e9 100644 --- a/scripts/register-hooks.ts +++ b/scripts/register-hooks.ts @@ -12,7 +12,12 @@ import { existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs"; import { join, resolve } from "node:path"; -import { stripLegacyBetterdbHooks } from "../src/hook-migration.js"; +import { + HOOK_SPECS, + buildHookMap, + formatHookSummary, + type HookSpec, +} from "../src/hook-spec.js"; const pluginRoot = process.argv[2]; if (!pluginRoot) { @@ -24,16 +29,11 @@ const resolvedRoot = resolve(pluginRoot); const hooksDir = join(resolvedRoot, "src", "hooks"); // Verify hook source files exist -const hookFiles = [ - "session-start.ts", - "pre-tool.ts", - "post-tool.ts", - "session-end.ts", - "stop-checkpoint.ts", -]; -for (const file of hookFiles) { - if (!existsSync(join(hooksDir, file))) { - console.error(`ERROR: Hook source not found: ${join(hooksDir, file)}`); +for (const spec of HOOK_SPECS) { + if (!existsSync(join(hooksDir, spec.source))) { + console.error( + `ERROR: Hook source not found: ${join(hooksDir, spec.source)}`, + ); process.exit(1); } } @@ -61,34 +61,13 @@ if (existsSync(settingsPath)) { } } -function cmd(hookFile: string): string { - return `bash -c 'bun run "${join(hooksDir, hookFile)}"'`; +function cmd(spec: HookSpec): string { + return `bash -c 'bun run "${join(hooksDir, spec.source)}"'`; } // Merge hooks — replaces BetterDB entries per event, preserves all others. -// The loop below only visits events in betterdbHooks, so the legacy Stop -// registration must be stripped explicitly or it survives forever. -const existingHooks = stripLegacyBetterdbHooks( - (settings["hooks"] ?? {}) as Record, - ["betterdb", hooksDir], -); -const betterdbHooks: Record = { - SessionStart: [ - { hooks: [{ type: "command", command: cmd("session-start.ts") }] }, - ], - PreToolUse: [ - { matcher: "", hooks: [{ type: "command", command: cmd("pre-tool.ts") }] }, - ], - PostToolUse: [ - { matcher: "", hooks: [{ type: "command", command: cmd("post-tool.ts") }] }, - ], - SessionEnd: [ - { hooks: [{ type: "command", command: cmd("session-end.ts") }] }, - ], - Stop: [ - { hooks: [{ type: "command", command: cmd("stop-checkpoint.ts") }] }, - ], -}; +const existingHooks = (settings["hooks"] ?? {}) as Record; +const betterdbHooks = buildHookMap(cmd); for (const [event, entries] of Object.entries(betterdbHooks)) { const prev = Array.isArray(existingHooks[event]) ? existingHooks[event] : []; @@ -103,10 +82,8 @@ settings["hooks"] = existingHooks; writeFileSync(settingsPath, JSON.stringify(settings, null, 2) + "\n"); console.log("BetterDB Memory — Hooks registered in ~/.claude/settings.json\n"); -console.log(" SessionStart → session-start.ts"); -console.log(" PreToolUse → pre-tool.ts"); -console.log(" PostToolUse → post-tool.ts"); -console.log(" SessionEnd → session-end.ts"); -console.log(" Stop → stop-checkpoint.ts"); +for (const line of formatHookSummary()) { + console.log(line); +} console.log(`\n Plugin root: ${resolvedRoot}`); console.log("\n Restart Claude Code for hooks to take effect."); diff --git a/src/hook-migration.ts b/src/hook-migration.ts deleted file mode 100644 index 84c8d1c..0000000 --- a/src/hook-migration.ts +++ /dev/null @@ -1,38 +0,0 @@ -// Events this plugin used to register on and no longer does. mergeHooks only -// touches events present in the current hook map, so without an explicit strip -// a stale registration would survive upgrades forever in a user's -// ~/.claude/settings.json. -const LEGACY_EVENTS = ["Stop"]; - -/** - * `markers` identify an entry as ours. The default catches installed binaries - * under ~/.betterdb; dev registrations point at a plugin checkout whose path - * may not contain "betterdb", so callers pass that path as an extra marker. - */ -export function stripLegacyBetterdbHooks( - hooks: Record, - markers: string[] = ["betterdb"], -): Record { - const out: Record = { ...hooks }; - - for (const event of LEGACY_EVENTS) { - const entries = out[event]; - if (!Array.isArray(entries)) { - continue; - } - // Only ours — a third party's hook on the same event must survive. - const kept = entries.filter((entry) => { - const json = JSON.stringify(entry); - return !markers.some((m) => { - return m.length > 0 && json.includes(m); - }); - }); - if (kept.length > 0) { - out[event] = kept; - } else { - delete out[event]; - } - } - - return out; -} diff --git a/src/hook-spec.ts b/src/hook-spec.ts new file mode 100644 index 0000000..0ba7092 --- /dev/null +++ b/src/hook-spec.ts @@ -0,0 +1,58 @@ +export type HookEvent = + | "SessionStart" + | "PreToolUse" + | "PostToolUse" + | "SessionEnd" + | "Stop"; + +export interface HookSpec { + readonly event: HookEvent; + readonly source: string; + readonly binary: string; + readonly matcher?: string; +} + +export interface HookCommand { + readonly type: "command"; + readonly command: string; +} + +export interface HookEntry { + readonly matcher?: string; + readonly hooks: readonly HookCommand[]; +} + +export const HOOK_SPECS: readonly HookSpec[] = [ + { event: "SessionStart", source: "session-start.ts", binary: "session-start" }, + { event: "PreToolUse", source: "pre-tool.ts", binary: "pre-tool", matcher: "" }, + { + event: "PostToolUse", + source: "post-tool.ts", + binary: "post-tool", + matcher: "", + }, + { event: "SessionEnd", source: "session-end.ts", binary: "session-end" }, + { event: "Stop", source: "stop-checkpoint.ts", binary: "stop-checkpoint" }, +]; + +export const HOOK_COUNT = HOOK_SPECS.length; + +export function buildHookMap( + toCommand: (spec: HookSpec) => string, +): Record { + const map: Record = {}; + for (const spec of HOOK_SPECS) { + const hooks: HookCommand[] = [{ type: "command", command: toCommand(spec) }]; + const entry: HookEntry = + spec.matcher === undefined ? { hooks } : { matcher: spec.matcher, hooks }; + (map[spec.event] ??= []).push(entry); + } + return map; +} + +export function formatHookSummary(): string[] { + const width = Math.max(...HOOK_SPECS.map((spec) => spec.event.length)); + return HOOK_SPECS.map( + (spec) => ` ${spec.event.padEnd(width)} → ${spec.source}`, + ); +} diff --git a/src/index.ts b/src/index.ts index cd78d63..244c13b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -19,7 +19,7 @@ import { rmSync, } from "node:fs"; import { join, resolve } from "node:path"; -import { stripLegacyBetterdbHooks } from "./hook-migration.js"; +import { HOOK_COUNT, HOOK_SPECS, buildHookMap } from "./hook-spec.js"; const VERSION = "0.5.0"; const HOME = process.env["HOME"] ?? process.env["USERPROFILE"] ?? ""; @@ -29,15 +29,14 @@ const CONFIG_PATH = join(BETTERDB_DIR, "memory.json"); const MANIFEST_PATH = join(BETTERDB_DIR, "install-manifest.json"); const PKG_ROOT = resolve(import.meta.dir, ".."); -const BINARIES = [ - { src: "src/hooks/session-start.ts", out: "session-start" }, - { src: "src/hooks/session-end.ts", out: "session-end" }, - { src: "src/hooks/stop-checkpoint.ts", out: "stop-checkpoint" }, - { src: "src/hooks/pre-tool.ts", out: "pre-tool" }, - { src: "src/hooks/post-tool.ts", out: "post-tool" }, +const BINARIES: readonly { src: string; out: string }[] = [ + ...HOOK_SPECS.map((spec) => ({ + src: `src/hooks/${spec.source}`, + out: spec.binary, + })), { src: "src/hooks/drain.ts", out: "drain" }, { src: "src/mcp/server.ts", out: "mcp-server" }, -] as const; +]; const USAGE = ` BetterDB Memory for Claude Code v${VERSION} @@ -236,38 +235,12 @@ async function runInstall() { } } - // mergeHooks only touches events present in betterdbHooks, so the legacy - // Stop registration must be stripped explicitly or it survives upgrades. - const existingHooks = stripLegacyBetterdbHooks( - (settings["hooks"] ?? {}) as Record, - ); - const betterdbHooks: Record = { - SessionStart: [ - { hooks: [{ type: "command", command: join(BIN_DIR, "session-start") }] }, - ], - PreToolUse: [ - { - matcher: "", - hooks: [{ type: "command", command: join(BIN_DIR, "pre-tool") }], - }, - ], - PostToolUse: [ - { - matcher: "", - hooks: [{ type: "command", command: join(BIN_DIR, "post-tool") }], - }, - ], - SessionEnd: [ - { hooks: [{ type: "command", command: join(BIN_DIR, "session-end") }] }, - ], - Stop: [ - { hooks: [{ type: "command", command: join(BIN_DIR, "stop-checkpoint") }] }, - ], - }; + const existingHooks = (settings["hooks"] ?? {}) as Record; + const betterdbHooks = buildHookMap((spec) => join(BIN_DIR, spec.binary)); settings["hooks"] = mergeHooks(existingHooks, betterdbHooks); writeFileSync(settingsPath, JSON.stringify(settings, null, 2) + "\n"); - console.log(" Registered 5 hooks in ~/.claude/settings.json"); + console.log(` Registered ${HOOK_COUNT} hooks in ~/.claude/settings.json`); // Register MCP server globally (-s user) so it's available in all projects const mcpBin = join(BIN_DIR, "mcp-server"); @@ -363,7 +336,7 @@ async function runInstall() { // 7. PRINT SUMMARY console.log("\n=== Installation Complete ===\n"); console.log(` ✅ Compiled ${BINARIES.length} binaries to ${BIN_DIR}/`); - console.log(" ✅ Registered 5 hooks with Claude Code"); + console.log(` ✅ Registered ${HOOK_COUNT} hooks with Claude Code`); console.log(" ✅ Registered MCP server: betterdb-memory"); console.log(" ✅ Valkey index ready"); console.log(` ✅ Config saved to ${CONFIG_PATH}`); diff --git a/tests/unit/hook-migration.test.ts b/tests/unit/hook-migration.test.ts deleted file mode 100644 index c2895fd..0000000 --- a/tests/unit/hook-migration.test.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { stripLegacyBetterdbHooks } from "../../src/hook-migration.js"; - -describe("stripLegacyBetterdbHooks", () => { - test("removes a betterdb Stop entry", () => { - const hooks = { - Stop: [ - { - hooks: [ - { type: "command", command: "/Users/x/.betterdb/bin/session-end" }, - ], - }, - ], - }; - const result = stripLegacyBetterdbHooks(hooks); - expect(result["Stop"]).toBeUndefined(); - }); - - test("preserves a third-party Stop hook", () => { - const hooks = { - Stop: [ - { - hooks: [ - { type: "command", command: "/Users/x/.betterdb/bin/session-end" }, - ], - }, - { hooks: [{ type: "command", command: "/other/tool/hook" }] }, - ], - }; - const result = stripLegacyBetterdbHooks(hooks); - expect(result["Stop"]).toHaveLength(1); - expect(JSON.stringify(result["Stop"])).toContain("/other/tool/hook"); - }); - - test("leaves unrelated events untouched", () => { - const hooks = { - PreCompact: [{ hooks: [{ type: "command", command: "/other/hook" }] }], - }; - const result = stripLegacyBetterdbHooks(hooks); - expect(result["PreCompact"]).toHaveLength(1); - }); - - test("is a no-op on settings with no Stop event", () => { - const hooks = { SessionStart: [{ hooks: [] }] }; - expect(stripLegacyBetterdbHooks(hooks)).toEqual(hooks); - }); - - test("strips a dev registration via an extra marker (path lacks 'betterdb')", () => { - const hooksDir = "/Users/x/dev/memory/src/hooks"; - const hooks = { - Stop: [ - { - hooks: [ - { - type: "command", - command: `bash -c 'bun run "${hooksDir}/session-end.ts"'`, - }, - ], - }, - { hooks: [{ type: "command", command: "/other/tool/hook" }] }, - ], - }; - const result = stripLegacyBetterdbHooks(hooks, ["betterdb", hooksDir]); - expect(result["Stop"]).toHaveLength(1); - expect(JSON.stringify(result["Stop"])).toContain("/other/tool/hook"); - }); - - test("ignores empty markers so they cannot match everything", () => { - const hooks = { - Stop: [{ hooks: [{ type: "command", command: "/other/tool/hook" }] }], - }; - const result = stripLegacyBetterdbHooks(hooks, [""]); - expect(result["Stop"]).toHaveLength(1); - }); - - test("does not mutate its input", () => { - const hooks = { - Stop: [ - { - hooks: [ - { type: "command", command: "/Users/x/.betterdb/bin/session-end" }, - ], - }, - ], - }; - stripLegacyBetterdbHooks(hooks); - expect(hooks["Stop"]).toHaveLength(1); - }); -}); From 3fe219d13309a7c24ca3543e221a941fd5a2affd Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 17 Jul 2026 12:53:34 +0300 Subject: [PATCH 11/30] fix: chunk the session-end tail instead of capping it once A session whose Stop hook never checkpointed (fresh install before a restart, Valkey unreachable) arrives at session end whole. A single 8K selectTranscript cap silently discarded everything past it, reinstating the truncation checkpoint capture exists to remove. planTailSegments now chunks the tail exactly as Stop does and selects down only the final sub-threshold remainder. Also gate the event-file fallback on segment 0: after a checkpoint an empty tail means the transcript is already queued, so re-queueing the whole event file duplicated earlier segments. --- src/hooks/session-end.ts | 45 +++++++++++++++---------- src/memory/checkpoint.ts | 37 +++++++++++++++++++- tests/unit/checkpoint-tail.test.ts | 54 ++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 18 deletions(-) create mode 100644 tests/unit/checkpoint-tail.test.ts diff --git a/src/hooks/session-end.ts b/src/hooks/session-end.ts index 7158bd4..fc6070a 100644 --- a/src/hooks/session-end.ts +++ b/src/hooks/session-end.ts @@ -6,9 +6,10 @@ import { getCwdProject, } from "../memory/capture.js"; import { SessionEventSchema } from "../memory/schema.js"; -import { selectTranscript } from "../memory/transcript.js"; import { + CHECKPOINT_THRESHOLD, parseTurnsFrom, + planTailSegments, readCheckpoint, removeCheckpoint, type OffsetTurn, @@ -52,10 +53,13 @@ runHook(async () => { turns = await parseTurnsFrom(transcriptPath, checkpoint.byteOffset); } - // Fall back to JSONL event file (tool calls captured by PostToolUse hook). - // Event lines rank as tool turns; with no user turns present the selector - // keeps them, so a tool-only fallback transcript is never emptied. - if (turns.length === 0) { + // Fall back to JSONL event file (tool calls captured by PostToolUse hook) + // only when no checkpoint ever ran. Event lines rank as tool turns; with no + // user turns present the selector keeps them, so a tool-only fallback + // transcript is never emptied. After a checkpoint an empty tail means the + // transcript is already fully queued, and the event file covers the whole + // session — re-queueing it would duplicate earlier segments. + if (turns.length === 0 && checkpoint.segment === 0) { const eventFile = Bun.file(eventFilePath); if (await eventFile.exists()) { const raw = await eventFile.text(); @@ -76,14 +80,19 @@ runHook(async () => { } } - // Cap to ~8K chars for the summarizer via priority-based selection (user - // turns > assistant turns near user turns > tool lines) instead of the old - // head+tail slice, which dropped the middle of long sessions wholesale. + // Chunk the tail the way Stop does rather than capping it once: a session + // whose Stop hook never checkpointed arrives here whole, and a single cap + // would discard everything past it. Only the sub-threshold remainder is + // selected down. const MAX_TRANSCRIPT = 8000; - const transcript = selectTranscript(turns, MAX_TRANSCRIPT); + const segments = planTailSegments( + turns, + CHECKPOINT_THRESHOLD, + MAX_TRANSCRIPT, + ).filter((text) => text.length >= 20); // Nothing to store - if (!transcript || transcript.length < 20) { + if (segments.length === 0) { await cleanup(eventFilePath, sessionId); return; } @@ -99,13 +108,15 @@ runHook(async () => { const project = getCwdProject(); const branch = getGitBranch(); - await valkeyClient.pushIngestQueue(transcript, { - project, - branch, - timestamp: new Date().toISOString(), - sessionId, - segment: checkpoint.segment, - }); + for (let i = 0; i < segments.length; i++) { + await valkeyClient.pushIngestQueue(segments[i]!, { + project, + branch, + timestamp: new Date().toISOString(), + sessionId, + segment: checkpoint.segment + i, + }); + } await spawnDrain(); diff --git a/src/memory/checkpoint.ts b/src/memory/checkpoint.ts index 465e0c3..0309269 100644 --- a/src/memory/checkpoint.ts +++ b/src/memory/checkpoint.ts @@ -1,5 +1,5 @@ import { rename, unlink } from "node:fs/promises"; -import { parseTranscriptLine } from "./transcript.js"; +import { parseTranscriptLine, selectTranscript } from "./transcript.js"; export interface OffsetTurn { role: "user" | "assistant" | "tool"; @@ -28,6 +28,41 @@ export function nextChunk( return null; } +/** + * Split a session-end tail into the segments to queue, in order. + * + * The Stop hook is an optimization, not a correctness requirement: when it + * never ran (fresh install before a restart, Valkey unreachable) the whole + * transcript arrives here. Capping that with a single `maxChars` selection + * would silently discard everything past the cap — the truncation checkpoint + * capture exists to remove. So chunk the tail exactly as Stop would, and let + * only the final sub-threshold remainder go through `selectTranscript`. + */ +export function planTailSegments( + turns: OffsetTurn[], + threshold: number, + maxChars: number, +): string[] { + const segments: string[] = []; + let remaining = turns; + + for (;;) { + const result = nextChunk(remaining, threshold); + if (result === null) { + break; + } + segments.push(result.chunk); + remaining = remaining.slice(result.consumedTurns); + } + + const tail = selectTranscript(remaining, maxChars); + if (tail.length > 0) { + segments.push(tail); + } + + return segments; +} + export interface Checkpoint { byteOffset: number; segment: number; diff --git a/tests/unit/checkpoint-tail.test.ts b/tests/unit/checkpoint-tail.test.ts new file mode 100644 index 0000000..1b550c5 --- /dev/null +++ b/tests/unit/checkpoint-tail.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, test } from "bun:test"; +import { planTailSegments, type OffsetTurn } from "../../src/memory/checkpoint.js"; + +const THRESHOLD = 8000; +const MAX_CHARS = 8000; + +function turn(text: string): OffsetTurn { + return { role: "user", text, endByte: 0 }; +} + +describe("planTailSegments", () => { + test("returns nothing for no turns", () => { + expect(planTailSegments([], THRESHOLD, MAX_CHARS)).toEqual([]); + }); + + test("emits a sub-threshold tail as a single segment", () => { + const segments = planTailSegments([turn("a".repeat(100))], THRESHOLD, MAX_CHARS); + expect(segments).toEqual(["a".repeat(100)]); + }); + + test("chunks a large tail instead of truncating it", () => { + // 10 turns x 2500 chars = ~25K: far past a single 8K cap. The old + // single-selectTranscript path kept 8K and dropped the rest. + const turns = Array.from({ length: 10 }, (_, i) => turn(`${i} ` + "x".repeat(2500))); + const segments = planTailSegments(turns, THRESHOLD, MAX_CHARS); + + expect(segments.length).toBeGreaterThan(1); + const totalKept = segments.reduce((n, s) => n + s.length, 0); + const totalInput = turns.reduce((n, t) => n + t.text.length, 0); + // Only newline joins separate them, so essentially nothing is discarded. + expect(totalKept).toBeGreaterThanOrEqual(totalInput); + }); + + test("every full chunk except the last reaches the threshold", () => { + const turns = Array.from({ length: 10 }, (_, i) => turn(`${i} ` + "x".repeat(2500))); + const segments = planTailSegments(turns, THRESHOLD, MAX_CHARS); + for (const segment of segments.slice(0, -1)) { + expect(segment.length).toBeGreaterThanOrEqual(THRESHOLD); + } + }); + + test("emits a single over-threshold turn as its own segment, never split", () => { + const huge = "x".repeat(20000); + const segments = planTailSegments([turn(huge)], THRESHOLD, MAX_CHARS); + expect(segments).toEqual([huge]); + }); + + test("does not append an empty trailing segment when the tail divides evenly", () => { + const turns = [turn("x".repeat(8000)), turn("y".repeat(8000))]; + const segments = planTailSegments(turns, THRESHOLD, MAX_CHARS); + expect(segments).toEqual(["x".repeat(8000), "y".repeat(8000)]); + expect(segments.every((s) => s.length > 0)).toBe(true); + }); +}); From 33349e46ad05ba26184712ada6eac9edf10cb824 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 17 Jul 2026 15:56:04 +0300 Subject: [PATCH 12/30] fix: spawn the drain when only Stop queued segments A fully-checkpointed session leaves an empty tail, so session-end took the nothing-to-store path and returned before spawning the drainer. Stop enqueues segments but never drains, so everything it queued sat unsummarized until a later session happened to drain it. --- src/hooks/session-end.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/hooks/session-end.ts b/src/hooks/session-end.ts index fc6070a..01bac00 100644 --- a/src/hooks/session-end.ts +++ b/src/hooks/session-end.ts @@ -91,8 +91,13 @@ runHook(async () => { MAX_TRANSCRIPT, ).filter((text) => text.length >= 20); - // Nothing to store + // Nothing new to store. The Stop hook may still have queued segments this + // session, and it never spawns a drainer — so returning here without one + // would leave them unsummarized until some later session happened to drain. if (segments.length === 0) { + if (checkpoint.segment > 0) { + await spawnDrain(); + } await cleanup(eventFilePath, sessionId); return; } From 0a028826ffec169059ed2999ee6345c6c0d6adfd Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 17 Jul 2026 16:03:29 +0300 Subject: [PATCH 13/30] fix: identify our hook entries by HOOK_SPECS, not a path substring install matched existing entries on BIN_DIR or the literal "betterdb", so a registration written by register-hooks.ts from a checkout not named betterdb went unrecognized: it survived the merge and kept firing next to the new entry. Derive the match from HOOK_SPECS' source filenames, which hold wherever the checkout lives, and keep install paths as extra markers. Both install paths now share one definition of ours. --- scripts/register-hooks.ts | 4 +-- src/hook-spec.ts | 26 +++++++++++++++++ src/index.ts | 11 +++++--- tests/unit/hook-spec.test.ts | 54 ++++++++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 6 deletions(-) create mode 100644 tests/unit/hook-spec.test.ts diff --git a/scripts/register-hooks.ts b/scripts/register-hooks.ts index 9a264e9..1e3fd63 100644 --- a/scripts/register-hooks.ts +++ b/scripts/register-hooks.ts @@ -16,6 +16,7 @@ import { HOOK_SPECS, buildHookMap, formatHookSummary, + isOwnHookEntry, type HookSpec, } from "../src/hook-spec.js"; @@ -72,8 +73,7 @@ const betterdbHooks = buildHookMap(cmd); for (const [event, entries] of Object.entries(betterdbHooks)) { const prev = Array.isArray(existingHooks[event]) ? existingHooks[event] : []; const filtered = prev.filter((entry) => { - const json = JSON.stringify(entry); - return !json.includes("betterdb") && !json.includes(hooksDir); + return !isOwnHookEntry(entry, [hooksDir, "betterdb"]); }); existingHooks[event] = [...filtered, ...entries]; } diff --git a/src/hook-spec.ts b/src/hook-spec.ts index 0ba7092..2914d94 100644 --- a/src/hook-spec.ts +++ b/src/hook-spec.ts @@ -37,6 +37,32 @@ export const HOOK_SPECS: readonly HookSpec[] = [ export const HOOK_COUNT = HOOK_SPECS.length; +const HOOK_SOURCES: readonly string[] = HOOK_SPECS.map((spec) => spec.source); + +/** + * Whether a hook entry already in ~/.claude/settings.json is one this plugin + * wrote, and so may be replaced. + * + * Matching an install path alone is not enough: register-hooks.ts points hooks + * at a checkout whose path need not contain "betterdb", so such an entry + * survived a later install and kept firing alongside the new registration. The + * source filenames come from HOOK_SPECS, so they identify our entries wherever + * the checkout lives. `markers` adds the caller's own install location. + * + * Deliberately conservative — a false positive deletes a third party's hook, + * which is worse than leaving one of ours behind. + */ +export function isOwnHookEntry( + entry: unknown, + markers: readonly string[] = [], +): boolean { + const json = JSON.stringify(entry) ?? ""; + if (HOOK_SOURCES.some((source) => json.includes(source))) { + return true; + } + return markers.some((marker) => marker.length > 0 && json.includes(marker)); +} + export function buildHookMap( toCommand: (spec: HookSpec) => string, ): Record { diff --git a/src/index.ts b/src/index.ts index 244c13b..213d5b3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -19,7 +19,12 @@ import { rmSync, } from "node:fs"; import { join, resolve } from "node:path"; -import { HOOK_COUNT, HOOK_SPECS, buildHookMap } from "./hook-spec.js"; +import { + HOOK_COUNT, + HOOK_SPECS, + buildHookMap, + isOwnHookEntry, +} from "./hook-spec.js"; const VERSION = "0.5.0"; const HOME = process.env["HOME"] ?? process.env["USERPROFILE"] ?? ""; @@ -897,10 +902,8 @@ function mergeHooks( const merged = { ...existing }; for (const [event, entries] of Object.entries(ours)) { const prev = Array.isArray(merged[event]) ? merged[event] : []; - // Filter out previous BetterDB entries (contain our BIN_DIR or betterdb path) const filtered = prev.filter((entry) => { - const json = JSON.stringify(entry); - return !json.includes(BIN_DIR) && !json.includes("betterdb"); + return !isOwnHookEntry(entry, [BIN_DIR, "betterdb"]); }); merged[event] = [...filtered, ...entries]; } diff --git a/tests/unit/hook-spec.test.ts b/tests/unit/hook-spec.test.ts new file mode 100644 index 0000000..e42ec57 --- /dev/null +++ b/tests/unit/hook-spec.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, test } from "bun:test"; +import { isOwnHookEntry, HOOK_SPECS, buildHookMap } from "../../src/hook-spec.js"; + +const BIN_DIR = "/Users/x/.betterdb/bin"; + +function entry(command: string): unknown { + return { hooks: [{ type: "command", command }] }; +} + +describe("isOwnHookEntry", () => { + test("recognizes an installed binary entry via the BIN_DIR marker", () => { + expect(isOwnHookEntry(entry(`${BIN_DIR}/session-end`), [BIN_DIR])).toBe(true); + }); + + test("recognizes a dev entry whose checkout path has no betterdb in it", () => { + // The gap this predicate exists to close: register-hooks.ts points at a + // checkout that need not be named betterdb, so a path-substring match + // missed it and the stale entry survived install. + const dev = entry(`bash -c 'bun run "/work/memory/src/hooks/session-end.ts"'`); + expect(isOwnHookEntry(dev, [BIN_DIR, "betterdb"])).toBe(true); + }); + + test("recognizes a legacy Stop entry pointing at session-end from any path", () => { + const legacy = entry(`bash -c 'bun run "/srv/clone/src/hooks/session-end.ts"'`); + expect(isOwnHookEntry(legacy, [BIN_DIR, "betterdb"])).toBe(true); + }); + + test("does not claim a third party's hook", () => { + expect(isOwnHookEntry(entry("/opt/other-tool/bin/their-hook"), [BIN_DIR])).toBe( + false, + ); + }); + + test("does not claim a third party hook merely near our install", () => { + expect(isOwnHookEntry(entry("/usr/local/bin/lint"), [BIN_DIR, "betterdb"])).toBe( + false, + ); + }); + + test("ignores empty markers rather than matching everything", () => { + expect(isOwnHookEntry(entry("/opt/other/hook"), [""])).toBe(false); + }); + + test("recognizes every hook this plugin registers", () => { + const map = buildHookMap((spec) => `bash -c 'bun run "/work/memory/src/hooks/${spec.source}"'`); + for (const spec of HOOK_SPECS) { + const entries = map[spec.event] ?? []; + expect(entries.length).toBeGreaterThan(0); + for (const e of entries) { + expect(isOwnHookEntry(e, [BIN_DIR, "betterdb"])).toBe(true); + } + } + }); +}); From 0da0df22829989a027e6af0f856e9a0d359ecc88 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 17 Jul 2026 17:08:57 +0300 Subject: [PATCH 14/30] refactor: derive install-hooks.sh registration from HOOK_SPECS - Build the settings map with buildHookMap instead of a third hardcoded copy of the five lifecycle hooks - Print the summary via formatHookSummary, now parameterized so the shell path shows dist binary paths and the dev path source files - Verify each spec's binary is actually registered instead of counting keys, which reported success even with a hook missing --- scripts/install-hooks.sh | 33 +++++++++++++++++++-------------- src/hook-spec.ts | 6 ++++-- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/scripts/install-hooks.sh b/scripts/install-hooks.sh index de90677..b01ad77 100755 --- a/scripts/install-hooks.sh +++ b/scripts/install-hooks.sh @@ -50,6 +50,7 @@ ENV_FILE="$PROJECT_DIR/.env" bun -e " const fs = require('fs'); +const { buildHookMap } = require('$PROJECT_DIR/src/hook-spec.ts'); const settingsPath = '$GLOBAL_SETTINGS'; const envFile = '$ENV_FILE'; const distDir = '$DIST_DIR'; @@ -59,13 +60,7 @@ const existing = JSON.parse(fs.readFileSync(settingsPath, 'utf8')); const wrap = (bin) => 'bash -c ' + JSON.stringify('set -a; [ -f ' + envFile + ' ] && . ' + envFile + '; set +a; ' + distDir + '/' + bin); -const hooks = { - SessionStart: [{ hooks: [{ type: 'command', command: wrap('session-start') }] }], - PreToolUse: [{ matcher: '', hooks: [{ type: 'command', command: wrap('pre-tool') }] }], - PostToolUse: [{ matcher: '', hooks: [{ type: 'command', command: wrap('post-tool') }] }], - SessionEnd: [{ hooks: [{ type: 'command', command: wrap('session-end') }] }], - Stop: [{ hooks: [{ type: 'command', command: wrap('stop-checkpoint') }] }], -}; +const hooks = buildHookMap((spec) => wrap(spec.binary)); fs.writeFileSync(settingsPath, JSON.stringify({ ...existing, hooks }, null, 2)); console.log('Hook configuration written to: ' + settingsPath); " @@ -81,9 +76,17 @@ echo "" echo "Verifying global settings..." bun -e " const fs = require('fs'); +const { HOOK_SPECS } = require('$PROJECT_DIR/src/hook-spec.ts'); const settings = JSON.parse(fs.readFileSync('$HOME/.claude/settings.json', 'utf8')); -const hookCount = Object.keys(settings.hooks || {}).length; -console.log('Hooks registered: ' + hookCount + ' lifecycle events'); +const missing = HOOK_SPECS.filter((spec) => { + const entries = (settings.hooks || {})[spec.event] || []; + return !JSON.stringify(entries).includes(spec.binary); +}); +console.log('Hooks registered: ' + (HOOK_SPECS.length - missing.length) + '/' + HOOK_SPECS.length + ' lifecycle events'); +if (missing.length > 0) { + console.error('ERROR: not registered: ' + missing.map((spec) => spec.event).join(', ')); + process.exit(1); +} " # Summary @@ -91,11 +94,13 @@ echo "" echo "=== Installation Complete ===" echo "" echo "Hooks written to: ~/.claude/settings.json" -echo " SessionStart → $DIST_DIR/session-start" -echo " SessionEnd → $DIST_DIR/session-end" -echo " Stop → $DIST_DIR/stop-checkpoint" -echo " PreToolUse → $DIST_DIR/pre-tool" -echo " PostToolUse → $DIST_DIR/post-tool" +bun -e " +const { formatHookSummary } = require('$PROJECT_DIR/src/hook-spec.ts'); +const distDir = '$DIST_DIR'; +for (const line of formatHookSummary((spec) => distDir + '/' + spec.binary)) { + console.log(line); +} +" echo "" echo "MCP server: betterdb-memory (stdio)" echo "" diff --git a/src/hook-spec.ts b/src/hook-spec.ts index 2914d94..e7b573c 100644 --- a/src/hook-spec.ts +++ b/src/hook-spec.ts @@ -76,9 +76,11 @@ export function buildHookMap( return map; } -export function formatHookSummary(): string[] { +export function formatHookSummary( + describe: (spec: HookSpec) => string = (spec) => spec.source, +): string[] { const width = Math.max(...HOOK_SPECS.map((spec) => spec.event.length)); return HOOK_SPECS.map( - (spec) => ` ${spec.event.padEnd(width)} → ${spec.source}`, + (spec) => ` ${spec.event.padEnd(width)} → ${describe(spec)}`, ); } From 10b8cf224e28d62d0c94f5c98dc9f541be1f5bf9 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 17 Jul 2026 17:13:17 +0300 Subject: [PATCH 15/30] fix: stop install-hooks.sh clobbering third-party hooks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Writing `{ ...existing, hooks }` replaced the whole hooks block, so installing deleted every hook this plugin did not write — including third-party entries on events we never register. Merge per event and drop only our own entries, matching the behaviour of the other two install paths. Reinstalling stays idempotent. --- scripts/install-hooks.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/install-hooks.sh b/scripts/install-hooks.sh index b01ad77..b2ed482 100755 --- a/scripts/install-hooks.sh +++ b/scripts/install-hooks.sh @@ -42,7 +42,8 @@ if [ ! -f "$GLOBAL_SETTINGS" ]; then echo '{}' > "$GLOBAL_SETTINGS" fi -# Merge hooks into existing settings using Bun (preserves other fields, overwrites hooks block) +# Merge hooks into existing settings using Bun — replaces our own entries per +# event and preserves every other field, including third-party hooks # Each hook command sources the .env file first so compiled binaries get the right env vars # (bun build --compile binaries don't auto-load .env like `bun run` does) DIST_DIR="$PROJECT_DIR/dist/hooks" @@ -50,7 +51,7 @@ ENV_FILE="$PROJECT_DIR/.env" bun -e " const fs = require('fs'); -const { buildHookMap } = require('$PROJECT_DIR/src/hook-spec.ts'); +const { buildHookMap, isOwnHookEntry } = require('$PROJECT_DIR/src/hook-spec.ts'); const settingsPath = '$GLOBAL_SETTINGS'; const envFile = '$ENV_FILE'; const distDir = '$DIST_DIR'; @@ -60,7 +61,12 @@ const existing = JSON.parse(fs.readFileSync(settingsPath, 'utf8')); const wrap = (bin) => 'bash -c ' + JSON.stringify('set -a; [ -f ' + envFile + ' ] && . ' + envFile + '; set +a; ' + distDir + '/' + bin); -const hooks = buildHookMap((spec) => wrap(spec.binary)); +const hooks = { ...(existing.hooks || {}) }; +for (const [event, entries] of Object.entries(buildHookMap((spec) => wrap(spec.binary)))) { + const prev = Array.isArray(hooks[event]) ? hooks[event] : []; + const kept = prev.filter((entry) => !isOwnHookEntry(entry, [distDir, 'betterdb'])); + hooks[event] = [...kept, ...entries]; +} fs.writeFileSync(settingsPath, JSON.stringify({ ...existing, hooks }, null, 2)); console.log('Hook configuration written to: ' + settingsPath); " From 5c4af0e28c0d56168c180d049655b8377b8e71ec Mon Sep 17 00:00:00 2001 From: jamby77 Date: Tue, 21 Jul 2026 14:31:19 +0300 Subject: [PATCH 16/30] Add build:hooks script derived from HOOK_SPECS - Define the missing build:hooks script referenced by install-hooks.sh and CLAUDE.md, compiling each hook entry point to dist/hooks/ - Derive the build list from HOOK_SPECS so a new hook needs no separate build wiring - Mark optional provider openai as external so --compile does not fail on the undeclared dynamic import - Sweep bun build temp files and ignore *.bun-build --- .gitignore | 1 + package.json | 1 + scripts/build-hooks.ts | 54 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 scripts/build-hooks.ts diff --git a/.gitignore b/.gitignore index 9957289..3501e6f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,6 @@ dist/ .betterdb_context.md .mcp.json *.log +*.bun-build bun.lockb .idea/ diff --git a/package.json b/package.json index deaf6de..0fac555 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "test": "bun test", "test:unit": "bun test tests/unit", "test:integration": "bun test tests/integration", + "build:hooks": "bun run scripts/build-hooks.ts", "validate": "bash scripts/validate-pack.sh", "prepublishOnly": "bash scripts/validate-pack.sh", "setup-index": "bun run scripts/setup-index.ts", diff --git a/scripts/build-hooks.ts b/scripts/build-hooks.ts new file mode 100644 index 0000000..0adc94a --- /dev/null +++ b/scripts/build-hooks.ts @@ -0,0 +1,54 @@ +#!/usr/bin/env bun + +/** + * Compile each hook entry point in HOOK_SPECS to a standalone binary. + * + * Usage: + * bun run scripts/build-hooks.ts + * + * Sources live in src/hooks/; binaries are written to + * dist/hooks/, the paths install-hooks.sh registers. + */ + +import { mkdirSync, readdirSync, rmSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { HOOK_SPECS } from "../src/hook-spec.js"; + +const projectRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const hooksDir = join(projectRoot, "src", "hooks"); +const outDir = join(projectRoot, "dist", "hooks"); + +const OPTIONAL_PROVIDERS = ["openai"]; + +mkdirSync(outDir, { recursive: true }); + +for (const spec of HOOK_SPECS) { + const source = join(hooksDir, spec.source); + const outfile = join(outDir, spec.binary); + console.log(`Compiling ${spec.source} → dist/hooks/${spec.binary}`); + const result = Bun.spawnSync( + [ + "bun", + "build", + "--compile", + ...OPTIONAL_PROVIDERS.flatMap((pkg) => ["--external", pkg]), + source, + "--outfile", + outfile, + ], + { stdout: "inherit", stderr: "inherit" }, + ); + if (!result.success) { + console.error(`ERROR: failed to compile ${spec.source}`); + process.exit(1); + } +} + +for (const name of readdirSync(projectRoot)) { + if (name.endsWith(".bun-build")) { + rmSync(join(projectRoot, name), { force: true }); + } +} + +console.log(`Compiled ${HOOK_SPECS.length} hook binaries to dist/hooks/`); From a981447977aab52c31f0e137e70260cadf674970 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 24 Jul 2026 09:51:25 +0300 Subject: [PATCH 17/30] fix: resolve the drain next to the running hook binary - Compiled hooks resolved drain.ts via import.meta.dir, a bunfs virtual path where the source never exists, so the dev-shape install never drained the ingest queue - Extract locateDrainCommand: sibling binary first, then ~/.betterdb/bin, then bun-running the source - Ship drain as a companion binary from build:hooks via COMPANION_BINARIES, shared with the CLI install --- scripts/build-hooks.ts | 11 ++++-- src/hook-spec.ts | 14 +++++++ src/hooks/locate-drain.ts | 36 ++++++++++++++++++ src/hooks/session-end.ts | 43 ++++++++-------------- src/index.ts | 4 +- tests/unit/locate-drain.test.ts | 65 +++++++++++++++++++++++++++++++++ 6 files changed, 140 insertions(+), 33 deletions(-) create mode 100644 src/hooks/locate-drain.ts create mode 100644 tests/unit/locate-drain.test.ts diff --git a/scripts/build-hooks.ts b/scripts/build-hooks.ts index 0adc94a..c678cf4 100644 --- a/scripts/build-hooks.ts +++ b/scripts/build-hooks.ts @@ -1,7 +1,8 @@ #!/usr/bin/env bun /** - * Compile each hook entry point in HOOK_SPECS to a standalone binary. + * Compile each hook entry point in HOOK_SPECS, plus the companion binaries + * hooks spawn at runtime, to standalone binaries. * * Usage: * bun run scripts/build-hooks.ts @@ -13,7 +14,7 @@ import { mkdirSync, readdirSync, rmSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { HOOK_SPECS } from "../src/hook-spec.js"; +import { COMPANION_BINARIES, HOOK_SPECS } from "../src/hook-spec.js"; const projectRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const hooksDir = join(projectRoot, "src", "hooks"); @@ -23,7 +24,9 @@ const OPTIONAL_PROVIDERS = ["openai"]; mkdirSync(outDir, { recursive: true }); -for (const spec of HOOK_SPECS) { +const specs = [...HOOK_SPECS, ...COMPANION_BINARIES]; + +for (const spec of specs) { const source = join(hooksDir, spec.source); const outfile = join(outDir, spec.binary); console.log(`Compiling ${spec.source} → dist/hooks/${spec.binary}`); @@ -51,4 +54,4 @@ for (const name of readdirSync(projectRoot)) { } } -console.log(`Compiled ${HOOK_SPECS.length} hook binaries to dist/hooks/`); +console.log(`Compiled ${specs.length} binaries to dist/hooks/`); diff --git a/src/hook-spec.ts b/src/hook-spec.ts index e7b573c..bfe72ca 100644 --- a/src/hook-spec.ts +++ b/src/hook-spec.ts @@ -35,6 +35,20 @@ export const HOOK_SPECS: readonly HookSpec[] = [ { event: "Stop", source: "stop-checkpoint.ts", binary: "stop-checkpoint" }, ]; +export interface BinarySpec { + readonly source: string; + readonly binary: string; +} + +/** + * Non-hook binaries that must ship next to the hook binaries: the SessionEnd + * hook resolves the drainer as a sibling of its own executable, so every + * shape that compiles hooks must compile these too. + */ +export const COMPANION_BINARIES: readonly BinarySpec[] = [ + { source: "drain.ts", binary: "drain" }, +]; + export const HOOK_COUNT = HOOK_SPECS.length; const HOOK_SOURCES: readonly string[] = HOOK_SPECS.map((spec) => spec.source); diff --git a/src/hooks/locate-drain.ts b/src/hooks/locate-drain.ts new file mode 100644 index 0000000..fb56cac --- /dev/null +++ b/src/hooks/locate-drain.ts @@ -0,0 +1,36 @@ +import { join } from "node:path"; + +export interface DrainLocations { + readonly execDir: string; + readonly installBinDir: string; + readonly sourceDir: string; +} + +/** + * Resolve the drainer to spawn, in order of preference: the binary compiled + * next to the running hook (dev builds via build:hooks), the binary the CLI + * install placed in ~/.betterdb/bin, then bun-running the source (the + * register-hooks.ts shape). Inside a compiled executable sourceDir is a bunfs + * virtual path where drain.ts never exists, which is why the sibling binary + * must be checked first. + */ +export async function locateDrainCommand( + locations: DrainLocations, +): Promise { + const siblingBin = join(locations.execDir, "drain"); + if (await Bun.file(siblingBin).exists()) { + return [siblingBin]; + } + + const installedBin = join(locations.installBinDir, "drain"); + if (await Bun.file(installedBin).exists()) { + return [installedBin]; + } + + const source = join(locations.sourceDir, "drain.ts"); + if (await Bun.file(source).exists()) { + return ["bun", "run", source]; + } + + return null; +} diff --git a/src/hooks/session-end.ts b/src/hooks/session-end.ts index 01bac00..f6095c9 100644 --- a/src/hooks/session-end.ts +++ b/src/hooks/session-end.ts @@ -15,8 +15,9 @@ import { type OffsetTurn, } from "../memory/checkpoint.js"; import { config, isConfigured } from "../config.js"; +import { locateDrainCommand } from "./locate-drain.js"; import { unlink } from "node:fs/promises"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; /** * SessionEnd hook: captures the session transcript and queues it. @@ -132,39 +133,27 @@ runHook(async () => { /** * Spawn the detached drainer. unref() releases it from this process's event * loop, so the hook exits immediately while summarization continues. - * - * Both install shapes must work: `install` compiles binaries into - * ~/.betterdb/bin, while register-hooks.ts registers `bun run ` and - * compiles nothing. Resolving only the compiled path left that second shape - * with no drainer at all — and the exists() guard made it silent. */ async function spawnDrain(): Promise { // HOME is unset on Windows, where install and config both fall back to // USERPROFILE. const home = process.env["HOME"] ?? process.env["USERPROFILE"] ?? ""; - const drainBin = join(home, ".betterdb", "bin", "drain"); - if (await Bun.file(drainBin).exists()) { - Bun.spawn([drainBin], { - stdin: "ignore", - stdout: "ignore", - stderr: "ignore", - }).unref(); + const cmd = await locateDrainCommand({ + execDir: dirname(process.execPath), + installBinDir: join(home, ".betterdb", "bin"), + sourceDir: import.meta.dir, + }); + if (!cmd) { + console.error( + "[betterdb] no drain binary or source found — queued transcripts stay queued until `betterdb-memory drain` runs", + ); return; } - - const drainSrc = join(import.meta.dir, "drain.ts"); - if (await Bun.file(drainSrc).exists()) { - Bun.spawn(["bun", "run", drainSrc], { - stdin: "ignore", - stdout: "ignore", - stderr: "ignore", - }).unref(); - return; - } - - console.error( - "[betterdb] no drain binary or source found — queued transcripts stay queued until `betterdb-memory drain` runs", - ); + Bun.spawn(cmd, { + stdin: "ignore", + stdout: "ignore", + stderr: "ignore", + }).unref(); } async function cleanup( diff --git a/src/index.ts b/src/index.ts index 213d5b3..f01e103 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,6 +20,7 @@ import { } from "node:fs"; import { join, resolve } from "node:path"; import { + COMPANION_BINARIES, HOOK_COUNT, HOOK_SPECS, buildHookMap, @@ -35,11 +36,10 @@ const MANIFEST_PATH = join(BETTERDB_DIR, "install-manifest.json"); const PKG_ROOT = resolve(import.meta.dir, ".."); const BINARIES: readonly { src: string; out: string }[] = [ - ...HOOK_SPECS.map((spec) => ({ + ...[...HOOK_SPECS, ...COMPANION_BINARIES].map((spec) => ({ src: `src/hooks/${spec.source}`, out: spec.binary, })), - { src: "src/hooks/drain.ts", out: "drain" }, { src: "src/mcp/server.ts", out: "mcp-server" }, ]; diff --git a/tests/unit/locate-drain.test.ts b/tests/unit/locate-drain.test.ts new file mode 100644 index 0000000..ba59633 --- /dev/null +++ b/tests/unit/locate-drain.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll } from "bun:test"; +import { locateDrainCommand } from "../../src/hooks/locate-drain.js"; + +const root = mkdtempSync(join(tmpdir(), "locate-drain-")); + +function dir(name: string, files: string[] = []): string { + const d = join(root, name); + mkdirSync(d, { recursive: true }); + for (const f of files) { + writeFileSync(join(d, f), ""); + } + return d; +} + +afterAll(() => { + rmSync(root, { recursive: true, force: true }); +}); + +describe("locateDrainCommand", () => { + test("prefers the drain binary sitting next to the running executable", async () => { + const execDir = dir("exec-with-drain", ["drain"]); + const installBinDir = dir("bin-with-drain", ["drain"]); + const sourceDir = dir("src-with-drain", ["drain.ts"]); + + const cmd = await locateDrainCommand({ execDir, installBinDir, sourceDir }); + + expect(cmd).toEqual([join(execDir, "drain")]); + }); + + test("falls back to the install bin dir when the executable has no sibling", async () => { + const execDir = dir("exec-empty-1"); + const installBinDir = dir("bin-with-drain-2", ["drain"]); + const sourceDir = dir("src-with-drain-2", ["drain.ts"]); + + const cmd = await locateDrainCommand({ execDir, installBinDir, sourceDir }); + + expect(cmd).toEqual([join(installBinDir, "drain")]); + }); + + test("falls back to bun-running the source in the dev shape", async () => { + const execDir = dir("exec-empty-2"); + const installBinDir = dir("bin-empty-2"); + const sourceDir = dir("src-with-drain-3", ["drain.ts"]); + + const cmd = await locateDrainCommand({ execDir, installBinDir, sourceDir }); + + expect(cmd).toEqual(["bun", "run", join(sourceDir, "drain.ts")]); + }); + + test("returns null when no shape provides a drain", async () => { + // The compiled-binary trap this function exists to close: sourceDir is a + // bunfs virtual path inside the executable, so drain.ts never exists there. + const execDir = dir("exec-empty-3"); + const installBinDir = dir("bin-empty-3"); + const sourceDir = join(root, "does-not-exist"); + + const cmd = await locateDrainCommand({ execDir, installBinDir, sourceDir }); + + expect(cmd).toBeNull(); + }); +}); From 102d8104543aa8768ff39b27dcfc88c2e38689a3 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 24 Jul 2026 10:06:41 +0300 Subject: [PATCH 18/30] fix: re-queue the whole unprocessed batch when the drain fails popIngestQueue is destructive; on a summarize failure only the failed item went back and the rest of the popped batch was lost when the drain process exited --- src/memory/aging.ts | 14 ++++++-- tests/unit/aging-guard.test.ts | 59 ++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 3 deletions(-) diff --git a/src/memory/aging.ts b/src/memory/aging.ts index 5eeaa01..80b2ec5 100644 --- a/src/memory/aging.ts +++ b/src/memory/aging.ts @@ -137,7 +137,8 @@ export class AgingPipeline { let processed = 0; let skipped = 0; - for (const item of items) { + for (let i = 0; i < items.length; i++) { + const item = items[i]!; try { const summary = await this.modelClient.summarize(item.transcript); @@ -169,8 +170,15 @@ export class AgingPipeline { processed++; } catch (err) { console.error("[betterdb] Failed to process queued transcript:", err); - // Re-queue on failure - await this.valkeyClient.pushIngestQueue(item.transcript, item.meta); + // The pop was destructive: everything not yet processed must go back, + // not just the item that failed, or the rest of the batch is lost + // when this process exits. + for (const unprocessed of items.slice(i)) { + await this.valkeyClient.pushIngestQueue( + unprocessed.transcript, + unprocessed.meta, + ); + } break; } } diff --git a/tests/unit/aging-guard.test.ts b/tests/unit/aging-guard.test.ts index 9ef6b89..c4fb7d0 100644 --- a/tests/unit/aging-guard.test.ts +++ b/tests/unit/aging-guard.test.ts @@ -37,6 +37,14 @@ function fakeModel(summary: SessionSummary) { }; } +function failingModel() { + return { + summarize: async () => { + throw new Error("summarizer unavailable"); + }, + }; +} + describe("processIngestQueue empty-summary guard", () => { test("does not store a husk", async () => { const husk = SessionSummarySchema.parse({}); @@ -87,3 +95,54 @@ describe("processIngestQueue empty-summary guard", () => { expect(result.skipped).toBe(0); }); }); + +describe("processIngestQueue failure handling", () => { + test("re-queues the failed item and every remaining popped item", async () => { + // The pop is destructive: anything popped but neither stored nor + // re-queued is lost forever when the drain process exits. + const vk = fakeValkey([ + { transcript: "first", meta: {} }, + { transcript: "second", meta: {} }, + { transcript: "third", meta: {} }, + ]); + + const pipeline = new AgingPipeline( + vk.client as never, + fakeStore([]) as never, + failingModel() as never, + ); + const result = await pipeline.processIngestQueue(); + + expect(result.processed).toBe(0); + expect(vk.requeued.sort()).toEqual(["first", "second", "third"]); + }); + + test("re-queues only the unprocessed tail when a later item fails", async () => { + const real = SessionSummarySchema.parse({ oneLineSummary: "Did a thing" }); + let calls = 0; + const flaky = { + summarize: async () => { + calls++; + if (calls > 1) throw new Error("summarizer died mid-batch"); + return real; + }, + }; + const stored: SessionSummary[] = []; + const vk = fakeValkey([ + { transcript: "first", meta: {} }, + { transcript: "second", meta: {} }, + { transcript: "third", meta: {} }, + ]); + + const pipeline = new AgingPipeline( + vk.client as never, + fakeStore(stored) as never, + flaky as never, + ); + const result = await pipeline.processIngestQueue(); + + expect(result.processed).toBe(1); + expect(stored.length).toBe(1); + expect(vk.requeued.sort()).toEqual(["second", "third"]); + }); +}); From 63cb5a12533ea0e198b7e025415ce6fe2ecdb418 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 24 Jul 2026 10:06:41 +0300 Subject: [PATCH 19/30] fix: stream Ollama summaries so idle timeouts cannot kill them A non-streaming generate sends no bytes until the whole summary is done; long generations died on Bun's fetch idle timeout as TimeoutError. Chunks keep the connection alive while the model is producing. Transport is injectable for tests --- src/client/providers/ollama.ts | 40 ++++++++++++++++++++++++++++---- tests/unit/ollama-stream.test.ts | 38 ++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 tests/unit/ollama-stream.test.ts diff --git a/src/client/providers/ollama.ts b/src/client/providers/ollama.ts index dd2b81f..1b6ce1e 100644 --- a/src/client/providers/ollama.ts +++ b/src/client/providers/ollama.ts @@ -4,13 +4,31 @@ import { SessionSummarySchema, type SessionSummary } from "../../memory/schema.j import type { ModelClient, ModelPreset } from "../model.js"; import { buildSummarizePrompt, stripCodeFences } from "./_prompt.js"; +export interface OllamaTransport { + chat(request: { + model: string; + messages: { role: string; content: string }[]; + format: string; + stream: true; + }): Promise>; + embed(request: { + model: string; + input: string; + }): Promise<{ embeddings: number[][] }>; +} + export class OllamaModelClient implements ModelClient { - private ollama: Ollama; + private ollama: OllamaTransport; readonly preset: ModelPreset; readonly embedDim: number; - constructor(preset: ModelPreset, ollamaUrl?: string) { - this.ollama = new Ollama({ host: ollamaUrl ?? config.ollama.url }); + constructor( + preset: ModelPreset, + ollamaUrl?: string, + transport?: OllamaTransport, + ) { + this.ollama = + transport ?? new Ollama({ host: ollamaUrl ?? config.ollama.url }); this.preset = preset; this.embedDim = preset.embedDim; } @@ -27,17 +45,29 @@ export class OllamaModelClient implements ModelClient { return first; } + /** + * Streamed rather than awaited whole: Bun's fetch enforces an idle timeout, + * and a non-streaming generate sends no bytes until the entire summary is + * done — long generations died as TimeoutError. Chunks keep the connection + * alive for as long as the model keeps producing. + */ async summarize(transcript: string): Promise { - const response = await this.ollama.chat({ + const stream = await this.ollama.chat({ model: this.preset.summarizeModel, messages: [ { role: "user", content: buildSummarizePrompt(transcript) }, ], format: "json", + stream: true, }); + let content = ""; + for await (const chunk of stream) { + content += chunk.message.content; + } + const parsed = SessionSummarySchema.safeParse( - JSON.parse(stripCodeFences(response.message.content)), + JSON.parse(stripCodeFences(content)), ); if (!parsed.success) { diff --git a/tests/unit/ollama-stream.test.ts b/tests/unit/ollama-stream.test.ts new file mode 100644 index 0000000..b103e40 --- /dev/null +++ b/tests/unit/ollama-stream.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, test } from "bun:test"; +import { OllamaModelClient } from "../../src/client/providers/ollama.js"; + +const PRESET = { + embedModel: "mxbai-embed-large", + summarizeModel: "mistral:7b", + embedDim: 1024, +}; + +function chunked(pieces: string[]) { + return { + chat: async (request: { stream?: boolean }) => { + expect(request.stream).toBe(true); + async function* stream() { + for (const piece of pieces) { + yield { message: { content: piece } }; + } + } + return stream(); + }, + embed: async () => ({ embeddings: [[0]] }), + }; +} + +describe("OllamaModelClient.summarize", () => { + test("assembles the summary from streamed chunks", async () => { + // Streaming keeps bytes flowing during generation so Bun's fetch idle + // timeout cannot fire while the model is still producing output. + const json = JSON.stringify({ oneLineSummary: "Did a thing" }); + const mid = Math.floor(json.length / 2); + const transport = chunked([json.slice(0, mid), json.slice(mid)]); + + const client = new OllamaModelClient(PRESET, undefined, transport); + const summary = await client.summarize("User: did we do a thing?"); + + expect(summary.oneLineSummary).toBe("Did a thing"); + }); +}); From 3d688a57cd5ac33351ddd1dc0f0ed752813f0aa4 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 24 Jul 2026 10:23:52 +0300 Subject: [PATCH 20/30] fix: retry a timed-out summarize once and keep the model warm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A cold Ollama model load is silent for longer than the client idle timeout (measured 388s TTFB vs 3.6s warm), but the load continues server-side after the client gives up — one retry lands on a warm model. keep_alive=60m holds it in memory between calls --- src/client/providers/ollama.ts | 44 ++++++++++++++++++++-------- tests/unit/ollama-stream.test.ts | 49 ++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 12 deletions(-) diff --git a/src/client/providers/ollama.ts b/src/client/providers/ollama.ts index 1b6ce1e..8d21462 100644 --- a/src/client/providers/ollama.ts +++ b/src/client/providers/ollama.ts @@ -10,6 +10,7 @@ export interface OllamaTransport { messages: { role: string; content: string }[]; format: string; stream: true; + keep_alive: string; }): Promise>; embed(request: { model: string; @@ -50,20 +51,21 @@ export class OllamaModelClient implements ModelClient { * and a non-streaming generate sends no bytes until the entire summary is * done — long generations died as TimeoutError. Chunks keep the connection * alive for as long as the model keeps producing. + * + * A cold model load is silent for longer than the idle timeout allows, but + * the load keeps going server-side after the client gives up — so one + * timed-out attempt is retried against the by-then warm model, and + * keep_alive holds the model in memory between calls. */ async summarize(transcript: string): Promise { - const stream = await this.ollama.chat({ - model: this.preset.summarizeModel, - messages: [ - { role: "user", content: buildSummarizePrompt(transcript) }, - ], - format: "json", - stream: true, - }); - - let content = ""; - for await (const chunk of stream) { - content += chunk.message.content; + let content: string; + try { + content = await this.chatSummary(transcript); + } catch (err) { + if (!(err instanceof DOMException && err.name === "TimeoutError")) { + throw err; + } + content = await this.chatSummary(transcript); } const parsed = SessionSummarySchema.safeParse( @@ -80,4 +82,22 @@ export class OllamaModelClient implements ModelClient { return parsed.data; } + + private async chatSummary(transcript: string): Promise { + const stream = await this.ollama.chat({ + model: this.preset.summarizeModel, + messages: [ + { role: "user", content: buildSummarizePrompt(transcript) }, + ], + format: "json", + stream: true, + keep_alive: "60m", + }); + + let content = ""; + for await (const chunk of stream) { + content += chunk.message.content; + } + return content; + } } diff --git a/tests/unit/ollama-stream.test.ts b/tests/unit/ollama-stream.test.ts index b103e40..d87b220 100644 --- a/tests/unit/ollama-stream.test.ts +++ b/tests/unit/ollama-stream.test.ts @@ -22,7 +22,56 @@ function chunked(pieces: string[]) { }; } +function timeoutOnce(pieces: string[]) { + let calls = 0; + const requests: { keep_alive?: string }[] = []; + return { + requests, + transport: { + chat: async (request: { stream?: boolean; keep_alive?: string }) => { + requests.push(request); + calls++; + if (calls === 1) { + throw new DOMException("The operation timed out.", "TimeoutError"); + } + async function* stream() { + for (const piece of pieces) { + yield { message: { content: piece } }; + } + } + return stream(); + }, + embed: async () => ({ embeddings: [[0]] }), + }, + }; +} + describe("OllamaModelClient.summarize", () => { + test("retries once when the cold model load outlives the fetch timeout", async () => { + // A cold Ollama model load can exceed the client idle timeout; the load + // continues server-side, so a single retry lands on a warm model. + const json = JSON.stringify({ oneLineSummary: "Recovered" }); + const fake = timeoutOnce([json]); + + const client = new OllamaModelClient(PRESET, undefined, fake.transport); + const summary = await client.summarize("User: transcript"); + + expect(summary.oneLineSummary).toBe("Recovered"); + expect(fake.requests.length).toBe(2); + }); + + test("keeps the model warm across calls via keep_alive", async () => { + const json = JSON.stringify({ oneLineSummary: "Recovered" }); + const fake = timeoutOnce([json]); + + const client = new OllamaModelClient(PRESET, undefined, fake.transport); + await client.summarize("User: transcript"); + + for (const request of fake.requests) { + expect(request.keep_alive).toBe("60m"); + } + }); + test("assembles the summary from streamed chunks", async () => { // Streaming keeps bytes flowing during generation so Bun's fetch idle // timeout cannot fire while the model is still producing output. From 9baa5a5b360db7002bd83ce8cb0983929495329a Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 24 Jul 2026 11:09:55 +0300 Subject: [PATCH 21/30] fix: fail fast when Valkey cannot be reached connectTimeout only bounds the TCP connect; a proxy that accepts for a missing backend passed it and hung the ready check until the 60s hook timeout, costing minutes per tool call. Every attempt now races a 1s deadline; worst case is ~3.4s across retries --- src/client/valkey.ts | 54 +++++++++++++++++++++++++------ tests/unit/valkey-connect.test.ts | 30 +++++++++++++++++ 2 files changed, 75 insertions(+), 9 deletions(-) create mode 100644 tests/unit/valkey-connect.test.ts diff --git a/src/client/valkey.ts b/src/client/valkey.ts index bf3826b..f0d999f 100644 --- a/src/client/valkey.ts +++ b/src/client/valkey.ts @@ -429,21 +429,20 @@ let clientInstance: ValkeyClient | null = null; export async function getValkeyClient(): Promise { if (clientInstance) return clientInstance; - const retryDelays = [100, 500, 2000]; + // Hooks run on every tool call, so a dead or unresponsive Valkey must cost + // milliseconds, not minutes. + const retryDelays = [100, 300]; let lastError: Error | null = null; - for (const delay of retryDelays) { + for (const delay of [...retryDelays, 0]) { try { - const redis = new Redis(config.valkey.url, { - maxRetriesPerRequest: 3, - lazyConnect: true, - }); - await redis.connect(); - clientInstance = new ValkeyClient(redis); + clientInstance = new ValkeyClient(await connectValkey(config.valkey.url)); return clientInstance; } catch (err) { lastError = err instanceof Error ? err : new Error(String(err)); - await new Promise((resolve) => setTimeout(resolve, delay)); + if (delay > 0) { + await new Promise((resolve) => setTimeout(resolve, delay)); + } } } @@ -455,3 +454,40 @@ export async function getValkeyClient(): Promise { export function resetValkeyClient(): void { clientInstance = null; } + +/** + * One connection attempt under a hard deadline. connectTimeout only bounds + * the TCP connect — a proxy that accepts for a missing backend passes it and + * then hangs the ready check — so the whole attempt races the deadline, and + * the null retryStrategy stops ioredis from retrying forever underneath the + * caller's retry loop. + */ +export async function connectValkey( + url: string, + deadlineMs = 1000, +): Promise { + const redis = new Redis(url, { + maxRetriesPerRequest: 3, + lazyConnect: true, + connectTimeout: deadlineMs, + retryStrategy: () => null, + }); + let deadline: ReturnType | undefined; + try { + await Promise.race([ + redis.connect(), + new Promise((_, reject) => { + deadline = setTimeout( + () => reject(new Error("connection attempt deadline exceeded")), + deadlineMs, + ); + }), + ]); + return redis; + } catch (err) { + redis.disconnect(); + throw err; + } finally { + clearTimeout(deadline); + } +} diff --git a/tests/unit/valkey-connect.test.ts b/tests/unit/valkey-connect.test.ts new file mode 100644 index 0000000..d82f0c1 --- /dev/null +++ b/tests/unit/valkey-connect.test.ts @@ -0,0 +1,30 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { connectValkey } from "../../src/client/valkey.js"; + +// A listener that accepts connections and never speaks RESP — the same shape +// as Docker's proxy accepting for a backend that is not there. Hooks run on +// every tool call, so a connect against this must fail fast, not hang until +// the hook harness kills the process. +const blackhole = Bun.listen({ + hostname: "127.0.0.1", + port: 0, + socket: { + data() {}, + }, +}); + +afterAll(() => { + blackhole.stop(true); +}); + +describe("connectValkey against an unresponsive server", () => { + test("fails within the deadline instead of hanging", async () => { + const started = Date.now(); + + await expect( + connectValkey(`redis://127.0.0.1:${blackhole.port}`, 1000), + ).rejects.toThrow(/deadline exceeded/); + + expect(Date.now() - started).toBeLessThan(3000); + }, 30_000); +}); From d97b7213eac6470cda43d269add025ea292cf017 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 24 Jul 2026 11:09:56 +0300 Subject: [PATCH 22/30] feat: register hooks with explicit timeouts Backstop over the fail-fast connect: pre/post-tool get 10s so no future hook bug can make every tool call hang for the 60s default --- src/hook-spec.ts | 42 +++++++++++++++++++++++++++++++----- tests/unit/hook-spec.test.ts | 17 +++++++++++++++ 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/src/hook-spec.ts b/src/hook-spec.ts index bfe72ca..8990860 100644 --- a/src/hook-spec.ts +++ b/src/hook-spec.ts @@ -10,11 +10,13 @@ export interface HookSpec { readonly source: string; readonly binary: string; readonly matcher?: string; + readonly timeout: number; } export interface HookCommand { readonly type: "command"; readonly command: string; + readonly timeout: number; } export interface HookEntry { @@ -22,17 +24,45 @@ export interface HookEntry { readonly hooks: readonly HookCommand[]; } +/** + * Timeouts are seconds, enforced by Claude Code per hook invocation. They are + * a backstop over the client-side fail-fast connect: pre/post fire on every + * tool call and must never make a tool call feel slow; the session-boundary + * hooks may parse large transcripts and get more room. + */ export const HOOK_SPECS: readonly HookSpec[] = [ - { event: "SessionStart", source: "session-start.ts", binary: "session-start" }, - { event: "PreToolUse", source: "pre-tool.ts", binary: "pre-tool", matcher: "" }, + { + event: "SessionStart", + source: "session-start.ts", + binary: "session-start", + timeout: 30, + }, + { + event: "PreToolUse", + source: "pre-tool.ts", + binary: "pre-tool", + matcher: "", + timeout: 10, + }, { event: "PostToolUse", source: "post-tool.ts", binary: "post-tool", matcher: "", + timeout: 10, + }, + { + event: "SessionEnd", + source: "session-end.ts", + binary: "session-end", + timeout: 60, + }, + { + event: "Stop", + source: "stop-checkpoint.ts", + binary: "stop-checkpoint", + timeout: 30, }, - { event: "SessionEnd", source: "session-end.ts", binary: "session-end" }, - { event: "Stop", source: "stop-checkpoint.ts", binary: "stop-checkpoint" }, ]; export interface BinarySpec { @@ -82,7 +112,9 @@ export function buildHookMap( ): Record { const map: Record = {}; for (const spec of HOOK_SPECS) { - const hooks: HookCommand[] = [{ type: "command", command: toCommand(spec) }]; + const hooks: HookCommand[] = [ + { type: "command", command: toCommand(spec), timeout: spec.timeout }, + ]; const entry: HookEntry = spec.matcher === undefined ? { hooks } : { matcher: spec.matcher, hooks }; (map[spec.event] ??= []).push(entry); diff --git a/tests/unit/hook-spec.test.ts b/tests/unit/hook-spec.test.ts index e42ec57..d677869 100644 --- a/tests/unit/hook-spec.test.ts +++ b/tests/unit/hook-spec.test.ts @@ -41,6 +41,23 @@ describe("isOwnHookEntry", () => { expect(isOwnHookEntry(entry("/opt/other/hook"), [""])).toBe(false); }); + test("every registered hook carries an explicit timeout", () => { + // Without one, Claude Code waits 60s per hook — a wedged backend held + // every tool call hostage for minutes. Pre/post fire on each tool call + // and must be the tightest. + const map = buildHookMap((spec) => spec.binary); + for (const [event, entries] of Object.entries(map)) { + for (const entry of entries) { + for (const cmd of entry.hooks) { + expect(cmd.timeout).toBeGreaterThan(0); + if (event === "PreToolUse" || event === "PostToolUse") { + expect(cmd.timeout).toBeLessThanOrEqual(10); + } + } + } + } + }); + test("recognizes every hook this plugin registers", () => { const map = buildHookMap((spec) => `bash -c 'bun run "/work/memory/src/hooks/${spec.source}"'`); for (const spec of HOOK_SPECS) { From b00d508c7f98e35eb1c07ff1954db6a990b5ab52 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 24 Jul 2026 11:09:56 +0300 Subject: [PATCH 23/30] feat: make Ollama keep_alive configurable, default 5m Native Ollama reloads in seconds, so pinning the model for an hour trades gigabytes of RAM for nothing on memory-tight machines. BETTERDB_OLLAMA_KEEP_ALIVE overrides --- src/client/providers/ollama.ts | 2 +- src/config.ts | 1 + tests/unit/ollama-stream.test.ts | 6 ++++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/client/providers/ollama.ts b/src/client/providers/ollama.ts index 8d21462..dceda15 100644 --- a/src/client/providers/ollama.ts +++ b/src/client/providers/ollama.ts @@ -91,7 +91,7 @@ export class OllamaModelClient implements ModelClient { ], format: "json", stream: true, - keep_alive: "60m", + keep_alive: config.ollama.keepAlive, }); let content = ""; diff --git a/src/config.ts b/src/config.ts index fb486a0..14bf05a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -43,6 +43,7 @@ export const config = { embedModel: env("BETTERDB_EMBED_MODEL") ?? "mxbai-embed-large", summarizeModel: env("BETTERDB_SUMMARIZE_MODEL") ?? "mistral:7b", embedDim: Number(env("BETTERDB_EMBED_DIM") ?? 1024), + keepAlive: env("BETTERDB_OLLAMA_KEEP_ALIVE") ?? "5m", }, memory: { maxContextMemories: Number(env("BETTERDB_MAX_CONTEXT_MEMORIES") ?? 5), diff --git a/tests/unit/ollama-stream.test.ts b/tests/unit/ollama-stream.test.ts index d87b220..2ddb079 100644 --- a/tests/unit/ollama-stream.test.ts +++ b/tests/unit/ollama-stream.test.ts @@ -60,7 +60,9 @@ describe("OllamaModelClient.summarize", () => { expect(fake.requests.length).toBe(2); }); - test("keeps the model warm across calls via keep_alive", async () => { + test("passes the configured keep_alive on every chat call", async () => { + // Default is a modest 5m: native Ollama reloads in seconds, so pinning + // the model in RAM for longer trades gigabytes for nothing. const json = JSON.stringify({ oneLineSummary: "Recovered" }); const fake = timeoutOnce([json]); @@ -68,7 +70,7 @@ describe("OllamaModelClient.summarize", () => { await client.summarize("User: transcript"); for (const request of fake.requests) { - expect(request.keep_alive).toBe("60m"); + expect(request.keep_alive).toBe("5m"); } }); From b839055531e8e2e47dc8a3ffb051cb8d74ef8c52 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Fri, 24 Jul 2026 11:09:56 +0300 Subject: [PATCH 24/30] feat: add macOS compose file without Ollama MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docker on macOS has no GPU passthrough — measured 388s to first token vs 10s native — so macOS runs only Valkey in Docker (with AOF persistence) and uses the native Ollama.app --- CLAUDE.md | 18 ++++++++++++++++++ docker-compose.mac.yml | 22 ++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 docker-compose.mac.yml diff --git a/CLAUDE.md b/CLAUDE.md index 70a04fa..47b72a6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,6 +48,24 @@ bunx @betterdb/memory install ## Setup (for development) +### macOS + +**Never run Ollama in Docker on macOS.** Docker on macOS has no GPU +passthrough, so models run CPU-only inside the VM — measured 388s to first +token vs 10s native. Use the native Ollama.app and the Mac compose file, +which runs only Valkey (with AOF persistence enabled): + +```bash +docker compose -f docker-compose.mac.yml up -d # Valkey only +open -a Ollama # Native Ollama on :11434 +ollama pull mxbai-embed-large +ollama pull qwen2.5:3b +bun install +bun run setup-index +``` + +### Linux + ```bash docker compose up -d # Start Valkey (with search module) + Ollama bun install # Install dependencies diff --git a/docker-compose.mac.yml b/docker-compose.mac.yml new file mode 100644 index 0000000..10dd2db --- /dev/null +++ b/docker-compose.mac.yml @@ -0,0 +1,22 @@ +services: + valkey: + image: valkey/valkey-bundle:9.1-alpine + command: + - valkey-server + - --dir + - /data + - --appendonly + - "yes" + - --save + - "60 1" + container_name: betterdb-valkey + ports: + - "6390:6379" + volumes: + - valkey-data:/data + restart: unless-stopped + +volumes: + valkey-data: + name: betterdb-valkey-data + external: true From e492ee5b3118c534a0e3b8ba74f995d507805541 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Mon, 27 Jul 2026 11:58:13 +0300 Subject: [PATCH 25/30] fix: let compose create the Mac Valkey volume on first run --- docker-compose.mac.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.mac.yml b/docker-compose.mac.yml index 10dd2db..6e0f642 100644 --- a/docker-compose.mac.yml +++ b/docker-compose.mac.yml @@ -19,4 +19,3 @@ services: volumes: valkey-data: name: betterdb-valkey-data - external: true From 0d910260b060b44f78e3d3154c5dada3c004af12 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Mon, 27 Jul 2026 11:58:13 +0300 Subject: [PATCH 26/30] fix: recognize compiled binary entries in isOwnHookEntry --- src/hook-spec.ts | 13 +++++++++++-- tests/unit/hook-spec.test.ts | 25 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/hook-spec.ts b/src/hook-spec.ts index 8990860..547ec4b 100644 --- a/src/hook-spec.ts +++ b/src/hook-spec.ts @@ -83,6 +83,10 @@ export const HOOK_COUNT = HOOK_SPECS.length; const HOOK_SOURCES: readonly string[] = HOOK_SPECS.map((spec) => spec.source); +const HOOK_BINARY_PATHS: readonly string[] = HOOK_SPECS.map( + (spec) => `dist/hooks/${spec.binary}`, +); + /** * Whether a hook entry already in ~/.claude/settings.json is one this plugin * wrote, and so may be replaced. @@ -90,8 +94,10 @@ const HOOK_SOURCES: readonly string[] = HOOK_SPECS.map((spec) => spec.source); * Matching an install path alone is not enough: register-hooks.ts points hooks * at a checkout whose path need not contain "betterdb", so such an entry * survived a later install and kept firing alongside the new registration. The - * source filenames come from HOOK_SPECS, so they identify our entries wherever - * the checkout lives. `markers` adds the caller's own install location. + * source filenames and dist/hooks binary paths come from HOOK_SPECS, so they + * identify our entries wherever the checkout lives — install-hooks.sh + * registers compiled binaries that contain no source filename. `markers` adds + * the caller's own install location. * * Deliberately conservative — a false positive deletes a third party's hook, * which is worse than leaving one of ours behind. @@ -104,6 +110,9 @@ export function isOwnHookEntry( if (HOOK_SOURCES.some((source) => json.includes(source))) { return true; } + if (HOOK_BINARY_PATHS.some((path) => json.includes(path))) { + return true; + } return markers.some((marker) => marker.length > 0 && json.includes(marker)); } diff --git a/tests/unit/hook-spec.test.ts b/tests/unit/hook-spec.test.ts index d677869..602d6ae 100644 --- a/tests/unit/hook-spec.test.ts +++ b/tests/unit/hook-spec.test.ts @@ -25,6 +25,31 @@ describe("isOwnHookEntry", () => { expect(isOwnHookEntry(legacy, [BIN_DIR, "betterdb"])).toBe(true); }); + test("recognizes a compiled binary entry from a foreign checkout", () => { + // install-hooks.sh registers dist/hooks/ wrapped in a bash -c + // env loader — no .ts source name anywhere. A later installer passes its + // own markers, which need not cover the old checkout's path. + const bin = entry( + `bash -c "set -a; [ -f /work/memory/.env ] && . /work/memory/.env; set +a; /work/memory/dist/hooks/stop-checkpoint"`, + ); + expect(isOwnHookEntry(bin, [BIN_DIR, "betterdb"])).toBe(true); + }); + + test("recognizes every compiled hook binary this plugin registers", () => { + const map = buildHookMap((spec) => `/work/memory/dist/hooks/${spec.binary}`); + for (const entries of Object.values(map)) { + for (const e of entries) { + expect(isOwnHookEntry(e, [BIN_DIR, "betterdb"])).toBe(true); + } + } + }); + + test("does not claim a third party binary under its own dist/hooks", () => { + expect( + isOwnHookEntry(entry("/opt/tool/dist/hooks/lint"), [BIN_DIR, "betterdb"]), + ).toBe(false); + }); + test("does not claim a third party's hook", () => { expect(isOwnHookEntry(entry("/opt/other-tool/bin/their-hook"), [BIN_DIR])).toBe( false, From c398024d343ca27b683b548b8a194f432c8e8a56 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Mon, 27 Jul 2026 11:58:13 +0300 Subject: [PATCH 27/30] fix: drain the ingest queue until dry instead of one capped pop --- src/hooks/drain.ts | 2 +- src/memory/aging.ts | 31 ++++++++++++++++- tests/unit/aging-guard.test.ts | 62 ++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/src/hooks/drain.ts b/src/hooks/drain.ts index 69eff47..7b02161 100644 --- a/src/hooks/drain.ts +++ b/src/hooks/drain.ts @@ -17,7 +17,7 @@ async function main(): Promise { const store = await getPluginMemoryStore((t) => modelClient.embed(t)); const pipeline = new AgingPipeline(valkeyClient, store, modelClient); - const { processed, skipped } = await pipeline.processIngestQueue(); + const { processed, skipped } = await pipeline.drainIngestQueue(); console.error(`[betterdb] drain: processed=${processed} skipped=${skipped}`); await store.close(); diff --git a/src/memory/aging.ts b/src/memory/aging.ts index 80b2ec5..e372f6b 100644 --- a/src/memory/aging.ts +++ b/src/memory/aging.ts @@ -132,10 +132,15 @@ export class AgingPipeline { // --- Ingest Queue Processing --- - async processIngestQueue(): Promise<{ processed: number; skipped: number }> { + async processIngestQueue(): Promise<{ + processed: number; + skipped: number; + halted: boolean; + }> { const items = await this.valkeyClient.popIngestQueue(20); let processed = 0; let skipped = 0; + let halted = false; for (let i = 0; i < items.length; i++) { const item = items[i]!; @@ -179,10 +184,34 @@ export class AgingPipeline { unprocessed.meta, ); } + halted = true; break; } } + return { processed, skipped, halted }; + } + + /** + * Drain the ingest queue completely. Each pop is capped, but a + * checkpointing session can enqueue far more segments than one cap; the + * drainer runs detached with nothing waiting on it, so it loops until a + * round comes back empty. A halted round means the failed batch was + * re-queued — looping again would spin against a dead summarizer. + */ + async drainIngestQueue( + maxRounds = 50, + ): Promise<{ processed: number; skipped: number }> { + let processed = 0; + let skipped = 0; + for (let round = 0; round < maxRounds; round++) { + const result = await this.processIngestQueue(); + processed += result.processed; + skipped += result.skipped; + if (result.halted || result.processed + result.skipped === 0) { + break; + } + } return { processed, skipped }; } diff --git a/tests/unit/aging-guard.test.ts b/tests/unit/aging-guard.test.ts index c4fb7d0..00c2b95 100644 --- a/tests/unit/aging-guard.test.ts +++ b/tests/unit/aging-guard.test.ts @@ -96,6 +96,68 @@ describe("processIngestQueue empty-summary guard", () => { }); }); +function countedValkey(items: Array<{ transcript: string; meta: object }>) { + const requeued: string[] = []; + let pops = 0; + return { + client: { + popIngestQueue: async (count: number) => { + pops++; + return items.splice(0, count); + }, + pushIngestQueue: async (t: string) => { + requeued.push(t); + }, + }, + requeued, + pops: () => pops, + }; +} + +describe("drainIngestQueue", () => { + test("drains the whole queue across multiple pops", async () => { + // A checkpointing session can enqueue more segments than one pop cap; + // the drainer is detached and unclocked, so it keeps going until dry. + const real = SessionSummarySchema.parse({ oneLineSummary: "Did a thing" }); + const items = Array.from({ length: 45 }, (_, i) => ({ + transcript: `segment ${i}`, + meta: {}, + })); + const stored: SessionSummary[] = []; + const vk = countedValkey(items); + + const pipeline = new AgingPipeline( + vk.client as never, + fakeStore(stored) as never, + fakeModel(real) as never, + ); + const result = await pipeline.drainIngestQueue(); + + expect(result.processed).toBe(45); + expect(stored.length).toBe(45); + expect(vk.pops()).toBeGreaterThan(2); + }); + + test("stops after a failing round instead of spinning on a dead summarizer", async () => { + const items = Array.from({ length: 25 }, (_, i) => ({ + transcript: `segment ${i}`, + meta: {}, + })); + const vk = countedValkey(items); + + const pipeline = new AgingPipeline( + vk.client as never, + fakeStore([]) as never, + failingModel() as never, + ); + const result = await pipeline.drainIngestQueue(); + + expect(result.processed).toBe(0); + expect(vk.pops()).toBe(1); + expect(vk.requeued.length).toBe(20); + }); +}); + describe("processIngestQueue failure handling", () => { test("re-queues the failed item and every remaining popped item", async () => { // The pop is destructive: anything popped but neither stored nor From ae6045e7ed070bb6daba55d082f02debb1418217 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Mon, 27 Jul 2026 11:58:13 +0300 Subject: [PATCH 28/30] fix: survive a mid-flush failure and still drain queued segments --- src/hooks/flush-segments.ts | 44 +++++++++++++++++++++++++++++++ src/hooks/session-end.ts | 26 +++++++++--------- tests/unit/flush-segments.test.ts | 41 ++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 13 deletions(-) create mode 100644 src/hooks/flush-segments.ts create mode 100644 tests/unit/flush-segments.test.ts diff --git a/src/hooks/flush-segments.ts b/src/hooks/flush-segments.ts new file mode 100644 index 0000000..bfdcd6a --- /dev/null +++ b/src/hooks/flush-segments.ts @@ -0,0 +1,44 @@ +export interface SegmentSink { + pushIngestQueue(transcript: string, meta: object): Promise; +} + +export interface FlushMeta { + readonly project: string; + readonly branch: string; + readonly sessionId: string; + readonly baseSegment: number; +} + +/** + * Push tail segments to the ingest queue, numbering them after the segments + * the Stop hook already queued. Swallows a mid-loop failure and reports how + * far it got: SessionEnd fires exactly once, so an escaped exception here + * would skip both the drain spawn and cleanup, stranding everything already + * queued this session. + */ +export async function flushSegments( + client: SegmentSink, + segments: readonly string[], + meta: FlushMeta, +): Promise<{ pushed: number; failed: boolean }> { + let pushed = 0; + for (const segment of segments) { + try { + await client.pushIngestQueue(segment, { + project: meta.project, + branch: meta.branch, + timestamp: new Date().toISOString(), + sessionId: meta.sessionId, + segment: meta.baseSegment + pushed, + }); + } catch (err) { + console.error( + `[betterdb] flush failed after ${pushed}/${segments.length} segments:`, + err instanceof Error ? err.message : String(err), + ); + return { pushed, failed: true }; + } + pushed++; + } + return { pushed, failed: false }; +} diff --git a/src/hooks/session-end.ts b/src/hooks/session-end.ts index f6095c9..c044c94 100644 --- a/src/hooks/session-end.ts +++ b/src/hooks/session-end.ts @@ -15,6 +15,7 @@ import { type OffsetTurn, } from "../memory/checkpoint.js"; import { config, isConfigured } from "../config.js"; +import { flushSegments } from "./flush-segments.js"; import { locateDrainCommand } from "./locate-drain.js"; import { unlink } from "node:fs/promises"; import { dirname, join } from "node:path"; @@ -111,22 +112,21 @@ runHook(async () => { return; // Valkey unreachable — skip silently } - const project = getCwdProject(); - const branch = getGitBranch(); + const { pushed } = await flushSegments(valkeyClient, segments, { + project: getCwdProject(), + branch: getGitBranch(), + sessionId, + baseSegment: checkpoint.segment, + }); - for (let i = 0; i < segments.length; i++) { - await valkeyClient.pushIngestQueue(segments[i]!, { - project, - branch, - timestamp: new Date().toISOString(), - sessionId, - segment: checkpoint.segment + i, - }); + // Drain even after a partial flush: the Stop hook's segments and whatever + // was pushed before the failure are already queued and nothing else will + // summarize them this session. + if (pushed > 0 || checkpoint.segment > 0) { + await spawnDrain(); } - await spawnDrain(); - - await valkeyClient.quit(); + await valkeyClient.quit().catch(() => {}); await cleanup(eventFilePath, sessionId); }); diff --git a/tests/unit/flush-segments.test.ts b/tests/unit/flush-segments.test.ts new file mode 100644 index 0000000..6089d50 --- /dev/null +++ b/tests/unit/flush-segments.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, test } from "bun:test"; +import { flushSegments } from "../../src/hooks/flush-segments.js"; + +const META = { + project: "p", + branch: "b", + sessionId: "s", + baseSegment: 3, +}; + +describe("flushSegments", () => { + test("pushes every segment with sequential segment numbers", async () => { + const pushed: Array<{ text: string; segment: number }> = []; + const client = { + pushIngestQueue: async (text: string, meta: { segment: number }) => { + pushed.push({ text, segment: meta.segment }); + }, + }; + + const result = await flushSegments(client, ["one", "two"], META); + + expect(result).toEqual({ pushed: 2, failed: false }); + expect(pushed.map((p) => p.segment)).toEqual([3, 4]); + }); + + test("a mid-loop failure reports partial progress instead of throwing", async () => { + // SessionEnd runs once; an escaped exception here skipped both the drain + // spawn and cleanup, stranding the already-queued segments unsummarized. + let calls = 0; + const client = { + pushIngestQueue: async () => { + calls++; + if (calls > 1) throw new Error("connection dropped"); + }, + }; + + const result = await flushSegments(client, ["one", "two", "three"], META); + + expect(result).toEqual({ pushed: 1, failed: true }); + }); +}); From 397c939f80e6ac3de829ccb0581d8bebaa6ae55c Mon Sep 17 00:00:00 2001 From: jamby77 Date: Mon, 27 Jul 2026 12:02:27 +0300 Subject: [PATCH 29/30] test: pin that the losing connect promise is absorbed by the race --- tests/unit/valkey-connect.test.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/unit/valkey-connect.test.ts b/tests/unit/valkey-connect.test.ts index d82f0c1..0f8f221 100644 --- a/tests/unit/valkey-connect.test.ts +++ b/tests/unit/valkey-connect.test.ts @@ -27,4 +27,24 @@ describe("connectValkey against an unresponsive server", () => { expect(Date.now() - started).toBeLessThan(3000); }, 30_000); + + test("does not leak an unhandled rejection from the losing connect", async () => { + // disconnect() rejects the still-pending connect() after the deadline + // already won the race; unabsorbed, that surfaces as an unhandled + // rejection a tick later. + const rejections: unknown[] = []; + const handler = (err: unknown) => { + rejections.push(err); + }; + process.on("unhandledRejection", handler); + try { + await connectValkey(`redis://127.0.0.1:${blackhole.port}`, 50).catch( + () => {}, + ); + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(rejections).toEqual([]); + } finally { + process.off("unhandledRejection", handler); + } + }, 30_000); }); From 707fd81c112f8b0bd4d303ea9f705f804bf0ee08 Mon Sep 17 00:00:00 2001 From: jamby77 Date: Mon, 27 Jul 2026 12:03:43 +0300 Subject: [PATCH 30/30] fix: drain until dry from the CLI drain command too --- src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index f01e103..7ea92d5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -554,7 +554,7 @@ async function runDrain() { const store = await getPluginMemoryStore((t) => modelClient.embed(t)); const pipeline = new AgingPipeline(valkeyClient, store, modelClient); - const { processed, skipped } = await pipeline.processIngestQueue(); + const { processed, skipped } = await pipeline.drainIngestQueue(); console.log( `Processed ${processed} queued transcript(s), skipped ${skipped} empty.`, );