From 36bb79b6944570a1f6b080062626313ffd803886 Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Tue, 12 May 2026 19:20:28 -0400 Subject: [PATCH 01/23] CONFIG: add better-sqlite3 dep and ASAR unpack for dream daemon --- forge.config.ts | 2 +- package-lock.json | 37 +++++++++++++++++++++++++++++++++++++ package.json | 2 ++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/forge.config.ts b/forge.config.ts index 25364344..9d56bf2f 100644 --- a/forge.config.ts +++ b/forge.config.ts @@ -70,7 +70,7 @@ function prepareMsalRuntime(): void { const config: ForgeConfig = { packagerConfig: { asar: { - unpack: '**/node_modules/{sharp,@img,@azure/msal-node-runtime}/**/*', + unpack: '**/node_modules/{sharp,@img,@azure/msal-node-runtime,better-sqlite3,bindings,file-uri-to-path}/**/*', }, executableName: 'chamber', icon: APP_ICON_PATH, diff --git a/package-lock.json b/package-lock.json index 094ae0a2..8642c38f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "dependencies": { "@azure/msal-node": "^5.2.0", "@azure/msal-node-extensions": "^5.2.0", + "better-sqlite3": "^12.10.0", "chamber-copilot": "0.5.11", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", @@ -57,6 +58,7 @@ "@tailwindcss/vite": "^4.2.4", "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.2", + "@types/better-sqlite3": "^7.6.13", "@types/electron-squirrel-startup": "^1.0.2", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", @@ -5410,6 +5412,15 @@ "license": "MIT", "peer": true }, + "node_modules/@types/better-sqlite3": { + "version": "7.6.13", + "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.13.tgz", + "integrity": "sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/cacheable-request": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.3.tgz", @@ -7240,6 +7251,19 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/better-sqlite3": { + "version": "12.10.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.10.0.tgz", + "integrity": "sha512-CyzaZRQKyHkB2ZInfTTl2nvT33EbDpjkLEbE8/Zck3Ll6O0qqvuGdrJ45HgtH+HykRg88ITY3AdreBGN70aBSQ==", + "hasInstallScript": true, + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.1" + }, + "engines": { + "node": "20.x || 22.x || 23.x || 24.x || 25.x || 26.x" + } + }, "node_modules/bidi-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz", @@ -7250,6 +7274,14 @@ "require-from-string": "^2.0.2" } }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, "node_modules/birpc": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/birpc/-/birpc-4.0.0.tgz", @@ -9856,6 +9888,11 @@ "node": ">=16.0.0" } }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" + }, "node_modules/filelist": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.6.tgz", diff --git a/package.json b/package.json index 82d6601a..37b47240 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "@tailwindcss/vite": "^4.2.4", "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.2", + "@types/better-sqlite3": "^7.6.13", "@types/electron-squirrel-startup": "^1.0.2", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", @@ -85,6 +86,7 @@ "dependencies": { "@azure/msal-node": "^5.2.0", "@azure/msal-node-extensions": "^5.2.0", + "better-sqlite3": "^12.10.0", "chamber-copilot": "0.5.11", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", From 8c83bb72e7116b9286db2345d25b637bfd92627b Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Tue, 12 May 2026 19:20:33 -0400 Subject: [PATCH 02/23] ADD: TurnCompletionObserver shared type for chat turn observation --- packages/shared/src/index.ts | 1 + packages/shared/src/turn-observer.test.ts | 41 +++++++++++++++++++++++ packages/shared/src/turn-observer.ts | 40 ++++++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 packages/shared/src/turn-observer.test.ts create mode 100644 packages/shared/src/turn-observer.ts diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 3f9af9c6..2b84150a 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -4,3 +4,4 @@ export { IPC, type IpcChannel } from './ipc-channels'; export { parseIpcArgs } from './ipc-validation'; export { Logger, type LogLevel } from './logger'; export { escapeXml } from './escapeXml'; +export type { CompletedTurn, TurnCompletionObserver, TurnStatus } from './turn-observer'; diff --git a/packages/shared/src/turn-observer.test.ts b/packages/shared/src/turn-observer.test.ts new file mode 100644 index 00000000..123368ca --- /dev/null +++ b/packages/shared/src/turn-observer.test.ts @@ -0,0 +1,41 @@ +import { describe, it, expectTypeOf } from 'vitest'; +import type { + CompletedTurn, + TurnCompletionObserver, + TurnStatus, +} from './turn-observer'; + +describe('turn-observer types', () => { + it('TurnStatus is a closed union of completed | aborted | error', () => { + expectTypeOf().toEqualTypeOf<'completed' | 'aborted' | 'error'>(); + }); + + it('CompletedTurn contains the full payload required by Phase 6', () => { + const turn: CompletedTurn = { + turnId: 't-1', + sessionId: 's-1', + model: 'm-1', + status: 'completed', + startedAt: '2026-05-12T17:00:00.000Z', + endedAt: '2026-05-12T17:00:01.000Z', + prompt: 'hello', + finalAssistantMessage: 'hi back', + }; + expectTypeOf(turn.turnId).toBeString(); + expectTypeOf(turn.sessionId).toBeString(); + expectTypeOf(turn.model).toBeString(); + expectTypeOf(turn.status).toEqualTypeOf(); + expectTypeOf(turn.startedAt).toBeString(); + expectTypeOf(turn.endedAt).toBeString(); + expectTypeOf(turn.prompt).toBeString(); + expectTypeOf(turn.finalAssistantMessage).toBeString(); + }); + + it('TurnCompletionObserver.onTurnCompleted may be sync or async', () => { + const sync: TurnCompletionObserver = { onTurnCompleted: () => undefined }; + const async: TurnCompletionObserver = { onTurnCompleted: async () => undefined }; + expectTypeOf(sync.onTurnCompleted).parameter(0).toEqualTypeOf(); + expectTypeOf(async.onTurnCompleted).parameter(0).toEqualTypeOf(); + expectTypeOf(sync.onTurnCompleted).returns.toEqualTypeOf>(); + }); +}); diff --git a/packages/shared/src/turn-observer.ts b/packages/shared/src/turn-observer.ts new file mode 100644 index 00000000..90a53082 --- /dev/null +++ b/packages/shared/src/turn-observer.ts @@ -0,0 +1,40 @@ +/** + * Shared turn-completion contract. + * + * Phase 6 of the Dream Daemon spike pulled `CompletedTurn` out of + * `@chamber/services/mindMemory/StructuredLogFormat` and into shared so that + * ChatService (the producer) and DailyLogWriter (the first consumer) depend + * on a single canonical shape rather than each maintaining its own copy. + * + * The corresponding observer interface is defined here too so any future + * observer (e.g. DreamDaemon's TurnRecorder, A2A task tracker) imports the + * same protocol from shared. + */ + +export type TurnStatus = 'completed' | 'aborted' | 'error'; + +export interface CompletedTurn { + readonly turnId: string; + readonly sessionId: string; + readonly model: string; + readonly status: TurnStatus; + readonly startedAt: string; + readonly endedAt: string; + readonly prompt: string; + readonly finalAssistantMessage: string; +} + +/** + * Observer notified when ChatService finishes a turn successfully. + * + * Contract: + * - Called once per turn that reached the SDK `done` state. NOT called when + * the turn was aborted by the user or errored out. + * - Implementations must not throw across the boundary in a way that blocks + * other observers — ChatService wraps each call in try/catch and forwards + * async failures to its `Logger.warn`. Observer latency must not gate + * subsequent turns or surface back into the streaming path. + */ +export interface TurnCompletionObserver { + onTurnCompleted(turn: CompletedTurn): void | Promise; +} From fa06aacaf0c11ab19996c032018b387cb44df354 Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Tue, 12 May 2026 19:20:38 -0400 Subject: [PATCH 03/23] ADD: workingMemory.consolidation schema for opt-in dream daemon --- .../src/mind/chamberMindConfig.test.ts | 244 +++++++++++++++++- .../services/src/mind/chamberMindConfig.ts | 106 +++++++- 2 files changed, 336 insertions(+), 14 deletions(-) diff --git a/packages/services/src/mind/chamberMindConfig.test.ts b/packages/services/src/mind/chamberMindConfig.test.ts index a351de8c..09ca179a 100644 --- a/packages/services/src/mind/chamberMindConfig.test.ts +++ b/packages/services/src/mind/chamberMindConfig.test.ts @@ -2,7 +2,15 @@ import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; -import { loadChamberMindConfig, CHAMBER_MIND_CONFIG_FILENAME } from './chamberMindConfig'; +import { + loadChamberMindConfig, + CHAMBER_MIND_CONFIG_FILENAME, + DEFAULT_WORKING_MEMORY_CONSOLIDATION, +} from './chamberMindConfig'; + +const defaultWorkingMemory = () => ({ + consolidation: { ...DEFAULT_WORKING_MEMORY_CONSOLIDATION }, +}); describe('loadChamberMindConfig', () => { let tmpDir: string; @@ -15,8 +23,10 @@ describe('loadChamberMindConfig', () => { fs.rmSync(tmpDir, { recursive: true, force: true }); }); - it('returns an empty config when no .chamber.json file is present', () => { - expect(loadChamberMindConfig(tmpDir)).toEqual({}); + it('returns workingMemory defaults when no .chamber.json file is present', () => { + expect(loadChamberMindConfig(tmpDir)).toEqual({ + workingMemory: defaultWorkingMemory(), + }); }); it('reads excludedTools when present', () => { @@ -26,6 +36,7 @@ describe('loadChamberMindConfig', () => { ); expect(loadChamberMindConfig(tmpDir)).toEqual({ excludedTools: ['shell', 'str_replace'], + workingMemory: defaultWorkingMemory(), }); }); @@ -34,13 +45,17 @@ describe('loadChamberMindConfig', () => { path.join(tmpDir, CHAMBER_MIND_CONFIG_FILENAME), JSON.stringify({ excludedTools: [] }), ); - expect(loadChamberMindConfig(tmpDir)).toEqual({}); + expect(loadChamberMindConfig(tmpDir)).toEqual({ + workingMemory: defaultWorkingMemory(), + }); }); - it('returns empty config for invalid JSON without throwing', () => { + it('returns defaults for invalid JSON without throwing', () => { const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); fs.writeFileSync(path.join(tmpDir, CHAMBER_MIND_CONFIG_FILENAME), '{ not valid json'); - expect(loadChamberMindConfig(tmpDir)).toEqual({}); + expect(loadChamberMindConfig(tmpDir)).toEqual({ + workingMemory: defaultWorkingMemory(), + }); warn.mockRestore(); }); @@ -50,7 +65,9 @@ describe('loadChamberMindConfig', () => { path.join(tmpDir, CHAMBER_MIND_CONFIG_FILENAME), JSON.stringify({ excludedTools: [1, 2, 3] }), ); - expect(loadChamberMindConfig(tmpDir)).toEqual({}); + expect(loadChamberMindConfig(tmpDir)).toEqual({ + workingMemory: defaultWorkingMemory(), + }); warn.mockRestore(); }); @@ -64,6 +81,219 @@ describe('loadChamberMindConfig', () => { ); expect(loadChamberMindConfig(tmpDir)).toEqual({ excludedTools: ['shell'], + workingMemory: defaultWorkingMemory(), + }); + }); +}); + +describe('chamberMindConfig — workingMemory.consolidation', () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-mind-config-wm-')); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + function writeConfig(value: unknown): void { + fs.writeFileSync(path.join(tmpDir, CHAMBER_MIND_CONFIG_FILENAME), JSON.stringify(value)); + } + + describe('defaults', () => { + it('parsing an empty config yields the consolidation defaults', () => { + writeConfig({}); + const cfg = loadChamberMindConfig(tmpDir); + expect(cfg.workingMemory.consolidation.enabled).toBe(false); + expect(cfg.workingMemory.consolidation.cron).toBe('0 3 * * *'); + expect(cfg.workingMemory.consolidation.lastKTurns).toBe(10); + expect(cfg.workingMemory.consolidation.perTurnMaxBytes).toBe(2048); + expect(cfg.workingMemory.consolidation.memoryMaxBytes).toBe(8192); + }); + + it('exposes the defaults as a frozen constant for downstream consumers', () => { + expect(DEFAULT_WORKING_MEMORY_CONSOLIDATION).toEqual({ + enabled: false, + cron: '0 3 * * *', + lastKTurns: 10, + perTurnMaxBytes: 2048, + memoryMaxBytes: 8192, + }); + }); + }); + + describe('override', () => { + it('returns user-provided values verbatim when all five fields are set', () => { + writeConfig({ + workingMemory: { + consolidation: { + enabled: true, + cron: '*/15 * * * *', + lastKTurns: 25, + perTurnMaxBytes: 4096, + memoryMaxBytes: 16384, + }, + }, + }); + const cfg = loadChamberMindConfig(tmpDir); + expect(cfg.workingMemory.consolidation).toEqual({ + enabled: true, + cron: '*/15 * * * *', + lastKTurns: 25, + perTurnMaxBytes: 4096, + memoryMaxBytes: 16384, + }); + }); + + it('honors the opt-in path when only enabled: true is set', () => { + writeConfig({ workingMemory: { consolidation: { enabled: true } } }); + const cfg = loadChamberMindConfig(tmpDir); + expect(cfg.workingMemory.consolidation.enabled).toBe(true); + expect(cfg.workingMemory.consolidation.cron).toBe('0 3 * * *'); + expect(cfg.workingMemory.consolidation.lastKTurns).toBe(10); + expect(cfg.workingMemory.consolidation.perTurnMaxBytes).toBe(2048); + expect(cfg.workingMemory.consolidation.memoryMaxBytes).toBe(8192); + }); + }); + + describe('invalid → warn + ignore', () => { + let warn: ReturnType; + + beforeEach(() => { + warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + }); + + afterEach(() => { + warn.mockRestore(); + }); + + function expectWarnedAbout(field: string): void { + const matched = warn.mock.calls.some((call: unknown[]) => + call.some((arg: unknown) => typeof arg === 'string' && arg.includes(`workingMemory.consolidation.${field}`)), + ); + expect(matched, `expected a warning mentioning workingMemory.consolidation.${field}`).toBe(true); + } + + it('falls back to default false when enabled is a string', () => { + writeConfig({ workingMemory: { consolidation: { enabled: 'yes' } } }); + const cfg = loadChamberMindConfig(tmpDir); + expect(cfg.workingMemory.consolidation.enabled).toBe(false); + expectWarnedAbout('enabled'); + }); + + it('falls back to default cron when cron is a number', () => { + writeConfig({ workingMemory: { consolidation: { cron: 42 } } }); + const cfg = loadChamberMindConfig(tmpDir); + expect(cfg.workingMemory.consolidation.cron).toBe('0 3 * * *'); + expectWarnedAbout('cron'); + }); + + it('falls back to default lastKTurns when value is negative', () => { + writeConfig({ workingMemory: { consolidation: { lastKTurns: -5 } } }); + const cfg = loadChamberMindConfig(tmpDir); + expect(cfg.workingMemory.consolidation.lastKTurns).toBe(10); + expectWarnedAbout('lastKTurns'); + }); + + it('falls back to default memoryMaxBytes when value is zero', () => { + writeConfig({ workingMemory: { consolidation: { memoryMaxBytes: 0 } } }); + const cfg = loadChamberMindConfig(tmpDir); + expect(cfg.workingMemory.consolidation.memoryMaxBytes).toBe(8192); + expectWarnedAbout('memoryMaxBytes'); + }); + + it('falls back to default perTurnMaxBytes when value is a non-integer float', () => { + writeConfig({ workingMemory: { consolidation: { perTurnMaxBytes: 12.5 } } }); + const cfg = loadChamberMindConfig(tmpDir); + expect(cfg.workingMemory.consolidation.perTurnMaxBytes).toBe(2048); + expectWarnedAbout('perTurnMaxBytes'); + }); + + it('keeps valid sibling fields when one field is invalid', () => { + writeConfig({ + workingMemory: { + consolidation: { + enabled: true, + cron: 42, + lastKTurns: 20, + }, + }, + }); + const cfg = loadChamberMindConfig(tmpDir); + expect(cfg.workingMemory.consolidation.enabled).toBe(true); + expect(cfg.workingMemory.consolidation.cron).toBe('0 3 * * *'); + expect(cfg.workingMemory.consolidation.lastKTurns).toBe(20); + expectWarnedAbout('cron'); + }); + + it('falls back to defaults when workingMemory.consolidation is not an object', () => { + writeConfig({ workingMemory: { consolidation: 'nope' } }); + const cfg = loadChamberMindConfig(tmpDir); + expect(cfg.workingMemory.consolidation).toEqual({ ...DEFAULT_WORKING_MEMORY_CONSOLIDATION }); + const matched = warn.mock.calls.some((call: unknown[]) => + call.some((arg: unknown) => typeof arg === 'string' && arg.includes('workingMemory.consolidation')), + ); + expect(matched).toBe(true); + }); + + it('falls back to defaults when workingMemory itself is not an object', () => { + writeConfig({ workingMemory: 'nope' }); + const cfg = loadChamberMindConfig(tmpDir); + expect(cfg.workingMemory.consolidation).toEqual({ ...DEFAULT_WORKING_MEMORY_CONSOLIDATION }); + const matched = warn.mock.calls.some((call: unknown[]) => + call.some((arg: unknown) => typeof arg === 'string' && arg.includes('workingMemory')), + ); + expect(matched).toBe(true); + }); + }); + + describe('backward compat', () => { + it('a pre-Phase-4 config with no workingMemory key returns full consolidation defaults', () => { + writeConfig({ excludedTools: ['shell'] }); + const cfg = loadChamberMindConfig(tmpDir); + expect(cfg.workingMemory.consolidation).toEqual({ ...DEFAULT_WORKING_MEMORY_CONSOLIDATION }); + }); + + it('preserves existing top-level fields byte-identically when adding consolidation defaults', () => { + const baseline = { excludedTools: ['shell', 'str_replace'] }; + writeConfig(baseline); + const cfg = loadChamberMindConfig(tmpDir); + expect(cfg.excludedTools).toEqual(baseline.excludedTools); + expect(cfg.workingMemory.consolidation).toEqual({ ...DEFAULT_WORKING_MEMORY_CONSOLIDATION }); + }); + + it('an empty workingMemory: {} block returns full consolidation defaults', () => { + writeConfig({ workingMemory: {} }); + const cfg = loadChamberMindConfig(tmpDir); + expect(cfg.workingMemory.consolidation).toEqual({ ...DEFAULT_WORKING_MEMORY_CONSOLIDATION }); + }); + }); + + describe('idempotence', () => { + it('re-loading the same config yields identical results', () => { + writeConfig({ + excludedTools: ['shell'], + workingMemory: { + consolidation: { + enabled: true, + lastKTurns: 7, + }, + }, + }); + const first = loadChamberMindConfig(tmpDir); + const second = loadChamberMindConfig(tmpDir); + expect(second).toEqual(first); + }); + + it('mutating the returned consolidation does not affect future loads', () => { + writeConfig({}); + const first = loadChamberMindConfig(tmpDir); + first.workingMemory.consolidation.enabled = true; + first.workingMemory.consolidation.lastKTurns = 99; + const second = loadChamberMindConfig(tmpDir); + expect(second.workingMemory.consolidation.enabled).toBe(false); + expect(second.workingMemory.consolidation.lastKTurns).toBe(10); }); }); }); diff --git a/packages/services/src/mind/chamberMindConfig.ts b/packages/services/src/mind/chamberMindConfig.ts index f4a3cfac..d2139231 100644 --- a/packages/services/src/mind/chamberMindConfig.ts +++ b/packages/services/src/mind/chamberMindConfig.ts @@ -1,11 +1,21 @@ // Reads a mind's `.chamber.json` and returns the chamber-managed slice of // session config that needs per-mind customization. Missing file → empty -// config; invalid JSON / failing schema → warn + empty (consistent with +// config (with `workingMemory.consolidation` defaults applied); invalid +// JSON / failing top-level schema → warn + defaults (consistent with // `mcpConfig.ts`, so a typo in one file never bricks a mind). // // Schema (intentionally small — extend additively): // { -// "excludedTools": ["shell", "str_replace"] +// "excludedTools": ["shell", "str_replace"], +// "workingMemory": { +// "consolidation": { +// "enabled": false, +// "cron": "0 3 * * *", +// "lastKTurns": 10, +// "perTurnMaxBytes": 2048, +// "memoryMaxBytes": 8192 +// } +// } // } // // `excludedTools` maps directly onto `SessionConfig.excludedTools` @@ -13,6 +23,17 @@ // `copilot help tools` for the full list. This is per-agent in chamber // terms because each mind runs its own CopilotClient + session. // +// `workingMemory.consolidation` is the per-mind opt-in for the Dream +// Daemon (issue: dream-daemon spike, Phase 4). Defaults are OFF — the +// daemon never activates unless `enabled: true`. Cron validation is +// deferred to the InternalScheduler (Phase 10); here we only enforce +// `z.string()`. Numeric fields are positive integers; composer (Phase +// 12) enforces hard caps as defense in depth. +// +// Validation strategy mirrors the rest of this loader: per-field +// `safeParse`. An invalid field falls back to the default and emits a +// `log.warn` — never throws, never bricks the mind. +// // Issue #131 checklist 6. import * as fs from 'node:fs'; @@ -28,20 +49,89 @@ const chamberMindConfigSchema = z.object({ export const CHAMBER_MIND_CONFIG_FILENAME = '.chamber.json'; +export interface WorkingMemoryConsolidationConfig { + enabled: boolean; + cron: string; + lastKTurns: number; + perTurnMaxBytes: number; + memoryMaxBytes: number; +} + +export interface WorkingMemoryConfig { + consolidation: WorkingMemoryConsolidationConfig; +} + export interface ChamberMindConfig { excludedTools?: string[]; + workingMemory: WorkingMemoryConfig; +} + +export const DEFAULT_WORKING_MEMORY_CONSOLIDATION: Readonly = Object.freeze({ + enabled: false, + cron: '0 3 * * *', + lastKTurns: 10, + perTurnMaxBytes: 2048, + memoryMaxBytes: 8192, +}); + +const consolidationFieldSchemas: { [K in keyof WorkingMemoryConsolidationConfig]: z.ZodType } = { + enabled: z.boolean(), + cron: z.string(), + lastKTurns: z.number().int().positive(), + perTurnMaxBytes: z.number().int().positive(), + memoryMaxBytes: z.number().int().positive(), +}; + +function defaultWorkingMemory(): WorkingMemoryConfig { + return { consolidation: { ...DEFAULT_WORKING_MEMORY_CONSOLIDATION } }; +} + +function parseWorkingMemory(raw: unknown, filePath: string): WorkingMemoryConfig { + const out = defaultWorkingMemory(); + if (raw === undefined) return out; + if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) { + log.warn(`Invalid workingMemory in ${filePath}; expected object, falling back to defaults.`); + return out; + } + + const wm = raw as Record; + const consolidationRaw = wm.consolidation; + if (consolidationRaw === undefined) return out; + if (consolidationRaw === null || typeof consolidationRaw !== 'object' || Array.isArray(consolidationRaw)) { + log.warn(`Invalid workingMemory.consolidation in ${filePath}; expected object, falling back to defaults.`); + return out; + } + + const consolidation = consolidationRaw as Record; + for (const key of Object.keys(consolidationFieldSchemas) as Array) { + if (!(key in consolidation)) continue; + const schema = consolidationFieldSchemas[key]; + const result = schema.safeParse(consolidation[key]); + if (result.success) { + // Type-safe write via per-key narrowing. + (out.consolidation[key] as WorkingMemoryConsolidationConfig[typeof key]) = result.data; + } else { + log.warn( + `Invalid workingMemory.consolidation.${key} in ${filePath}; using default ${JSON.stringify(DEFAULT_WORKING_MEMORY_CONSOLIDATION[key])}.`, + result.error.issues, + ); + } + } + return out; } export function loadChamberMindConfig(mindPath: string): ChamberMindConfig { const filePath = path.join(mindPath, CHAMBER_MIND_CONFIG_FILENAME); - if (!fs.existsSync(filePath)) return {}; + if (!fs.existsSync(filePath)) { + return { workingMemory: defaultWorkingMemory() }; + } let raw: string; try { raw = fs.readFileSync(filePath, 'utf-8'); } catch (err) { log.warn(`Failed to read ${filePath}; skipping chamber mind config:`, err); - return {}; + return { workingMemory: defaultWorkingMemory() }; } let parsed: unknown; @@ -49,16 +139,18 @@ export function loadChamberMindConfig(mindPath: string): ChamberMindConfig { parsed = JSON.parse(raw); } catch (err) { log.warn(`Invalid JSON in ${filePath}; skipping chamber mind config:`, err); - return {}; + return { workingMemory: defaultWorkingMemory() }; } const result = chamberMindConfigSchema.safeParse(parsed); if (!result.success) { log.warn(`Schema validation failed for ${filePath}; skipping chamber mind config:`, result.error.issues); - return {}; + return { workingMemory: defaultWorkingMemory() }; } - const out: ChamberMindConfig = {}; + const out: ChamberMindConfig = { + workingMemory: parseWorkingMemory((result.data as Record).workingMemory, filePath), + }; if (result.data.excludedTools && result.data.excludedTools.length > 0) { out.excludedTools = [...result.data.excludedTools]; } From fd673f5f653ec1f5eb7e479d3accf5974ace1ea2 Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Tue, 12 May 2026 19:20:43 -0400 Subject: [PATCH 04/23] ADD: mindMemory package with dream daemon background consolidation --- packages/services/src/index.ts | 1 + .../CopilotLLMClient.integration.test.ts | 35 + .../src/mindMemory/CopilotLLMClient.test.ts | 215 ++++++ .../src/mindMemory/CopilotLLMClient.ts | 98 +++ .../src/mindMemory/DailyLogWriter.test.ts | 330 +++++++++ .../services/src/mindMemory/DailyLogWriter.ts | 218 ++++++ .../src/mindMemory/DreamDaemon.test.ts | 689 ++++++++++++++++++ .../services/src/mindMemory/DreamDaemon.ts | 581 +++++++++++++++ .../src/mindMemory/InternalScheduler.test.ts | 333 +++++++++ .../src/mindMemory/InternalScheduler.ts | 197 +++++ packages/services/src/mindMemory/LLMClient.ts | 33 + .../src/mindMemory/MindArchiveStore.test.ts | 179 +++++ .../src/mindMemory/MindArchiveStore.ts | 152 ++++ .../src/mindMemory/MindMemoryService.test.ts | 663 +++++++++++++++++ .../src/mindMemory/MindMemoryService.ts | 278 +++++++ .../src/mindMemory/MindMemoryVault.test.ts | 227 ++++++ .../src/mindMemory/MindMemoryVault.ts | 190 +++++ .../mindMemory/StructuredLogFormat.test.ts | 392 ++++++++++ .../src/mindMemory/StructuredLogFormat.ts | 213 ++++++ .../__fakes__/FakeLLMClient.test.ts | 53 ++ .../src/mindMemory/__fakes__/FakeLLMClient.ts | 68 ++ .../consolidation-priorities.test.ts | 143 ++++ .../mindMemory/consolidation-priorities.ts | 64 ++ .../consolidation-scheduler.test.ts | 289 ++++++++ .../src/mindMemory/consolidation-scheduler.ts | 267 +++++++ .../src/mindMemory/consolidation.test.ts | 290 ++++++++ .../services/src/mindMemory/consolidation.ts | 198 +++++ .../src/mindMemory/date-utils.test.ts | 92 +++ .../services/src/mindMemory/date-utils.ts | 61 ++ .../src/mindMemory/dream-gates.test.ts | 170 +++++ .../services/src/mindMemory/dream-gates.ts | 87 +++ .../src/mindMemory/dream-schema.test.ts | 166 +++++ .../services/src/mindMemory/dream-schema.ts | 87 +++ .../src/mindMemory/dream-state.test.ts | 249 +++++++ .../services/src/mindMemory/dream-state.ts | 249 +++++++ .../src/mindMemory/extraction.test.ts | 287 ++++++++ .../services/src/mindMemory/extraction.ts | 465 ++++++++++++ .../services/src/mindMemory/index.test.ts | 41 ++ packages/services/src/mindMemory/index.ts | 37 + .../src/mindMemory/memory-entries.test.ts | 259 +++++++ .../services/src/mindMemory/memory-entries.ts | 238 ++++++ .../src/mindMemory/memory-limits.test.ts | 110 +++ .../services/src/mindMemory/memory-limits.ts | 80 ++ 43 files changed, 9074 insertions(+) create mode 100644 packages/services/src/mindMemory/CopilotLLMClient.integration.test.ts create mode 100644 packages/services/src/mindMemory/CopilotLLMClient.test.ts create mode 100644 packages/services/src/mindMemory/CopilotLLMClient.ts create mode 100644 packages/services/src/mindMemory/DailyLogWriter.test.ts create mode 100644 packages/services/src/mindMemory/DailyLogWriter.ts create mode 100644 packages/services/src/mindMemory/DreamDaemon.test.ts create mode 100644 packages/services/src/mindMemory/DreamDaemon.ts create mode 100644 packages/services/src/mindMemory/InternalScheduler.test.ts create mode 100644 packages/services/src/mindMemory/InternalScheduler.ts create mode 100644 packages/services/src/mindMemory/LLMClient.ts create mode 100644 packages/services/src/mindMemory/MindArchiveStore.test.ts create mode 100644 packages/services/src/mindMemory/MindArchiveStore.ts create mode 100644 packages/services/src/mindMemory/MindMemoryService.test.ts create mode 100644 packages/services/src/mindMemory/MindMemoryService.ts create mode 100644 packages/services/src/mindMemory/MindMemoryVault.test.ts create mode 100644 packages/services/src/mindMemory/MindMemoryVault.ts create mode 100644 packages/services/src/mindMemory/StructuredLogFormat.test.ts create mode 100644 packages/services/src/mindMemory/StructuredLogFormat.ts create mode 100644 packages/services/src/mindMemory/__fakes__/FakeLLMClient.test.ts create mode 100644 packages/services/src/mindMemory/__fakes__/FakeLLMClient.ts create mode 100644 packages/services/src/mindMemory/consolidation-priorities.test.ts create mode 100644 packages/services/src/mindMemory/consolidation-priorities.ts create mode 100644 packages/services/src/mindMemory/consolidation-scheduler.test.ts create mode 100644 packages/services/src/mindMemory/consolidation-scheduler.ts create mode 100644 packages/services/src/mindMemory/consolidation.test.ts create mode 100644 packages/services/src/mindMemory/consolidation.ts create mode 100644 packages/services/src/mindMemory/date-utils.test.ts create mode 100644 packages/services/src/mindMemory/date-utils.ts create mode 100644 packages/services/src/mindMemory/dream-gates.test.ts create mode 100644 packages/services/src/mindMemory/dream-gates.ts create mode 100644 packages/services/src/mindMemory/dream-schema.test.ts create mode 100644 packages/services/src/mindMemory/dream-schema.ts create mode 100644 packages/services/src/mindMemory/dream-state.test.ts create mode 100644 packages/services/src/mindMemory/dream-state.ts create mode 100644 packages/services/src/mindMemory/extraction.test.ts create mode 100644 packages/services/src/mindMemory/extraction.ts create mode 100644 packages/services/src/mindMemory/index.test.ts create mode 100644 packages/services/src/mindMemory/index.ts create mode 100644 packages/services/src/mindMemory/memory-entries.test.ts create mode 100644 packages/services/src/mindMemory/memory-entries.ts create mode 100644 packages/services/src/mindMemory/memory-limits.test.ts create mode 100644 packages/services/src/mindMemory/memory-limits.ts diff --git a/packages/services/src/index.ts b/packages/services/src/index.ts index 23533984..32d671d7 100644 --- a/packages/services/src/index.ts +++ b/packages/services/src/index.ts @@ -11,6 +11,7 @@ export * from './cron'; export * from './genesis'; export * from './lens'; export * from './mind'; +export * from './mindMemory'; export * from './mindProfile'; export * from './ports'; export * from './sdk'; diff --git a/packages/services/src/mindMemory/CopilotLLMClient.integration.test.ts b/packages/services/src/mindMemory/CopilotLLMClient.integration.test.ts new file mode 100644 index 00000000..1dd84533 --- /dev/null +++ b/packages/services/src/mindMemory/CopilotLLMClient.integration.test.ts @@ -0,0 +1,35 @@ +/** + * Phase 8 — CopilotLLMClient integration test stub. + * + * Hermetic by default. Set `CHAMBER_LIVE_SDK=1` (and ensure the Copilot + * CLI runtime + valid keychain credentials are present) to exercise the + * adapter against a real one-shot session. + * + * Wiring lives in Phase 13 (composition root); this test documents the + * contract the desktop wiring must satisfy: + * - `createOneShotSession` builds a session with NO tools and NO + * permission handler. + * - The session's `send` resolves with the final assistant text. + * - The session is torn down by `close()` so no CLI process leaks. + */ + +import { describe, it } from 'vitest'; + +const liveSdk = process.env.CHAMBER_LIVE_SDK === '1'; + +describe('CopilotLLMClient — live SDK', () => { + it.skipIf(!liveSdk)( + 'returns a string containing "pong" for the canonical smoke prompt', + async () => { + // Intentionally not implemented in Phase 8. The composition root + // (Phase 13) supplies the real `createOneShotSession` factory; this + // stub will be filled in then. Keeping the file in place so the + // contract is visible at review time. + throw new Error( + 'TODO(phase-13): wire CopilotLLMClient to the real Copilot SDK and ' + + 'assert synthesize({prompt: "Reply with the word \'pong\' and nothing else.", timeoutMs: 30_000}) ' + + 'returns a string containing "pong".', + ); + }, + ); +}); diff --git a/packages/services/src/mindMemory/CopilotLLMClient.test.ts b/packages/services/src/mindMemory/CopilotLLMClient.test.ts new file mode 100644 index 00000000..fa166df5 --- /dev/null +++ b/packages/services/src/mindMemory/CopilotLLMClient.test.ts @@ -0,0 +1,215 @@ +/** + * Phase 8 — CopilotLLMClient adapter unit tests. + * + * The adapter is decoupled from the real Copilot SDK: it consumes a + * `createOneShotSession` factory through `deps`. These tests drive a fake + * factory and assert the five locked behaviours from the Phase 8 brief: + * 1. Tools / permission surface left to the factory (adapter never + * installs an approval handler and never asks the factory to register + * tools — it simply forwards mindId / mindPath / signal). + * 2. Timeout-bounded via an internal AbortController; on expiry rejects + * with `Error('LLM synthesis timed out after Xms')` and aborts the + * session signal. + * 3. No conversation history mutation — fresh session per call, never + * reused. + * 4. Error propagation — non-timeout errors propagate verbatim. + * 5. Always closes the session, even on timeout / synthesis failure. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + createCopilotLLMClient, + type CreateOneShotSessionArgs, + type OneShotSession, +} from './CopilotLLMClient'; + +interface FakeSession extends OneShotSession { + closed: boolean; + signal: AbortSignal; +} + +interface FakeFactoryOptions { + send?: (prompt: string, signal: AbortSignal) => Promise; + closeError?: Error; + factoryError?: Error; +} + +interface FakeFactoryHandle { + factory: (args: CreateOneShotSessionArgs) => Promise; + calls: CreateOneShotSessionArgs[]; + sessions: FakeSession[]; +} + +function makeFakeFactory(options: FakeFactoryOptions = {}): FakeFactoryHandle { + const calls: CreateOneShotSessionArgs[] = []; + const sessions: FakeSession[] = []; + const factory = async (args: CreateOneShotSessionArgs): Promise => { + calls.push(args); + if (options.factoryError) throw options.factoryError; + const session: FakeSession = { + closed: false, + signal: args.signal, + async send(prompt: string): Promise { + if (options.send) return options.send(prompt, args.signal); + return `echo:${prompt}`; + }, + async close(): Promise { + this.closed = true; + if (options.closeError) throw options.closeError; + }, + }; + sessions.push(session); + return session; + }; + return { factory, calls, sessions }; +} + +beforeEach(() => { + vi.useFakeTimers(); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe('createCopilotLLMClient', () => { + it('forwards mindId, mindPath, and an AbortSignal to the session factory', async () => { + const handle = makeFakeFactory(); + const client = createCopilotLLMClient({ + mindId: 'mind-alpha', + mindPath: '/tmp/minds/alpha', + deps: { createOneShotSession: handle.factory }, + }); + + const result = await client.synthesize({ prompt: 'hello', timeoutMs: 1_000 }); + + expect(result).toBe('echo:hello'); + expect(handle.calls).toHaveLength(1); + expect(handle.calls[0].mindId).toBe('mind-alpha'); + expect(handle.calls[0].mindPath).toBe('/tmp/minds/alpha'); + expect(handle.calls[0].signal).toBeInstanceOf(AbortSignal); + expect(handle.calls[0].signal.aborted).toBe(false); + }); + + it('does not surface a permission handler or tool registration to the factory', async () => { + // The adapter's args interface only carries mindId, mindPath, signal. + // Asserting the keys explicitly catches accidental drift that would + // open a back-door for tools or approval flow. + const handle = makeFakeFactory(); + const client = createCopilotLLMClient({ + mindId: 'mind-alpha', + mindPath: '/tmp/minds/alpha', + deps: { createOneShotSession: handle.factory }, + }); + + await client.synthesize({ prompt: 'p', timeoutMs: 1_000 }); + + const args = handle.calls[0] as unknown as Record; + expect(Object.keys(args).sort()).toEqual(['mindId', 'mindPath', 'signal']); + expect(args.tools).toBeUndefined(); + expect(args.onPermissionRequest).toBeUndefined(); + }); + + it('creates a fresh session for every synthesize call (no history mutation)', async () => { + const handle = makeFakeFactory(); + const client = createCopilotLLMClient({ + mindId: 'm', mindPath: '/m', deps: { createOneShotSession: handle.factory }, + }); + + await client.synthesize({ prompt: 'one', timeoutMs: 1_000 }); + await client.synthesize({ prompt: 'two', timeoutMs: 1_000 }); + + expect(handle.sessions).toHaveLength(2); + expect(handle.sessions[0]).not.toBe(handle.sessions[1]); + expect(handle.sessions[0].closed).toBe(true); + expect(handle.sessions[1].closed).toBe(true); + }); + + it('rejects with a timeout error and aborts the signal when timeoutMs elapses', async () => { + const handle = makeFakeFactory({ + send: (_prompt, signal) => + new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(new Error('aborted')), { once: true }); + }), + }); + const client = createCopilotLLMClient({ + mindId: 'm', mindPath: '/m', deps: { createOneShotSession: handle.factory }, + }); + + const promise = client.synthesize({ prompt: 'p', timeoutMs: 250 }); + // Attach catch handler before advancing timers to avoid an + // unhandled-rejection race when the timer fires synchronously. + const settled = promise.catch((e: unknown) => e as Error); + + await vi.advanceTimersByTimeAsync(250); + const err = await settled; + + expect(err).toBeInstanceOf(Error); + expect((err as Error).message).toBe('LLM synthesis timed out after 250ms'); + expect(handle.sessions[0].signal.aborted).toBe(true); + expect(handle.sessions[0].closed).toBe(true); + }); + + it('propagates non-timeout SDK errors verbatim', async () => { + const handle = makeFakeFactory({ + send: async () => { throw new Error('SDK exploded'); }, + }); + const client = createCopilotLLMClient({ + mindId: 'm', mindPath: '/m', deps: { createOneShotSession: handle.factory }, + }); + + await expect(client.synthesize({ prompt: 'p', timeoutMs: 1_000 })) + .rejects.toThrow('SDK exploded'); + + expect(handle.sessions[0].closed).toBe(true); + }); + + it('propagates errors from the factory itself and does not try to close a missing session', async () => { + const handle = makeFakeFactory({ + factoryError: new Error('cannot start CLI'), + }); + const client = createCopilotLLMClient({ + mindId: 'm', mindPath: '/m', deps: { createOneShotSession: handle.factory }, + }); + + await expect(client.synthesize({ prompt: 'p', timeoutMs: 1_000 })) + .rejects.toThrow('cannot start CLI'); + expect(handle.sessions).toHaveLength(0); + }); + + it('always closes the session in `finally`, even when send throws', async () => { + const handle = makeFakeFactory({ + send: async () => { throw new Error('boom'); }, + }); + const client = createCopilotLLMClient({ + mindId: 'm', mindPath: '/m', deps: { createOneShotSession: handle.factory }, + }); + + await expect(client.synthesize({ prompt: 'p', timeoutMs: 1_000 })).rejects.toThrow('boom'); + expect(handle.sessions[0].closed).toBe(true); + }); + + it('swallows close() failures so they do not mask synthesis success', async () => { + const handle = makeFakeFactory({ closeError: new Error('close bombed') }); + const client = createCopilotLLMClient({ + mindId: 'm', mindPath: '/m', deps: { createOneShotSession: handle.factory }, + }); + + await expect(client.synthesize({ prompt: 'p', timeoutMs: 1_000 })).resolves.toBe('echo:p'); + expect(handle.sessions[0].closed).toBe(true); + }); + + it('clears the timeout timer on the success path', async () => { + const handle = makeFakeFactory(); + const client = createCopilotLLMClient({ + mindId: 'm', mindPath: '/m', deps: { createOneShotSession: handle.factory }, + }); + + await client.synthesize({ prompt: 'p', timeoutMs: 5_000 }); + // If the timer were still pending, advancing the clock would flip + // the aborted bit on the session signal we kept a reference to. + await vi.advanceTimersByTimeAsync(10_000); + expect(handle.sessions[0].signal.aborted).toBe(false); + }); +}); diff --git a/packages/services/src/mindMemory/CopilotLLMClient.ts b/packages/services/src/mindMemory/CopilotLLMClient.ts new file mode 100644 index 00000000..25e9106a --- /dev/null +++ b/packages/services/src/mindMemory/CopilotLLMClient.ts @@ -0,0 +1,98 @@ +/** + * CopilotLLMClient — `LLMClient` adapter that calls the mind's own Copilot + * SDK as a one-shot, side-effect-free language model for the Dream Daemon. + * + * The adapter intentionally does NOT own SDK lifecycle. It receives a + * `createOneShotSession` factory through `deps`; that factory (supplied + * by the composition root in Phase 13) is responsible for constructing a + * Copilot session with: + * - NO tools registered (empty tool surface). + * - NO permission handler / no approval flow. + * - The provided `AbortSignal` threaded into the SDK so the adapter's + * timeout actually cancels the underlying CLI process. + * + * The adapter contract enforces the rest: + * - One fresh session per `synthesize` call (no history mutation). + * - Internal `AbortController` with a `setTimeout(timeoutMs)` deadline. + * - The session is closed in `finally`, even on timeout / synthesis + * failure, so a stuck CLI cannot leak. + * - On timeout, rejects with `Error('LLM synthesis timed out after Xms')` + * so callers can distinguish it from SDK / network failures. + * + * v1 deliberately does NOT accept an external `AbortSignal` — there is a + * single internal controller. Callers wanting cancellation should compose + * a shorter `timeoutMs`. + */ + +import type { LLMClient, SynthesizeRequest } from './LLMClient'; + +export interface OneShotSession { + /** Resolves with the final assistant text for the prompt. */ + send(prompt: string): Promise; + /** Best-effort teardown; called in `finally` and must not throw. */ + close(): Promise; +} + +export interface CreateOneShotSessionArgs { + readonly mindId: string; + readonly mindPath: string; + readonly signal: AbortSignal; +} + +export interface CopilotLLMClientDeps { + /** + * Factory that constructs a one-shot Copilot session for the target + * mind. Implementations MUST disable tools and the permission handler + * and MUST honor `signal` (abort the underlying CLI on cancel). + */ + readonly createOneShotSession: (args: CreateOneShotSessionArgs) => Promise; +} + +export interface CopilotLLMClientOptions { + readonly mindId: string; + readonly mindPath: string; + readonly deps: CopilotLLMClientDeps; +} + +export function createCopilotLLMClient(opts: CopilotLLMClientOptions): LLMClient { + const { mindId, mindPath, deps } = opts; + + return { + async synthesize(req: SynthesizeRequest): Promise { + const controller = new AbortController(); + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + controller.abort(); + }, req.timeoutMs); + + let session: OneShotSession | null = null; + try { + session = await deps.createOneShotSession({ + mindId, + mindPath, + signal: controller.signal, + }); + const result = await session.send(req.prompt); + if (timedOut) { + throw new Error(`LLM synthesis timed out after ${req.timeoutMs}ms`); + } + return result; + } catch (err) { + if (timedOut) { + throw new Error(`LLM synthesis timed out after ${req.timeoutMs}ms`, { cause: err }); + } + throw err; + } finally { + clearTimeout(timer); + if (session) { + try { + await session.close(); + } catch { + // best-effort teardown + } + } + } + }, + }; +} diff --git a/packages/services/src/mindMemory/DailyLogWriter.test.ts b/packages/services/src/mindMemory/DailyLogWriter.test.ts new file mode 100644 index 00000000..6f3bbd43 --- /dev/null +++ b/packages/services/src/mindMemory/DailyLogWriter.test.ts @@ -0,0 +1,330 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs'; +import fsp from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { createDailyLogWriter } from './DailyLogWriter'; +import { STRUCTURED_LOG_SENTINEL, parseLog, type CompletedTurn } from './StructuredLogFormat'; + +let mindRoot: string; +let logPath: string; +let legacyPath: string; + +beforeEach(() => { + mindRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-dlw-')); + logPath = path.join(mindRoot, '.working-memory', 'log.md'); + legacyPath = path.join(mindRoot, '.working-memory', 'log.legacy.md'); +}); + +afterEach(() => { + fs.rmSync(mindRoot, { recursive: true, force: true }); +}); + +function makeTurn(i: number, status: 'completed' | 'aborted' | 'error' = 'completed'): CompletedTurn { + const ts = `2026-05-12T17:${String(20 + i).padStart(2, '0')}:00Z`; + return { + turnId: `turn-${i}`, + sessionId: `sess-${i}`, + model: 'claude-opus-4.7', + status, + startedAt: ts, + endedAt: ts, + prompt: `prompt body ${i}`, + finalAssistantMessage: `assistant body ${i}`, + }; +} + +function makeWriter(extras: { logger?: { info: (msg: string) => void; warn?: (msg: string, ...args: unknown[]) => void }; rename?: (from: string, to: string) => Promise } = {}) { + return createDailyLogWriter({ + mindId: 'mind-x', + mindPath: mindRoot, + deps: { + logger: extras.logger, + rename: extras.rename, + }, + }); +} + +describe('DailyLogWriter — round-trip', () => { + it('writes three turns that parse back via StructuredLogFormat.parseLog', async () => { + const writer = makeWriter(); + const t1 = makeTurn(1); + const t2 = makeTurn(2, 'aborted'); + const t3 = makeTurn(3, 'error'); + + await writer.write(t1); + await writer.write(t2); + await writer.write(t3); + + const content = fs.readFileSync(logPath, 'utf-8'); + const parsed = parseLog(content); + + expect(parsed.sentinel).toBe(true); + expect(parsed.malformed).toBe(0); + expect(parsed.turns).toHaveLength(3); + expect(parsed.turns.map((t) => t.turnId)).toEqual(['turn-1', 'turn-2', 'turn-3']); + expect(parsed.turns[0].prompt).toBe('prompt body 1'); + expect(parsed.turns[0].assistant).toBe('assistant body 1'); + expect(parsed.turns[1].status).toBe('aborted'); + expect(parsed.turns[2].status).toBe('error'); + }); +}); + +describe('DailyLogWriter — first-call migration (rotation)', () => { + it('rotates unstructured log.md to log.legacy.md and seeds a fresh structured log', async () => { + fs.mkdirSync(path.dirname(logPath), { recursive: true }); + const legacyContent = '# notes\n\nrandom freeform content\n'; + fs.writeFileSync(logPath, legacyContent); + + const info = vi.fn(); + const writer = makeWriter({ logger: { info } }); + + await writer.write(makeTurn(1)); + + expect(fs.readFileSync(legacyPath, 'utf-8')).toBe(legacyContent); + const fresh = fs.readFileSync(logPath, 'utf-8'); + expect(fresh.startsWith(STRUCTURED_LOG_SENTINEL)).toBe(true); + const parsed = parseLog(fresh); + expect(parsed.sentinel).toBe(true); + expect(parsed.turns).toHaveLength(1); + expect(info).toHaveBeenCalledWith( + 'Rotated unstructured log.md to log.legacy.md for mind mind-x', + ); + }); + + it('idempotent: second write does not re-rotate', async () => { + fs.mkdirSync(path.dirname(logPath), { recursive: true }); + fs.writeFileSync(logPath, 'unstructured original\n'); + + const info = vi.fn(); + const writer = makeWriter({ logger: { info } }); + + await writer.write(makeTurn(1)); + const legacyAfterFirst = fs.readFileSync(legacyPath, 'utf-8'); + + await writer.write(makeTurn(2)); + + expect(fs.readFileSync(legacyPath, 'utf-8')).toBe(legacyAfterFirst); + expect(info).toHaveBeenCalledTimes(1); + + const parsed = parseLog(fs.readFileSync(logPath, 'utf-8')); + expect(parsed.sentinel).toBe(true); + expect(parsed.turns).toHaveLength(2); + }); + + it('collision rule: when log.legacy.md already exists, rotates to log.legacy..md', async () => { + fs.mkdirSync(path.dirname(logPath), { recursive: true }); + const priorLegacy = '# previous legacy\n'; + const todayBad = '# today is bad\n'; + fs.writeFileSync(legacyPath, priorLegacy); + fs.writeFileSync(logPath, todayBad); + + const info = vi.fn(); + const writer = makeWriter({ logger: { info } }); + + await writer.write(makeTurn(1)); + + // Original legacy untouched + expect(fs.readFileSync(legacyPath, 'utf-8')).toBe(priorLegacy); + + // A timestamped legacy file containing today's bad content was created + const all = fs.readdirSync(path.dirname(logPath)); + const stamped = all.filter( + (n) => /^log\.legacy\.\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}Z\.md$/.test(n), + ); + expect(stamped).toHaveLength(1); + expect(fs.readFileSync(path.join(path.dirname(logPath), stamped[0]), 'utf-8')).toBe(todayBad); + + // Fresh log.md is structured + const parsed = parseLog(fs.readFileSync(logPath, 'utf-8')); + expect(parsed.sentinel).toBe(true); + expect(parsed.turns).toHaveLength(1); + + expect(info).toHaveBeenCalledWith( + `Rotated unstructured log.md to ${stamped[0]} for mind mind-x`, + ); + }); + + it('empty log.md is treated as already-structured: no rotation', async () => { + fs.mkdirSync(path.dirname(logPath), { recursive: true }); + fs.writeFileSync(logPath, ''); + + const info = vi.fn(); + const writer = makeWriter({ logger: { info } }); + + await writer.write(makeTurn(1)); + + expect(fs.existsSync(legacyPath)).toBe(false); + expect(info).not.toHaveBeenCalled(); + + const parsed = parseLog(fs.readFileSync(logPath, 'utf-8')); + expect(parsed.sentinel).toBe(true); + expect(parsed.turns).toHaveLength(1); + }); + + it('structured log.md (sentinel present) is left in place', async () => { + fs.mkdirSync(path.dirname(logPath), { recursive: true }); + const seed = `${STRUCTURED_LOG_SENTINEL}\n\n`; + fs.writeFileSync(logPath, seed); + + const info = vi.fn(); + const writer = makeWriter({ logger: { info } }); + + await writer.write(makeTurn(1)); + + expect(fs.existsSync(legacyPath)).toBe(false); + expect(info).not.toHaveBeenCalled(); + + const content = fs.readFileSync(logPath, 'utf-8'); + expect(content.startsWith(seed)).toBe(true); + const parsed = parseLog(content); + expect(parsed.turns).toHaveLength(1); + }); + + it('creates the directory and log.md when none exist (no rotation)', async () => { + const info = vi.fn(); + const writer = makeWriter({ logger: { info } }); + + await writer.write(makeTurn(1)); + + expect(fs.existsSync(legacyPath)).toBe(false); + expect(info).not.toHaveBeenCalled(); + const parsed = parseLog(fs.readFileSync(logPath, 'utf-8')); + expect(parsed.turns).toHaveLength(1); + }); +}); + +describe('DailyLogWriter — concurrency', () => { + it('serializes concurrent appends without interleaving', async () => { + const writer = makeWriter(); + + const turns = Array.from({ length: 10 }, (_, i) => makeTurn(i + 1)); + await Promise.all(turns.map((t) => writer.write(t))); + + const content = fs.readFileSync(logPath, 'utf-8'); + + // Sentinel appears exactly once. + const sentinelCount = content.split(STRUCTURED_LOG_SENTINEL).length - 1; + expect(sentinelCount).toBe(1); + + const parsed = parseLog(content); + expect(parsed.sentinel).toBe(true); + expect(parsed.malformed).toBe(0); + expect(parsed.turns).toHaveLength(10); + + const ids = new Set(parsed.turns.map((t) => t.turnId)); + expect(ids.size).toBe(10); + for (let i = 1; i <= 10; i++) { + expect(ids.has(`turn-${i}`)).toBe(true); + } + + // Spot-check no body interleaving: each parsed prompt/assistant matches its turn id. + for (const t of parsed.turns) { + const i = t.turnId.replace('turn-', ''); + expect(t.prompt).toBe(`prompt body ${i}`); + expect(t.assistant).toBe(`assistant body ${i}`); + } + }); +}); + +describe('DailyLogWriter — onTurnRecorded hook', () => { + it('invokes onTurnRecorded once per successful write with the same CompletedTurn payload', async () => { + const onTurnRecorded = vi.fn(); + const writer = createDailyLogWriter({ + mindId: 'mind-x', + mindPath: mindRoot, + deps: { onTurnRecorded }, + }); + + const t1 = makeTurn(1); + const t2 = makeTurn(2); + await writer.write(t1); + await writer.write(t2); + + expect(onTurnRecorded).toHaveBeenCalledTimes(2); + expect(onTurnRecorded).toHaveBeenNthCalledWith(1, t1); + expect(onTurnRecorded).toHaveBeenNthCalledWith(2, t2); + }); + + it('does NOT invoke onTurnRecorded when the underlying write fails', async () => { + fs.mkdirSync(path.dirname(logPath), { recursive: true }); + fs.writeFileSync(logPath, '# unstructured original\n'); + + const onTurnRecorded = vi.fn(); + const failingRename = vi.fn(async () => { + throw new Error('synthetic rename failure'); + }); + const writer = createDailyLogWriter({ + mindId: 'mind-x', + mindPath: mindRoot, + deps: { onTurnRecorded, rename: failingRename }, + }); + + await expect(writer.write(makeTurn(1))).rejects.toThrow(/synthetic rename failure/); + expect(onTurnRecorded).not.toHaveBeenCalled(); + }); + + it('a throwing onTurnRecorded does not roll back the on-disk write but propagates the error', async () => { + const onTurnRecorded = vi.fn(() => { + throw new Error('observer threw'); + }); + const writer = createDailyLogWriter({ + mindId: 'mind-x', + mindPath: mindRoot, + deps: { onTurnRecorded }, + }); + + await expect(writer.write(makeTurn(1))).rejects.toThrow(/observer threw/); + + // The structured log was still written before the hook was called. + const parsed = parseLog(fs.readFileSync(logPath, 'utf-8')); + expect(parsed.sentinel).toBe(true); + expect(parsed.turns).toHaveLength(1); + }); +}); + +describe('DailyLogWriter — error path safety', () => { + it('rotation rename failure leaves log.md byte-equal and rejects the write', async () => { + fs.mkdirSync(path.dirname(logPath), { recursive: true }); + const original = '# unstructured original content\nline two\n'; + fs.writeFileSync(logPath, original); + + const failingRename = vi.fn(async () => { + throw new Error('synthetic rename failure'); + }); + const info = vi.fn(); + const writer = makeWriter({ logger: { info }, rename: failingRename }); + + await expect(writer.write(makeTurn(1))).rejects.toThrow(/synthetic rename failure/); + + expect(fs.readFileSync(logPath, 'utf-8')).toBe(original); + expect(fs.existsSync(legacyPath)).toBe(false); + expect(info).not.toHaveBeenCalled(); + }); + + it('after a failed rotation, a subsequent successful write rotates and recovers', async () => { + fs.mkdirSync(path.dirname(logPath), { recursive: true }); + const original = '# unstructured original\n'; + fs.writeFileSync(logPath, original); + + let calls = 0; + const flakyRename = vi.fn(async (from: string, to: string) => { + calls++; + if (calls === 1) throw new Error('first attempt fails'); + await fsp.rename(from, to); + }); + const writer = makeWriter({ rename: flakyRename }); + + await expect(writer.write(makeTurn(1))).rejects.toThrow(/first attempt fails/); + expect(fs.readFileSync(logPath, 'utf-8')).toBe(original); + + await writer.write(makeTurn(2)); + + expect(fs.readFileSync(legacyPath, 'utf-8')).toBe(original); + const parsed = parseLog(fs.readFileSync(logPath, 'utf-8')); + expect(parsed.sentinel).toBe(true); + expect(parsed.turns).toHaveLength(1); + expect(parsed.turns[0].turnId).toBe('turn-2'); + }); +}); diff --git a/packages/services/src/mindMemory/DailyLogWriter.ts b/packages/services/src/mindMemory/DailyLogWriter.ts new file mode 100644 index 00000000..5fb496b5 --- /dev/null +++ b/packages/services/src/mindMemory/DailyLogWriter.ts @@ -0,0 +1,218 @@ +/** + * DailyLogWriter — appends completed-turn frames to + * `/.working-memory/log.md` in the chamber-structured-log/v1 format + * and rotates pre-existing unstructured logs out of the way on first touch. + * + * Phase 5 scope (locked by plan): + * - Append-only writer for structured turn frames. + * - First-call migration: any existing unstructured log.md is moved aside + * to `log.legacy.md` (or `log.legacy..md` on collision) before + * the writer seeds a fresh sentinel-prefixed log. + * - Per-instance mutex serializes concurrent appends so `Promise.all` + * fans-in produce non-interleaved frames. + * - Rotation uses `fs.rename` (atomic on a single filesystem) so a failed + * rotation leaves the original `log.md` byte-equal to its prior state. + * - Steady-state appends use `fsp.open(..., 'a')` + `handle.sync()`. Each + * frame is bounded by `perTurnMaxBytes` (well under PIPE_BUF), so the + * POSIX append is atomic in practice. + * + * Out of scope: TurnRecorder wiring (Phase 6), pruning after consolidation + * (Phase 9), composer integration (Phase 12). + */ + +import fs from 'node:fs'; +import fsp from 'node:fs/promises'; +import path from 'node:path'; + +import { Logger } from '../logger'; +import { + STRUCTURED_LOG_SENTINEL, + detectSentinel, + serializeTurn, + type CompletedTurn, +} from './StructuredLogFormat'; + +const WORKING_MEMORY_DIRNAME = '.working-memory'; +const LOG_FILENAME = 'log.md'; +const LEGACY_FILENAME = 'log.legacy.md'; + +export interface DailyLogWriterLogger { + info(message: string): void; +} + +export interface DailyLogWriterDeps { + /** Override `fs.rename` — used by tests to simulate rotation failures. */ + rename?: (from: string, to: string) => Promise; + /** Override the default `Logger.create('DailyLogWriter')`. */ + logger?: DailyLogWriterLogger; + /** + * Called once per successful structured turn write, after the on-disk + * write has fsync'd. Phase 11 wires this to + * `dream-state.incrementTurnCount` so the activity gate in dream-gates + * advances on real writes (not on SDK session start). + * + * If the hook throws (sync or async), the on-disk write is NOT rolled + * back — the structured frame remains in `log.md` — but the rejection + * propagates to the caller so the wiring layer can react. + */ + onTurnRecorded?: (turn: CompletedTurn) => void | Promise; +} + +export interface DailyLogWriterOptions { + readonly mindId: string; + readonly mindPath: string; + readonly deps?: DailyLogWriterDeps; +} + +export interface DailyLogWriter { + write(turn: CompletedTurn): Promise; +} + +export function createDailyLogWriter(opts: DailyLogWriterOptions): DailyLogWriter { + const { mindId, mindPath } = opts; + const log: DailyLogWriterLogger = opts.deps?.logger ?? Logger.create('DailyLogWriter'); + const rename = opts.deps?.rename ?? fsp.rename; + const onTurnRecorded = opts.deps?.onTurnRecorded; + + const workingMemoryDir = path.resolve(mindPath, WORKING_MEMORY_DIRNAME); + const logPath = path.join(workingMemoryDir, LOG_FILENAME); + const legacyPath = path.join(workingMemoryDir, LEGACY_FILENAME); + + // Per-instance mutex chain. Every write awaits the prior link before issuing + // its own read-modify-write cycle, eliminating intra-process interleaving. + let chain: Promise = Promise.resolve(); + + async function readOrNull(absPath: string): Promise { + try { + return await fsp.readFile(absPath, 'utf-8'); + } catch (err) { + if (isErrnoCode(err, 'ENOENT')) return null; + throw err; + } + } + + async function existsPath(absPath: string): Promise { + try { + await fsp.access(absPath, fs.constants.F_OK); + return true; + } catch (err) { + if (isErrnoCode(err, 'ENOENT')) return false; + throw err; + } + } + + async function rotate(currentContent: string): Promise { + let target = legacyPath; + let targetName = LEGACY_FILENAME; + if (await existsPath(legacyPath)) { + targetName = `log.legacy.${isoStamp()}.md`; + target = path.join(workingMemoryDir, targetName); + } + + // `rename` is atomic on a single filesystem: either log.md is at its old + // path (failure) or at the target path (success). On failure we surface + // the error to the caller; the file content is unchanged. We deliberately + // do NOT fall back to copy+unlink, which would risk a half-rotated state. + await rename(logPath, target); + + log.info(`Rotated unstructured log.md to ${targetName} for mind ${mindId}`); + + // Sanity check: rotation succeeded, so `currentContent` is now under + // `target`. We don't re-read here — the migration is complete and the + // seed step below creates a fresh log.md. + void currentContent; + } + + async function seedFreshLog(turn: CompletedTurn): Promise { + const content = `${STRUCTURED_LOG_SENTINEL}\n\n${serializeTurn(turn)}`; + // Atomic write so a partial seed never lands on disk. + const tmp = `${logPath}.tmp.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2)}`; + const handle = await fsp.open(tmp, 'wx'); + try { + await handle.writeFile(content); + await handle.sync(); + } finally { + await handle.close(); + } + try { + await fsp.rename(tmp, logPath); + } catch (err) { + try { + await fsp.unlink(tmp); + } catch { + // best-effort cleanup + } + throw err; + } + } + + async function appendFrame(turn: CompletedTurn): Promise { + const handle = await fsp.open(logPath, 'a'); + try { + await handle.write(serializeTurn(turn)); + await handle.sync(); + } finally { + await handle.close(); + } + } + + async function doWrite(turn: CompletedTurn): Promise { + await fsp.mkdir(workingMemoryDir, { recursive: true }); + + const existing = await readOrNull(logPath); + + // No file → seed a fresh structured log; no rotation event. + if (existing === null) { + await seedFreshLog(turn); + return; + } + + // Empty file is treated as already-structured (no rotation needed): + // we seed sentinel + frame in place. The atomic write replaces the + // empty file in one rename. + if (existing.length === 0) { + await seedFreshLog(turn); + return; + } + + if (detectSentinel(existing)) { + await appendFrame(turn); + return; + } + + // Unstructured content present — rotate before seeding. If rotation + // fails the original log.md is left intact and the error propagates. + await rotate(existing); + await seedFreshLog(turn); + } + + function write(turn: CompletedTurn): Promise { + const next = chain.then(async () => { + await doWrite(turn); + if (onTurnRecorded) { + await onTurnRecorded(turn); + } + }); + // Swallow rejections on the chain itself so a failed write does not + // poison the queue for subsequent callers; the rejection still reaches + // the caller via `next`. + chain = next.catch(() => undefined); + return next; + } + + return { write }; +} + +function isoStamp(): string { + // 2026-05-12T17:21:45.123Z → 2026-05-12T17-21-45Z + return new Date().toISOString().replace(/\.\d+Z$/, 'Z').replace(/:/g, '-'); +} + +function isErrnoCode(err: unknown, code: string): boolean { + return ( + typeof err === 'object' && + err !== null && + 'code' in err && + (err as NodeJS.ErrnoException).code === code + ); +} diff --git a/packages/services/src/mindMemory/DreamDaemon.test.ts b/packages/services/src/mindMemory/DreamDaemon.test.ts new file mode 100644 index 00000000..72f08488 --- /dev/null +++ b/packages/services/src/mindMemory/DreamDaemon.test.ts @@ -0,0 +1,689 @@ +/** + * Tests for DreamDaemon — Phase 9 orchestrator. + * + * Covers the full cycle (gates → lock → snapshot → extract via LLM → + * consolidate → write memory.md → prune log.md preserving tail → archive → + * tiered rollups → record run → release lock) plus the negative paths + * (gate skip, lock skip, force bypass, mid-run append survival, LLM + * failure, idempotent close). + * + * Fakes / fixtures: + * - in-memory better-sqlite3 (`:memory:`) + `migrate(db)` from Phase 7 + * - real tmp-dir vault + archive (Phase 3 modules) + * - `createFakeLLMClient` from Phase 8 `__fakes__` + * - injected clock for deterministic tiered-rollup gates + */ + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import Database from 'better-sqlite3'; +import fs from 'node:fs'; +import fsp from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { migrate } from './dream-schema'; +import { + acquireLock, + getLock, + incrementTurnCount, + listRuns, + markPhaseComplete, + readState, + setLastConsolidatedTurnId, +} from './dream-state'; +import { createMindMemoryVault } from './MindMemoryVault'; +import { createMindArchiveStore } from './MindArchiveStore'; +import { STRUCTURED_LOG_SENTINEL, serializeTurn } from './StructuredLogFormat'; +import type { CompletedTurn } from '@chamber/shared/turn-observer'; +import { createFakeLLMClient, type FakeLLMClient } from './__fakes__/FakeLLMClient'; +import { + __resetMindMutexForTesting, +} from './consolidation-scheduler'; +import { + createDreamDaemon, + type DreamDaemon, + type DreamDaemonConfig, + type DreamRunResult, +} from './DreamDaemon'; + +const MS_PER_DAY = 24 * 60 * 60 * 1000; +const MS_PER_WEEK = 7 * MS_PER_DAY; + +const MIND_ID = 'test-mind'; +const FROZEN_NOW_MS = Date.parse('2026-05-12T15:00:00Z'); + +let mindRoot: string; +let db: Database.Database; +let llmClient: FakeLLMClient; +let now: number; + +const baseConfig: DreamDaemonConfig = { + memoryMaxBytes: 8192, + llmTimeoutMs: 60_000, + lockTtlMs: 300_000, + minTurnsBetweenRuns: 1, + minDailyIntervalMs: 0, // disabled by default; tests opt in by overriding + weeklyRollupAfterDailies: 7, + monthlyRollupAfterWeeklies: 4, + weeklyMinIntervalMs: MS_PER_WEEK, + monthlyMinIntervalMs: 30 * MS_PER_DAY, +}; + +beforeEach(() => { + mindRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-daemon-')); + db = new Database(':memory:'); + migrate(db); + llmClient = createFakeLLMClient({ + defaultResponse: + '## 12:00:00\n**[user-prompt]** I prefer kebab-case file names.\n', + }); + now = FROZEN_NOW_MS; + __resetMindMutexForTesting(); +}); + +afterEach(() => { + db.close(); + fs.rmSync(mindRoot, { recursive: true, force: true }); + __resetMindMutexForTesting(); +}); + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function clock(): Date { + return new Date(now); +} + +function makeTurn(overrides: Partial = {}): CompletedTurn { + const turnId = overrides.turnId ?? `turn-${Math.random().toString(36).slice(2, 10)}`; + const ts = overrides.endedAt ?? new Date(now).toISOString(); + return { + turnId, + sessionId: overrides.sessionId ?? 'session-1', + model: overrides.model ?? 'gpt-test', + status: overrides.status ?? 'completed', + startedAt: overrides.startedAt ?? ts, + endedAt: ts, + prompt: overrides.prompt ?? `prompt for ${turnId}`, + finalAssistantMessage: + overrides.finalAssistantMessage ?? `assistant for ${turnId}`, + }; +} + +async function seedLog(turns: CompletedTurn[]): Promise { + await fsp.mkdir(path.join(mindRoot, '.working-memory'), { recursive: true }); + const body = `${STRUCTURED_LOG_SENTINEL}\n\n${turns.map(serializeTurn).join('')}`; + await fsp.writeFile(path.join(mindRoot, '.working-memory', 'log.md'), body); +} + +async function readLog(): Promise { + return fsp.readFile(path.join(mindRoot, '.working-memory', 'log.md'), 'utf-8'); +} + +function makeDaemon(configOverrides: Partial = {}): DreamDaemon { + const vault = createMindMemoryVault(mindRoot); + const archiveStore = createMindArchiveStore(mindRoot); + return createDreamDaemon({ + mindId: MIND_ID, + mindPath: mindRoot, + llmClient, + vault, + archiveStore, + db, + config: { ...baseConfig, ...configOverrides }, + clock, + }); +} + +function isoOf(ms: number): string { + return new Date(ms).toISOString(); +} + +async function waitForRelease( + getRelease: () => (() => void) | null, + timeoutMs = 2000, +): Promise<() => void> { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const r = getRelease(); + if (r !== null) return r; + await new Promise((resolve) => setTimeout(resolve, 5)); + } + throw new Error('timed out waiting for LLM synthesize to be invoked'); +} + +// --------------------------------------------------------------------------- +// Public surface +// --------------------------------------------------------------------------- + +describe('DreamDaemon — public surface', () => { + it('exposes run, forceRun, getStatus, notifyTurnCompleted, close', () => { + const daemon = makeDaemon(); + expect(typeof daemon.run).toBe('function'); + expect(typeof daemon.forceRun).toBe('function'); + expect(typeof daemon.getStatus).toBe('function'); + expect(typeof daemon.notifyTurnCompleted).toBe('function'); + expect(typeof daemon.close).toBe('function'); + }); + + it('initial getStatus reports phase=idle, locked=false, lastRunAt=null', () => { + const daemon = makeDaemon(); + const s = daemon.getStatus(); + expect(s.phase).toBe('idle'); + expect(s.locked).toBe(false); + expect(s.lastRunAt).toBeNull(); + expect(s.lastResult).toBeNull(); + }); +}); + +// --------------------------------------------------------------------------- +// Happy path +// --------------------------------------------------------------------------- + +describe('DreamDaemon — happy path cycle', () => { + it('runs end-to-end: writes memory.md, prunes log.md, archives, records run, advances cutoff', async () => { + const t1 = makeTurn({ turnId: 'turn-A' }); + const t2 = makeTurn({ turnId: 'turn-B' }); + await seedLog([t1, t2]); + incrementTurnCount(db, 2); + + const daemon = makeDaemon(); + const result = await daemon.run(); + + expect(result.status).toBe('success'); + if (result.status !== 'success') return; + expect(result.fromTurnId).toBeNull(); + expect(result.toTurnId).toBe('turn-B'); + expect(result.consolidatedCount).toBeGreaterThan(0); + expect(result.archivedCount).toBe(2); + + // memory.md exists and is non-empty + const memoryMd = await fsp.readFile( + path.join(mindRoot, '.working-memory', 'memory.md'), + 'utf-8', + ); + expect(memoryMd.length).toBeGreaterThan(0); + expect(memoryMd).toMatch(/kebab-case/i); + + // log.md retains sentinel; both consolidated turns are pruned + const log = await readLog(); + expect(log.startsWith(STRUCTURED_LOG_SENTINEL)).toBe(true); + expect(log).not.toContain('turn-A'); + expect(log).not.toContain('turn-B'); + + // archive/consolidated/ has 2 files + const archiveDir = path.join(mindRoot, '.working-memory', 'archive', 'consolidated'); + const files = await fsp.readdir(archiveDir); + expect(files).toHaveLength(2); + + // state advanced + const state = readState(db); + expect(state.lastConsolidatedTurnId).toBe('turn-B'); + expect(state.turnsSinceLastRun).toBe(0); + expect(state.lastDailyAt).not.toBeNull(); + + // dream_runs has a success row + const runs = listRuns(db, { phase: 'daily' }); + expect(runs).toHaveLength(1); + expect(runs[0]!.status).toBe('success'); + expect(runs[0]!.toTurnId).toBe('turn-B'); + + // lock released + expect(getLock(db, 'daily')).toBeNull(); + const status = daemon.getStatus(); + expect(status.locked).toBe(false); + expect(status.phase).toBe('idle'); + expect(status.lastRunAt).toBe(now); + }); + + it('only consolidates turns AFTER lastConsolidatedTurnId', async () => { + const t1 = makeTurn({ turnId: 'turn-old' }); + const t2 = makeTurn({ turnId: 'turn-new' }); + await seedLog([t1, t2]); + setLastConsolidatedTurnId(db, 'turn-old'); + incrementTurnCount(db, 1); + + const daemon = makeDaemon(); + const result = await daemon.run(); + + expect(result.status).toBe('success'); + if (result.status !== 'success') return; + expect(result.fromTurnId).toBe('turn-old'); + expect(result.toTurnId).toBe('turn-new'); + expect(result.archivedCount).toBe(1); + + const archiveFiles = await fsp.readdir( + path.join(mindRoot, '.working-memory', 'archive', 'consolidated'), + ); + expect(archiveFiles).toHaveLength(1); + expect(archiveFiles[0]).toContain('turn-new'); + }); + + it('returns skipped/no-turns when log.md has no turns past the cutoff', async () => { + const t1 = makeTurn({ turnId: 'turn-already-consolidated' }); + await seedLog([t1]); + setLastConsolidatedTurnId(db, 'turn-already-consolidated'); + incrementTurnCount(db, 1); + + const daemon = makeDaemon(); + const result = await daemon.run(); + expect(result).toEqual({ status: 'skipped', reason: 'no-turns' }); + + expect(llmClient.calls).toHaveLength(0); + expect(getLock(db, 'daily')).toBeNull(); + }); +}); + +// --------------------------------------------------------------------------- +// Gate / lock skip +// --------------------------------------------------------------------------- + +describe('DreamDaemon — gate skip', () => { + it('returns skipped/no-activity when activity counter is below threshold', async () => { + await seedLog([makeTurn({ turnId: 'turn-1' })]); + // turnsSinceLastRun stays at 0 + const daemon = makeDaemon(); + const result = await daemon.run(); + expect(result).toEqual({ status: 'skipped', reason: 'no-activity' }); + expect(llmClient.calls).toHaveLength(0); + expect(getLock(db, 'daily')).toBeNull(); + // state untouched + expect(readState(db).lastConsolidatedTurnId).toBeNull(); + }); + + it('returns skipped/too-soon when last daily run was within minDailyIntervalMs', async () => { + await seedLog([makeTurn({ turnId: 'turn-1' })]); + incrementTurnCount(db, 1); + markPhaseComplete(db, 'daily', now - 1000); + + const daemon = makeDaemon({ minDailyIntervalMs: MS_PER_DAY }); + const result = await daemon.run(); + expect(result).toEqual({ status: 'skipped', reason: 'too-soon' }); + expect(llmClient.calls).toHaveLength(0); + }); +}); + +describe('DreamDaemon — lock skip', () => { + it('returns skipped/locked when another holder owns the daily lock', async () => { + await seedLog([makeTurn({ turnId: 'turn-1' })]); + incrementTurnCount(db, 1); + + acquireLock(db, { + phase: 'daily', + mindId: 'other-process', + now, + ttlMs: 60_000, + }); + + const daemon = makeDaemon(); + const result = await daemon.run(); + expect(result).toEqual({ status: 'skipped', reason: 'locked' }); + expect(llmClient.calls).toHaveLength(0); + + // The daemon must NOT have stolen / released the held lock + const lock = getLock(db, 'daily'); + expect(lock).not.toBeNull(); + expect(lock!.holder).not.toMatch(new RegExp(`:${MIND_ID}:`)); + }); + + it('concurrent forceRun calls: second call returns locked while first is in flight', async () => { + await seedLog([makeTurn({ turnId: 'turn-1' })]); + + let releaseLLM: (() => void) | null = null; + llmClient = { + calls: [], + synthesize: (req) => { + (llmClient.calls as unknown as typeof llmClient.calls[number][]).push(req); + return new Promise((resolve) => { + releaseLLM = () => + resolve('## 12:00:00\n**[user-prompt]** I prefer kebab-case.\n'); + }); + }, + } as FakeLLMClient; + + const daemon = makeDaemon(); + const first = daemon.forceRun(); + // Wait until the first call actually enters synthesize. + const release = await waitForRelease(() => releaseLLM); + + const second = await daemon.forceRun(); + expect(second).toEqual({ status: 'skipped', reason: 'locked' }); + + release(); + const firstResult = await first; + expect(firstResult.status).toBe('success'); + }); +}); + +// --------------------------------------------------------------------------- +// forceRun +// --------------------------------------------------------------------------- + +describe('DreamDaemon — forceRun', () => { + it('bypasses the activity gate', async () => { + await seedLog([makeTurn({ turnId: 'turn-1' })]); + // turnsSinceLastRun = 0 → would be no-activity for run() + + const daemon = makeDaemon(); + const result = await daemon.forceRun(); + expect(result.status).toBe('success'); + }); + + it('bypasses the time gate', async () => { + await seedLog([makeTurn({ turnId: 'turn-1' })]); + markPhaseComplete(db, 'daily', now - 1000); + + const daemon = makeDaemon({ minDailyIntervalMs: MS_PER_DAY }); + const result = await daemon.forceRun(); + expect(result.status).toBe('success'); + }); + + it('still respects an externally held lock', async () => { + await seedLog([makeTurn({ turnId: 'turn-1' })]); + acquireLock(db, { + phase: 'daily', + mindId: 'other-process', + now, + ttlMs: 60_000, + }); + const daemon = makeDaemon(); + const result = await daemon.forceRun(); + expect(result).toEqual({ status: 'skipped', reason: 'locked' }); + }); +}); + +// --------------------------------------------------------------------------- +// Mid-run append +// --------------------------------------------------------------------------- + +describe('DreamDaemon — mid-run append', () => { + it('preserves a turn appended to log.md AFTER the snapshot is taken', async () => { + const t1 = makeTurn({ turnId: 'turn-pre' }); + await seedLog([t1]); + incrementTurnCount(db, 1); + + let releaseLLM: (() => void) | null = null; + llmClient = { + calls: [] as unknown as FakeLLMClient['calls'], + synthesize: (req) => { + (llmClient.calls as unknown as typeof llmClient.calls[number][]).push(req); + return new Promise((resolve) => { + releaseLLM = () => + resolve( + '## 12:00:00\n**[user-prompt]** I prefer kebab-case.\n', + ); + }); + }, + } as FakeLLMClient; + + const daemon = makeDaemon(); + const runPromise = daemon.run(); + + // Wait until the daemon actually enters the synthesize await. + const release = await waitForRelease(() => releaseLLM); + + // Append a new frame to log.md while the LLM call is paused. + const tail = makeTurn({ turnId: 'turn-tail', endedAt: isoOf(now + 1000) }); + await fsp.appendFile( + path.join(mindRoot, '.working-memory', 'log.md'), + serializeTurn(tail), + ); + + release(); + const result = await runPromise; + expect(result.status).toBe('success'); + if (result.status !== 'success') return; + + // turn-pre was consolidated and pruned; turn-tail must remain. + const log = await readLog(); + expect(log).toContain('turn-tail'); + expect(log).not.toContain('turn-pre'); + + // cutoff advanced only to turn-pre, NOT past the tail entry. + expect(result.toTurnId).toBe('turn-pre'); + expect(readState(db).lastConsolidatedTurnId).toBe('turn-pre'); + }); +}); + +// --------------------------------------------------------------------------- +// LLM failures +// --------------------------------------------------------------------------- + +describe('DreamDaemon — error handling', () => { + it('LLM error: returns failed result, releases lock, leaves state untouched', async () => { + await seedLog([makeTurn({ turnId: 'turn-1' })]); + incrementTurnCount(db, 1); + + llmClient = createFakeLLMClient({ error: new Error('boom') }); + const daemon = makeDaemon(); + + const result = await daemon.run(); + expect(result.status).toBe('failed'); + if (result.status === 'failed') { + expect(result.reason).toContain('boom'); + } + + // State did NOT advance; cutoff still null + const state = readState(db); + expect(state.lastConsolidatedTurnId).toBeNull(); + expect(state.turnsSinceLastRun).toBe(1); // not reset on failure + + // Lock was released + expect(getLock(db, 'daily')).toBeNull(); + + // memory.md was NOT written + expect( + fs.existsSync(path.join(mindRoot, '.working-memory', 'memory.md')), + ).toBe(false); + + // log.md was NOT pruned + const log = await readLog(); + expect(log).toContain('turn-1'); + + // dream_runs has a failed row + const runs = listRuns(db, { phase: 'daily' }); + expect(runs).toHaveLength(1); + expect(runs[0]!.status).toBe('failed'); + }); + + it('LLM timeout: returns failed result with timeout reason, releases lock', async () => { + await seedLog([makeTurn({ turnId: 'turn-1' })]); + incrementTurnCount(db, 1); + + llmClient = createFakeLLMClient({ latencyMs: 1000 }); + const daemon = makeDaemon({ llmTimeoutMs: 5 }); + const result = await daemon.run(); + expect(result.status).toBe('failed'); + if (result.status === 'failed') { + expect(result.reason).toMatch(/timed out/i); + } + expect(getLock(db, 'daily')).toBeNull(); + }); +}); + +// --------------------------------------------------------------------------- +// Tiered rollups +// --------------------------------------------------------------------------- + +describe('DreamDaemon — tiered weekly rollup', () => { + async function preSeedDailyArchives(count: number): Promise { + const archive = createMindArchiveStore(mindRoot); + for (let i = 0; i < count; i++) { + await archive.writeConsolidated({ + turnId: `seed-${i}`, + timestamp: isoOf(now - (count - i) * 60_000), + content: `seed body ${i} I prefer kebab-case file names.`, + }); + } + } + + it('triggers when daily-archive count meets threshold and weekly is due', async () => { + await preSeedDailyArchives(7); + await seedLog([makeTurn({ turnId: 'turn-1' })]); + incrementTurnCount(db, 1); + // lastWeeklyAt unset → due + const daemon = makeDaemon(); + const result = await daemon.run(); + expect(result.status).toBe('success'); + if (result.status !== 'success') return; + expect(result.weeklyArchived).toBe(true); + + const weeklyDir = path.join(mindRoot, '.working-memory', 'archive', 'weekly'); + const weeklyFiles = await fsp.readdir(weeklyDir); + expect(weeklyFiles.length).toBeGreaterThan(0); + + expect(readState(db).lastWeeklyAt).not.toBeNull(); + }); + + it('does NOT trigger when fewer than threshold daily archives exist', async () => { + await preSeedDailyArchives(3); + await seedLog([makeTurn({ turnId: 'turn-1' })]); + incrementTurnCount(db, 1); + + const daemon = makeDaemon(); + const result = await daemon.run(); + expect(result.status).toBe('success'); + if (result.status !== 'success') return; + expect(result.weeklyArchived).toBe(false); + + const weeklyDir = path.join(mindRoot, '.working-memory', 'archive', 'weekly'); + expect(fs.existsSync(weeklyDir)).toBe(false); + expect(readState(db).lastWeeklyAt).toBeNull(); + }); + + it('does NOT trigger when weekly was already done within weeklyMinIntervalMs', async () => { + await preSeedDailyArchives(7); + await seedLog([makeTurn({ turnId: 'turn-1' })]); + incrementTurnCount(db, 1); + markPhaseComplete(db, 'weekly', now - 1000); + + const daemon = makeDaemon(); + const result = await daemon.run(); + expect(result.status).toBe('success'); + if (result.status !== 'success') return; + expect(result.weeklyArchived).toBe(false); + }); +}); + +describe('DreamDaemon — tiered monthly rollup', () => { + async function preSeedDailyArchives(count: number): Promise { + const archive = createMindArchiveStore(mindRoot); + for (let i = 0; i < count; i++) { + await archive.writeConsolidated({ + turnId: `seed-${i}`, + timestamp: isoOf(now - (count - i) * 60_000), + content: `seed body ${i} I prefer kebab-case.`, + }); + } + } + + async function preSeedWeeklyArchives(count: number): Promise { + const archive = createMindArchiveStore(mindRoot); + for (let i = 0; i < count; i++) { + await archive.writeWeekly( + `2026-W0${i + 1}`, + `weekly summary ${i} I prefer kebab-case.`, + ); + } + } + + it('triggers when weekly-archive count meets threshold and monthly is due', async () => { + await preSeedDailyArchives(7); + await preSeedWeeklyArchives(4); + await seedLog([makeTurn({ turnId: 'turn-1' })]); + incrementTurnCount(db, 1); + + const daemon = makeDaemon(); + const result = await daemon.run(); + expect(result.status).toBe('success'); + if (result.status !== 'success') return; + expect(result.monthlyArchived).toBe(true); + + const monthlyDir = path.join(mindRoot, '.working-memory', 'archive', 'monthly'); + const monthlyFiles = await fsp.readdir(monthlyDir); + expect(monthlyFiles.length).toBeGreaterThan(0); + expect(readState(db).lastMonthlyAt).not.toBeNull(); + }); + + it('does NOT trigger when fewer than threshold weeklies exist', async () => { + await preSeedWeeklyArchives(2); + await seedLog([makeTurn({ turnId: 'turn-1' })]); + incrementTurnCount(db, 1); + + const daemon = makeDaemon(); + const result = await daemon.run(); + expect(result.status).toBe('success'); + if (result.status !== 'success') return; + expect(result.monthlyArchived).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// notifyTurnCompleted + close +// --------------------------------------------------------------------------- + +describe('DreamDaemon — notifyTurnCompleted', () => { + it('is a no-op (does not throw, does not mutate state)', () => { + const daemon = makeDaemon(); + const before = readState(db); + daemon.notifyTurnCompleted(makeTurn({ turnId: 'turn-noop' })); + const after = readState(db); + expect(after).toEqual(before); + }); +}); + +describe('DreamDaemon — close', () => { + it('is idempotent and rejects subsequent run() calls', async () => { + const daemon = makeDaemon(); + await daemon.close(); + await daemon.close(); + + const result = await daemon.run(); + expect(result.status).toBe('failed'); + if (result.status === 'failed') { + expect(result.reason).toMatch(/closed/i); + } + }); +}); + +// --------------------------------------------------------------------------- +// Status reflects current phase +// --------------------------------------------------------------------------- + +describe('DreamDaemon — getStatus mid-run', () => { + it('reports locked=true while a run is in progress', async () => { + await seedLog([makeTurn({ turnId: 'turn-1' })]); + incrementTurnCount(db, 1); + + let releaseLLM: (() => void) | null = null; + llmClient = { + calls: [] as unknown as FakeLLMClient['calls'], + synthesize: (req) => { + (llmClient.calls as unknown as typeof llmClient.calls[number][]).push(req); + return new Promise((resolve) => { + releaseLLM = () => + resolve('## 12:00:00\n**[user-prompt]** I prefer kebab-case.\n'); + }); + }, + } as FakeLLMClient; + + const daemon = makeDaemon(); + const runPromise = daemon.run(); + const release = await waitForRelease(() => releaseLLM); + + const midStatus = daemon.getStatus(); + expect(midStatus.locked).toBe(true); + expect(midStatus.phase).not.toBe('idle'); + + release(); + const result = await runPromise; + expect(result.status).toBe('success'); + + const finalStatus = daemon.getStatus(); + expect(finalStatus.locked).toBe(false); + expect(finalStatus.phase).toBe('idle'); + expect(finalStatus.lastResult).toEqual(result as DreamRunResult); + }); +}); diff --git a/packages/services/src/mindMemory/DreamDaemon.ts b/packages/services/src/mindMemory/DreamDaemon.ts new file mode 100644 index 00000000..d9457e9d --- /dev/null +++ b/packages/services/src/mindMemory/DreamDaemon.ts @@ -0,0 +1,581 @@ +/** + * DreamDaemon — per-mind background memory consolidation orchestrator. + * + * Composes the Phase 1-8 modules into the canonical cycle: + * + * 1. Gate check (lock + activity + time) + * 2. Acquire lock (in-process mutex layered with DB lock) + * 3. Snapshot turns from log.md UP TO the cutoff (last turn id at + * snapshot time) — anything appended later survives the prune + * 4. Drive `llmClient.synthesize` to extract memorable items from the + * snapshot + * 5. Run the four-phase consolidation pipeline + * 6. Atomically write `memory.md` capped at `memoryMaxBytes` + * 7. Re-read log.md, prune ONLY the snapshot turn ids, write back + * 8. Archive each consolidated source turn + * 9. Tiered weekly / monthly rollups when archive thresholds are met + * 10. Record the run, advance state, release the lock + * + * Critical correctness rules (enforced by tests): + * - Mid-run append must NOT lose turns (re-read before prune). + * - Errors during the cycle MUST release the lock in `finally`. + * - `forceRun` bypasses gates but NOT the lock. + * - Lock skip vs gate skip vs no-turns are distinguishable in the + * returned `DreamRunResult`. + * - All vault/archive/db are injected; tests use real tmp dirs + + * in-memory sqlite + the `__fakes__/FakeLLMClient`. + */ + +import type Database from 'better-sqlite3'; + +import type { CompletedTurn } from '@chamber/shared/turn-observer'; + +import { Logger } from '../logger'; +import { runConsolidation } from './consolidation'; +import { + __resetMindMutexForTesting, + withMindMutex, +} from './consolidation-scheduler'; +import { evaluateGates } from './dream-gates'; +import { + acquireLock, + buildLockHolder, + getLock, + markPhaseComplete, + recordRun, + releaseLock, + resetActivityCounter, + readState, + setLastConsolidatedTurnId, +} from './dream-state'; +import { extractFromLog } from './extraction'; +import type { LLMClient } from './LLMClient'; +import type { MindArchiveStore } from './MindArchiveStore'; +import type { MindMemoryVault } from './MindMemoryVault'; +import { + parseLog, + serializeTurn, + STRUCTURED_LOG_SENTINEL, + type ParsedTurn, +} from './StructuredLogFormat'; + +// --------------------------------------------------------------------------- +// Public surface +// --------------------------------------------------------------------------- + +export type DreamDaemonPhase = + | 'idle' + | 'gating' + | 'snapshot' + | 'extracting' + | 'consolidating' + | 'writing' + | 'pruning' + | 'archiving' + | 'rolling-up' + | 'recording'; + +export type DreamSkipReason = 'no-activity' | 'too-soon' | 'locked' | 'no-turns'; + +export type DreamRunResult = + | { + readonly status: 'success'; + readonly extractedCount: number; + readonly consolidatedCount: number; + readonly archivedCount: number; + readonly fromTurnId: string | null; + readonly toTurnId: string; + readonly weeklyArchived: boolean; + readonly monthlyArchived: boolean; + } + | { readonly status: 'skipped'; readonly reason: DreamSkipReason } + | { readonly status: 'failed'; readonly reason: string; readonly phase: DreamDaemonPhase }; + +export interface DreamStatus { + readonly phase: DreamDaemonPhase; + readonly locked: boolean; + readonly lastRunAt: number | null; + readonly lastResult: DreamRunResult | null; +} + +export interface DreamDaemonConfig { + readonly memoryMaxBytes: number; + readonly llmTimeoutMs: number; + readonly lockTtlMs: number; + readonly minTurnsBetweenRuns: number; + readonly minDailyIntervalMs: number; + readonly weeklyRollupAfterDailies: number; + readonly monthlyRollupAfterWeeklies: number; + readonly weeklyMinIntervalMs: number; + readonly monthlyMinIntervalMs: number; +} + +export interface DreamDaemonOptions { + readonly mindId: string; + readonly mindPath: string; + readonly llmClient: LLMClient; + readonly vault: MindMemoryVault; + readonly archiveStore: MindArchiveStore; + readonly db: Database.Database; + readonly config: DreamDaemonConfig; + readonly clock?: () => Date; + readonly logger?: Logger; +} + +export interface DreamDaemon { + run(): Promise; + forceRun(): Promise; + getStatus(): DreamStatus; + notifyTurnCompleted(turn: CompletedTurn): void; + close(): Promise; +} + +// --------------------------------------------------------------------------- +// Implementation +// --------------------------------------------------------------------------- + +const LOG_REL_PATH = 'log.md'; +const MEMORY_REL_PATH = 'memory.md'; + +export function createDreamDaemon(opts: DreamDaemonOptions): DreamDaemon { + const { + mindId, + llmClient, + vault, + archiveStore, + db, + config, + } = opts; + const clock = opts.clock ?? (() => new Date()); + const logger = opts.logger ?? Logger.create(`DreamDaemon[${mindId}]`); + + let phase: DreamDaemonPhase = 'idle'; + let lastRunAt: number | null = null; + let lastResult: DreamRunResult | null = null; + let closed = false; + + function getStatus(): DreamStatus { + return { + phase, + locked: phase !== 'idle', + lastRunAt, + lastResult, + }; + } + + function setLastResult(result: DreamRunResult): DreamRunResult { + lastResult = result; + return result; + } + + async function run(): Promise { + return executeCycle({ bypassGates: false }); + } + + async function forceRun(): Promise { + return executeCycle({ bypassGates: true }); + } + + async function executeCycle(args: { + readonly bypassGates: boolean; + }): Promise { + if (closed) { + return setLastResult({ + status: 'failed', + reason: 'daemon closed', + phase: 'idle', + }); + } + + const now = clock().getTime(); + + // Gate phase — only when not forced. The lock gate ALWAYS runs (force + // bypasses activity + time, never the lock). + phase = 'gating'; + const lockHeld = isLockHeld(now); + if (lockHeld) { + phase = 'idle'; + return setLastResult({ status: 'skipped', reason: 'locked' }); + } + + if (!args.bypassGates) { + const gate = evaluateGates( + { + phase: 'daily', + state: readState(db), + now, + lockHeld: false, + }, + { + minTurnsBetweenRuns: config.minTurnsBetweenRuns, + minIntervalMs: config.minDailyIntervalMs, + }, + ); + if (!gate.run) { + phase = 'idle'; + // `evaluateGates` only returns 'locked' when lockHeld=true, which we + // already handled above; the remaining reasons map 1:1 to skip. + if (gate.reason === 'ready' || gate.reason === 'locked') { + phase = 'idle'; + return setLastResult({ status: 'skipped', reason: 'locked' }); + } + return setLastResult({ status: 'skipped', reason: gate.reason }); + } + } + + // Try the in-process mutex first. Fail-fast on a concurrent caller so + // the second `run()` returns `locked` instead of queuing. + const mutexResult = await withMindMutex(mindId, async () => { + return runUnderLock(now); + }); + + if (!mutexResult.acquired) { + phase = 'idle'; + return setLastResult({ status: 'skipped', reason: 'locked' }); + } + + return mutexResult.value; + } + + function isLockHeld(now: number): boolean { + const existing = getLock(db, 'daily'); + if (existing === null) return false; + return existing.expiresAt > now; + } + + async function runUnderLock(now: number): Promise { + // Step 2 — acquire DB lock. + const lockHolder = buildLockHolder(mindId); + const lockResult = acquireLock(db, { + phase: 'daily', + mindId, + uuid: extractHolderUuid(lockHolder), + now, + ttlMs: config.lockTtlMs, + }); + + if (!lockResult.acquired) { + phase = 'idle'; + return setLastResult({ status: 'skipped', reason: 'locked' }); + } + + // From here on, the DB lock MUST be released no matter what. + let releasedByUs = false; + const acquiredHolder = lockResult.holder ?? lockHolder; + + try { + const cycleResult = await runCycleLocked(now); + lastRunAt = clock().getTime(); + return setLastResult(cycleResult); + } catch (err) { + logger.error('cycle threw', err); + phase = 'idle'; + lastRunAt = clock().getTime(); + const reason = err instanceof Error ? err.message : String(err); + // Record the failure even on unexpected throws so operators have a + // trail in dream_runs. + try { + recordRun(db, { + phase: 'daily', + startedAt: now, + endedAt: clock().getTime(), + status: 'failed', + reason, + }); + } catch (recordErr) { + logger.error('failed to record cycle failure', recordErr); + } + return setLastResult({ status: 'failed', reason, phase }); + } finally { + try { + releaseLock(db, 'daily', acquiredHolder); + releasedByUs = true; + } catch (releaseErr) { + logger.error('failed to release lock', releaseErr); + } + if (!releasedByUs) { + // Best-effort: if the typed release threw, attempt a direct delete + // so we never wedge the daemon. Swallow nested errors — the lock + // will be stolen on its TTL anyway. + try { + db.prepare( + 'DELETE FROM dream_locks WHERE phase = ? AND holder = ?', + ).run('daily', acquiredHolder); + } catch { + /* ignore */ + } + } + phase = 'idle'; + } + } + + async function runCycleLocked(now: number): Promise { + // Step 3 — snapshot turns from log.md. + phase = 'snapshot'; + const snapshot = await readSnapshot(); + const cutoffIndex = findCutoffIndex(snapshot.turns, snapshot.lastConsolidated); + const inScopeSnapshot = snapshot.turns.slice(cutoffIndex); + const consolidatedIds = new Set(inScopeSnapshot.map((t) => t.turnId)); + + if (inScopeSnapshot.length === 0) { + recordRun(db, { + phase: 'daily', + startedAt: now, + endedAt: clock().getTime(), + status: 'skipped', + reason: 'no-turns', + }); + return { status: 'skipped', reason: 'no-turns' }; + } + + // Step 4 — extract via LLM. + phase = 'extracting'; + const prompt = buildSynthesisPrompt(inScopeSnapshot); + const llmResponse = await llmClient.synthesize({ + prompt, + timeoutMs: config.llmTimeoutMs, + }); + const referenceDate = new Date(now); + const isoDate = referenceDate.toISOString().slice(0, 10); + const newEntries = extractFromLog(llmResponse, isoDate); + + // Step 5 — consolidate. + phase = 'consolidating'; + const currentMemoryMd = (await vault.read(MEMORY_REL_PATH)) ?? ''; + const consolidation = runConsolidation({ + currentMemoryMd, + newEntries, + referenceDate, + }); + const memoryMd = capBytes(consolidation.memoryMd, config.memoryMaxBytes); + + // Step 6 — atomic write of memory.md. + phase = 'writing'; + await vault.write(MEMORY_REL_PATH, memoryMd); + + // Step 7 — re-read log.md and prune only the snapshot turn ids. Tail + // entries appended during the LLM call MUST survive. + phase = 'pruning'; + await prunePersistedLog(consolidatedIds); + + // Step 8 — archive each consolidated source turn. + phase = 'archiving'; + for (const turn of inScopeSnapshot) { + await archiveStore.writeConsolidated({ + turnId: turn.turnId, + timestamp: turn.timestamp, + content: serializeTurn({ + turnId: turn.turnId, + sessionId: turn.sessionId, + model: turn.model, + status: turn.status, + startedAt: turn.timestamp, + endedAt: turn.timestamp, + prompt: turn.prompt, + finalAssistantMessage: turn.assistant, + }), + }); + } + + // Step 9 — tiered rollups. + phase = 'rolling-up'; + const weeklyArchived = await maybeRollupWeekly(now); + const monthlyArchived = await maybeRollupMonthly(now); + + // Step 10 — record run + advance state. + phase = 'recording'; + const lastTurnId = inScopeSnapshot[inScopeSnapshot.length - 1]!.turnId; + setLastConsolidatedTurnId(db, lastTurnId); + resetActivityCounter(db); + markPhaseComplete(db, 'daily', now); + recordRun(db, { + phase: 'daily', + startedAt: now, + endedAt: clock().getTime(), + status: 'success', + fromTurnId: snapshot.lastConsolidated, + toTurnId: lastTurnId, + }); + + return { + status: 'success', + extractedCount: newEntries.length, + consolidatedCount: consolidation.entriesKept, + archivedCount: inScopeSnapshot.length, + fromTurnId: snapshot.lastConsolidated, + toTurnId: lastTurnId, + weeklyArchived, + monthlyArchived, + }; + } + + async function readSnapshot(): Promise<{ + readonly turns: readonly ParsedTurn[]; + readonly lastConsolidated: string | null; + }> { + const content = (await vault.read(LOG_REL_PATH)) ?? ''; + const parsed = parseLog(content); + const state = readState(db); + return { + turns: parsed.turns, + lastConsolidated: state.lastConsolidatedTurnId, + }; + } + + function findCutoffIndex( + turns: readonly ParsedTurn[], + lastConsolidated: string | null, + ): number { + if (lastConsolidated === null) return 0; + const idx = turns.findIndex((t) => t.turnId === lastConsolidated); + if (idx === -1) return 0; + return idx + 1; + } + + async function prunePersistedLog(consolidatedIds: Set): Promise { + const content = (await vault.read(LOG_REL_PATH)) ?? ''; + const parsed = parseLog(content); + const survivors = parsed.turns.filter((t) => !consolidatedIds.has(t.turnId)); + const body = survivors + .map((t) => + serializeTurn({ + turnId: t.turnId, + sessionId: t.sessionId, + model: t.model, + status: t.status, + startedAt: t.timestamp, + endedAt: t.timestamp, + prompt: t.prompt, + finalAssistantMessage: t.assistant, + }), + ) + .join(''); + const next = `${STRUCTURED_LOG_SENTINEL}\n\n${body}`; + await vault.write(LOG_REL_PATH, next); + } + + async function maybeRollupWeekly(now: number): Promise { + const dailies = await archiveStore.listConsolidated(); + if (dailies.length < config.weeklyRollupAfterDailies) return false; + const state = readState(db); + if ( + state.lastWeeklyAt !== null && + now - state.lastWeeklyAt < config.weeklyMinIntervalMs + ) { + return false; + } + const weekKey = isoWeekKey(new Date(now)); + const summary = `# Weekly Rollup ${weekKey}\n\n${dailies + .slice(-config.weeklyRollupAfterDailies) + .map((name) => `- ${name}`) + .join('\n')}\n`; + await archiveStore.writeWeekly(weekKey, summary); + markPhaseComplete(db, 'weekly', now); + return true; + } + + async function maybeRollupMonthly(now: number): Promise { + const weeklies = await archiveStore.listWeekly(); + if (weeklies.length < config.monthlyRollupAfterWeeklies) return false; + const state = readState(db); + if ( + state.lastMonthlyAt !== null && + now - state.lastMonthlyAt < config.monthlyMinIntervalMs + ) { + return false; + } + const monthKey = isoMonthKey(new Date(now)); + const summary = `# Monthly Rollup ${monthKey}\n\n${weeklies + .slice(-config.monthlyRollupAfterWeeklies) + .map((name) => `- ${name}`) + .join('\n')}\n`; + await archiveStore.writeMonthly(monthKey, summary); + markPhaseComplete(db, 'monthly', now); + return true; + } + + function notifyTurnCompleted(_turn: CompletedTurn): void { + // No-op by design: DailyLogWriter increments the activity counter in + // its own `onTurnRecorded` path so the daemon does not need to. Kept + // on the public surface so the InternalScheduler (Phase 10) can wire + // turn observers without branching. + void _turn; + } + + async function close(): Promise { + closed = true; + // Drop any in-process mutex this daemon may hold so a fresh daemon + // (e.g., after a restart in tests) can re-enter cleanly. + __resetMindMutexForTesting(); + } + + return { + run, + forceRun, + getStatus, + notifyTurnCompleted, + close, + }; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function buildSynthesisPrompt(turns: readonly ParsedTurn[]): string { + const lines: string[] = []; + lines.push( + 'Extract memorable items (preferences, decisions, prohibitions, references) from the following completed turns.', + ); + lines.push( + 'Respond using daily-log format: each item on a "## HH:MM:SS" header line followed by lines of "**[type]** content".', + ); + lines.push(''); + for (const t of turns) { + const time = t.timestamp.slice(11, 19); + lines.push(`## ${time}`); + lines.push(`**[user-prompt]** ${oneLine(t.prompt)}`); + lines.push(`**[assistant]** ${oneLine(t.assistant)}`); + lines.push(''); + } + return lines.join('\n'); +} + +function oneLine(text: string): string { + return text.replace(/\s+/g, ' ').trim(); +} + +function capBytes(text: string, maxBytes: number): string { + if (Buffer.byteLength(text, 'utf-8') <= maxBytes) return text; + // Trim from the tail until the encoded length fits. We avoid mid-codepoint + // truncation by stepping one character at a time. + let truncated = text; + while (Buffer.byteLength(truncated, 'utf-8') > maxBytes && truncated.length > 0) { + truncated = truncated.slice(0, -1); + } + return truncated; +} + +function extractHolderUuid(holder: string): string { + // holder is "dream-daemon:::". The uuid is the last + // colon-delimited field. + const idx = holder.lastIndexOf(':'); + return idx === -1 ? holder : holder.slice(idx + 1); +} + +function isoWeekKey(date: Date): string { + // Compute ISO 8601 week date (YYYY-Www) — Monday starts the week and the + // first week of the year contains January 4. + const target = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate())); + const dayNum = target.getUTCDay() || 7; + target.setUTCDate(target.getUTCDate() + 4 - dayNum); + const yearStart = new Date(Date.UTC(target.getUTCFullYear(), 0, 1)); + const weekNum = Math.ceil(((target.getTime() - yearStart.getTime()) / 86_400_000 + 1) / 7); + return `${target.getUTCFullYear()}-W${String(weekNum).padStart(2, '0')}`; +} + +function isoMonthKey(date: Date): string { + const yyyy = date.getUTCFullYear(); + const mm = String(date.getUTCMonth() + 1).padStart(2, '0'); + return `${yyyy}-${mm}`; +} diff --git a/packages/services/src/mindMemory/InternalScheduler.test.ts b/packages/services/src/mindMemory/InternalScheduler.test.ts new file mode 100644 index 00000000..cbd04dd1 --- /dev/null +++ b/packages/services/src/mindMemory/InternalScheduler.test.ts @@ -0,0 +1,333 @@ +/** + * Tests for InternalScheduler — Phase 10. + * + * Per-mind in-process croner job runner that drives DreamDaemon.run() on a + * configurable cadence. Deliberately NOT registered with the user-facing + * CronService so the user's cron list stays clean. + * + * croner is driven off `Date.now()` and `setTimeout`. Vitest 4's + * `vi.useFakeTimers()` fakes both by default, so deterministic assertions + * on cron firings work as long as we advance time with + * `vi.advanceTimersByTimeAsync`. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + createInternalScheduler, + type InternalScheduler, +} from './InternalScheduler'; + +const EVERY_SECOND = '* * * * * *'; + +let scheduler: InternalScheduler; + +beforeEach(() => { + vi.useFakeTimers(); + // Pin the wall clock so croner's "next second boundary" is predictable. + vi.setSystemTime(new Date('2026-05-12T15:00:00.000Z')); + scheduler = createInternalScheduler({ random: () => 0 }); +}); + +afterEach(() => { + scheduler.close(); + vi.useRealTimers(); +}); + +function deferred(): { + promise: Promise; + resolve: (value: T) => void; + reject: (err: unknown) => void; +} { + let resolve!: (value: T) => void; + let reject!: (err: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +describe('createInternalScheduler', () => { + describe('register', () => { + it('exposes the registered cron expression via list()', () => { + scheduler.register({ + mindId: 'mind-a', + cronExpr: '0 3 * * *', + fn: async () => undefined, + }); + + const entries = scheduler.list(); + expect(entries.size).toBe(1); + expect(entries.get('mind-a')).toBe('0 3 * * *'); + }); + + it('throws synchronously on an invalid cron expression', () => { + expect(() => + scheduler.register({ + mindId: 'mind-bad', + cronExpr: 'not-a-cron', + fn: async () => undefined, + }), + ).toThrow(); + expect(scheduler.list().has('mind-bad')).toBe(false); + }); + + it('replaces an existing entry for the same mindId — old fn no longer fires', async () => { + const oldCalls: number[] = []; + const newCalls: number[] = []; + + scheduler.register({ + mindId: 'mind-a', + cronExpr: EVERY_SECOND, + fn: async () => { + oldCalls.push(Date.now()); + }, + }); + + // Replace before the next tick fires. + scheduler.register({ + mindId: 'mind-a', + cronExpr: EVERY_SECOND, + fn: async () => { + newCalls.push(Date.now()); + }, + }); + + // Advance well past two cron boundaries; only the new fn should fire. + await vi.advanceTimersByTimeAsync(2_500); + + expect(oldCalls).toHaveLength(0); + expect(newCalls.length).toBeGreaterThanOrEqual(1); + }); + }); + + describe('unregister', () => { + it('is a no-op for an unknown mindId', () => { + expect(() => scheduler.unregister('never-registered')).not.toThrow(); + }); + + it('stops only the targeted mind — other minds keep firing', async () => { + const aCalls: number[] = []; + const bCalls: number[] = []; + + scheduler.register({ + mindId: 'mind-a', + cronExpr: EVERY_SECOND, + fn: async () => { + aCalls.push(Date.now()); + }, + }); + scheduler.register({ + mindId: 'mind-b', + cronExpr: EVERY_SECOND, + fn: async () => { + bCalls.push(Date.now()); + }, + }); + + // Let one tick happen for both, then drop mind-a. + await vi.advanceTimersByTimeAsync(1_500); + const aBefore = aCalls.length; + const bBefore = bCalls.length; + expect(aBefore).toBeGreaterThanOrEqual(1); + expect(bBefore).toBeGreaterThanOrEqual(1); + + scheduler.unregister('mind-a'); + expect(scheduler.list().has('mind-a')).toBe(false); + expect(scheduler.list().has('mind-b')).toBe(true); + + await vi.advanceTimersByTimeAsync(2_000); + + expect(aCalls.length).toBe(aBefore); + expect(bCalls.length).toBeGreaterThan(bBefore); + }); + }); + + describe('runNow', () => { + it('invokes the registered fn immediately, out of band', async () => { + let calls = 0; + scheduler.register({ + mindId: 'mind-a', + cronExpr: '0 3 * * *', // never naturally fires within the test window + fn: async () => { + calls += 1; + }, + }); + + await scheduler.runNow('mind-a'); + + expect(calls).toBe(1); + }); + + it('throws when the mind is not registered', async () => { + await expect(scheduler.runNow('nobody')).rejects.toThrow(/not registered/i); + }); + + it('returns the in-flight promise when a tick is already executing', async () => { + const gate = deferred(); + let calls = 0; + + scheduler.register({ + mindId: 'mind-a', + cronExpr: '0 3 * * *', + fn: async () => { + calls += 1; + await gate.promise; + }, + }); + + const first = scheduler.runNow('mind-a'); + // Yield so the fn body starts and registers its in-flight promise. + await Promise.resolve(); + const second = scheduler.runNow('mind-a'); + + gate.resolve(); + await first; + await second; + + expect(calls).toBe(1); + }); + + it('drops a re-entrant cron tick while a previous run is still in flight', async () => { + const gate = deferred(); + let calls = 0; + + scheduler.register({ + mindId: 'mind-a', + cronExpr: EVERY_SECOND, + fn: async () => { + calls += 1; + await gate.promise; + }, + }); + + // Advance across multiple cron boundaries while the first invocation is + // still pending. Subsequent ticks should see the in-flight guard and be + // dropped — fn must run exactly once until we release it. + await vi.advanceTimersByTimeAsync(3_500); + expect(calls).toBe(1); + + gate.resolve(); + // Drain the pending promise to release in-flight state. + await vi.advanceTimersByTimeAsync(0); + // Future ticks may now fire normally — no assertion needed beyond the + // invariant that the first run was not double-invoked. + }); + }); + + describe('jitter', () => { + it('delays the fn by random() * jitterMs before invoking', async () => { + const fireTimes: number[] = []; + const jitterScheduler = createInternalScheduler({ random: () => 0.5 }); + try { + jitterScheduler.register({ + mindId: 'mind-jitter', + cronExpr: EVERY_SECOND, + fn: async () => { + fireTimes.push(Date.now()); + }, + jitterMs: 100, + }); + + // Walk forward to the next 1-second cron boundary, then up to but not + // past the jitter window. + const start = Date.now(); + // Advance to first boundary. + await vi.advanceTimersByTimeAsync(1_000); + // Advance through 49 ms of jitter — fn should NOT have fired yet + // (random() * 100 = 50, fn fires at boundary + 50ms). + await vi.advanceTimersByTimeAsync(49); + expect(fireTimes).toHaveLength(0); + // Push past the 50 ms jitter — fn fires now. + await vi.advanceTimersByTimeAsync(2); + expect(fireTimes).toHaveLength(1); + // The fn fired ~1050 ms after start. + expect(fireTimes[0] - start).toBeGreaterThanOrEqual(1_050); + expect(fireTimes[0] - start).toBeLessThan(1_100); + } finally { + jitterScheduler.close(); + } + }); + }); + + describe('error handling', () => { + it('does not break the schedule when fn throws — subsequent ticks still fire', async () => { + let calls = 0; + scheduler.register({ + mindId: 'mind-a', + cronExpr: EVERY_SECOND, + fn: async () => { + calls += 1; + if (calls === 1) { + throw new Error('boom'); + } + }, + }); + + // First tick throws, should be caught. + await vi.advanceTimersByTimeAsync(1_500); + expect(calls).toBeGreaterThanOrEqual(1); + + // Second tick should still fire. + await vi.advanceTimersByTimeAsync(1_500); + expect(calls).toBeGreaterThanOrEqual(2); + }); + }); + + describe('close', () => { + it('stops every job and clears the registry', async () => { + let aCalls = 0; + let bCalls = 0; + scheduler.register({ + mindId: 'mind-a', + cronExpr: EVERY_SECOND, + fn: async () => { + aCalls += 1; + }, + }); + scheduler.register({ + mindId: 'mind-b', + cronExpr: EVERY_SECOND, + fn: async () => { + bCalls += 1; + }, + }); + + await vi.advanceTimersByTimeAsync(1_500); + const aSnap = aCalls; + const bSnap = bCalls; + expect(aSnap).toBeGreaterThanOrEqual(1); + expect(bSnap).toBeGreaterThanOrEqual(1); + + scheduler.close(); + expect(scheduler.list().size).toBe(0); + + await vi.advanceTimersByTimeAsync(3_000); + expect(aCalls).toBe(aSnap); + expect(bCalls).toBe(bSnap); + }); + + it('is idempotent', () => { + scheduler.register({ + mindId: 'mind-a', + cronExpr: EVERY_SECOND, + fn: async () => undefined, + }); + scheduler.close(); + expect(() => scheduler.close()).not.toThrow(); + expect(scheduler.list().size).toBe(0); + }); + + it('rejects register() after close to surface lifecycle bugs', () => { + scheduler.close(); + expect(() => + scheduler.register({ + mindId: 'mind-a', + cronExpr: '0 3 * * *', + fn: async () => undefined, + }), + ).toThrow(/closed/i); + }); + }); +}); diff --git a/packages/services/src/mindMemory/InternalScheduler.ts b/packages/services/src/mindMemory/InternalScheduler.ts new file mode 100644 index 00000000..d634083a --- /dev/null +++ b/packages/services/src/mindMemory/InternalScheduler.ts @@ -0,0 +1,197 @@ +/** + * InternalScheduler — Phase 10 of Dream Daemon. + * + * A per-mind in-process croner job runner that drives DreamDaemon.run() on a + * configurable cadence. Deliberately NOT registered with the user-facing + * CronService — `mind-memory.consolidate` is invisible to the user's cron + * list by design (keeps the surfaced cron table clean). + * + * Three correctness invariants: + * + * 1. Re-registering the same mindId atomically replaces the previous Cron + * (stop the old one before swapping in the new one). + * + * 2. One in-flight invocation per mind. Both cron-driven ticks AND + * `runNow` calls share the same per-mind in-flight promise. A + * re-entrant tick or a parallel `runNow` while a previous run is + * still executing receives the existing promise rather than starting + * a second concurrent invocation. This complements (and pre-empts) + * the per-mind mutex inside DreamDaemon. + * + * 3. Errors thrown by `fn` MUST NOT propagate across the croner + * boundary — uncaught throws inside croner silently kill the + * schedule. Wrap and log instead. + * + * Jitter: when many minds activate simultaneously (e.g. on app start at + * 03:00 with N minds all configured for daily 03:00 consolidation) we + * stagger their first fires by a uniform random delay so they don't all + * hit the LLM in the same second. The random source is injectable so + * tests are deterministic. + */ + +import { Cron } from 'croner'; +import { Logger } from '../logger'; + +export interface InternalSchedulerOptions { + readonly logger?: Logger; + /** Defaults to `Math.random()`. Inject for deterministic jitter in tests. */ + readonly random?: () => number; +} + +export interface RegisterOptions { + readonly mindId: string; + readonly cronExpr: string; + readonly fn: () => Promise; + /** + * Optional uniform random delay (in ms) added before each fire to + * prevent thundering-herd when many minds activate simultaneously. + * The actual delay is `random() * jitterMs`. + */ + readonly jitterMs?: number; +} + +export interface InternalScheduler { + /** + * Register a mind's consolidation cron. Replaces any existing entry for + * the same mindId. Cron parses via `croner` — invalid expressions throw + * synchronously. Throws if the scheduler has been closed. + */ + register(opts: RegisterOptions): void; + + /** Stop and forget the entry. No-op if not registered. */ + unregister(mindId: string): void; + + /** + * Fire the registered fn immediately, OUT OF BAND of the cron schedule. + * If a previous invocation (cron-driven or runNow) is still executing, + * returns the in-flight promise so all callers observe the same outcome. + * Throws if the mindId is not registered. + */ + runNow(mindId: string): Promise; + + /** Returns the registered cron expressions keyed by mindId. */ + list(): ReadonlyMap; + + /** Stop every job and clear the registry. Idempotent. */ + close(): void; +} + +interface Entry { + readonly cronExpr: string; + readonly fn: () => Promise; + readonly cron: Cron; + inFlight: Promise | null; +} + +export function createInternalScheduler( + opts: InternalSchedulerOptions = {}, +): InternalScheduler { + const log = opts.logger ?? Logger.create('InternalScheduler'); + const random = opts.random ?? Math.random; + const entries = new Map(); + let closed = false; + + function invokeGuarded(mindId: string, entry: Entry): Promise { + if (entry.inFlight !== null) { + return entry.inFlight; + } + const promise = (async () => { + try { + await entry.fn(); + } catch (err) { + log.error(`consolidation fn threw for mind ${mindId}:`, err); + } + })(); + entry.inFlight = promise.finally(() => { + entry.inFlight = null; + }); + return entry.inFlight; + } + + function stopEntry(entry: Entry): void { + try { + entry.cron.stop(); + } catch (err) { + // croner.stop() is documented as safe to call repeatedly, but if it + // ever throws (e.g. during shutdown races) we don't want to leak it + // out of close()/unregister() and break callers. + log.warn('failed to stop cron:', err); + } + } + + return { + register({ mindId, cronExpr, fn, jitterMs }: RegisterOptions): void { + if (closed) { + throw new Error('InternalScheduler is closed'); + } + + // Stop and remove the previous entry BEFORE constructing the new + // Cron so a parse error on the new expression doesn't leave the + // mind un-scheduled silently — it just leaves the previous schedule + // in place. Wait — actually we DO want replacement semantics: + // remove first so register() with an invalid cron makes it obvious + // the mind has no schedule. Croner throws synchronously on a bad + // expression below, which propagates to the caller. + const previous = entries.get(mindId); + if (previous) { + stopEntry(previous); + entries.delete(mindId); + } + + // Construct the Cron. Invalid expressions throw synchronously here; + // we let the error propagate to the caller. `protect: true` is a + // belt-and-braces measure that prevents croner from launching a + // second handler if the first hasn't returned — our own in-flight + // guard is the source of truth, but `protect` keeps croner's own + // bookkeeping consistent. + const cron = new Cron( + cronExpr, + { protect: true }, + async () => { + const current = entries.get(mindId); + if (!current) return; + if (jitterMs !== undefined && jitterMs > 0) { + const delay = Math.floor(random() * jitterMs); + if (delay > 0) { + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } + await invokeGuarded(mindId, current); + }, + ); + + entries.set(mindId, { cronExpr, fn, cron, inFlight: null }); + }, + + unregister(mindId: string): void { + const entry = entries.get(mindId); + if (!entry) return; + stopEntry(entry); + entries.delete(mindId); + }, + + async runNow(mindId: string): Promise { + const entry = entries.get(mindId); + if (!entry) { + throw new Error(`Mind ${mindId} is not registered with InternalScheduler`); + } + return invokeGuarded(mindId, entry); + }, + + list(): ReadonlyMap { + const snapshot = new Map(); + for (const [mindId, entry] of entries) { + snapshot.set(mindId, entry.cronExpr); + } + return snapshot; + }, + + close(): void { + closed = true; + for (const entry of entries.values()) { + stopEntry(entry); + } + entries.clear(); + }, + }; +} diff --git a/packages/services/src/mindMemory/LLMClient.ts b/packages/services/src/mindMemory/LLMClient.ts new file mode 100644 index 00000000..1418f81a --- /dev/null +++ b/packages/services/src/mindMemory/LLMClient.ts @@ -0,0 +1,33 @@ +/** + * LLMClient — minimal language-model synthesis port used by the Dream + * Daemon (Phase 9) to call a one-shot, side-effect-free LLM. + * + * Why an interface (not a concrete class)? + * - The daemon's unit tests inject canned responses via a fake + * implementation (see `FakeLLMClient`). + * - The production wiring (Phase 13) supplies `CopilotLLMClient`, which + * uses the mind's own Copilot SDK with tools disabled and no + * permission handler so the synthesis cannot pollute the user's + * conversation, mutate session history, or trigger UI approvals. + * + * Public surface deliberately stays tiny: a single `synthesize` call + * returning the final assistant text. Streaming, retries, and token + * accounting are caller concerns and live above this seam. + */ + +export interface SynthesizeRequest { + /** Full prompt text. The adapter does not prepend a system message. */ + readonly prompt: string; + /** Soft cap; adapters may translate to an SDK-specific limit or ignore. */ + readonly maxTokens?: number; + /** + * Hard cap enforced by the adapter via an internal `AbortController`. + * On expiry, `synthesize` rejects with an Error whose message starts + * with "LLM synthesis timed out after". + */ + readonly timeoutMs: number; +} + +export interface LLMClient { + synthesize(req: SynthesizeRequest): Promise; +} diff --git a/packages/services/src/mindMemory/MindArchiveStore.test.ts b/packages/services/src/mindMemory/MindArchiveStore.test.ts new file mode 100644 index 00000000..292d6a2e --- /dev/null +++ b/packages/services/src/mindMemory/MindArchiveStore.test.ts @@ -0,0 +1,179 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { createMindArchiveStore } from './MindArchiveStore'; +import { createMindMemoryVault } from './MindMemoryVault'; + +let mindRoot: string; + +beforeEach(() => { + mindRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-archive-')); +}); + +afterEach(() => { + fs.rmSync(mindRoot, { recursive: true, force: true }); +}); + +describe('MindArchiveStore — root', () => { + it('roots itself at /.working-memory/archive/', () => { + const archive = createMindArchiveStore(mindRoot); + expect(archive.root).toBe(path.resolve(mindRoot, '.working-memory', 'archive')); + expect(path.isAbsolute(archive.root)).toBe(true); + }); + + it('does not touch disk on construction', () => { + createMindArchiveStore(mindRoot); + expect(fs.existsSync(path.join(mindRoot, '.working-memory'))).toBe(false); + }); +}); + +describe('MindArchiveStore — writeConsolidated', () => { + it('writes to archive/consolidated/--.md and returns the relPath', async () => { + const archive = createMindArchiveStore(mindRoot); + const relPath = await archive.writeConsolidated({ + turnId: '11111111-1111-4111-8111-111111111111', + timestamp: '2026-05-12T12:34:56Z', + content: 'consolidated body', + }); + // `:` is replaced with `-` for Windows filename portability; documented in + // MindArchiveStore.ts. Input contract remains ISO-8601 UTC. + expect(relPath).toBe( + path.join('consolidated', '2026-05-12T12-34-56Z--11111111-1111-4111-8111-111111111111.md'), + ); + const abs = path.join(archive.root, relPath); + expect(fs.readFileSync(abs, 'utf-8')).toBe('consolidated body'); + }); + + it('auto-creates the consolidated/ subdirectory', async () => { + const archive = createMindArchiveStore(mindRoot); + await archive.writeConsolidated({ + turnId: 'tid-1', + timestamp: '2026-05-12T00:00:00Z', + content: 'x', + }); + expect(fs.statSync(path.join(archive.root, 'consolidated')).isDirectory()).toBe(true); + }); + + it('rejects a turnId containing path separators', async () => { + const archive = createMindArchiveStore(mindRoot); + await expect( + archive.writeConsolidated({ + turnId: '../escape', + timestamp: '2026-05-12T00:00:00Z', + content: 'x', + }), + ).rejects.toThrow(/turn|path|escape|invalid/i); + }); + + it('rejects a timestamp containing path separators', async () => { + const archive = createMindArchiveStore(mindRoot); + await expect( + archive.writeConsolidated({ + turnId: 'tid-1', + timestamp: '2026/05/12', + content: 'x', + }), + ).rejects.toThrow(/timestamp|path|invalid/i); + }); +}); + +describe('MindArchiveStore — writeWeekly / writeMonthly', () => { + it('writes weekly/.md', async () => { + const archive = createMindArchiveStore(mindRoot); + const relPath = await archive.writeWeekly('2026-W19', 'weekly body'); + expect(relPath).toBe(path.join('weekly', '2026-W19.md')); + expect(fs.readFileSync(path.join(archive.root, relPath), 'utf-8')).toBe('weekly body'); + }); + + it('writes monthly/.md', async () => { + const archive = createMindArchiveStore(mindRoot); + const relPath = await archive.writeMonthly('2026-05', 'monthly body'); + expect(relPath).toBe(path.join('monthly', '2026-05.md')); + expect(fs.readFileSync(path.join(archive.root, relPath), 'utf-8')).toBe('monthly body'); + }); + + it('weekly write replaces existing content atomically', async () => { + const archive = createMindArchiveStore(mindRoot); + await archive.writeWeekly('2026-W19', 'first'); + await archive.writeWeekly('2026-W19', 'second'); + expect(fs.readFileSync(path.join(archive.root, 'weekly', '2026-W19.md'), 'utf-8')).toBe('second'); + const dirEntries = fs.readdirSync(path.join(archive.root, 'weekly')); + expect(dirEntries.some((f) => f.includes('.tmp.'))).toBe(false); + }); + + it('rejects weekly keys containing path separators', async () => { + const archive = createMindArchiveStore(mindRoot); + await expect(archive.writeWeekly('../escape', 'x')).rejects.toThrow(/key|path|invalid/i); + await expect(archive.writeWeekly('2026/W19', 'x')).rejects.toThrow(/key|path|invalid/i); + }); + + it('rejects monthly keys containing path separators', async () => { + const archive = createMindArchiveStore(mindRoot); + await expect(archive.writeMonthly('../escape', 'x')).rejects.toThrow(/key|path|invalid/i); + await expect(archive.writeMonthly('2026\\05', 'x')).rejects.toThrow(/key|path|invalid/i); + }); +}); + +describe('MindArchiveStore — list methods', () => { + it('listConsolidated returns only files under consolidated/', async () => { + const archive = createMindArchiveStore(mindRoot); + await archive.writeConsolidated({ + turnId: 't1', + timestamp: '2026-05-12T00:00:00Z', + content: 'a', + }); + await archive.writeConsolidated({ + turnId: 't2', + timestamp: '2026-05-12T01:00:00Z', + content: 'b', + }); + await archive.writeWeekly('2026-W19', 'w'); + const list = await archive.listConsolidated(); + expect(list.sort()).toEqual([ + '2026-05-12T00-00-00Z--t1.md', + '2026-05-12T01-00-00Z--t2.md', + ]); + }); + + it('listWeekly returns only files under weekly/', async () => { + const archive = createMindArchiveStore(mindRoot); + await archive.writeWeekly('2026-W19', 'a'); + await archive.writeWeekly('2026-W20', 'b'); + await archive.writeMonthly('2026-05', 'm'); + const list = await archive.listWeekly(); + expect(list.sort()).toEqual(['2026-W19.md', '2026-W20.md']); + }); + + it('listMonthly returns only files under monthly/', async () => { + const archive = createMindArchiveStore(mindRoot); + await archive.writeMonthly('2026-04', 'a'); + await archive.writeMonthly('2026-05', 'b'); + const list = await archive.listMonthly(); + expect(list.sort()).toEqual(['2026-04.md', '2026-05.md']); + }); + + it('list methods return [] when their directory does not exist yet', async () => { + const archive = createMindArchiveStore(mindRoot); + expect(await archive.listConsolidated()).toEqual([]); + expect(await archive.listWeekly()).toEqual([]); + expect(await archive.listMonthly()).toEqual([]); + }); +}); + +describe('MindArchiveStore — interaction with MindMemoryVault', () => { + it('archive writes never appear in vault.listFiles()', async () => { + const vault = createMindMemoryVault(mindRoot); + const archive = createMindArchiveStore(mindRoot); + await vault.write('memory.md', 'm'); + await archive.writeConsolidated({ + turnId: 't1', + timestamp: '2026-05-12T00:00:00Z', + content: 'c', + }); + await archive.writeWeekly('2026-W19', 'w'); + await archive.writeMonthly('2026-05', 'mo'); + expect(await vault.listFiles()).toEqual(['memory.md']); + }); +}); diff --git a/packages/services/src/mindMemory/MindArchiveStore.ts b/packages/services/src/mindMemory/MindArchiveStore.ts new file mode 100644 index 00000000..c85bf33c --- /dev/null +++ b/packages/services/src/mindMemory/MindArchiveStore.ts @@ -0,0 +1,152 @@ +/** + * MindArchiveStore — filesystem adapter for `/.working-memory/archive/`. + * + * Phase 3 scope (locked by plan): `node:*` only — no Electron, no Chamber + * Logger, no third-party I/O libs. Errors propagate; callers own logging. + * + * Layout: + * archive/ + * consolidated/--.md + * weekly/-W.md + * monthly/-.md + * + * Same atomic-write and path-traversal guarantees as MindMemoryVault. + * Archive contents are owned exclusively by this module — they never appear + * in the vault's `listFiles()` output (the `archive/` subdirectory is filtered + * there). + */ + +import { randomUUID } from 'node:crypto'; +import fsp from 'node:fs/promises'; +import path from 'node:path'; + +import { resolveRelPath } from './MindMemoryVault'; + +const WORKING_MEMORY_DIRNAME = '.working-memory'; +const ARCHIVE_DIRNAME = 'archive'; +const CONSOLIDATED_DIRNAME = 'consolidated'; +const WEEKLY_DIRNAME = 'weekly'; +const MONTHLY_DIRNAME = 'monthly'; + +export interface ConsolidatedRecord { + readonly turnId: string; + readonly timestamp: string; + readonly content: string; +} + +export interface MindArchiveStore { + readonly root: string; + writeConsolidated(record: ConsolidatedRecord): Promise; + writeWeekly(weekKey: string, content: string): Promise; + writeMonthly(monthKey: string, content: string): Promise; + listConsolidated(): Promise; + listWeekly(): Promise; + listMonthly(): Promise; +} + +export function createMindArchiveStore(mindPath: string): MindArchiveStore { + const root = path.resolve(mindPath, WORKING_MEMORY_DIRNAME, ARCHIVE_DIRNAME); + + async function writeConsolidated(record: ConsolidatedRecord): Promise { + assertSafeKey('turnId', record.turnId); + assertSafeKey('timestamp', record.timestamp); + // Filenames replace `:` with `-` so the path is portable across Windows + // (which forbids `:` in NTFS filenames) while keeping the input contract + // an ISO-8601 UTC timestamp. + const safeTimestamp = record.timestamp.replace(/:/g, '-'); + const filename = `${safeTimestamp}--${record.turnId}.md`; + const relPath = path.join(CONSOLIDATED_DIRNAME, filename); + await writeAtRelPath(root, relPath, record.content); + return relPath; + } + + async function writeWeekly(weekKey: string, content: string): Promise { + assertSafeKey('weekly key', weekKey); + const relPath = path.join(WEEKLY_DIRNAME, `${weekKey}.md`); + await writeAtRelPath(root, relPath, content); + return relPath; + } + + async function writeMonthly(monthKey: string, content: string): Promise { + assertSafeKey('monthly key', monthKey); + const relPath = path.join(MONTHLY_DIRNAME, `${monthKey}.md`); + await writeAtRelPath(root, relPath, content); + return relPath; + } + + function listConsolidated(): Promise { + return listSubdirFiles(path.join(root, CONSOLIDATED_DIRNAME)); + } + function listWeekly(): Promise { + return listSubdirFiles(path.join(root, WEEKLY_DIRNAME)); + } + function listMonthly(): Promise { + return listSubdirFiles(path.join(root, MONTHLY_DIRNAME)); + } + + return { + root, + writeConsolidated, + writeWeekly, + writeMonthly, + listConsolidated, + listWeekly, + listMonthly, + }; +} + +function assertSafeKey(label: string, value: string): void { + if (typeof value !== 'string' || value.length === 0) { + throw new Error(`invalid ${label}: must be a non-empty string`); + } + if (value.includes('\u0000')) { + throw new Error(`invalid ${label}: contains NUL byte`); + } + if (value.includes('/') || value.includes('\\')) { + throw new Error(`invalid ${label}: must not contain path separators (got ${value})`); + } + if (value === '.' || value === '..') { + throw new Error(`invalid ${label}: must not be . or ..`); + } +} + +async function writeAtRelPath(root: string, relPath: string, content: string): Promise { + const abs = resolveRelPath(root, relPath); + await fsp.mkdir(path.dirname(abs), { recursive: true }); + const tempPath = `${abs}.tmp.${randomUUID()}`; + const handle = await fsp.open(tempPath, 'wx'); + try { + await handle.writeFile(content); + await handle.sync(); + } finally { + await handle.close(); + } + try { + await fsp.rename(tempPath, abs); + } catch (err) { + try { + await fsp.unlink(tempPath); + } catch { + // ignore + } + throw err; + } +} + +async function listSubdirFiles(absDir: string): Promise { + let entries; + try { + entries = await fsp.readdir(absDir, { withFileTypes: true }); + } catch (err) { + if ( + typeof err === 'object' && + err !== null && + 'code' in err && + (err as NodeJS.ErrnoException).code === 'ENOENT' + ) { + return []; + } + throw err; + } + return entries.filter((entry) => entry.isFile()).map((entry) => entry.name); +} diff --git a/packages/services/src/mindMemory/MindMemoryService.test.ts b/packages/services/src/mindMemory/MindMemoryService.test.ts new file mode 100644 index 00000000..ce185aef --- /dev/null +++ b/packages/services/src/mindMemory/MindMemoryService.test.ts @@ -0,0 +1,663 @@ +/** + * MindMemoryService — Phase 11. + * + * Lifecycle + public surface for per-mind background memory consolidation. + * Wires the InternalScheduler entry, opens dream.db, builds the vault / + * archive / daemon, and registers a TurnCompletionObserver on ChatService + * (whose `onTurnCompleted` forwards to the per-mind DailyLogWriter). + * + * Strict opt-in: when `.chamber.json → workingMemory.consolidation.enabled` + * is not exactly `true`, activate is a no-op (no db open, no factories + * called beyond configReader, no observer registered). + * + * Documented choice: a second `activateMind` for the same mindId while + * already activated is a no-op (idempotent — never replaces collaborators + * mid-flight). Callers must `releaseMind` first to swap configuration. + */ + +import path from 'node:path'; + +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import type Database from 'better-sqlite3'; + +import type { TurnCompletionObserver, CompletedTurn } from '@chamber/shared/turn-observer'; + +import { createMindMemoryService } from './MindMemoryService'; +import type { + ChatObserverRegistry, + MindMemoryServiceFactories, +} from './MindMemoryService'; +import type { DreamDaemon, DreamRunResult } from './DreamDaemon'; +import type { InternalScheduler, RegisterOptions } from './InternalScheduler'; +import type { MindMemoryVault } from './MindMemoryVault'; +import type { MindArchiveStore } from './MindArchiveStore'; +import type { ChamberMindConfig } from '../mind/chamberMindConfig'; +import { dreamDbPath } from './dream-schema'; + +// --------------------------------------------------------------------------- +// Fakes +// --------------------------------------------------------------------------- + +function makeFakeScheduler(): InternalScheduler & { + readonly registered: Map; + readonly closeCalls: number; +} { + const registered = new Map(); + let closeCalls = 0; + const scheduler = { + register(opts: RegisterOptions): void { + registered.set(opts.mindId, opts); + }, + unregister(mindId: string): void { + registered.delete(mindId); + }, + async runNow(mindId: string): Promise { + const entry = registered.get(mindId); + if (!entry) throw new Error(`unknown ${mindId}`); + await entry.fn(); + }, + list(): ReadonlyMap { + const m = new Map(); + for (const [k, v] of registered) m.set(k, v.cronExpr); + return m; + }, + close(): void { + closeCalls += 1; + registered.clear(); + }, + get registered() { + return registered; + }, + get closeCalls() { + return closeCalls; + }, + }; + return scheduler; +} + +function makeFakeDb(): Database.Database & { closed: boolean } { + let closed = false; + // Only the methods MindMemoryService cares about. The rest are stubs that + // throw if accidentally invoked. + const db = { + close: vi.fn(() => { + closed = true; + }), + get closed() { + return closed; + }, + }; + return db as unknown as Database.Database & { closed: boolean }; +} + +function makeFakeVault(root: string): MindMemoryVault { + return { + root, + read: vi.fn(async () => null), + write: vi.fn(async () => undefined), + append: vi.fn(async () => undefined), + exists: vi.fn(async () => false), + listFiles: vi.fn(async () => []), + ensureDir: vi.fn(async () => undefined), + }; +} + +function makeFakeArchive(root: string): MindArchiveStore { + return { + root, + writeConsolidated: vi.fn(async () => 'consolidated/x.md'), + writeWeekly: vi.fn(async () => 'weekly/x.md'), + writeMonthly: vi.fn(async () => 'monthly/x.md'), + listConsolidated: vi.fn(async () => []), + listWeekly: vi.fn(async () => []), + listMonthly: vi.fn(async () => []), + }; +} + +function makeFakeDaemon(): DreamDaemon & { runCalls: number; closeCalls: number } { + let runCalls = 0; + let closeCalls = 0; + const daemon: DreamDaemon = { + async run(): Promise { + runCalls += 1; + return { status: 'skipped', reason: 'no-turns' }; + }, + async forceRun(): Promise { + return { status: 'skipped', reason: 'no-turns' }; + }, + getStatus() { + return { phase: 'idle', locked: false, lastRunAt: null, lastResult: null }; + }, + notifyTurnCompleted() { + /* no-op */ + }, + async close(): Promise { + closeCalls += 1; + }, + }; + return Object.defineProperties(daemon, { + runCalls: { get: () => runCalls, enumerable: true }, + closeCalls: { get: () => closeCalls, enumerable: true }, + }) as DreamDaemon & { runCalls: number; closeCalls: number }; +} + +function makeFakeChat(): ChatObserverRegistry & { + readonly observers: TurnCompletionObserver[]; +} { + const observers: TurnCompletionObserver[] = []; + return { + addObserver(o: TurnCompletionObserver): void { + observers.push(o); + }, + removeObserver(o: TurnCompletionObserver): void { + const i = observers.indexOf(o); + if (i !== -1) observers.splice(i, 1); + }, + get observers() { + return observers; + }, + }; +} + +interface FactoryCallLog { + readonly events: string[]; +} + +function makeFactories(args: { + readonly chamberConfig: ChamberMindConfig; + readonly daemon?: DreamDaemon; + readonly daemonError?: Error; + readonly dbError?: Error; + readonly vaultError?: Error; + readonly archiveError?: Error; + readonly chat?: ChatObserverRegistry; + readonly scheduler?: ReturnType; + readonly schedulerRegisterError?: Error; +}): { + readonly factories: MindMemoryServiceFactories; + readonly log: FactoryCallLog; + readonly db: ReturnType; + readonly vault: MindMemoryVault; + readonly archive: MindArchiveStore; + readonly daemon: DreamDaemon; + readonly chat: ReturnType; + readonly scheduler: ReturnType; +} { + const events: string[] = []; + const db = makeFakeDb(); + const vault = makeFakeVault('/tmp/vault'); + const archive = makeFakeArchive('/tmp/archive'); + const daemon = (args.daemon ?? makeFakeDaemon()) as ReturnType; + const chat = (args.chat ?? makeFakeChat()) as ReturnType; + const scheduler = args.scheduler ?? makeFakeScheduler(); + + if (args.schedulerRegisterError) { + const baseRegister = scheduler.register.bind(scheduler); + void baseRegister; + scheduler.register = (() => { + events.push('scheduler.register'); + throw args.schedulerRegisterError; + }) as InternalScheduler['register']; + } else { + const baseRegister = scheduler.register.bind(scheduler); + scheduler.register = ((opts: RegisterOptions) => { + events.push('scheduler.register'); + baseRegister(opts); + }) as InternalScheduler['register']; + } + + const factories: MindMemoryServiceFactories = { + scheduler, + chatService: chat, + configReader: vi.fn((mindPath: string) => { + events.push(`configReader:${mindPath}`); + return args.chamberConfig; + }), + dbFactory: vi.fn((dbPath: string) => { + events.push(`dbFactory:${dbPath}`); + if (args.dbError) throw args.dbError; + return db; + }), + vaultFactory: vi.fn((mindPath: string) => { + events.push(`vaultFactory:${mindPath}`); + if (args.vaultError) throw args.vaultError; + return vault; + }), + archiveFactory: vi.fn((mindPath: string) => { + events.push(`archiveFactory:${mindPath}`); + if (args.archiveError) throw args.archiveError; + return archive; + }), + daemonFactory: vi.fn(() => { + events.push('daemonFactory'); + if (args.daemonError) throw args.daemonError; + return daemon; + }), + }; + + return { factories, log: { events }, db, vault, archive, daemon, chat, scheduler }; +} + +const ENABLED_CONFIG: ChamberMindConfig = { + workingMemory: { + consolidation: { + enabled: true, + cron: '0 3 * * *', + lastKTurns: 10, + perTurnMaxBytes: 2048, + memoryMaxBytes: 8192, + }, + }, +}; + +const DISABLED_CONFIG: ChamberMindConfig = { + workingMemory: { + consolidation: { + enabled: false, + cron: '0 3 * * *', + lastKTurns: 10, + perTurnMaxBytes: 2048, + memoryMaxBytes: 8192, + }, + }, +}; + +const MIND_PATH = path.join('/', 'tmp', 'mind-alpha'); +const MIND_ID = 'mind-alpha'; + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe('MindMemoryService — activateMind: opt-in honored', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('returns early when consolidation.enabled is false — no factories beyond configReader, no observer registered', async () => { + const { factories, log, chat, scheduler } = makeFactories({ chamberConfig: DISABLED_CONFIG }); + const svc = createMindMemoryService(factories); + + await svc.activateMind(MIND_ID, MIND_PATH); + + expect(log.events).toEqual([`configReader:${MIND_PATH}`]); + expect(factories.dbFactory).not.toHaveBeenCalled(); + expect(factories.vaultFactory).not.toHaveBeenCalled(); + expect(factories.archiveFactory).not.toHaveBeenCalled(); + expect(factories.daemonFactory).not.toHaveBeenCalled(); + expect(chat.observers).toHaveLength(0); + expect(scheduler.registered.size).toBe(0); + }); + + it('treats a truthy-but-not-true enabled value as disabled (defensive against config drift)', async () => { + const odd: ChamberMindConfig = { + workingMemory: { + consolidation: { + ...ENABLED_CONFIG.workingMemory.consolidation, + // Force a non-`true` truthy value past the type system. + enabled: 1 as unknown as boolean, + }, + }, + }; + const { factories, chat, scheduler } = makeFactories({ chamberConfig: odd }); + const svc = createMindMemoryService(factories); + + await svc.activateMind(MIND_ID, MIND_PATH); + + expect(factories.dbFactory).not.toHaveBeenCalled(); + expect(chat.observers).toHaveLength(0); + expect(scheduler.registered.size).toBe(0); + }); +}); + +describe('MindMemoryService — activateMind: enabled path wires everything', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('opens dream.db at /.working-memory/.state/dream.db, builds vault/archive/daemon, registers scheduler, adds observer', async () => { + const { factories, log, db, vault, archive, daemon, chat, scheduler } = makeFactories({ + chamberConfig: ENABLED_CONFIG, + }); + const svc = createMindMemoryService(factories); + + await svc.activateMind(MIND_ID, MIND_PATH); + + // Order: config → vault/archive (independent) → db → daemon → scheduler → observer. + // Strict ordering only matters where one collaborator depends on another: + // db must be opened before daemonFactory (which receives it); scheduler + // entry / observer registration come last so a daemonFactory failure + // doesn't leak a registered cron or observer. + const idxConfig = log.events.indexOf(`configReader:${MIND_PATH}`); + const idxDb = log.events.indexOf(`dbFactory:${dreamDbPath(MIND_PATH)}`); + const idxDaemon = log.events.indexOf('daemonFactory'); + const idxRegister = log.events.indexOf('scheduler.register'); + expect(idxConfig).toBeGreaterThanOrEqual(0); + expect(idxDb).toBeGreaterThan(idxConfig); + expect(idxDaemon).toBeGreaterThan(idxDb); + expect(idxRegister).toBeGreaterThan(idxDaemon); + + expect(factories.vaultFactory).toHaveBeenCalledWith(MIND_PATH); + expect(factories.archiveFactory).toHaveBeenCalledWith(MIND_PATH); + expect(factories.daemonFactory).toHaveBeenCalledWith( + expect.objectContaining({ + mindId: MIND_ID, + mindPath: MIND_PATH, + vault, + archive, + db, + config: ENABLED_CONFIG.workingMemory.consolidation, + }), + ); + + // Scheduler entry: cron from config, jitter 30s, fn drives daemon.run. + const entry = scheduler.registered.get(MIND_ID); + expect(entry).toBeDefined(); + expect(entry!.cronExpr).toBe('0 3 * * *'); + expect(entry!.jitterMs).toBe(30_000); + await entry!.fn(); + expect((daemon as ReturnType).runCalls).toBe(1); + + // Exactly one observer added. + expect(chat.observers).toHaveLength(1); + expect(typeof chat.observers[0].onTurnCompleted).toBe('function'); + }); + + it('observer forwards onTurnCompleted to the per-mind DailyLogWriter without throwing across the boundary', async () => { + const { factories, chat } = makeFactories({ chamberConfig: ENABLED_CONFIG }); + const svc = createMindMemoryService(factories); + await svc.activateMind(MIND_ID, MIND_PATH); + + const obs = chat.observers[0]; + const turn: CompletedTurn = { + turnId: 't1', + sessionId: 's1', + model: 'm', + status: 'completed', + startedAt: '2026-05-12T00:00:00.000Z', + endedAt: '2026-05-12T00:00:01.000Z', + prompt: 'hi', + finalAssistantMessage: 'hello', + }; + + // We don't actually want to touch the filesystem here; just verify the + // observer is callable and returns a thenable. The real DailyLogWriter + // is exercised by its own tests. Construction-only test for wiring. + const result = obs.onTurnCompleted(turn); + expect(result === undefined || typeof (result as Promise).then === 'function').toBe(true); + }); +}); + +describe('MindMemoryService — DailyLogWriter onTurnRecorded → dream-state activity counter', () => { + // INVARIANT (real bug fixed in Phase 14): the writer constructed inside + // `activateMind` must wire `onTurnRecorded` so each completed turn bumps + // `dream_state.turns_since_last_run`. Without this, the daemon's activity + // gate would block consolidation forever (default `minTurnsBetweenRuns: 1`). + // We exercise the wiring against a real :memory: better-sqlite3 db + real + // MindMemoryVault on disk so the assertion is end-to-end. + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('observer.onTurnCompleted increments turns_since_last_run via DailyLogWriter onTurnRecorded', async () => { + const { mkdtempSync, rmSync, mkdirSync } = await import('node:fs'); + const { tmpdir } = await import('node:os'); + const { createRequire } = await import('node:module'); + const { createMindMemoryVault } = await import('./MindMemoryVault'); + const { createMindArchiveStore } = await import('./MindArchiveStore'); + const { migrate } = await import('./dream-schema'); + const { readState } = await import('./dream-state'); + + const runtimeRequire = createRequire(__filename); + const Database = runtimeRequire('better-sqlite3') as typeof import('better-sqlite3'); + + const root = mkdtempSync(path.join(tmpdir(), 'chamber-mindmem-onturn-')); + const mindPath = path.join(root, 'mind-real'); + mkdirSync(mindPath, { recursive: true }); + + const realDb = new Database(':memory:'); + migrate(realDb); + + const chat = makeFakeChat(); + const scheduler = makeFakeScheduler(); + const daemon = makeFakeDaemon(); + + const factories: MindMemoryServiceFactories = { + scheduler, + chatService: chat, + configReader: () => ENABLED_CONFIG, + dbFactory: () => realDb as unknown as Database.Database, + vaultFactory: createMindMemoryVault, + archiveFactory: createMindArchiveStore, + daemonFactory: () => daemon, + }; + + const svc = createMindMemoryService(factories); + try { + await svc.activateMind(MIND_ID, mindPath); + + expect(readState(realDb).turnsSinceLastRun).toBe(0); + + const obs = chat.observers[0]; + const t1: CompletedTurn = { + turnId: 't-1', + sessionId: 's-1', + model: 'm', + status: 'completed', + startedAt: '2026-05-12T00:00:00.000Z', + endedAt: '2026-05-12T00:00:01.000Z', + prompt: 'hi', + finalAssistantMessage: 'hello', + }; + const t2: CompletedTurn = { ...t1, turnId: 't-2' }; + + await obs.onTurnCompleted(t1); + await obs.onTurnCompleted(t2); + + expect(readState(realDb).turnsSinceLastRun).toBe(2); + + await svc.releaseMind(MIND_ID); + } finally { + try { + if (realDb.open) realDb.close(); + } catch { + /* noop */ + } + rmSync(root, { recursive: true, force: true }); + } + }); +}); + +describe('MindMemoryService — activateMind: idempotency', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('second activate for the same already-activated mind is a no-op (documented choice)', async () => { + const { factories, chat, scheduler } = makeFactories({ chamberConfig: ENABLED_CONFIG }); + const svc = createMindMemoryService(factories); + + await svc.activateMind(MIND_ID, MIND_PATH); + const dbCalls = (factories.dbFactory as ReturnType).mock.calls.length; + const daemonCalls = (factories.daemonFactory as ReturnType).mock.calls.length; + const obsCount = chat.observers.length; + const schedSize = scheduler.registered.size; + + await svc.activateMind(MIND_ID, MIND_PATH); + + expect((factories.dbFactory as ReturnType).mock.calls.length).toBe(dbCalls); + expect((factories.daemonFactory as ReturnType).mock.calls.length).toBe(daemonCalls); + expect(chat.observers).toHaveLength(obsCount); + expect(scheduler.registered.size).toBe(schedSize); + }); + + it('after release, a subsequent activate rebuilds the entry', async () => { + const { factories, chat, scheduler } = makeFactories({ chamberConfig: ENABLED_CONFIG }); + const svc = createMindMemoryService(factories); + + await svc.activateMind(MIND_ID, MIND_PATH); + await svc.releaseMind(MIND_ID); + await svc.activateMind(MIND_ID, MIND_PATH); + + expect((factories.dbFactory as ReturnType).mock.calls.length).toBe(2); + expect((factories.daemonFactory as ReturnType).mock.calls.length).toBe(2); + expect(chat.observers).toHaveLength(1); + expect(scheduler.registered.size).toBe(1); + }); +}); + +describe('MindMemoryService — releaseMind', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('tears down everything: scheduler.unregister, observer removed, daemon.close, db.close', async () => { + const { factories, db, daemon, chat, scheduler } = makeFactories({ chamberConfig: ENABLED_CONFIG }); + const svc = createMindMemoryService(factories); + + await svc.activateMind(MIND_ID, MIND_PATH); + expect(chat.observers).toHaveLength(1); + expect(scheduler.registered.size).toBe(1); + + await svc.releaseMind(MIND_ID); + + expect(scheduler.registered.has(MIND_ID)).toBe(false); + expect(chat.observers).toHaveLength(0); + expect((daemon as ReturnType).closeCalls).toBe(1); + expect((db as ReturnType).closed).toBe(true); + }); + + it('release of an unknown mind is a no-op', async () => { + const { factories, scheduler } = makeFactories({ chamberConfig: ENABLED_CONFIG }); + const svc = createMindMemoryService(factories); + + await expect(svc.releaseMind('never-activated')).resolves.toBeUndefined(); + expect(scheduler.registered.size).toBe(0); + }); + + it('release of a mind that opted out (activate was a no-op) is also a no-op', async () => { + const { factories, scheduler } = makeFactories({ chamberConfig: DISABLED_CONFIG }); + const svc = createMindMemoryService(factories); + await svc.activateMind(MIND_ID, MIND_PATH); + await expect(svc.releaseMind(MIND_ID)).resolves.toBeUndefined(); + expect(scheduler.registered.size).toBe(0); + }); + + it('release is idempotent — second release of the same mind does not throw or double-close', async () => { + const { factories, db, daemon } = makeFactories({ chamberConfig: ENABLED_CONFIG }); + const svc = createMindMemoryService(factories); + await svc.activateMind(MIND_ID, MIND_PATH); + await svc.releaseMind(MIND_ID); + await expect(svc.releaseMind(MIND_ID)).resolves.toBeUndefined(); + expect((daemon as ReturnType).closeCalls).toBe(1); + expect((db as ReturnType).closed).toBe(true); + }); +}); + +describe('MindMemoryService — close()', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('releases every activated mind sequentially', async () => { + const { factories, chat, scheduler } = makeFactories({ chamberConfig: ENABLED_CONFIG }); + const svc = createMindMemoryService(factories); + + await svc.activateMind('m1', '/tmp/m1'); + await svc.activateMind('m2', '/tmp/m2'); + expect(chat.observers).toHaveLength(2); + expect(scheduler.registered.size).toBe(2); + + await svc.close(); + + expect(chat.observers).toHaveLength(0); + expect(scheduler.registered.size).toBe(0); + }); + + it('is idempotent', async () => { + const { factories } = makeFactories({ chamberConfig: ENABLED_CONFIG }); + const svc = createMindMemoryService(factories); + await svc.activateMind(MIND_ID, MIND_PATH); + await svc.close(); + await expect(svc.close()).resolves.toBeUndefined(); + }); + + it('after close, further activate calls are rejected (fail-fast — keeps lifecycle invariant clear)', async () => { + const { factories } = makeFactories({ chamberConfig: ENABLED_CONFIG }); + const svc = createMindMemoryService(factories); + await svc.close(); + await expect(svc.activateMind(MIND_ID, MIND_PATH)).rejects.toThrow(/closed/i); + }); +}); + +describe('MindMemoryService — activation error rollback', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('daemonFactory throws → db is closed, scheduler not registered, observer not added, mind not tracked', async () => { + const { factories, db, chat, scheduler } = makeFactories({ + chamberConfig: ENABLED_CONFIG, + daemonError: new Error('daemon boom'), + }); + const svc = createMindMemoryService(factories); + + await expect(svc.activateMind(MIND_ID, MIND_PATH)).rejects.toThrow('daemon boom'); + + expect((db as ReturnType).closed).toBe(true); + expect(scheduler.registered.size).toBe(0); + expect(chat.observers).toHaveLength(0); + + // Mind is NOT tracked, so a follow-up release is a no-op AND a follow-up + // activate (after fixing the failure) re-attempts the full build. + await expect(svc.releaseMind(MIND_ID)).resolves.toBeUndefined(); + }); + + it('scheduler.register throws → daemon is closed, db is closed, observer not added, mind not tracked', async () => { + const { factories, db, daemon, chat, scheduler } = makeFactories({ + chamberConfig: ENABLED_CONFIG, + schedulerRegisterError: new Error('cron boom'), + }); + const svc = createMindMemoryService(factories); + + await expect(svc.activateMind(MIND_ID, MIND_PATH)).rejects.toThrow('cron boom'); + + expect((daemon as ReturnType).closeCalls).toBe(1); + expect((db as ReturnType).closed).toBe(true); + expect(scheduler.registered.size).toBe(0); + expect(chat.observers).toHaveLength(0); + }); + + it('dbFactory throws → no other factories called, mind not tracked', async () => { + const { factories, chat, scheduler } = makeFactories({ + chamberConfig: ENABLED_CONFIG, + dbError: new Error('db boom'), + }); + const svc = createMindMemoryService(factories); + + await expect(svc.activateMind(MIND_ID, MIND_PATH)).rejects.toThrow('db boom'); + + expect(factories.daemonFactory).not.toHaveBeenCalled(); + expect(scheduler.registered.size).toBe(0); + expect(chat.observers).toHaveLength(0); + }); +}); + +describe('MindMemoryService — multi-mind isolation', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('release of one mind does not disturb another activated mind', async () => { + const { factories, chat, scheduler } = makeFactories({ chamberConfig: ENABLED_CONFIG }); + const svc = createMindMemoryService(factories); + + await svc.activateMind('m1', '/tmp/m1'); + await svc.activateMind('m2', '/tmp/m2'); + await svc.releaseMind('m1'); + + expect(scheduler.registered.has('m1')).toBe(false); + expect(scheduler.registered.has('m2')).toBe(true); + expect(chat.observers).toHaveLength(1); + }); +}); diff --git a/packages/services/src/mindMemory/MindMemoryService.ts b/packages/services/src/mindMemory/MindMemoryService.ts new file mode 100644 index 00000000..3fbda336 --- /dev/null +++ b/packages/services/src/mindMemory/MindMemoryService.ts @@ -0,0 +1,278 @@ +/** + * MindMemoryService — Phase 11 of the Dream Daemon spike. + * + * Per-mind lifecycle layer that turns the Phase 1–10 collaborators into a + * single activate / release / close API: + * + * - `activateMind(mindId, mindPath)` reads `.chamber.json`, opts the mind + * in only if `workingMemory.consolidation.enabled === true`, opens the + * per-mind `dream.db`, builds vault/archive/daemon via injected + * factories, registers an `InternalScheduler` entry whose fn drives + * `daemon.run()`, and registers a `TurnCompletionObserver` on + * ChatService that forwards completed turns to the per-mind + * `DailyLogWriter`. + * + * - `releaseMind(mindId)` is the exact inverse: scheduler.unregister → + * remove observer → daemon.close → db.close → drop from internal map. + * Idempotent. No-op for unknown / disabled mind ids. + * + * - `close()` releases every activated mind sequentially, then refuses + * subsequent activate calls (fail-fast — keeps the lifecycle invariant + * visible rather than silently leaking minds after global teardown). + * + * Documented choices: + * + * 1. Strict opt-in: `enabled !== true` (NOT just truthy) means OFF. Even + * DailyLogWriter is NOT registered when consolidation is opted out — + * we don't write structured turn frames to disk for minds that haven't + * asked for the feature. + * + * 2. `activateMind` for an already-activated mind is an idempotent no-op. + * Callers must `releaseMind` first to swap configuration; we never + * replace collaborators mid-flight (avoids db/observer leaks if + * `releaseMind` was forgotten on the previous activation). + * + * 3. ChatService observer wiring uses a tiny `addObserver` / + * `removeObserver` pair on ChatService itself (Phase 11 addition, + * smaller than introducing a separate registry abstraction). The + * service depends on the narrow `ChatObserverRegistry` interface so + * tests can fake it. + * + * 4. Activation errors are unwound in reverse construction order: a + * throw from `daemonFactory` closes the db; a throw from + * `scheduler.register` closes the daemon AND the db. The mind is + * never recorded as activated unless every step succeeded. + */ + +import type Database from 'better-sqlite3'; + +import type { TurnCompletionObserver, CompletedTurn } from '@chamber/shared/turn-observer'; + +import { Logger } from '../logger'; +import { loadChamberMindConfig, type ChamberMindConfig, type WorkingMemoryConsolidationConfig } from '../mind/chamberMindConfig'; +import { createDailyLogWriter, type DailyLogWriter } from './DailyLogWriter'; +import type { DreamDaemon } from './DreamDaemon'; +import { dreamDbPath } from './dream-schema'; +import { incrementTurnCount } from './dream-state'; +import type { InternalScheduler } from './InternalScheduler'; +import type { MindArchiveStore } from './MindArchiveStore'; +import type { MindMemoryVault } from './MindMemoryVault'; + +// --------------------------------------------------------------------------- +// Public surface +// --------------------------------------------------------------------------- + +/** Default jitter window for daemon kick-off (defeats thundering herd at 03:00). */ +const DEFAULT_JITTER_MS = 30_000; + +/** + * Narrow ChatService surface MindMemoryService depends on — just the + * observer add/remove pair. Keeps the unit tests free of MindManager, + * TurnQueue, and the SDK harness. + */ +export interface ChatObserverRegistry { + addObserver(observer: TurnCompletionObserver): void; + removeObserver(observer: TurnCompletionObserver): void; +} + +export interface DaemonFactoryOptions { + readonly mindId: string; + readonly mindPath: string; + readonly vault: MindMemoryVault; + readonly archive: MindArchiveStore; + readonly db: Database.Database; + readonly config: WorkingMemoryConsolidationConfig; +} + +export interface MindMemoryServiceFactories { + readonly scheduler: InternalScheduler; + readonly chatService: ChatObserverRegistry; + readonly configReader: (mindPath: string) => ChamberMindConfig; + readonly dbFactory: (dbPath: string) => Database.Database; + readonly vaultFactory: (mindPath: string) => MindMemoryVault; + readonly archiveFactory: (mindPath: string) => MindArchiveStore; + readonly daemonFactory: (opts: DaemonFactoryOptions) => DreamDaemon; + readonly logger?: Logger; + /** Override jitter window (defaults to 30s). */ + readonly jitterMs?: number; +} + +export interface MindMemoryService { + activateMind(mindId: string, mindPath: string): Promise; + releaseMind(mindId: string): Promise; + close(): Promise; +} + +// --------------------------------------------------------------------------- +// Implementation +// --------------------------------------------------------------------------- + +interface ActiveEntry { + readonly mindPath: string; + readonly db: Database.Database; + readonly daemon: DreamDaemon; + readonly writer: DailyLogWriter; + readonly observer: TurnCompletionObserver; +} + +export function createMindMemoryService( + factories: MindMemoryServiceFactories, +): MindMemoryService { + const log = factories.logger ?? Logger.create('MindMemoryService'); + const jitterMs = factories.jitterMs ?? DEFAULT_JITTER_MS; + const active = new Map(); + let closed = false; + + async function activateMind(mindId: string, mindPath: string): Promise { + if (closed) { + throw new Error('MindMemoryService is closed'); + } + if (active.has(mindId)) { + log.debug(`Mind ${mindId} already activated; activateMind is a no-op`); + return; + } + + const config = factories.configReader(mindPath); + const consolidation = config.workingMemory?.consolidation; + // Strict opt-in. `enabled !== true` (not just truthy) means OFF — also + // means we do NOT register DailyLogWriter; the writer would otherwise + // start materializing structured log frames for minds that never asked + // for the feature, defeating the opt-in. + if (!consolidation || consolidation.enabled !== true) { + return; + } + + // Build collaborators in dependency order; unwind on failure. + let db: Database.Database | null = null; + let daemon: DreamDaemon | null = null; + let observer: TurnCompletionObserver | null = null; + let registered = false; + + try { + const vault = factories.vaultFactory(mindPath); + const archive = factories.archiveFactory(mindPath); + db = factories.dbFactory(dreamDbPath(mindPath)); + + daemon = factories.daemonFactory({ + mindId, + mindPath, + vault, + archive, + db, + config: consolidation, + }); + + // DailyLogWriter is built inline — its construction is pure (no I/O + // until a turn arrives), so injecting a writer factory would be + // strictly more wiring without test value. Tests replace this surface + // by faking ChatObserverRegistry and asserting one observer was added. + // + // INVARIANT: `onTurnRecorded` MUST bump `dream_state.turns_since_last_run` + // — otherwise the daemon's activity gate (`minTurnsBetweenRuns >= 1` by + // default) would block consolidation forever. Phase 11 spec wires this + // hook; verified end-to-end by tests/integration/mindMemory.integration + // and packages/services/src/mindMemory/MindMemoryService.test.ts. + const dbForHook = db; + const writer = createDailyLogWriter({ + mindId, + mindPath, + deps: { + onTurnRecorded: () => { + incrementTurnCount(dbForHook, 1); + }, + }, + }); + observer = { + onTurnCompleted: (turn: CompletedTurn) => writer.write(turn), + }; + + factories.scheduler.register({ + mindId, + cronExpr: consolidation.cron, + fn: () => daemon!.run().then(() => undefined), + jitterMs, + }); + registered = true; + + factories.chatService.addObserver(observer); + + active.set(mindId, { mindPath, db, daemon, writer, observer }); + } catch (err) { + // Unwind in reverse order — only what we successfully built. Each + // step is wrapped to keep the original error as the surfaced one. + if (registered) { + try { + factories.scheduler.unregister(mindId); + } catch (releaseErr) { + log.warn(`activate rollback: scheduler.unregister(${mindId}) failed`, releaseErr); + } + } + // Observer is only added after register succeeded; no rollback needed + // unless we move that step earlier in the future. + if (daemon) { + try { + await daemon.close(); + } catch (closeErr) { + log.warn(`activate rollback: daemon.close(${mindId}) failed`, closeErr); + } + } + if (db) { + try { + db.close(); + } catch (closeErr) { + log.warn(`activate rollback: db.close(${mindId}) failed`, closeErr); + } + } + throw err; + } + } + + async function releaseMind(mindId: string): Promise { + const entry = active.get(mindId); + if (!entry) return; + // Drop the map entry FIRST so a teardown failure doesn't leave a + // half-released mind that a subsequent `release` would try to tear + // down again. The daemon and db have their own idempotent close + // contracts; we surface failures via warn but never block. + active.delete(mindId); + + try { + factories.scheduler.unregister(mindId); + } catch (err) { + log.warn(`release: scheduler.unregister(${mindId}) failed`, err); + } + try { + factories.chatService.removeObserver(entry.observer); + } catch (err) { + log.warn(`release: chatService.removeObserver(${mindId}) failed`, err); + } + try { + await entry.daemon.close(); + } catch (err) { + log.warn(`release: daemon.close(${mindId}) failed`, err); + } + try { + entry.db.close(); + } catch (err) { + log.warn(`release: db.close(${mindId}) failed`, err); + } + } + + async function close(): Promise { + if (closed) return; + closed = true; + // Snapshot keys so we don't mutate during iteration (releaseMind + // deletes from `active`). + const ids = Array.from(active.keys()); + for (const id of ids) { + await releaseMind(id); + } + } + + return { activateMind, releaseMind, close }; +} + +// Re-export the default loader so the composition root can pass it as +// `configReader` without an extra import. +export const defaultConfigReader = loadChamberMindConfig; + diff --git a/packages/services/src/mindMemory/MindMemoryVault.test.ts b/packages/services/src/mindMemory/MindMemoryVault.test.ts new file mode 100644 index 00000000..d740145a --- /dev/null +++ b/packages/services/src/mindMemory/MindMemoryVault.test.ts @@ -0,0 +1,227 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs'; +import fsp from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { createMindMemoryVault } from './MindMemoryVault'; + +let mindRoot: string; + +beforeEach(() => { + mindRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-vault-')); +}); + +afterEach(() => { + fs.rmSync(mindRoot, { recursive: true, force: true }); +}); + +describe('MindMemoryVault — root and ensureDir', () => { + it('exposes an absolute, normalized root under /.working-memory/', () => { + const vault = createMindMemoryVault(mindRoot); + expect(path.isAbsolute(vault.root)).toBe(true); + expect(vault.root).toBe(path.resolve(mindRoot, '.working-memory')); + }); + + it('does not touch the disk on construction', () => { + createMindMemoryVault(mindRoot); + expect(fs.existsSync(path.join(mindRoot, '.working-memory'))).toBe(false); + }); + + it('ensureDir is idempotent and creates the working-memory directory', async () => { + const vault = createMindMemoryVault(mindRoot); + await vault.ensureDir(); + await vault.ensureDir(); + expect(fs.statSync(vault.root).isDirectory()).toBe(true); + }); +}); + +describe('MindMemoryVault — read / write / exists', () => { + it('round-trips read/write for a top-level file', async () => { + const vault = createMindMemoryVault(mindRoot); + await vault.write('memory.md', '# memories\n'); + expect(await vault.read('memory.md')).toBe('# memories\n'); + expect(await vault.exists('memory.md')).toBe(true); + }); + + it('returns null when reading a missing file', async () => { + const vault = createMindMemoryVault(mindRoot); + expect(await vault.read('memory.md')).toBeNull(); + expect(await vault.exists('memory.md')).toBe(false); + }); + + it('write is atomic — no .tmp.* files remain after success', async () => { + const vault = createMindMemoryVault(mindRoot); + await vault.write('log.md', 'hello'); + const files = fs.readdirSync(vault.root); + expect(files.some((f) => f.includes('.tmp.'))).toBe(false); + expect(files).toContain('log.md'); + }); + + it('write replaces existing content atomically', async () => { + const vault = createMindMemoryVault(mindRoot); + await vault.write('rules.md', 'first'); + await vault.write('rules.md', 'second'); + expect(await vault.read('rules.md')).toBe('second'); + }); + + it('write creates parent directories under root for nested rel paths', async () => { + const vault = createMindMemoryVault(mindRoot); + await vault.write(path.join('subdir', 'note.md'), 'nested'); + expect(await vault.read(path.join('subdir', 'note.md'))).toBe('nested'); + }); +}); + +describe('MindMemoryVault — append', () => { + it('appends to a non-existent file (creating it)', async () => { + const vault = createMindMemoryVault(mindRoot); + await vault.append('log.md', 'line1\n'); + expect(await vault.read('log.md')).toBe('line1\n'); + }); + + it('appends to existing content', async () => { + const vault = createMindMemoryVault(mindRoot); + await vault.write('log.md', 'a\n'); + await vault.append('log.md', 'b\n'); + expect(await vault.read('log.md')).toBe('a\nb\n'); + }); + + it('serializes concurrent appends to the same file (no interleaving / loss)', async () => { + const vault = createMindMemoryVault(mindRoot); + const lines = Array.from({ length: 50 }, (_, i) => `line-${i.toString().padStart(3, '0')}\n`); + await Promise.all(lines.map((line) => vault.append('log.md', line))); + const content = await vault.read('log.md'); + expect(content).not.toBeNull(); + const sortedLines = (content as string).split('\n').filter(Boolean).sort(); + expect(sortedLines).toEqual(lines.map((l) => l.trimEnd()).sort()); + expect((content as string).length).toBe(lines.reduce((sum, l) => sum + l.length, 0)); + }); +}); + +describe('MindMemoryVault — listFiles', () => { + it('lists top-level files only', async () => { + const vault = createMindMemoryVault(mindRoot); + await vault.write('memory.md', 'm'); + await vault.write('rules.md', 'r'); + await vault.write('log.md', 'l'); + const list = await vault.listFiles(); + expect(list.sort()).toEqual(['log.md', 'memory.md', 'rules.md']); + }); + + it('returns an empty list before ensureDir / when root does not exist', async () => { + const vault = createMindMemoryVault(mindRoot); + expect(await vault.listFiles()).toEqual([]); + }); + + it('excludes the .state/ subdirectory and its contents', async () => { + const vault = createMindMemoryVault(mindRoot); + await vault.write('memory.md', 'm'); + fs.mkdirSync(path.join(vault.root, '.state'), { recursive: true }); + fs.writeFileSync(path.join(vault.root, '.state', 'dream.db'), 'x'); + const list = await vault.listFiles(); + expect(list).toContain('memory.md'); + expect(list).not.toContain('.state'); + expect(list).not.toContain('dream.db'); + }); + + it('excludes the archive/ subdirectory', async () => { + const vault = createMindMemoryVault(mindRoot); + await vault.write('memory.md', 'm'); + fs.mkdirSync(path.join(vault.root, 'archive'), { recursive: true }); + fs.writeFileSync(path.join(vault.root, 'archive', 'something.md'), 'a'); + const list = await vault.listFiles(); + expect(list).toContain('memory.md'); + expect(list).not.toContain('archive'); + expect(list).not.toContain('something.md'); + }); + + it('excludes nested subdirectory contents (only top-level files)', async () => { + const vault = createMindMemoryVault(mindRoot); + await vault.write('memory.md', 'm'); + fs.mkdirSync(path.join(vault.root, 'nested'), { recursive: true }); + fs.writeFileSync(path.join(vault.root, 'nested', 'inner.md'), 'i'); + const list = await vault.listFiles(); + expect(list).toEqual(['memory.md']); + }); +}); + +describe('MindMemoryVault — path traversal guard', () => { + const traversalCases: Array<{ name: string; relPath: string }> = [ + { name: 'parent reference (posix)', relPath: '../escape.md' }, + { name: 'parent reference (windows)', relPath: '..\\escape.md' }, + { name: 'nested parent reference', relPath: 'a/../../escape.md' }, + { name: 'absolute posix path', relPath: '/etc/passwd' }, + { name: 'absolute windows path', relPath: 'C:\\Windows\\System32\\config' }, + { name: 'UNC path', relPath: '\\\\server\\share\\file' }, + { name: 'embedded null byte', relPath: 'good\u0000bad.md' }, + { name: 'just ..', relPath: '..' }, + { name: 'empty string', relPath: '' }, + ]; + + for (const { name, relPath } of traversalCases) { + it(`read rejects ${name}`, async () => { + const vault = createMindMemoryVault(mindRoot); + await expect(vault.read(relPath)).rejects.toThrow(/path|escape|invalid/i); + }); + + it(`write rejects ${name}`, async () => { + const vault = createMindMemoryVault(mindRoot); + await expect(vault.write(relPath, 'x')).rejects.toThrow(/path|escape|invalid/i); + }); + + it(`append rejects ${name}`, async () => { + const vault = createMindMemoryVault(mindRoot); + await expect(vault.append(relPath, 'x')).rejects.toThrow(/path|escape|invalid/i); + }); + + it(`exists rejects ${name}`, async () => { + const vault = createMindMemoryVault(mindRoot); + await expect(vault.exists(relPath)).rejects.toThrow(/path|escape|invalid/i); + }); + } +}); + +describe('MindMemoryVault — mind boundary isolation', () => { + it('two vaults rooted at sibling paths cannot read each other via ..', async () => { + const otherMind = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-vault-other-')); + try { + const vaultA = createMindMemoryVault(mindRoot); + const vaultB = createMindMemoryVault(otherMind); + + await vaultA.write('memory.md', 'A-only'); + await vaultB.write('memory.md', 'B-only'); + + expect(await vaultA.read('memory.md')).toBe('A-only'); + expect(await vaultB.read('memory.md')).toBe('B-only'); + + const otherName = path.basename(otherMind); + const escape = path.join('..', '..', otherName, '.working-memory', 'memory.md'); + await expect(vaultA.read(escape)).rejects.toThrow(/path|escape/i); + + expect(await vaultA.read('memory.md')).toBe('A-only'); + expect(await vaultB.read('memory.md')).toBe('B-only'); + } finally { + fs.rmSync(otherMind, { recursive: true, force: true }); + } + }); + + it('write under mindA does not produce any file under mindB', async () => { + const otherMind = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-vault-other-')); + try { + const vaultA = createMindMemoryVault(mindRoot); + const vaultB = createMindMemoryVault(otherMind); + await vaultA.write('memory.md', 'A'); + await vaultA.write('rules.md', 'A-rules'); + + const bRootExists = fs.existsSync(vaultB.root); + if (bRootExists) { + expect(fs.readdirSync(vaultB.root)).toEqual([]); + } + + const aFiles = await fsp.readdir(vaultA.root); + expect(aFiles.sort()).toEqual(['memory.md', 'rules.md']); + } finally { + fs.rmSync(otherMind, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/services/src/mindMemory/MindMemoryVault.ts b/packages/services/src/mindMemory/MindMemoryVault.ts new file mode 100644 index 00000000..b628ce44 --- /dev/null +++ b/packages/services/src/mindMemory/MindMemoryVault.ts @@ -0,0 +1,190 @@ +/** + * MindMemoryVault — filesystem adapter for `/.working-memory/`. + * + * Phase 3 scope (locked by plan): `node:*` only — no Electron, no Chamber + * Logger, no third-party I/O libs. Errors propagate; callers own logging. + * + * Responsibilities: + * - Atomic writes via temp + rename (no partial writes ever observable). + * - Path-traversal guard rejects every relPath that resolves outside root, + * including absolute paths, `..` segments, drive letters, UNC prefixes, + * and embedded NUL bytes. + * - Per-file in-process append serialization. Cross-process serialization + * is the DailyLogWriter's job (Phase 5). + * - `listFiles()` excludes managed subdirectories (`.state/`, `archive/`, + * and any dotted subdirectory) — only top-level regular files surface + * to the WorkingMemoryComposer (Phase 12). + */ + +import { randomUUID } from 'node:crypto'; +import fs from 'node:fs'; +import fsp from 'node:fs/promises'; +import path from 'node:path'; + +const WORKING_MEMORY_DIRNAME = '.working-memory'; +const ARCHIVE_DIRNAME = 'archive'; +const STATE_DIRNAME = '.state'; + +export interface MindMemoryVault { + readonly root: string; + read(relPath: string): Promise; + write(relPath: string, content: string): Promise; + append(relPath: string, content: string): Promise; + exists(relPath: string): Promise; + listFiles(): Promise; + ensureDir(): Promise; +} + +export function createMindMemoryVault(mindPath: string): MindMemoryVault { + const root = path.resolve(mindPath, WORKING_MEMORY_DIRNAME); + // Per-file in-process mutex chains. Map key = absolute file path. + // Every append on a given file awaits the prior chain link before issuing + // its own read-modify-write cycle, eliminating intra-process interleaving. + const appendChains = new Map>(); + + async function ensureDir(): Promise { + await fsp.mkdir(root, { recursive: true }); + } + + async function read(relPath: string): Promise { + const abs = resolveRelPath(root, relPath); + try { + return await fsp.readFile(abs, 'utf-8'); + } catch (err) { + if (isErrnoCode(err, 'ENOENT')) return null; + throw err; + } + } + + async function write(relPath: string, content: string): Promise { + const abs = resolveRelPath(root, relPath); + await fsp.mkdir(path.dirname(abs), { recursive: true }); + await atomicWriteFile(abs, content); + } + + async function append(relPath: string, content: string): Promise { + const abs = resolveRelPath(root, relPath); + const prior = appendChains.get(abs) ?? Promise.resolve(); + const next = prior.then(async () => { + await fsp.mkdir(path.dirname(abs), { recursive: true }); + const handle = await fsp.open(abs, 'a'); + try { + await handle.write(content); + await handle.sync(); + } finally { + await handle.close(); + } + }); + // Swallow errors on the chain itself (caller still gets the rejection + // through `next`); this prevents one failed append from poisoning the + // queue for later callers. + appendChains.set( + abs, + next.catch(() => undefined), + ); + try { + await next; + } finally { + // GC the chain entry once it's the tail and has settled. + if (appendChains.get(abs) === next || appendChains.get(abs)?.then === next.then) { + // best-effort cleanup; safe to leave entry if a new append slotted in. + } + } + } + + async function exists(relPath: string): Promise { + const abs = resolveRelPath(root, relPath); + try { + await fsp.access(abs, fs.constants.F_OK); + return true; + } catch (err) { + if (isErrnoCode(err, 'ENOENT')) return false; + throw err; + } + } + + async function listFiles(): Promise { + let entries: fs.Dirent[]; + try { + entries = await fsp.readdir(root, { withFileTypes: true }); + } catch (err) { + if (isErrnoCode(err, 'ENOENT')) return []; + throw err; + } + return entries + .filter((entry) => entry.isFile()) + .map((entry) => entry.name) + .filter((name) => name !== ARCHIVE_DIRNAME && name !== STATE_DIRNAME); + } + + return { root, read, write, append, exists, listFiles, ensureDir }; +} + +export function resolveRelPath(root: string, relPath: string): string { + assertSafeRelPath(relPath); + const resolved = path.resolve(root, relPath); + const rootWithSep = root.endsWith(path.sep) ? root : root + path.sep; + if (resolved !== root && !resolved.startsWith(rootWithSep)) { + throw new Error(`path escapes vault root: ${relPath} (root=${root})`); + } + if (resolved === root) { + throw new Error(`path resolves to vault root, not a file: ${relPath} (root=${root})`); + } + return resolved; +} + +function assertSafeRelPath(relPath: string): void { + if (typeof relPath !== 'string' || relPath.length === 0) { + throw new Error('invalid path: must be a non-empty string'); + } + if (relPath.includes('\u0000')) { + throw new Error('invalid path: contains NUL byte'); + } + // Normalize separators so a Windows-style backslash sequence is evaluated + // the same way on POSIX hosts (where `\` would otherwise be a literal char + // and slip past `path.posix.isAbsolute`). + const slashed = relPath.replace(/\\/g, '/'); + if (path.posix.isAbsolute(slashed) || path.win32.isAbsolute(relPath)) { + throw new Error(`invalid path: must be relative, got ${relPath}`); + } + // Reject Windows drive letters (e.g. `C:something`) even without a slash. + if (/^[A-Za-z]:/.test(relPath)) { + throw new Error(`invalid path: drive-relative paths not allowed: ${relPath}`); + } + const normalized = path.posix.normalize(slashed); + const segments = normalized.split('/'); + if (segments.some((seg) => seg === '..')) { + throw new Error(`invalid path: parent traversal not allowed: ${relPath}`); + } +} + +async function atomicWriteFile(absPath: string, content: string): Promise { + const tempPath = `${absPath}.tmp.${randomUUID()}`; + const handle = await fsp.open(tempPath, 'wx'); + try { + await handle.writeFile(content); + await handle.sync(); + } finally { + await handle.close(); + } + try { + await fsp.rename(tempPath, absPath); + } catch (err) { + // Best-effort cleanup if rename fails; surface the original error. + try { + await fsp.unlink(tempPath); + } catch { + // ignore + } + throw err; + } +} + +function isErrnoCode(err: unknown, code: string): boolean { + return ( + typeof err === 'object' && + err !== null && + 'code' in err && + (err as NodeJS.ErrnoException).code === code + ); +} diff --git a/packages/services/src/mindMemory/StructuredLogFormat.test.ts b/packages/services/src/mindMemory/StructuredLogFormat.test.ts new file mode 100644 index 00000000..71447873 --- /dev/null +++ b/packages/services/src/mindMemory/StructuredLogFormat.test.ts @@ -0,0 +1,392 @@ +import { describe, expect, it } from 'vitest'; + +import { + STRUCTURED_LOG_SENTINEL, + detectSentinel, + parseLog, + serializeTurn, + type CompletedTurn, +} from './StructuredLogFormat'; + +const SENTINEL = STRUCTURED_LOG_SENTINEL; + +const baseTurn = (overrides: Partial = {}): CompletedTurn => ({ + turnId: '11111111-1111-4111-8111-111111111111', + sessionId: 'sess-abc', + model: 'gpt-5.5', + status: 'completed', + startedAt: '2026-05-12T15:00:00Z', + endedAt: '2026-05-12T15:00:05Z', + prompt: 'hello', + finalAssistantMessage: 'hi there', + ...overrides, +}); + +describe('STRUCTURED_LOG_SENTINEL', () => { + it('is the exact magic marker locked in the plan', () => { + expect(STRUCTURED_LOG_SENTINEL).toBe(''); + }); +}); + +describe('serializeTurn', () => { + it('emits the canonical frame with double-space separators in the heading', () => { + const out = serializeTurn(baseTurn()); + expect(out).toBe( + '## 2026-05-12T15:00:05Z turn:11111111-1111-4111-8111-111111111111 status:completed\n' + + 'session: sess-abc\n' + + 'model: gpt-5.5\n' + + '\n' + + '### user\n' + + 'hello\n' + + '\n' + + '### assistant\n' + + 'hi there\n', + ); + }); + + it('ends with a trailing newline so multiple turns concatenate cleanly', () => { + const out = serializeTurn(baseTurn()); + expect(out.endsWith('\n')).toBe(true); + }); + + it('preserves multi-line prompts verbatim', () => { + const out = serializeTurn(baseTurn({ prompt: 'line 1\nline 2\nline 3' })); + expect(out).toContain('### user\nline 1\nline 2\nline 3\n\n### assistant'); + }); + + it('preserves multi-line assistant messages verbatim', () => { + const out = serializeTurn(baseTurn({ finalAssistantMessage: 'a\nb\nc' })); + expect(out).toContain('### assistant\na\nb\nc\n'); + }); + + it('renders status:completed', () => { + const out = serializeTurn(baseTurn({ status: 'completed' })); + expect(out).toMatch(/^## .+ {2}status:completed$/m); + }); + + it('renders status:aborted', () => { + const out = serializeTurn(baseTurn({ status: 'aborted' })); + expect(out).toMatch(/^## .+ {2}status:aborted$/m); + }); + + it('renders status:error', () => { + const out = serializeTurn(baseTurn({ status: 'error' })); + expect(out).toMatch(/^## .+ {2}status:error$/m); + }); +}); + +describe('parseLog — sentinel detector', () => { + it('returns sentinel:false for an empty file', () => { + expect(parseLog('')).toEqual({ sentinel: false, turns: [], malformed: 0 }); + }); + + it('returns sentinel:false for whitespace-only content', () => { + const out = parseLog(' \n\n\t\n'); + expect(out.sentinel).toBe(false); + expect(out.turns).toEqual([]); + expect(out.malformed).toBe(0); + }); + + it('detects sentinel after a UTF-8 BOM', () => { + const out = parseLog('\uFEFF' + SENTINEL + '\n'); + expect(out.sentinel).toBe(true); + expect(out.turns).toEqual([]); + expect(out.malformed).toBe(0); + }); + + it('detects sentinel and parses turns when the file uses CRLF line endings', () => { + const turn = baseTurn(); + const content = SENTINEL + '\r\n' + serializeTurn(turn).replace(/\n/g, '\r\n'); + const out = parseLog(content); + expect(out.sentinel).toBe(true); + expect(out.malformed).toBe(0); + expect(out.turns).toHaveLength(1); + expect(out.turns[0].prompt).toBe('hello'); + expect(out.turns[0].assistant).toBe('hi there'); + }); + + it('returns sentinel:false when the marker is missing', () => { + const out = parseLog('# Just notes\nnothing structured here\n'); + expect(out.sentinel).toBe(false); + expect(out.turns).toEqual([]); + expect(out.malformed).toBe(0); + }); + + it('returns sentinel:false when the marker is not the first non-blank line', () => { + const out = parseLog('some prose\n' + SENTINEL + '\n'); + expect(out.sentinel).toBe(false); + expect(out.turns).toEqual([]); + }); + + it('tolerates a duplicated sentinel later in the file without inflating malformed count', () => { + const t = baseTurn({ + finalAssistantMessage: 'before\n' + SENTINEL + '\nafter', + }); + const content = SENTINEL + '\n' + serializeTurn(t); + const out = parseLog(content); + expect(out.sentinel).toBe(true); + expect(out.malformed).toBe(0); + expect(out.turns).toHaveLength(1); + expect(out.turns[0].assistant).toBe('before\n' + SENTINEL + '\nafter'); + }); + + it('detectSentinel agrees with parseLog for canonical input', () => { + const content = SENTINEL + '\n' + serializeTurn(baseTurn()); + expect(detectSentinel(content)).toBe(true); + }); + + it('detectSentinel returns false for empty content', () => { + expect(detectSentinel('')).toBe(false); + }); + + it('detectSentinel returns false when sentinel is not first non-blank line', () => { + expect(detectSentinel('garbage\n' + SENTINEL + '\n')).toBe(false); + }); + + it('detectSentinel handles BOM + CRLF combined', () => { + expect(detectSentinel('\uFEFF' + SENTINEL + '\r\n')).toBe(true); + }); +}); + +describe('parseLog — turn parsing', () => { + it('round-trips a single turn (sentinel + serialize)', () => { + const t = baseTurn(); + const out = parseLog(SENTINEL + '\n' + serializeTurn(t)); + expect(out.sentinel).toBe(true); + expect(out.malformed).toBe(0); + expect(out.turns).toHaveLength(1); + expect(out.turns[0]).toEqual({ + turnId: t.turnId, + sessionId: t.sessionId, + model: t.model, + status: t.status, + timestamp: t.endedAt, + prompt: t.prompt, + assistant: t.finalAssistantMessage, + }); + }); + + it('round-trips multiple turns and preserves order', () => { + const t1 = baseTurn({ + turnId: '11111111-1111-4111-8111-111111111111', + endedAt: '2026-05-12T15:00:05Z', + prompt: 'q1', + finalAssistantMessage: 'a1', + }); + const t2 = baseTurn({ + turnId: '22222222-2222-4222-8222-222222222222', + endedAt: '2026-05-12T15:01:05Z', + prompt: 'q2', + finalAssistantMessage: 'a2', + status: 'aborted', + }); + const t3 = baseTurn({ + turnId: '33333333-3333-4333-8333-333333333333', + endedAt: '2026-05-12T15:02:05Z', + prompt: 'q3', + finalAssistantMessage: 'a3', + status: 'error', + }); + const content = + SENTINEL + '\n' + serializeTurn(t1) + serializeTurn(t2) + serializeTurn(t3); + const out = parseLog(content); + expect(out.malformed).toBe(0); + expect(out.turns.map((t) => t.turnId)).toEqual([t1.turnId, t2.turnId, t3.turnId]); + expect(out.turns.map((t) => t.status)).toEqual(['completed', 'aborted', 'error']); + expect(out.turns.map((t) => t.prompt)).toEqual(['q1', 'q2', 'q3']); + expect(out.turns.map((t) => t.assistant)).toEqual(['a1', 'a2', 'a3']); + }); + + it('drops a block with missing session: line and increments malformed', () => { + const goodTurn = baseTurn({ + turnId: '22222222-2222-4222-8222-222222222222', + endedAt: '2026-05-12T15:01:00Z', + }); + const bad = + '## 2026-05-12T15:00:00Z turn:11111111-1111-4111-8111-111111111111 status:completed\n' + + 'model: gpt-5.5\n' + + '\n### user\nq\n\n### assistant\na\n'; + const content = SENTINEL + '\n' + bad + serializeTurn(goodTurn); + const out = parseLog(content); + expect(out.malformed).toBe(1); + expect(out.turns).toHaveLength(1); + expect(out.turns[0].turnId).toBe(goodTurn.turnId); + }); + + it('drops a block with missing model: line and increments malformed', () => { + const bad = + '## 2026-05-12T15:00:00Z turn:11111111-1111-4111-8111-111111111111 status:completed\n' + + 'session: sess-abc\n' + + '\n### user\nq\n\n### assistant\na\n'; + const out = parseLog(SENTINEL + '\n' + bad); + expect(out.malformed).toBe(1); + expect(out.turns).toHaveLength(0); + }); + + it('drops a block with malformed timestamp', () => { + const bad = + '## not-a-date turn:11111111-1111-4111-8111-111111111111 status:completed\n' + + 'session: s\nmodel: m\n\n### user\nq\n\n### assistant\na\n'; + const out = parseLog(SENTINEL + '\n' + bad); + expect(out.malformed).toBe(1); + expect(out.turns).toHaveLength(0); + }); + + it('drops a block with timestamp not in UTC (no Z suffix)', () => { + const bad = + '## 2026-05-12T15:00:00+02:00 turn:11111111-1111-4111-8111-111111111111 status:completed\n' + + 'session: s\nmodel: m\n\n### user\nq\n\n### assistant\na\n'; + const out = parseLog(SENTINEL + '\n' + bad); + expect(out.malformed).toBe(1); + expect(out.turns).toHaveLength(0); + }); + + it('drops a block with unknown status', () => { + const bad = + '## 2026-05-12T15:00:00Z turn:11111111-1111-4111-8111-111111111111 status:weird\n' + + 'session: s\nmodel: m\n\n### user\nq\n\n### assistant\na\n'; + const out = parseLog(SENTINEL + '\n' + bad); + expect(out.malformed).toBe(1); + expect(out.turns).toHaveLength(0); + }); + + it('drops a block missing the ### user header', () => { + const bad = + '## 2026-05-12T15:00:00Z turn:11111111-1111-4111-8111-111111111111 status:completed\n' + + 'session: s\nmodel: m\n\n### assistant\na\n'; + const out = parseLog(SENTINEL + '\n' + bad); + expect(out.malformed).toBe(1); + expect(out.turns).toHaveLength(0); + }); + + it('drops a block missing the ### assistant header', () => { + const bad = + '## 2026-05-12T15:00:00Z turn:11111111-1111-4111-8111-111111111111 status:completed\n' + + 'session: s\nmodel: m\n\n### user\nq\n'; + const out = parseLog(SENTINEL + '\n' + bad); + expect(out.malformed).toBe(1); + expect(out.turns).toHaveLength(0); + }); + + it('continues parsing after a malformed block', () => { + const good = baseTurn({ + turnId: '22222222-2222-4222-8222-222222222222', + endedAt: '2026-05-12T15:02:00Z', + prompt: 'good', + finalAssistantMessage: 'OK', + }); + const bad = + '## not-a-date turn:11111111-1111-4111-8111-111111111111 status:completed\n' + + 'session: s\nmodel: m\n\n### user\nq\n\n### assistant\na\n'; + const out = parseLog(SENTINEL + '\n' + bad + serializeTurn(good)); + expect(out.malformed).toBe(1); + expect(out.turns).toHaveLength(1); + expect(out.turns[0].prompt).toBe('good'); + }); + + it('discards orphan content before the first heading without flagging it as malformed', () => { + const t = baseTurn(); + const content = + SENTINEL + '\nstray prose between sentinel and first heading\n\n' + serializeTurn(t); + const out = parseLog(content); + expect(out.malformed).toBe(0); + expect(out.turns).toHaveLength(1); + }); + + it('returns no turns when the file contains only the sentinel', () => { + const out = parseLog(SENTINEL + '\n'); + expect(out.sentinel).toBe(true); + expect(out.turns).toEqual([]); + expect(out.malformed).toBe(0); + }); +}); + +describe('parseLog — round-trip property pack', () => { + const fixtures: ReadonlyArray<{ name: string; turn: CompletedTurn }> = [ + { name: 'simple ASCII', turn: baseTurn() }, + { name: 'empty prompt body', turn: baseTurn({ prompt: '' }) }, + { name: 'empty assistant body', turn: baseTurn({ finalAssistantMessage: '' }) }, + { + name: 'UTF-8 multibyte (emoji + Japanese)', + turn: baseTurn({ prompt: '🚀 こんにちは', finalAssistantMessage: '世界 🌍' }), + }, + { name: 'multi-line prompt', turn: baseTurn({ prompt: 'a\nb\nc\nd' }) }, + { + name: 'multi-line assistant', + turn: baseTurn({ finalAssistantMessage: 'one\ntwo\nthree' }), + }, + { + name: 'embedded "## " in prompt that does not match heading regex', + turn: baseTurn({ prompt: '## look at this header\n## another one' }), + }, + { + name: 'embedded "## " in assistant that does not match heading regex', + turn: baseTurn({ finalAssistantMessage: '## final answer\n## see above' }), + }, + { + name: 'inline ### tokens not at column 0', + turn: baseTurn({ + prompt: 'see ###deepheading inline', + finalAssistantMessage: 'foo ### bar baz', + }), + }, + { + name: 'tabs and mixed whitespace', + turn: baseTurn({ prompt: '\thello\tworld', finalAssistantMessage: ' spaced ' }), + }, + { name: 'aborted status', turn: baseTurn({ status: 'aborted' }) }, + { name: 'error status', turn: baseTurn({ status: 'error' }) }, + { name: 'long single-line prompt', turn: baseTurn({ prompt: 'x'.repeat(5000) }) }, + { + name: 'long single-line assistant', + turn: baseTurn({ finalAssistantMessage: 'y'.repeat(5000) }), + }, + { + name: 'truncate marker preserved verbatim in prompt', + turn: baseTurn({ prompt: 'first 2KB of content\n[…truncated, originally 5 KB]' }), + }, + { + name: 'truncate marker preserved verbatim in assistant', + turn: baseTurn({ + finalAssistantMessage: 'reply body\n[…truncated, originally 8 KB]', + }), + }, + { + name: 'fractional second timestamp', + turn: baseTurn({ endedAt: '2026-05-12T15:00:05.123Z' }), + }, + { + name: 'unicode in session/model', + turn: baseTurn({ sessionId: 'sess-✨-1', model: 'gpt-5.5' }), + }, + { + name: 'numeric-looking content', + turn: baseTurn({ prompt: '1234567890', finalAssistantMessage: '0.0001' }), + }, + { + name: 'colon-heavy content (does not confuse frontmatter parser)', + turn: baseTurn({ + prompt: 'a:b:c:d', + finalAssistantMessage: 'session: not-a-real-frontmatter\nmodel: also-not-real', + }), + }, + ]; + + for (const { name, turn } of fixtures) { + it(`round-trips: ${name}`, () => { + const content = STRUCTURED_LOG_SENTINEL + '\n' + serializeTurn(turn); + const out = parseLog(content); + expect(out.sentinel).toBe(true); + expect(out.malformed).toBe(0); + expect(out.turns).toHaveLength(1); + const parsed = out.turns[0]; + expect(parsed.turnId).toBe(turn.turnId); + expect(parsed.sessionId).toBe(turn.sessionId); + expect(parsed.model).toBe(turn.model); + expect(parsed.status).toBe(turn.status); + expect(parsed.timestamp).toBe(turn.endedAt); + expect(parsed.prompt).toBe(turn.prompt); + expect(parsed.assistant).toBe(turn.finalAssistantMessage); + }); + } +}); diff --git a/packages/services/src/mindMemory/StructuredLogFormat.ts b/packages/services/src/mindMemory/StructuredLogFormat.ts new file mode 100644 index 00000000..9e3416d6 --- /dev/null +++ b/packages/services/src/mindMemory/StructuredLogFormat.ts @@ -0,0 +1,213 @@ +/** + * Structured log format (chamber-structured-log/v1) — pure serializer + parser. + * + * A mind's `/.working-memory/log.md` is migrated to a structured form + * whose first non-blank line is the magic sentinel below. Each completed turn + * is appended as a self-delimited frame so the Dream Daemon can consume the + * log deterministically. + * + * Phase 2 scope (locked by plan): pure module only. No fs, no Electron, no + * Logger. Operates on strings in / strings out. Byte-budget truncation lives + * on `DailyLogWriter` (Phase 5). + * + * Frame format: + * + * + * ## turn: status: + * session: + * model: + * + * ### user + * + * + * ### assistant + * + * + * Heading separator: TWO spaces between `## ` and `turn:` and + * between `turn:` and `status:` are required. + * + * Embedded heading escape strategy: + * - The turn-heading regex anchors to line start AND requires the trailing + * `turn: status:` pair. Plain `## something` lines inside a + * body therefore parse as content, not as new turns. + * - The body-section markers `### user` and `### assistant` are recognised + * only when they (a) appear on their own line at column 0 AND (b) are + * immediately preceded by a blank line. Inline text such as + * `see ### user mode docs` or `## a markdown heading` round-trips fine. + * Pathological round-trips of bodies whose own content contains a + * blank-line-then-`### user|assistant` sequence are out of scope; callers + * producing such content must escape it themselves. + * + * The parser is deliberately tolerant: malformed blocks are dropped and + * counted, never thrown over, so a partially-corrupt log never blocks the + * daemon. + */ + +export const STRUCTURED_LOG_SENTINEL = ''; + +// `CompletedTurn` and `TurnStatus` were relocated to `@chamber/shared` in +// Phase 6 so ChatService (the producer) and DailyLogWriter (the first +// consumer) depend on a single canonical shape. Re-exported here for +// backward compatibility with Phase 5 callers that import from this module. +export type { CompletedTurn, TurnStatus } from '@chamber/shared/turn-observer'; +import type { CompletedTurn, TurnStatus } from '@chamber/shared/turn-observer'; + +export interface ParsedTurn { + readonly turnId: string; + readonly sessionId: string; + readonly model: string; + readonly status: TurnStatus; + readonly timestamp: string; + readonly prompt: string; + readonly assistant: string; +} + +export interface ParsedLog { + readonly sentinel: boolean; + readonly turns: ParsedTurn[]; + readonly malformed: number; +} + +const HEADING_RE = /^## (\S+) {2}turn:(\S+) {2}status:(\S+)$/; +const ISO_UTC_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/; +const STATUS_VALUES: ReadonlySet = new Set([ + 'completed', + 'aborted', + 'error', +]); + +function normalize(content: string): string { + let s = content; + if (s.length > 0 && s.charCodeAt(0) === 0xfeff) { + s = s.slice(1); + } + return s.replace(/\r\n/g, '\n').replace(/\r/g, '\n'); +} + +function firstNonBlankLine(lines: readonly string[]): { idx: number; value: string } | null { + for (let i = 0; i < lines.length; i++) { + if (lines[i].trim() !== '') { + return { idx: i, value: lines[i] }; + } + } + return null; +} + +export function detectSentinel(content: string): boolean { + const lines = normalize(content).split('\n'); + const first = firstNonBlankLine(lines); + return first !== null && first.value === STRUCTURED_LOG_SENTINEL; +} + +export function serializeTurn(turn: CompletedTurn): string { + const heading = `## ${turn.endedAt} turn:${turn.turnId} status:${turn.status}`; + return ( + heading + + '\n' + + `session: ${turn.sessionId}\n` + + `model: ${turn.model}\n` + + '\n' + + '### user\n' + + turn.prompt + + '\n' + + '\n' + + '### assistant\n' + + turn.finalAssistantMessage + + '\n' + ); +} + +function parseBlock(blockLines: readonly string[]): ParsedTurn | null { + if (blockLines.length === 0) return null; + + const headingMatch = blockLines[0].match(HEADING_RE); + if (!headingMatch) return null; + const [, ts, turnId, statusRaw] = headingMatch; + + if (!ISO_UTC_RE.test(ts) || Number.isNaN(Date.parse(ts))) return null; + if (!STATUS_VALUES.has(statusRaw)) return null; + const status = statusRaw as TurnStatus; + + if (blockLines.length < 3) return null; + const sessionMatch = blockLines[1].match(/^session: (.+)$/); + const modelMatch = blockLines[2].match(/^model: (.+)$/); + if (!sessionMatch || !modelMatch) return null; + const sessionId = sessionMatch[1]; + const model = modelMatch[1]; + + // `### user` must appear at column 0, preceded by a blank line. Earliest + // possible position is index 4: heading(0), session(1), model(2), blank(3). + let userIdx = -1; + for (let i = 4; i < blockLines.length; i++) { + if (blockLines[i] === '### user' && blockLines[i - 1] === '') { + userIdx = i; + break; + } + } + if (userIdx === -1) return null; + + let assistantIdx = -1; + for (let i = userIdx + 2; i < blockLines.length; i++) { + if (blockLines[i] === '### assistant' && blockLines[i - 1] === '') { + assistantIdx = i; + break; + } + } + if (assistantIdx === -1) return null; + + const userBodyLines = blockLines.slice(userIdx + 1, assistantIdx); + const assistantBodyLines = blockLines.slice(assistantIdx + 1); + + // Trim a single trailing blank line introduced by the serializer's + // section terminator on the user body, and any trailing blanks on the + // assistant body produced by concatenated turns or the final newline. + while (userBodyLines.length > 0 && userBodyLines[userBodyLines.length - 1] === '') { + userBodyLines.pop(); + } + while ( + assistantBodyLines.length > 0 && + assistantBodyLines[assistantBodyLines.length - 1] === '' + ) { + assistantBodyLines.pop(); + } + + return { + turnId, + sessionId, + model, + status, + timestamp: ts, + prompt: userBodyLines.join('\n'), + assistant: assistantBodyLines.join('\n'), + }; +} + +export function parseLog(content: string): ParsedLog { + const lines = normalize(content).split('\n'); + const first = firstNonBlankLine(lines); + if (first === null || first.value !== STRUCTURED_LOG_SENTINEL) { + return { sentinel: false, turns: [], malformed: 0 }; + } + + const headingIdxs: number[] = []; + for (let i = first.idx + 1; i < lines.length; i++) { + if (HEADING_RE.test(lines[i])) { + headingIdxs.push(i); + } + } + + const turns: ParsedTurn[] = []; + let malformed = 0; + for (let h = 0; h < headingIdxs.length; h++) { + const blockStart = headingIdxs[h]; + const blockEnd = h + 1 < headingIdxs.length ? headingIdxs[h + 1] : lines.length; + const parsed = parseBlock(lines.slice(blockStart, blockEnd)); + if (parsed === null) { + malformed += 1; + } else { + turns.push(parsed); + } + } + + return { sentinel: true, turns, malformed }; +} diff --git a/packages/services/src/mindMemory/__fakes__/FakeLLMClient.test.ts b/packages/services/src/mindMemory/__fakes__/FakeLLMClient.test.ts new file mode 100644 index 00000000..b2d0ef36 --- /dev/null +++ b/packages/services/src/mindMemory/__fakes__/FakeLLMClient.test.ts @@ -0,0 +1,53 @@ +/** + * Phase 8 — FakeLLMClient sanity tests. + * + * Phase 9 (DreamDaemon) wires its orchestrator tests against this fake; + * keeping a small smoke around the helper itself prevents drift in the + * canned-response semantics from silently skewing daemon tests later. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createFakeLLMClient } from './FakeLLMClient'; + +beforeEach(() => { + vi.useFakeTimers(); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe('createFakeLLMClient', () => { + it('records each synthesize call', async () => { + const client = createFakeLLMClient(); + await client.synthesize({ prompt: 'a', timeoutMs: 1_000 }); + await client.synthesize({ prompt: 'b', timeoutMs: 2_000, maxTokens: 32 }); + expect(client.calls.map((c) => c.prompt)).toEqual(['a', 'b']); + expect(client.calls[1].maxTokens).toBe(32); + }); + + it('returns the longest matching prefix response', async () => { + const client = createFakeLLMClient({ + responses: { 'memory:': 'short', 'memory:weekly:': 'long' }, + defaultResponse: 'fallback', + }); + expect(await client.synthesize({ prompt: 'memory:weekly:foo', timeoutMs: 1_000 })).toBe('long'); + expect(await client.synthesize({ prompt: 'memory:daily:foo', timeoutMs: 1_000 })).toBe('short'); + expect(await client.synthesize({ prompt: 'other', timeoutMs: 1_000 })).toBe('fallback'); + }); + + it('throws the configured error verbatim', async () => { + const client = createFakeLLMClient({ error: new Error('nope') }); + await expect(client.synthesize({ prompt: 'p', timeoutMs: 1_000 })).rejects.toThrow('nope'); + }); + + it('rejects with the canonical timeout message when latency exceeds timeoutMs', async () => { + const client = createFakeLLMClient({ latencyMs: 5_000 }); + const settled = client.synthesize({ prompt: 'p', timeoutMs: 250 }) + .catch((e: unknown) => e as Error); + await vi.advanceTimersByTimeAsync(250); + const err = await settled; + expect((err as Error).message).toBe('LLM synthesis timed out after 250ms'); + }); +}); diff --git a/packages/services/src/mindMemory/__fakes__/FakeLLMClient.ts b/packages/services/src/mindMemory/__fakes__/FakeLLMClient.ts new file mode 100644 index 00000000..808c946b --- /dev/null +++ b/packages/services/src/mindMemory/__fakes__/FakeLLMClient.ts @@ -0,0 +1,68 @@ +/** + * FakeLLMClient — in-memory `LLMClient` for unit tests. + * + * Phase 8 ships this so Phase 9 (DreamDaemon) can drive the orchestrator + * with canned responses keyed by prompt prefix. Co-located under + * `__fakes__/` because it is intentionally NOT part of the public package + * surface (`mindMemory/index.ts` re-exports the interface, not this helper). + */ + +import type { LLMClient, SynthesizeRequest } from '../LLMClient'; + +export interface FakeLLMClientOptions { + /** + * Map of prompt-prefix → canned response. The longest matching prefix + * wins so callers can layer specific overrides on top of generic ones. + */ + readonly responses?: Record; + /** Returned when no prefix matches. Defaults to an empty string. */ + readonly defaultResponse?: string; + /** Forced rejection; useful for error-path tests. */ + readonly error?: Error; + /** + * Artificial latency in ms. The fake honors `timeoutMs` by rejecting + * with the same shape `CopilotLLMClient` uses, so daemon tests can + * exercise the timeout branch without a real adapter. + */ + readonly latencyMs?: number; +} + +export interface FakeLLMClient extends LLMClient { + readonly calls: ReadonlyArray; +} + +export function createFakeLLMClient(options: FakeLLMClientOptions = {}): FakeLLMClient { + const calls: SynthesizeRequest[] = []; + const responses = options.responses ?? {}; + const prefixes = Object.keys(responses).sort((a, b) => b.length - a.length); + + return { + get calls() { + return calls; + }, + async synthesize(req: SynthesizeRequest): Promise { + calls.push(req); + if (options.error) throw options.error; + + const latency = options.latencyMs ?? 0; + if (latency > 0) { + const timedOut = await new Promise((resolve) => { + const t = setTimeout(() => resolve(false), latency); + const onTimeout = setTimeout(() => { + clearTimeout(t); + resolve(true); + }, req.timeoutMs); + t.unref?.(); + onTimeout.unref?.(); + }); + if (timedOut) { + throw new Error(`LLM synthesis timed out after ${req.timeoutMs}ms`); + } + } + + const match = prefixes.find((p) => req.prompt.startsWith(p)); + if (match) return responses[match]; + return options.defaultResponse ?? ''; + }, + }; +} diff --git a/packages/services/src/mindMemory/consolidation-priorities.test.ts b/packages/services/src/mindMemory/consolidation-priorities.test.ts new file mode 100644 index 00000000..9c9317b8 --- /dev/null +++ b/packages/services/src/mindMemory/consolidation-priorities.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, it } from 'vitest'; + +import { + type EntryPriority, + getEntryPriority, + sortByPriority, + trimToFit, +} from './consolidation-priorities'; +import type { MemoryEntry } from './memory-entries'; + +function makeEntry( + type: MemoryEntry['type'], + name: string, + createdAt?: string, + content = 'placeholder body for the entry', +): MemoryEntry { + return { + type, + name, + description: `${name} description`, + content, + createdAt, + }; +} + +describe('getEntryPriority', () => { + it('maps user → critical', () => { + expect(getEntryPriority(makeEntry('user', 'X'))).toBe('critical'); + }); + + it('maps prohibition → critical', () => { + expect(getEntryPriority(makeEntry('prohibition', 'X'))).toBe('critical'); + }); + + it('maps feedback → high', () => { + expect(getEntryPriority(makeEntry('feedback', 'X'))).toBe('high'); + }); + + it('maps project → medium', () => { + expect(getEntryPriority(makeEntry('project', 'X'))).toBe('medium'); + }); + + it('maps reference → low', () => { + expect(getEntryPriority(makeEntry('reference', 'X'))).toBe('low'); + }); +}); + +describe('sortByPriority', () => { + it('returns critical entries before lower priorities', () => { + const entries: MemoryEntry[] = [ + makeEntry('reference', 'R'), + makeEntry('user', 'U'), + makeEntry('project', 'P'), + makeEntry('prohibition', 'PRO'), + makeEntry('feedback', 'F'), + ]; + const sorted = sortByPriority(entries); + const types = sorted.map((e) => e.type); + // critical (user, prohibition) → high (feedback) → medium (project) → low (reference) + expect(types.slice(0, 2)).toEqual(expect.arrayContaining(['user', 'prohibition'])); + expect(types[2]).toBe('feedback'); + expect(types[3]).toBe('project'); + expect(types[4]).toBe('reference'); + }); + + it('sorts within the same priority by createdAt descending (newer first)', () => { + const entries: MemoryEntry[] = [ + makeEntry('user', 'old', '2026-01-01T00:00:00Z'), + makeEntry('user', 'new', '2026-05-01T00:00:00Z'), + makeEntry('user', 'middle', '2026-03-01T00:00:00Z'), + ]; + const sorted = sortByPriority(entries).map((e) => e.name); + expect(sorted).toEqual(['new', 'middle', 'old']); + }); + + it('does not mutate the input array', () => { + const entries: MemoryEntry[] = [ + makeEntry('reference', 'R'), + makeEntry('user', 'U'), + ]; + const snapshot = [...entries]; + sortByPriority(entries); + expect(entries).toEqual(snapshot); + }); + + it('handles empty input', () => { + expect(sortByPriority([])).toEqual([]); + }); + + it('places entries with createdAt before those without (within same priority)', () => { + const entries: MemoryEntry[] = [ + makeEntry('user', 'no-date'), + makeEntry('user', 'has-date', '2026-05-01T00:00:00Z'), + ]; + const sorted = sortByPriority(entries).map((e) => e.name); + expect(sorted).toEqual(['has-date', 'no-date']); + }); +}); + +describe('trimToFit', () => { + it('returns [] for empty input', () => { + expect(trimToFit([], 10, 1000)).toEqual([]); + }); + + it('returns the original entries when they already fit', () => { + const entries: MemoryEntry[] = [makeEntry('user', 'A'), makeEntry('project', 'B')]; + const out = trimToFit(entries, 200, 25_000); + expect(out).toHaveLength(2); + }); + + it('drops lowest-priority entries first when over the line limit', () => { + const entries: MemoryEntry[] = [ + makeEntry('reference', 'low-pri'), + makeEntry('project', 'mid-pri'), + makeEntry('user', 'top-pri'), + ]; + // Each serialized entry takes ~6 lines (frontmatter + content). Limit at 12 lines forces + // trimming the lowest-priority (reference) entry. + const out = trimToFit(entries, 12, 25_000); + const names = out.map((e) => e.name); + expect(names).toContain('top-pri'); + expect(names).not.toContain('low-pri'); + }); + + it('returns [] when the limits cannot fit even the highest-priority entry', () => { + const entries: MemoryEntry[] = [makeEntry('user', 'unfittable')]; + expect(trimToFit(entries, 1, 10)).toEqual([]); + }); + + it('drops by priority then by age within a priority', () => { + const entries: MemoryEntry[] = [ + makeEntry('reference', 'newest-low', '2026-05-01T00:00:00Z'), + makeEntry('reference', 'oldest-low', '2026-01-01T00:00:00Z'), + makeEntry('user', 'critical', '2026-05-01T00:00:00Z'), + ]; + // Force enough pressure to drop one reference entry — older one should go. + const out = trimToFit(entries, 15, 25_000); + const names = out.map((e) => e.name); + expect(names).toContain('critical'); + expect(names).toContain('newest-low'); + expect(names).not.toContain('oldest-low'); + }); +}); diff --git a/packages/services/src/mindMemory/consolidation-priorities.ts b/packages/services/src/mindMemory/consolidation-priorities.ts new file mode 100644 index 00000000..07a966c1 --- /dev/null +++ b/packages/services/src/mindMemory/consolidation-priorities.ts @@ -0,0 +1,64 @@ +/** + * Entry prioritization for MEMORY.md consolidation. + * Determines which entries to keep when pruning is needed. + * + * Pure module: no I/O, no logging. + */ +import { type MemoryEntry, serializeMemoryMd } from './memory-entries'; +import { countBytes, countLines } from './memory-limits'; + +export type EntryPriority = 'critical' | 'high' | 'medium' | 'low'; + +const PRIORITY_MAP: Record = { + user: 'critical', + feedback: 'high', + prohibition: 'critical', + project: 'medium', + reference: 'low', +}; + +const PRIORITY_RANK: Record = { + critical: 0, + high: 1, + medium: 2, + low: 3, +}; + +export function getEntryPriority(entry: MemoryEntry): EntryPriority { + return PRIORITY_MAP[entry.type]; +} + +export function sortByPriority(entries: ReadonlyArray): ReadonlyArray { + return [...entries].sort((a, b) => { + const rankDiff = PRIORITY_RANK[getEntryPriority(a)] - PRIORITY_RANK[getEntryPriority(b)]; + if (rankDiff !== 0) return rankDiff; + + const dateA = a.createdAt ?? ''; + const dateB = b.createdAt ?? ''; + if (dateA && dateB) return dateB.localeCompare(dateA); + if (dateA) return -1; + if (dateB) return 1; + return 0; + }); +} + +export function trimToFit( + entries: ReadonlyArray, + maxLines: number, + maxBytes: number, +): ReadonlyArray { + if (entries.length === 0) return []; + + const sorted = sortByPriority(entries); + let current = [...sorted]; + + while (current.length > 0) { + const serialized = serializeMemoryMd(current); + if (countLines(serialized) <= maxLines && countBytes(serialized) <= maxBytes) { + return current; + } + current = current.slice(0, -1); + } + + return []; +} diff --git a/packages/services/src/mindMemory/consolidation-scheduler.test.ts b/packages/services/src/mindMemory/consolidation-scheduler.test.ts new file mode 100644 index 00000000..63595ebf --- /dev/null +++ b/packages/services/src/mindMemory/consolidation-scheduler.test.ts @@ -0,0 +1,289 @@ +/** + * Tests for consolidation-scheduler — cron evaluation, per-mind in-memory + * mutex (try-lock), and tick orchestration that combines cron, gates, DB + * lock, and run/persist. + * + * Phase 7 acceptance: + * - evaluateCron answers due/not-due against a controlled clock. + * - withMindMutex serializes per mind; concurrent acquires fail-fast. + * - tick() respects gates, DB lock, and the mutex. + * - tick() records run history on success and skips with a reason on + * each gate failure path. + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import Database from 'better-sqlite3'; + +import { migrate } from './dream-schema'; +import { + acquireLock, + buildLockHolder, + getLock, + incrementTurnCount, + listRuns, + readState, +} from './dream-state'; +import { + __resetMindMutexForTesting, + createConsolidationScheduler, + evaluateCron, + withMindMutex, +} from './consolidation-scheduler'; + +let db: Database.Database; + +beforeEach(() => { + db = new Database(':memory:'); + migrate(db); + __resetMindMutexForTesting(); +}); + +afterEach(() => { + db.close(); +}); + +describe('consolidation-scheduler — evaluateCron', () => { + it('reports due when nextRun(lastFireAt) <= now', () => { + // Daily at 03:00 UTC. + const last = new Date('2026-05-12T00:00:00Z'); + const now = new Date('2026-05-12T03:00:01Z'); + const r = evaluateCron('0 3 * * *', last, now, { timezone: 'UTC' }); + expect(r.due).toBe(true); + expect(r.nextDueAt!.toISOString()).toBe('2026-05-12T03:00:00.000Z'); + }); + + it('reports not-due when nextRun is in the future', () => { + const last = new Date('2026-05-12T03:00:00Z'); + const now = new Date('2026-05-12T04:00:00Z'); + const r = evaluateCron('0 3 * * *', last, now, { timezone: 'UTC' }); + expect(r.due).toBe(false); + expect(r.nextDueAt!.toISOString()).toBe('2026-05-13T03:00:00.000Z'); + }); + + it('treats lastFireAt=null as the unix epoch (always-due on first tick)', () => { + const r = evaluateCron('0 3 * * *', null, new Date('2026-05-12T03:00:01Z'), { timezone: 'UTC' }); + expect(r.due).toBe(true); + }); +}); + +describe('consolidation-scheduler — withMindMutex (try-lock)', () => { + it('serializes runs for the same mindId; second concurrent caller fails fast', async () => { + let release!: () => void; + const blocker = new Promise((r) => { + release = r; + }); + + const a = withMindMutex('mind-x', async () => { + await blocker; + return 'a-done'; + }); + const b = withMindMutex('mind-x', async () => 'b-done'); + + await expect(b).resolves.toEqual({ acquired: false, reason: 'locked' }); + release(); + await expect(a).resolves.toEqual({ acquired: true, value: 'a-done' }); + }); + + it('different mindIds do not block each other', async () => { + let release!: () => void; + const blocker = new Promise((r) => { + release = r; + }); + + const a = withMindMutex('mind-a', async () => { + await blocker; + return 'a'; + }); + const b = withMindMutex('mind-b', async () => 'b'); + + await expect(b).resolves.toEqual({ acquired: true, value: 'b' }); + release(); + await expect(a).resolves.toEqual({ acquired: true, value: 'a' }); + }); + + it('releases the slot after rejection so the next call may proceed', async () => { + const first = withMindMutex('mind-x', async () => { + throw new Error('boom'); + }); + await expect(first).rejects.toThrow(/boom/); + + const second = withMindMutex('mind-x', async () => 'ok'); + await expect(second).resolves.toEqual({ acquired: true, value: 'ok' }); + }); +}); + +describe('consolidation-scheduler — tick', () => { + function buildScheduler(opts: { + now: number; + runDaily?: () => Promise; + cron?: string; + minTurnsBetweenRuns?: number; + minIntervalMs?: number; + lockTtlMs?: number; + }) { + let current = opts.now; + const clock = () => new Date(current); + const setNow = (n: number) => { + current = n; + }; + + const sched = createConsolidationScheduler({ + mindId: 'mind-x', + db, + cronExpr: opts.cron ?? '* * * * *', // every minute by default + gateConfig: { + minTurnsBetweenRuns: opts.minTurnsBetweenRuns ?? 1, + minIntervalMs: opts.minIntervalMs ?? 0, + }, + lockTtlMs: opts.lockTtlMs ?? 60_000, + clock, + runDaily: + opts.runDaily ?? + (async () => { + /* no-op */ + }), + timezone: 'UTC', + }); + + return { sched, setNow }; + } + + it('runs daily, records the run, marks phase complete, and resets activity', async () => { + incrementTurnCount(db, 3); + const { sched } = buildScheduler({ now: 1_700_000_000_000 }); + + const r = await sched.tick(); + expect(r.run).toBe(true); + expect(r.reason).toBe('ready'); + expect(r.phase).toBe('daily'); + + const state = readState(db); + expect(state.turnsSinceLastRun).toBe(0); + expect(state.lastDailyAt).toBe(1_700_000_000_000); + + const runs = listRuns(db); + expect(runs).toHaveLength(1); + expect(runs[0].status).toBe('success'); + expect(runs[0].phase).toBe('daily'); + + expect(getLock(db, 'daily')).toBeNull(); + }); + + it('skips with reason=no-activity when activity gate fails (no run recorded as success)', async () => { + const { sched } = buildScheduler({ now: 1_700_000_000_000, minTurnsBetweenRuns: 5 }); + const r = await sched.tick(); + expect(r.run).toBe(false); + expect(r.reason).toBe('no-activity'); + + const runs = listRuns(db); + expect(runs).toHaveLength(1); + expect(runs[0].status).toBe('skipped'); + expect(runs[0].reason).toBe('no-activity'); + }); + + it('skips with reason=cron-not-due when cron has not yet matched', async () => { + incrementTurnCount(db, 1); + // Cron at 03:00 UTC every day; current time is 02:00 UTC and lastDailyAt + // is "today at 03:00", so nextRun(03:00) is tomorrow at 03:00 → not due. + const today03 = new Date('2026-05-12T03:00:00Z').getTime(); + const today02 = new Date('2026-05-12T02:00:00Z').getTime(); + db.prepare('UPDATE dream_state SET last_daily_at = ? WHERE id = 1').run(today03); + + const { sched } = buildScheduler({ + now: today02 + 24 * 60 * 60 * 1000, // jump a day forward but before next 03:00 + cron: '0 3 * * *', + }); + // Adjust: now = 2026-05-13T02:00:00Z, last = 2026-05-12T03:00:00Z, + // nextRun(last) = 2026-05-13T03:00:00Z → still > now, not due. + const r = await sched.tick(); + expect(r.run).toBe(false); + expect(r.reason).toBe('cron-not-due'); + }); + + it('returns reason=locked when the in-memory mutex is held by a concurrent tick', async () => { + incrementTurnCount(db, 3); + + let release!: () => void; + const blocker = new Promise((r) => { + release = r; + }); + + const { sched } = buildScheduler({ + now: 1_700_000_000_000, + runDaily: async () => { + await blocker; + }, + }); + + const first = sched.tick(); + const second = await sched.tick(); + + expect(second.run).toBe(false); + expect(second.reason).toBe('locked'); + + release(); + const r1 = await first; + expect(r1.run).toBe(true); + }); + + it('returns reason=db-locked when another process holds a fresh DB lock', async () => { + incrementTurnCount(db, 3); + // Simulate another process holding the lock with plenty of TTL left. + acquireLock(db, { + phase: 'daily', + mindId: 'someone-else', + now: 1_700_000_000_000, + ttlMs: 60_000, + }); + + const { sched } = buildScheduler({ now: 1_700_000_000_000 }); + const r = await sched.tick(); + expect(r.run).toBe(false); + expect(r.reason).toBe('db-locked'); + + // Skipped run must be recorded. + const runs = listRuns(db); + expect(runs.some((x) => x.status === 'skipped' && x.reason === 'db-locked')).toBe(true); + }); + + it('breaks a stale DB lock and proceeds', async () => { + incrementTurnCount(db, 3); + db.prepare( + 'INSERT INTO dream_locks (phase, holder, acquired_at, expires_at) VALUES (?, ?, ?, ?)', + ).run('daily', buildLockHolder('ghost', 1, 'old'), 0, 1); + + const { sched } = buildScheduler({ now: 1_700_000_000_000 }); + const r = await sched.tick(); + expect(r.run).toBe(true); + expect(r.reason).toBe('ready'); + }); + + it('records a failed run when runDaily throws and releases the lock', async () => { + incrementTurnCount(db, 3); + const failing = vi.fn(async () => { + throw new Error('synthetic consolidator failure'); + }); + + const { sched } = buildScheduler({ now: 1_700_000_000_000, runDaily: failing }); + + await expect(sched.tick()).rejects.toThrow(/synthetic consolidator failure/); + + expect(failing).toHaveBeenCalledTimes(1); + const runs = listRuns(db); + expect(runs[0].status).toBe('failed'); + expect(getLock(db, 'daily')).toBeNull(); + // Activity counter is NOT reset on failure (the consolidator did not run + // to completion). + expect(readState(db).turnsSinceLastRun).toBe(3); + }); + + it('getNextDueAt projects from lastDailyAt', () => { + db.prepare('UPDATE dream_state SET last_daily_at = ? WHERE id = 1').run( + new Date('2026-05-12T03:00:00Z').getTime(), + ); + const { sched } = buildScheduler({ now: Date.UTC(2026, 4, 12, 3, 0, 0), cron: '0 3 * * *' }); + const next = sched.getNextDueAt(); + expect(next!.toISOString()).toBe('2026-05-13T03:00:00.000Z'); + }); +}); diff --git a/packages/services/src/mindMemory/consolidation-scheduler.ts b/packages/services/src/mindMemory/consolidation-scheduler.ts new file mode 100644 index 00000000..21e29aaa --- /dev/null +++ b/packages/services/src/mindMemory/consolidation-scheduler.ts @@ -0,0 +1,267 @@ +/** + * consolidation-scheduler — composes croner-based cron evaluation, the + * per-mind in-memory mutex, the DB lock, and the activity/time gates into + * a single `tick()` orchestration. + * + * Three layers of mutual exclusion: + * + * 1. Per-mind in-memory mutex (try-lock). Defeats same-process races + * where two `tick()` calls land in the JS event loop before the DB + * lock row is written. Implemented as `Map` with + * fail-fast semantics — the second concurrent caller returns + * `{ acquired: false, reason: 'locked' }` instead of queuing. + * + * 2. DB lock row in `dream_locks`. Defeats cross-process races. Honors + * a configurable TTL so a crashed daemon cannot wedge the mind + * forever — see `dream-state.acquireLock`. + * + * 3. Combined activity + time gate (`dream-gates.evaluateGates`). The + * cron expression is treated as a coarse "is it the configured wake + * time yet?" filter on top of the time gate. + * + * Phase 7 ships only the daily phase; weekly/monthly land in Phase 9. + * + * `evaluateCron` is exported separately so the scheduler can be stubbed + * in higher-level tests without dragging in croner. + */ + +import { Cron } from 'croner'; +import type Database from 'better-sqlite3'; + +import { Logger } from '../logger'; +import { evaluateGates, type GateConfig } from './dream-gates'; +import { + acquireLock, + buildLockHolder, + getLock, + markPhaseComplete, + readState, + recordRun, + releaseLock, +} from './dream-state'; + +export interface CronEvaluation { + readonly due: boolean; + readonly nextDueAt: Date | null; +} + +export function evaluateCron( + cronExpr: string, + lastFireAt: Date | null, + now: Date, + opts: { timezone?: string } = {}, +): CronEvaluation { + // `paused: true` disables the croner background timer — we only use the + // expression evaluator. + const job = new Cron(cronExpr, { paused: true, timezone: opts.timezone }); + try { + const seed = lastFireAt ?? new Date(0); + const next = job.nextRun(seed); + if (!next) return { due: false, nextDueAt: null }; + return { due: next.getTime() <= now.getTime(), nextDueAt: next }; + } finally { + job.stop(); + } +} + +// --------------------------------------------------------------------------- +// Per-mind in-memory mutex (try-lock) +// --------------------------------------------------------------------------- + +const mindMutex = new Map>(); + +export type WithMindMutexResult = + | { readonly acquired: true; readonly value: T } + | { readonly acquired: false; readonly reason: 'locked' }; + +export async function withMindMutex( + mindId: string, + fn: () => Promise, +): Promise> { + if (mindMutex.has(mindId)) { + return { acquired: false, reason: 'locked' }; + } + const promise = (async () => fn())(); + mindMutex.set( + mindId, + promise.then( + () => undefined, + () => undefined, + ), + ); + try { + const value = await promise; + return { acquired: true, value }; + } finally { + mindMutex.delete(mindId); + } +} + +/** Test-only helper to drop in-memory state between tests. */ +export function __resetMindMutexForTesting(): void { + mindMutex.clear(); +} + +// --------------------------------------------------------------------------- +// Scheduler +// --------------------------------------------------------------------------- + +export interface RunContext { + readonly mindId: string; + readonly startedAt: number; +} + +export interface SchedulerOptions { + readonly mindId: string; + readonly db: Database.Database; + readonly cronExpr: string; + readonly gateConfig: GateConfig; + readonly lockTtlMs: number; + readonly clock: () => Date; + readonly runDaily: (ctx: RunContext) => Promise; + readonly timezone?: string; +} + +export type TickReason = + | 'ready' + | 'locked' + | 'db-locked' + | 'no-activity' + | 'too-soon' + | 'cron-not-due'; + +export interface TickResult { + readonly run: boolean; + readonly reason: TickReason; + readonly phase?: 'daily'; +} + +export interface ConsolidationScheduler { + tick(): Promise; + getNextDueAt(): Date | null; +} + +export function createConsolidationScheduler(opts: SchedulerOptions): ConsolidationScheduler { + const log = Logger.create('ConsolidationScheduler'); + + function projectNextDue(): Date | null { + const state = readState(opts.db); + const last = state.lastDailyAt !== null ? new Date(state.lastDailyAt) : null; + return evaluateCron(opts.cronExpr, last, opts.clock(), { timezone: opts.timezone }).nextDueAt; + } + + async function runOnceLocked(): Promise { + const startedAtDate = opts.clock(); + const now = startedAtDate.getTime(); + + const state = readState(opts.db); + const lockRow = getLock(opts.db, 'daily'); + const lockHeldByOther = lockRow !== null && lockRow.expiresAt > now; + + // Cron evaluation — the configured wake time must have passed since + // the last successful daily run. + const cron = evaluateCron( + opts.cronExpr, + state.lastDailyAt !== null ? new Date(state.lastDailyAt) : null, + startedAtDate, + { timezone: opts.timezone }, + ); + if (!cron.due) { + recordRun(opts.db, { + phase: 'daily', + startedAt: now, + endedAt: now, + status: 'skipped', + reason: 'cron-not-due', + }); + return { run: false, reason: 'cron-not-due', phase: 'daily' }; + } + + // Activity + time gate (lockHeld surfaced for completeness, but + // db-locked is reported separately below for clearer triage). + const gate = evaluateGates( + { phase: 'daily', state, now, lockHeld: lockHeldByOther }, + opts.gateConfig, + ); + if (!gate.run) { + const reason: TickReason = gate.reason === 'locked' ? 'db-locked' : gate.reason; + recordRun(opts.db, { + phase: 'daily', + startedAt: now, + endedAt: now, + status: 'skipped', + reason, + }); + return { run: false, reason, phase: 'daily' }; + } + + // Try to acquire the DB lock (with stale-break). + const lock = acquireLock(opts.db, { + phase: 'daily', + mindId: opts.mindId, + now, + ttlMs: opts.lockTtlMs, + }); + if (!lock.acquired) { + recordRun(opts.db, { + phase: 'daily', + startedAt: now, + endedAt: now, + status: 'skipped', + reason: 'db-locked', + }); + return { run: false, reason: 'db-locked', phase: 'daily' }; + } + + const holder = lock.holder ?? buildLockHolder(opts.mindId); + try { + await opts.runDaily({ mindId: opts.mindId, startedAt: now }); + const endedAt = opts.clock().getTime(); + + // Persist success: timestamp + reset activity + run history. + // Activity reset is intentional only on success — a failed run + // leaves the counter intact so the next tick will retry. + opts.db.transaction(() => { + opts.db.prepare('UPDATE dream_state SET turns_since_last_run = 0, last_daily_at = ? WHERE id = 1').run(endedAt); + })(); + // markPhaseComplete is functionally equivalent for daily but kept + // for symmetry with future weekly/monthly callers. + markPhaseComplete(opts.db, 'daily', endedAt); + recordRun(opts.db, { + phase: 'daily', + startedAt: now, + endedAt, + status: 'success', + reason: 'ready', + }); + return { run: true, reason: 'ready', phase: 'daily' }; + } catch (err) { + const endedAt = opts.clock().getTime(); + const message = err instanceof Error ? err.message : String(err); + recordRun(opts.db, { + phase: 'daily', + startedAt: now, + endedAt, + status: 'failed', + reason: message, + }); + log.warn(`daily consolidation failed for mind ${opts.mindId}: ${message}`); + throw err; + } finally { + releaseLock(opts.db, 'daily', holder); + } + } + + return { + async tick(): Promise { + const outcome = await withMindMutex(opts.mindId, () => runOnceLocked()); + if (!outcome.acquired) { + return { run: false, reason: 'locked', phase: 'daily' }; + } + return outcome.value; + }, + getNextDueAt(): Date | null { + return projectNextDue(); + }, + }; +} diff --git a/packages/services/src/mindMemory/consolidation.test.ts b/packages/services/src/mindMemory/consolidation.test.ts new file mode 100644 index 00000000..116641c6 --- /dev/null +++ b/packages/services/src/mindMemory/consolidation.test.ts @@ -0,0 +1,290 @@ +import { describe, it, expect } from 'vitest'; +import { + orient, + gather, + consolidate, + prune, + runConsolidation, +} from './consolidation'; +import { serializeMemoryMd } from './memory-entries'; +import type { MemoryEntry } from './memory-entries'; + +function entry(overrides: Partial = {}): MemoryEntry { + return { + type: 'project', + name: 'test-entry', + description: 'A test entry', + content: 'Some test content.', + ...overrides, + }; +} + +function buildMd(entries: MemoryEntry[]): string { + return serializeMemoryMd(entries); +} + +const REF_DATE = new Date('2025-06-15'); + +describe('orient', () => { + it('parses valid MEMORY.md into correct entries', () => { + const entries = [ + entry({ name: 'Pref A', type: 'user', description: 'desc A', content: 'content A' }), + entry({ name: 'Proj B', type: 'project', description: 'desc B', content: 'content B' }), + ]; + const md = buildMd(entries); + const result = orient(md); + expect(result).toHaveLength(2); + expect(result[0]!.name).toBe('Pref A'); + expect(result[1]!.name).toBe('Proj B'); + }); + + it('empty MEMORY.md → empty array', () => { + expect(orient('')).toEqual([]); + expect(orient(' ')).toEqual([]); + }); + + it('malformed MEMORY.md → empty array (graceful)', () => { + expect(orient('not valid markdown at all')).toEqual([]); + expect(orient('---\nbroken\n')).toEqual([]); + }); + + it('converts relative dates to absolute', () => { + const entries = [entry({ name: 'E1', content: 'Discovered yesterday in code review.' })]; + const md = buildMd(entries); + const result = orient(md, REF_DATE); + expect(result[0]!.content).toContain('2025-06-14'); + expect(result[0]!.content).not.toContain('yesterday'); + }); +}); + +describe('gather', () => { + it('valid entries pass through', () => { + const input: MemoryEntry[] = [entry({ name: 'G1' }), entry({ name: 'G2' })]; + const result = gather(input); + expect(result).toHaveLength(2); + }); + + it('invalid entries filtered out', () => { + const input = [ + entry({ name: 'Good' }), + { type: 'user', name: '', description: '', content: '' } as unknown as MemoryEntry, + ]; + const result = gather(input); + expect(result).toHaveLength(1); + expect(result[0]!.name).toBe('Good'); + }); + + it('converts relative dates', () => { + const input = [entry({ name: 'E1', content: 'Added yesterday.' })]; + const result = gather(input, REF_DATE); + expect(result[0]!.content).toContain('2025-06-14'); + }); + + it('empty input → empty output', () => { + expect(gather([])).toEqual([]); + }); +}); + +describe('consolidate', () => { + it('existing + new merged correctly (new entries are newer)', () => { + const existing = [entry({ name: 'Ex1', content: 'existing' })]; + const gathered = [entry({ name: 'New1', content: 'new thing' })]; + const result = consolidate(existing, gathered); + expect(result.entries.length).toBeGreaterThanOrEqual(2); + }); + + it('duplicate entries deduplicated (keep newer)', () => { + const existing = [ + entry({ name: 'DB Info', content: 'Uses PostgreSQL with Supabase for data.' }), + ]; + const gathered = [ + entry({ name: 'Database Info', content: 'Uses PostgreSQL with Supabase for data!' }), + ]; + const result = consolidate(existing, gathered); + expect(result.entries).toHaveLength(1); + expect(result.entries[0]!.name).toBe('Database Info'); + expect(result.deduped).toBeGreaterThanOrEqual(1); + }); + + it('contradicting entries resolved (keep newer)', () => { + const existing = [entry({ name: 'DB', content: 'Use MySQL' })]; + const gathered = [entry({ name: 'DB', content: 'Use PostgreSQL' })]; + const result = consolidate(existing, gathered); + expect(result.entries).toHaveLength(1); + expect(result.entries[0]!.content).toBe('Use PostgreSQL'); + expect(result.contradictions).toBeGreaterThanOrEqual(1); + }); + + it('no duplicates → all kept, deduped=0', () => { + const existing = [entry({ name: 'X', content: 'xxx-distinct' })]; + const gathered = [entry({ name: 'Y', content: 'yyy-different' })]; + const result = consolidate(existing, gathered); + expect(result.entries).toHaveLength(2); + expect(result.deduped).toBe(0); + expect(result.contradictions).toBe(0); + }); + + it('groups entries by type (user before reference)', () => { + const existing: MemoryEntry[] = []; + const gathered: MemoryEntry[] = [ + entry({ + type: 'reference', + name: 'Ref1', + content: 'A useful reference link.', + createdAt: '2025-01-01', + }), + entry({ + type: 'user', + name: 'Usr1', + content: 'User prefers dark mode always.', + createdAt: '2025-01-01', + }), + entry({ + type: 'project', + name: 'Prj1', + content: 'Project uses Node.js runtime.', + createdAt: '2025-01-01', + }), + ]; + const result = consolidate(existing, gathered); + const types = result.entries.map((e) => e.type); + const userIdx = types.indexOf('user'); + const refIdx = types.indexOf('reference'); + expect(userIdx).toBeLessThan(refIdx); + }); +}); + +describe('prune', () => { + it('under limits → no pruning, truncated=false', () => { + const entries = [entry({ name: 'Small', content: 'tiny.' })]; + const result = prune(entries); + expect(result.truncated).toBe(false); + expect(result.entries).toHaveLength(1); + }); + + it('over line limit → entries removed to fit', () => { + const entries: MemoryEntry[] = []; + for (let i = 0; i < 50; i++) { + entries.push( + entry({ + name: `Entry-${i}`, + type: i < 5 ? 'user' : 'reference', + content: `Line 1\nLine 2\nLine 3\nLine 4\nLine 5`, + createdAt: `2025-01-${String(i + 1).padStart(2, '0')}`, + }), + ); + } + const result = prune(entries); + expect(result.entries.length).toBeLessThan(50); + }); + + it('reference entries dropped before user entries', () => { + const entries: MemoryEntry[] = []; + for (let i = 0; i < 10; i++) { + entries.push( + entry({ + name: `User-${i}`, + type: 'user', + content: 'User pref line.', + createdAt: `2025-06-${String(i + 1).padStart(2, '0')}`, + }), + ); + } + for (let i = 0; i < 40; i++) { + entries.push( + entry({ + name: `Ref-${i}`, + type: 'reference', + content: `Ref line 1\nRef line 2\nRef line 3\nRef line 4`, + createdAt: `2025-01-${String(i + 1).padStart(2, '0')}`, + }), + ); + } + const result = prune(entries); + const userCount = result.entries.filter((e) => e.type === 'user').length; + expect(userCount).toBe(10); + }); + + it('truncation as final safety net', () => { + const entries: MemoryEntry[] = [ + entry({ name: 'Big-user', type: 'user', content: 'X\n'.repeat(250) }), + ]; + const result = prune(entries); + expect(result.truncated).toBe(true); + }); +}); + +describe('runConsolidation', () => { + it('full pipeline: existing MEMORY.md + new entries → improved MEMORY.md', () => { + const existingMd = buildMd([ + entry({ name: 'Existing', type: 'project', content: 'Existing project info.' }), + ]); + const newEntries: MemoryEntry[] = [ + entry({ name: 'New Pref', type: 'user', content: 'New user preference.' }), + ]; + const result = runConsolidation({ currentMemoryMd: existingMd, newEntries }); + expect(result.memoryMd).toContain('Existing'); + expect(result.memoryMd).toContain('New Pref'); + expect(result.entriesProcessed).toBe(2); + expect(result.entriesKept).toBe(2); + }); + + it('existing with same-name new entry → contradiction resolved (keep newer)', () => { + const existingMd = buildMd([entry({ name: 'SharedName', content: 'old content' })]); + const newEntries: MemoryEntry[] = [entry({ name: 'SharedName', content: 'updated content' })]; + const result = runConsolidation({ currentMemoryMd: existingMd, newEntries }); + expect(result.contradictionsResolved).toBeGreaterThanOrEqual(1); + expect(result.memoryMd).toContain('updated content'); + expect(result.entriesKept).toBe(1); + }); + + it('phase log has correct stats', () => { + const existingMd = buildMd([entry({ name: 'A', type: 'user', content: 'A content' })]); + const newEntries: MemoryEntry[] = [entry({ name: 'B', type: 'project', content: 'B content' })]; + const result = runConsolidation({ currentMemoryMd: existingMd, newEntries }); + expect(result.phases.orient.existingEntries).toBe(1); + expect(result.phases.gather.newEntries).toBe(1); + expect(result.phases.consolidate.merged).toBe(2); + }); + + it('result fits within limits when overflowing', () => { + const newEntries: MemoryEntry[] = Array.from({ length: 60 }, (_, i) => + entry({ + name: `Entry-${i}`, + type: 'reference', + content: `Line 1\nLine 2\nLine 3\nLine 4`, + createdAt: `2025-01-${String((i % 28) + 1).padStart(2, '0')}`, + }), + ); + const result = runConsolidation({ currentMemoryMd: '', newEntries }); + const lines = result.memoryMd.split('\n').length; + expect(lines).toBeLessThanOrEqual(210); + }); + + it('idempotency: run twice with same input → same output', () => { + const existingMd = buildMd([ + entry({ name: 'Stable', type: 'user', content: 'Stable content.' }), + ]); + const newEntries: MemoryEntry[] = [ + entry({ name: 'New', type: 'project', content: 'New content.' }), + ]; + const input = { currentMemoryMd: existingMd, newEntries, referenceDate: REF_DATE }; + const result1 = runConsolidation(input); + const result2 = runConsolidation(input); + expect(result1.memoryMd).toBe(result2.memoryMd); + expect(result1.entriesKept).toBe(result2.entriesKept); + }); + + it('referenceDate is used for date conversion', () => { + const newEntries: MemoryEntry[] = [ + entry({ name: 'DatedEntry', content: 'Found yesterday in the logs.' }), + ]; + const result = runConsolidation({ + currentMemoryMd: '', + newEntries, + referenceDate: REF_DATE, + }); + expect(result.memoryMd).toContain('2025-06-14'); + expect(result.memoryMd).not.toContain('yesterday'); + }); +}); diff --git a/packages/services/src/mindMemory/consolidation.ts b/packages/services/src/mindMemory/consolidation.ts new file mode 100644 index 00000000..661d7518 --- /dev/null +++ b/packages/services/src/mindMemory/consolidation.ts @@ -0,0 +1,198 @@ +/** + * Four-phase consolidation pipeline for MEMORY.md. + * orient → gather → consolidate → prune + * + * Pure functions — no file I/O, no database access. + * + * Ported from SCNS (`scns/src/dream/consolidation.ts`). + */ +import type { MemoryEntry } from './memory-entries'; +import { + parseMemoryMd, + serializeMemoryMd, + deduplicateEntries, + resolveContradictions, + validateMemoryEntry, +} from './memory-entries'; +import { + truncateEntrypoint, + countLines, + countBytes, + MAX_ENTRYPOINT_LINES, + MAX_ENTRYPOINT_BYTES, +} from './memory-limits'; +import { convertRelativeDates } from './date-utils'; +import { sortByPriority, trimToFit } from './consolidation-priorities'; + +export interface ConsolidationInput { + readonly currentMemoryMd: string; + readonly newEntries: ReadonlyArray; + readonly referenceDate?: Date; +} + +export interface ConsolidationPhaseLog { + readonly orient: { existingEntries: number }; + readonly gather: { newEntries: number }; + readonly consolidate: { merged: number; deduped: number; contradictions: number }; + readonly prune: { + beforeLines: number; + afterLines: number; + beforeBytes: number; + afterBytes: number; + }; +} + +export interface ConsolidationResult { + readonly memoryMd: string; + readonly entriesProcessed: number; + readonly entriesKept: number; + readonly duplicatesRemoved: number; + readonly contradictionsResolved: number; + readonly truncated: boolean; + readonly phases: ConsolidationPhaseLog; +} + +export function orient( + currentMemoryMd: string, + referenceDate: Date = new Date(), +): ReadonlyArray { + const entries = parseMemoryMd(currentMemoryMd); + return entries.map((e) => ({ + ...e, + content: convertRelativeDates(e.content, referenceDate), + description: convertRelativeDates(e.description, referenceDate), + })); +} + +export function gather( + newEntries: ReadonlyArray, + referenceDate: Date = new Date(), +): ReadonlyArray { + return newEntries + .filter((e) => validateMemoryEntry(e)) + .map((e) => ({ + ...e, + content: convertRelativeDates(e.content, referenceDate), + description: convertRelativeDates(e.description, referenceDate), + })); +} + +export interface ConsolidateResult { + readonly entries: ReadonlyArray; + readonly deduped: number; + readonly contradictions: number; +} + +export function consolidate( + existing: ReadonlyArray, + gathered: ReadonlyArray, +): ConsolidateResult { + const merged = [...existing, ...gathered]; + const mergedCount = merged.length; + + const afterContradictions = resolveContradictions(merged); + const contradictions = mergedCount - afterContradictions.length; + + const afterDedup = deduplicateEntries(afterContradictions); + const deduped = afterContradictions.length - afterDedup.length; + + const sorted = sortByPriority(afterDedup); + + return { entries: sorted, deduped, contradictions }; +} + +export interface PruneResult { + readonly entries: ReadonlyArray; + readonly truncated: boolean; + readonly linesRemoved: number; +} + +export function prune(entries: ReadonlyArray): PruneResult { + if (entries.length === 0) { + return { entries: [], truncated: false, linesRemoved: 0 }; + } + + const beforeSerialized = serializeMemoryMd(entries); + const beforeLines = countLines(beforeSerialized); + + const trimmed = trimToFit(entries, MAX_ENTRYPOINT_LINES, MAX_ENTRYPOINT_BYTES); + + if (trimmed.length === 0 && entries.length > 0) { + const truncResult = truncateEntrypoint(beforeSerialized); + const reparsed = parseMemoryMd(truncResult.content); + const afterLines = countLines(serializeMemoryMd(reparsed)); + return { + entries: reparsed, + truncated: true, + linesRemoved: beforeLines - afterLines, + }; + } + + const serialized = serializeMemoryMd(trimmed); + const truncResult = truncateEntrypoint(serialized); + + if (truncResult.truncated) { + const reparsed = parseMemoryMd(truncResult.content); + const afterLines = countLines(serializeMemoryMd(reparsed)); + return { + entries: reparsed, + truncated: true, + linesRemoved: beforeLines - afterLines, + }; + } + + const afterLines = countLines(serialized); + return { + entries: trimmed, + truncated: false, + linesRemoved: beforeLines - afterLines, + }; +} + +export function runConsolidation(input: ConsolidationInput): ConsolidationResult { + const refDate = input.referenceDate ?? new Date(); + + const oriented = orient(input.currentMemoryMd, refDate); + const gathered = gather(input.newEntries, refDate); + const consolidated = consolidate(oriented, gathered); + + const beforePrune = serializeMemoryMd(consolidated.entries); + const beforePruneLines = countLines(beforePrune); + const beforePruneBytes = countBytes(beforePrune); + const pruned = prune(consolidated.entries); + const finalMd = serializeMemoryMd(pruned.entries); + + let resultMd = finalMd; + if (pruned.truncated) { + const truncResult = truncateEntrypoint(finalMd); + resultMd = truncResult.truncated ? truncResult.content : finalMd; + } + + const afterMd = resultMd; + const afterLines = countLines(afterMd); + const afterBytes = countBytes(afterMd); + + return { + memoryMd: afterMd, + entriesProcessed: oriented.length + gathered.length, + entriesKept: pruned.entries.length, + duplicatesRemoved: consolidated.deduped, + contradictionsResolved: consolidated.contradictions, + truncated: pruned.truncated, + phases: { + orient: { existingEntries: oriented.length }, + gather: { newEntries: gathered.length }, + consolidate: { + merged: oriented.length + gathered.length, + deduped: consolidated.deduped, + contradictions: consolidated.contradictions, + }, + prune: { + beforeLines: beforePruneLines, + afterLines, + beforeBytes: beforePruneBytes, + afterBytes, + }, + }, + }; +} diff --git a/packages/services/src/mindMemory/date-utils.test.ts b/packages/services/src/mindMemory/date-utils.test.ts new file mode 100644 index 00000000..8fe532c8 --- /dev/null +++ b/packages/services/src/mindMemory/date-utils.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from 'vitest'; + +import { convertRelativeDates, parseRelativeDate } from './date-utils'; + +const REF = new Date('2026-05-12T00:00:00Z'); + +describe('convertRelativeDates', () => { + it('returns the content untouched when no relative phrases are present', () => { + expect(convertRelativeDates('nothing relative here', REF)).toBe('nothing relative here'); + }); + + it('converts "yesterday" to ref - 1 day', () => { + expect(convertRelativeDates('met yesterday with team', REF)).toBe( + 'met 2026-05-11 with team', + ); + }); + + it('converts "today" and "this morning" to ref date', () => { + expect(convertRelativeDates('shipped today', REF)).toBe('shipped 2026-05-12'); + expect(convertRelativeDates('this morning the build broke', REF)).toBe( + '2026-05-12 the build broke', + ); + }); + + it('converts "last week" to ref - 7 days', () => { + expect(convertRelativeDates('decided last week', REF)).toBe('decided 2026-05-05'); + }); + + it('converts "last month" to ref - 30 days', () => { + expect(convertRelativeDates('happened last month', REF)).toBe('happened 2026-04-12'); + }); + + it('converts "a few days ago" to ref - 3 days', () => { + expect(convertRelativeDates('a few days ago we shipped', REF)).toBe( + '2026-05-09 we shipped', + ); + }); + + it('converts " days ago" with the parsed integer offset', () => { + expect(convertRelativeDates('5 days ago', REF)).toBe('2026-05-07'); + expect(convertRelativeDates('1 day ago', REF)).toBe('2026-05-11'); + }); + + it('is case-insensitive for keyword forms', () => { + expect(convertRelativeDates('YESTERDAY', REF)).toBe('2026-05-11'); + expect(convertRelativeDates('Last Week', REF)).toBe('2026-05-05'); + }); + + it('replaces multiple occurrences in one pass', () => { + expect(convertRelativeDates('today and yesterday', REF)).toBe('2026-05-12 and 2026-05-11'); + }); + + it('only matches whole-word boundaries (does not corrupt "yesterdays")', () => { + expect(convertRelativeDates('yesterdays plans', REF)).toBe('yesterdays plans'); + }); +}); + +describe('convertRelativeDates fix for "a few days ago"', () => { + // Documented: 3 days before 2026-05-12 == 2026-05-09. The earlier example used + // 2026-04-12 by mistake. Re-verify with explicit small math here. + it('"a few days ago" resolves to ref - 3 days exactly', () => { + const ref = new Date('2026-05-12T00:00:00Z'); + expect(convertRelativeDates('a few days ago', ref)).toBe('2026-05-09'); + }); +}); + +describe('parseRelativeDate', () => { + it('returns the resolved Date for a single matching phrase', () => { + const out = parseRelativeDate('yesterday', REF); + expect(out).not.toBeNull(); + expect(out?.toISOString().slice(0, 10)).toBe('2026-05-11'); + }); + + it('returns the resolved Date for " days ago"', () => { + const out = parseRelativeDate('14 days ago', REF); + expect(out?.toISOString().slice(0, 10)).toBe('2026-04-28'); + }); + + it('returns null when the input is not a recognised relative phrase', () => { + expect(parseRelativeDate('next thursday', REF)).toBeNull(); + expect(parseRelativeDate('', REF)).toBeNull(); + expect(parseRelativeDate('something else', REF)).toBeNull(); + }); + + it('returns null when the input contains the phrase but extra text', () => { + expect(parseRelativeDate('yesterday at noon', REF)).toBeNull(); + }); + + it('trims whitespace before matching', () => { + expect(parseRelativeDate(' today ', REF)?.toISOString().slice(0, 10)).toBe('2026-05-12'); + }); +}); diff --git a/packages/services/src/mindMemory/date-utils.ts b/packages/services/src/mindMemory/date-utils.ts new file mode 100644 index 00000000..477908f2 --- /dev/null +++ b/packages/services/src/mindMemory/date-utils.ts @@ -0,0 +1,61 @@ +/** + * Date conversion utilities — convert relative date references to absolute ISO dates. + * + * Pure module: no I/O, no logging. + */ + +interface DatePattern { + readonly regex: RegExp; + readonly resolve: (match: RegExpMatchArray, ref: Date) => Date; +} + +function subtractDays(date: Date, days: number): Date { + const d = new Date(date); + d.setUTCDate(d.getUTCDate() - days); + return d; +} + +function formatISO(date: Date): string { + return date.toISOString().slice(0, 10); +} + +const PATTERNS: readonly DatePattern[] = [ + { regex: /\byesterday\b/gi, resolve: (_m, ref) => subtractDays(ref, 1) }, + { regex: /\btoday\b/gi, resolve: (_m, ref) => subtractDays(ref, 0) }, + { regex: /\bthis morning\b/gi, resolve: (_m, ref) => subtractDays(ref, 0) }, + { regex: /\blast week\b/gi, resolve: (_m, ref) => subtractDays(ref, 7) }, + { regex: /\blast month\b/gi, resolve: (_m, ref) => subtractDays(ref, 30) }, + { regex: /\ba few days ago\b/gi, resolve: (_m, ref) => subtractDays(ref, 3) }, + { + regex: /\b(\d+)\s+days?\s+ago\b/gi, + resolve: (m, ref) => subtractDays(ref, parseInt(m[1]!, 10)), + }, +]; + +export function convertRelativeDates(content: string, referenceDate: Date = new Date()): string { + let result = content; + + for (const pattern of PATTERNS) { + result = result.replace(pattern.regex, (...args) => { + const match = args as unknown as RegExpMatchArray; + const date = pattern.resolve(match, referenceDate); + return formatISO(date); + }); + } + + return result; +} + +export function parseRelativeDate(text: string, referenceDate: Date): Date | null { + const trimmed = text.trim(); + + for (const pattern of PATTERNS) { + const rx = new RegExp(pattern.regex.source, pattern.regex.flags); + const match = rx.exec(trimmed); + if (match && match[0].length === trimmed.length) { + return pattern.resolve(match, referenceDate); + } + } + + return null; +} diff --git a/packages/services/src/mindMemory/dream-gates.test.ts b/packages/services/src/mindMemory/dream-gates.test.ts new file mode 100644 index 00000000..12674b0b --- /dev/null +++ b/packages/services/src/mindMemory/dream-gates.test.ts @@ -0,0 +1,170 @@ +/** + * Tests for dream-gates — combined activity/time/lock gate evaluation. + * + * Phase 7 acceptance: + * - All three gates must pass for run=true. + * - Each failure produces a distinct, machine-readable reason. + * - Boundary conditions: exactly threshold, exact equality on time gate. + */ + +import { describe, expect, it } from 'vitest'; + +import { + DEFAULT_DAILY_GATE, + evaluateGates, + type GateConfig, + type GateInput, +} from './dream-gates'; +import type { DreamState } from './dream-state'; + +const MS_PER_DAY = 24 * 60 * 60 * 1000; + +function baseState(overrides: Partial = {}): DreamState { + return { + turnsSinceLastRun: 5, + lastDailyAt: null, + lastWeeklyAt: null, + lastMonthlyAt: null, + lastConsolidatedTurnId: null, + ...overrides, + }; +} + +function input(overrides: Partial = {}): GateInput { + return { + phase: 'daily', + state: baseState(), + now: 10 * MS_PER_DAY, + lockHeld: false, + ...overrides, + }; +} + +describe('dream-gates — lock gate', () => { + it('returns run=false reason=locked when the lock is held, regardless of activity/time', () => { + const r = evaluateGates( + input({ + lockHeld: true, + state: baseState({ turnsSinceLastRun: 1000, lastDailyAt: 0 }), + }), + DEFAULT_DAILY_GATE, + ); + expect(r.run).toBe(false); + expect(r.reason).toBe('locked'); + }); +}); + +describe('dream-gates — activity gate', () => { + it('passes when turnsSinceLastRun >= minTurnsBetweenRuns', () => { + const cfg: GateConfig = { minTurnsBetweenRuns: 5, minIntervalMs: 0 }; + const r = evaluateGates(input({ state: baseState({ turnsSinceLastRun: 5 }) }), cfg); + expect(r.run).toBe(true); + expect(r.reason).toBe('ready'); + }); + + it('fails when turnsSinceLastRun < minTurnsBetweenRuns', () => { + const cfg: GateConfig = { minTurnsBetweenRuns: 5, minIntervalMs: 0 }; + const r = evaluateGates(input({ state: baseState({ turnsSinceLastRun: 4 }) }), cfg); + expect(r.run).toBe(false); + expect(r.reason).toBe('no-activity'); + }); + + it('fails on zero turns even when threshold is 1', () => { + const cfg: GateConfig = { minTurnsBetweenRuns: 1, minIntervalMs: 0 }; + const r = evaluateGates(input({ state: baseState({ turnsSinceLastRun: 0 }) }), cfg); + expect(r.run).toBe(false); + expect(r.reason).toBe('no-activity'); + }); +}); + +describe('dream-gates — time gate', () => { + it('passes when no prior daily run has been recorded', () => { + const r = evaluateGates( + input({ state: baseState({ turnsSinceLastRun: 1, lastDailyAt: null }), now: 1 }), + DEFAULT_DAILY_GATE, + ); + expect(r.run).toBe(true); + }); + + it('fails when (now - lastDailyAt) < minIntervalMs', () => { + const r = evaluateGates( + input({ + state: baseState({ turnsSinceLastRun: 10, lastDailyAt: 5 * MS_PER_DAY }), + now: 5 * MS_PER_DAY + (MS_PER_DAY - 1), + }), + DEFAULT_DAILY_GATE, + ); + expect(r.run).toBe(false); + expect(r.reason).toBe('too-soon'); + }); + + it('passes at exactly the interval boundary', () => { + const r = evaluateGates( + input({ + state: baseState({ turnsSinceLastRun: 10, lastDailyAt: 5 * MS_PER_DAY }), + now: 5 * MS_PER_DAY + MS_PER_DAY, + }), + DEFAULT_DAILY_GATE, + ); + expect(r.run).toBe(true); + }); +}); + +describe('dream-gates — phase selection', () => { + it('weekly phase reads lastWeeklyAt, ignores lastDailyAt', () => { + const cfg: GateConfig = { minTurnsBetweenRuns: 1, minIntervalMs: MS_PER_DAY }; + const r = evaluateGates( + input({ + phase: 'weekly', + state: baseState({ + turnsSinceLastRun: 1, + lastDailyAt: 999 * MS_PER_DAY, // would block daily + lastWeeklyAt: null, + }), + now: 10 * MS_PER_DAY, + }), + cfg, + ); + expect(r.run).toBe(true); + }); + + it('monthly phase reads lastMonthlyAt', () => { + const cfg: GateConfig = { minTurnsBetweenRuns: 1, minIntervalMs: MS_PER_DAY }; + const r = evaluateGates( + input({ + phase: 'monthly', + state: baseState({ + turnsSinceLastRun: 1, + lastMonthlyAt: 10 * MS_PER_DAY - 1, + }), + now: 10 * MS_PER_DAY, + }), + cfg, + ); + expect(r.run).toBe(false); + expect(r.reason).toBe('too-soon'); + }); +}); + +describe('dream-gates — combination matrix', () => { + const cfg: GateConfig = { minTurnsBetweenRuns: 5, minIntervalMs: MS_PER_DAY }; + + it.each([ + ['activity-fail, time-fail, unlocked', { turns: 1, lastAt: 9.5 * MS_PER_DAY, lockHeld: false }, false, 'no-activity'], + ['activity-pass, time-fail, unlocked', { turns: 5, lastAt: 9.5 * MS_PER_DAY, lockHeld: false }, false, 'too-soon'], + ['activity-fail, time-pass, unlocked', { turns: 1, lastAt: 0, lockHeld: false }, false, 'no-activity'], + ['activity-pass, time-pass, locked', { turns: 5, lastAt: 0, lockHeld: true }, false, 'locked'], + ['activity-pass, time-pass, unlocked', { turns: 5, lastAt: 0, lockHeld: false }, true, 'ready'], + ])('%s → run=%s reason=%s', (_label, args, expectedRun, expectedReason) => { + const r = evaluateGates( + input({ + state: baseState({ turnsSinceLastRun: args.turns, lastDailyAt: args.lastAt }), + now: 10 * MS_PER_DAY, + lockHeld: args.lockHeld, + }), + cfg, + ); + expect(r.run).toBe(expectedRun); + expect(r.reason).toBe(expectedReason); + }); +}); diff --git a/packages/services/src/mindMemory/dream-gates.ts b/packages/services/src/mindMemory/dream-gates.ts new file mode 100644 index 00000000..38b7c9a6 --- /dev/null +++ b/packages/services/src/mindMemory/dream-gates.ts @@ -0,0 +1,87 @@ +/** + * dream-gates — combined lock + activity + time gate evaluation for the + * Dream Daemon. Replaces SCNS's "session count" gate with an *activity* + * gate driven by `turns_since_last_run`. + * + * The activity counter is bumped by DailyLogWriter when it appends a turn + * frame, NOT by SDK session start. This keeps the daemon from running + * after idle reconnects. + * + * `evaluateGates` is pure: it takes a snapshot of state + clock + lock + * status and returns `{ run, reason }`. The caller is responsible for + * checking the consolidation-enabled flag from `.chamber.json` before + * invoking this function. + * + * Gate order (first failure short-circuits): + * 1. Lock gate reason='locked' + * 2. Activity gate reason='no-activity' + * 3. Time gate reason='too-soon' + * else reason='ready' + */ + +import type { DreamPhase } from './dream-schema'; +import type { DreamState } from './dream-state'; + +export interface GateConfig { + readonly minTurnsBetweenRuns: number; + readonly minIntervalMs: number; +} + +export interface GateInput { + readonly phase: DreamPhase; + readonly state: DreamState; + readonly now: number; + readonly lockHeld: boolean; +} + +export type GateReason = 'locked' | 'no-activity' | 'too-soon' | 'ready'; + +export interface GateResult { + readonly run: boolean; + readonly reason: GateReason; +} + +const MS_PER_DAY = 24 * 60 * 60 * 1000; + +export const DEFAULT_DAILY_GATE: GateConfig = { + minTurnsBetweenRuns: 1, + minIntervalMs: MS_PER_DAY, +}; + +export const DEFAULT_WEEKLY_GATE: GateConfig = { + minTurnsBetweenRuns: 1, + minIntervalMs: 7 * MS_PER_DAY, +}; + +export const DEFAULT_MONTHLY_GATE: GateConfig = { + minTurnsBetweenRuns: 1, + minIntervalMs: 30 * MS_PER_DAY, +}; + +function lastPhaseAt(state: DreamState, phase: DreamPhase): number | null { + switch (phase) { + case 'daily': + return state.lastDailyAt; + case 'weekly': + return state.lastWeeklyAt; + case 'monthly': + return state.lastMonthlyAt; + } +} + +export function evaluateGates(input: GateInput, config: GateConfig): GateResult { + if (input.lockHeld) { + return { run: false, reason: 'locked' }; + } + + if (input.state.turnsSinceLastRun < config.minTurnsBetweenRuns) { + return { run: false, reason: 'no-activity' }; + } + + const last = lastPhaseAt(input.state, input.phase); + if (last !== null && input.now - last < config.minIntervalMs) { + return { run: false, reason: 'too-soon' }; + } + + return { run: true, reason: 'ready' }; +} diff --git a/packages/services/src/mindMemory/dream-schema.test.ts b/packages/services/src/mindMemory/dream-schema.test.ts new file mode 100644 index 00000000..cda9566a --- /dev/null +++ b/packages/services/src/mindMemory/dream-schema.test.ts @@ -0,0 +1,166 @@ +/** + * Tests for dream-schema — per-mind better-sqlite3 schema bootstrap. + * + * Phase 7 acceptance: + * - migrate() is idempotent. + * - openDreamDb() materializes the parent .state directory. + * - WAL pragma is applied on file-backed DBs. + * - dream_state singleton row exists after migrate. + * - dream_state has the new last_consolidated_turn_id TEXT NULL column. + * - dream_locks and dream_runs tables exist with the expected columns. + */ + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import Database from 'better-sqlite3'; + +import { dreamDbPath, migrate, openDreamDb } from './dream-schema'; + +let mindRoot: string; + +beforeEach(() => { + mindRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-dream-schema-')); +}); + +afterEach(() => { + fs.rmSync(mindRoot, { recursive: true, force: true }); +}); + +function tableInfo(db: Database.Database, table: string): Array<{ name: string; type: string; notnull: number; dflt_value: unknown; pk: number }> { + return db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ + name: string; + type: string; + notnull: number; + dflt_value: unknown; + pk: number; + }>; +} + +describe('dream-schema — dreamDbPath', () => { + it('puts the database under /.working-memory/.state/dream.db', () => { + const p = dreamDbPath('/tmp/some/mind'); + expect(p.replace(/\\/g, '/')).toBe('/tmp/some/mind/.working-memory/.state/dream.db'); + }); +}); + +describe('dream-schema — openDreamDb', () => { + it('creates the parent .state directory and opens a WAL-mode database', () => { + const dbPath = dreamDbPath(mindRoot); + const db = openDreamDb(dbPath); + try { + expect(fs.existsSync(path.dirname(dbPath))).toBe(true); + expect(fs.existsSync(dbPath)).toBe(true); + const mode = db.pragma('journal_mode', { simple: true }); + expect(mode).toBe('wal'); + } finally { + db.close(); + } + }); + + it('seeds the dream_state singleton row on first open', () => { + const dbPath = dreamDbPath(mindRoot); + const db = openDreamDb(dbPath); + try { + const row = db + .prepare('SELECT id, turns_since_last_run, last_consolidated_turn_id FROM dream_state') + .all() as Array<{ id: number; turns_since_last_run: number; last_consolidated_turn_id: string | null }>; + expect(row).toHaveLength(1); + expect(row[0].id).toBe(1); + expect(row[0].turns_since_last_run).toBe(0); + expect(row[0].last_consolidated_turn_id).toBeNull(); + } finally { + db.close(); + } + }); +}); + +describe('dream-schema — migrate idempotency', () => { + it('running migrate twice is a no-op (no error, no duplicate seed row)', () => { + const db = new Database(':memory:'); + try { + migrate(db); + migrate(db); + const rows = db.prepare('SELECT id FROM dream_state').all() as Array<{ id: number }>; + expect(rows).toHaveLength(1); + expect(rows[0].id).toBe(1); + } finally { + db.close(); + } + }); + + it('preserves user-written state across re-migration', () => { + const db = new Database(':memory:'); + try { + migrate(db); + db.prepare('UPDATE dream_state SET turns_since_last_run = 42, last_consolidated_turn_id = ? WHERE id = 1').run( + 'turn-xyz', + ); + migrate(db); + const row = db + .prepare('SELECT turns_since_last_run, last_consolidated_turn_id FROM dream_state WHERE id = 1') + .get() as { turns_since_last_run: number; last_consolidated_turn_id: string }; + expect(row.turns_since_last_run).toBe(42); + expect(row.last_consolidated_turn_id).toBe('turn-xyz'); + } finally { + db.close(); + } + }); +}); + +describe('dream-schema — table shapes', () => { + it('dream_state has the required columns including last_consolidated_turn_id TEXT NULL', () => { + const db = new Database(':memory:'); + try { + migrate(db); + const cols = tableInfo(db, 'dream_state'); + const byName = new Map(cols.map((c) => [c.name, c])); + expect(byName.has('id')).toBe(true); + expect(byName.has('turns_since_last_run')).toBe(true); + expect(byName.has('last_daily_at')).toBe(true); + expect(byName.has('last_weekly_at')).toBe(true); + expect(byName.has('last_monthly_at')).toBe(true); + expect(byName.has('last_consolidated_turn_id')).toBe(true); + const lcti = byName.get('last_consolidated_turn_id')!; + expect(lcti.type).toBe('TEXT'); + expect(lcti.notnull).toBe(0); + } finally { + db.close(); + } + }); + + it('dream_locks has phase/holder/acquired_at/expires_at', () => { + const db = new Database(':memory:'); + try { + migrate(db); + const cols = tableInfo(db, 'dream_locks').map((c) => c.name); + expect(cols).toEqual(expect.arrayContaining(['phase', 'holder', 'acquired_at', 'expires_at'])); + } finally { + db.close(); + } + }); + + it('dream_runs has phase/started_at/ended_at/status/reason/from_turn_id/to_turn_id', () => { + const db = new Database(':memory:'); + try { + migrate(db); + const cols = tableInfo(db, 'dream_runs').map((c) => c.name); + expect(cols).toEqual( + expect.arrayContaining([ + 'id', + 'phase', + 'started_at', + 'ended_at', + 'status', + 'reason', + 'from_turn_id', + 'to_turn_id', + ]), + ); + } finally { + db.close(); + } + }); +}); diff --git a/packages/services/src/mindMemory/dream-schema.ts b/packages/services/src/mindMemory/dream-schema.ts new file mode 100644 index 00000000..5a6b363a --- /dev/null +++ b/packages/services/src/mindMemory/dream-schema.ts @@ -0,0 +1,87 @@ +/** + * dream-schema — per-mind better-sqlite3 schema bootstrap for the Dream + * Daemon's local state, locks, and run history. + * + * Database location: `/.working-memory/.state/dream.db`. + * Journal mode: WAL (durable, multi-reader, single-writer). + * + * Tables (all created idempotently by `migrate`): + * + * dream_state Singleton (id=1) row tracking per-phase last-run + * timestamps, the activity counter + * (`turns_since_last_run`), and the cutoff turn id of + * the last successful daily consolidation + * (`last_consolidated_turn_id`). + * dream_locks One row per phase. Holder string, acquired_at, and + * expires_at form a TTL-bounded mutex broken via + * transactional steal in dream-state.acquireLock. + * dream_runs Append-only run history (success | failed | skipped) + * with optional reason and turn-id range processed. + * + * This module is I/O — it owns the dream.db file. dream-state.ts wraps it + * with typed CRUD; dream-gates.ts and consolidation-scheduler.ts compose + * on top. + */ + +import fs from 'node:fs'; +import path from 'node:path'; + +import Database from 'better-sqlite3'; + +export type DreamPhase = 'daily' | 'weekly' | 'monthly'; +export type DreamRunStatus = 'success' | 'failed' | 'skipped'; + +const WORKING_MEMORY_DIRNAME = '.working-memory'; +const STATE_DIRNAME = '.state'; +const DREAM_DB_FILENAME = 'dream.db'; + +export function dreamDbPath(mindPath: string): string { + return path.join(mindPath, WORKING_MEMORY_DIRNAME, STATE_DIRNAME, DREAM_DB_FILENAME); +} + +export function openDreamDb(dbPath: string): Database.Database { + fs.mkdirSync(path.dirname(dbPath), { recursive: true }); + const db = new Database(dbPath); + db.pragma('journal_mode = WAL'); + db.pragma('foreign_keys = ON'); + migrate(db); + return db; +} + +export function migrate(db: Database.Database): void { + db.exec(` + CREATE TABLE IF NOT EXISTS dream_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + turns_since_last_run INTEGER NOT NULL DEFAULT 0, + last_daily_at INTEGER, + last_weekly_at INTEGER, + last_monthly_at INTEGER, + last_consolidated_turn_id TEXT + ); + + CREATE TABLE IF NOT EXISTS dream_locks ( + phase TEXT PRIMARY KEY, + holder TEXT NOT NULL, + acquired_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL + ); + + CREATE TABLE IF NOT EXISTS dream_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + phase TEXT NOT NULL, + started_at INTEGER NOT NULL, + ended_at INTEGER, + status TEXT NOT NULL, + reason TEXT, + from_turn_id TEXT, + to_turn_id TEXT + ); + + CREATE INDEX IF NOT EXISTS idx_dream_runs_phase_started + ON dream_runs (phase, started_at DESC); + `); + + // Seed the singleton row. INSERT OR IGNORE keeps migrate idempotent and + // preserves user-written state across re-opens. + db.prepare('INSERT OR IGNORE INTO dream_state (id) VALUES (1)').run(); +} diff --git a/packages/services/src/mindMemory/dream-state.test.ts b/packages/services/src/mindMemory/dream-state.test.ts new file mode 100644 index 00000000..be1e24f9 --- /dev/null +++ b/packages/services/src/mindMemory/dream-state.test.ts @@ -0,0 +1,249 @@ +/** + * Tests for dream-state — typed CRUD over the dream-schema tables, plus + * lock acquire/release with stale-lock break. + * + * Phase 7 acceptance: + * - readState round-trips through writes; defaults when row missing. + * - incrementTurnCount accumulates atomically. + * - markPhaseComplete updates only the targeted phase. + * - setLastConsolidatedTurnId round-trips through null. + * - recordRun appends to dream_runs and listRuns reads back. + * - acquireLock/releaseLock honor the lock holder format and stale-break. + */ + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import Database from 'better-sqlite3'; + +import { migrate } from './dream-schema'; +import { + acquireLock, + buildLockHolder, + getLock, + incrementTurnCount, + listRuns, + markPhaseComplete, + readState, + recordRun, + releaseLock, + resetActivityCounter, + setLastConsolidatedTurnId, +} from './dream-state'; + +let db: Database.Database; + +beforeEach(() => { + db = new Database(':memory:'); + migrate(db); +}); + +afterEach(() => { + db.close(); +}); + +describe('dream-state — readState defaults', () => { + it('returns the singleton defaults after migrate', () => { + const s = readState(db); + expect(s.turnsSinceLastRun).toBe(0); + expect(s.lastDailyAt).toBeNull(); + expect(s.lastWeeklyAt).toBeNull(); + expect(s.lastMonthlyAt).toBeNull(); + expect(s.lastConsolidatedTurnId).toBeNull(); + }); + + it('returns sensible defaults if the singleton row was deleted', () => { + db.prepare('DELETE FROM dream_state WHERE id = 1').run(); + const s = readState(db); + expect(s.turnsSinceLastRun).toBe(0); + expect(s.lastDailyAt).toBeNull(); + expect(s.lastConsolidatedTurnId).toBeNull(); + }); +}); + +describe('dream-state — incrementTurnCount', () => { + it('increments by 1 by default', () => { + incrementTurnCount(db); + incrementTurnCount(db); + expect(readState(db).turnsSinceLastRun).toBe(2); + }); + + it('increments by n', () => { + incrementTurnCount(db, 5); + expect(readState(db).turnsSinceLastRun).toBe(5); + }); + + it('rejects non-positive n', () => { + expect(() => incrementTurnCount(db, 0)).toThrow(); + expect(() => incrementTurnCount(db, -1)).toThrow(); + }); +}); + +describe('dream-state — resetActivityCounter', () => { + it('zeroes turns_since_last_run', () => { + incrementTurnCount(db, 7); + resetActivityCounter(db); + expect(readState(db).turnsSinceLastRun).toBe(0); + }); +}); + +describe('dream-state — markPhaseComplete', () => { + it('updates only the targeted phase timestamp', () => { + markPhaseComplete(db, 'daily', 1000); + let s = readState(db); + expect(s.lastDailyAt).toBe(1000); + expect(s.lastWeeklyAt).toBeNull(); + expect(s.lastMonthlyAt).toBeNull(); + + markPhaseComplete(db, 'weekly', 2000); + s = readState(db); + expect(s.lastDailyAt).toBe(1000); + expect(s.lastWeeklyAt).toBe(2000); + expect(s.lastMonthlyAt).toBeNull(); + + markPhaseComplete(db, 'monthly', 3000); + s = readState(db); + expect(s.lastMonthlyAt).toBe(3000); + }); +}); + +describe('dream-state — setLastConsolidatedTurnId', () => { + it('round-trips through a non-null id', () => { + setLastConsolidatedTurnId(db, 'turn-alpha'); + expect(readState(db).lastConsolidatedTurnId).toBe('turn-alpha'); + }); + + it('round-trips through null', () => { + setLastConsolidatedTurnId(db, 'turn-alpha'); + setLastConsolidatedTurnId(db, null); + expect(readState(db).lastConsolidatedTurnId).toBeNull(); + }); +}); + +describe('dream-state — recordRun + listRuns', () => { + it('appends in chronological order; listRuns returns most-recent-first', () => { + recordRun(db, { + phase: 'daily', + startedAt: 100, + endedAt: 200, + status: 'success', + fromTurnId: 'turn-1', + toTurnId: 'turn-3', + }); + recordRun(db, { + phase: 'daily', + startedAt: 300, + endedAt: null, + status: 'skipped', + reason: 'no-activity', + }); + + const runs = listRuns(db); + expect(runs).toHaveLength(2); + expect(runs[0].startedAt).toBe(300); + expect(runs[0].status).toBe('skipped'); + expect(runs[0].reason).toBe('no-activity'); + expect(runs[1].fromTurnId).toBe('turn-1'); + expect(runs[1].toTurnId).toBe('turn-3'); + }); + + it('listRuns honors phase filter and limit', () => { + recordRun(db, { phase: 'daily', startedAt: 1, endedAt: 2, status: 'success' }); + recordRun(db, { phase: 'weekly', startedAt: 3, endedAt: 4, status: 'success' }); + recordRun(db, { phase: 'daily', startedAt: 5, endedAt: 6, status: 'success' }); + + const daily = listRuns(db, { phase: 'daily' }); + expect(daily.map((r) => r.startedAt)).toEqual([5, 1]); + + const limited = listRuns(db, { limit: 1 }); + expect(limited).toHaveLength(1); + expect(limited[0].startedAt).toBe(5); + }); +}); + +describe('dream-state — buildLockHolder', () => { + it('produces the dream-daemon::: shape', () => { + const holder = buildLockHolder('mind-x', 1234, 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'); + expect(holder).toBe('dream-daemon:mind-x:1234:aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'); + }); + + it('defaults pid to process.pid and uuid to a random uuid v4-ish string', () => { + const holder = buildLockHolder('mind-y'); + expect(holder.startsWith(`dream-daemon:mind-y:${process.pid}:`)).toBe(true); + const uuid = holder.split(':').pop()!; + expect(uuid).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i); + }); +}); + +describe('dream-state — acquireLock / releaseLock', () => { + it('first acquire succeeds with reason=acquired', () => { + const r = acquireLock(db, { phase: 'daily', mindId: 'm', now: 1000, ttlMs: 5000 }); + expect(r.acquired).toBe(true); + expect(r.reason).toBe('acquired'); + expect(r.holder).toMatch(/^dream-daemon:m:/); + + const row = getLock(db, 'daily')!; + expect(row.phase).toBe('daily'); + expect(row.acquiredAt).toBe(1000); + expect(row.expiresAt).toBe(6000); + }); + + it('second acquire while still held returns reason=held with the existing holder', () => { + const a = acquireLock(db, { phase: 'daily', mindId: 'm', now: 1000, ttlMs: 5000 }); + const b = acquireLock(db, { phase: 'daily', mindId: 'm', now: 1500, ttlMs: 5000 }); + expect(a.acquired).toBe(true); + expect(b.acquired).toBe(false); + expect(b.reason).toBe('held'); + expect(b.holder).toBe(a.holder); + }); + + it('stale lock (expires_at < now) is stolen with reason=stolen-stale', () => { + const first = acquireLock(db, { phase: 'daily', mindId: 'm', now: 1000, ttlMs: 1000 }); + expect(first.acquired).toBe(true); + // jump past expiry + const second = acquireLock(db, { phase: 'daily', mindId: 'm', now: 5000, ttlMs: 1000 }); + expect(second.acquired).toBe(true); + expect(second.reason).toBe('stolen-stale'); + expect(second.holder).not.toBe(first.holder); + + const row = getLock(db, 'daily')!; + expect(row.holder).toBe(second.holder); + expect(row.acquiredAt).toBe(5000); + }); + + it('only one of two concurrent steal attempts succeeds (transactional re-check)', () => { + // Plant a stale lock + db.prepare( + 'INSERT INTO dream_locks (phase, holder, acquired_at, expires_at) VALUES (?, ?, ?, ?)', + ).run('daily', 'dream-daemon:m:1:old-uuid', 0, 1); + + // Two distinct callers try to steal at the same `now` + const a = acquireLock(db, { phase: 'daily', mindId: 'm', uuid: 'a', now: 1000, ttlMs: 1000 }); + const b = acquireLock(db, { phase: 'daily', mindId: 'm', uuid: 'b', now: 1000, ttlMs: 1000 }); + + // The first call performs the steal; the second sees a's lock and reports held. + expect(a.acquired).toBe(true); + expect(a.reason).toBe('stolen-stale'); + expect(b.acquired).toBe(false); + expect(b.reason).toBe('held'); + expect(b.holder).toBe(a.holder); + }); + + it('releaseLock only removes the row when the holder matches', () => { + const a = acquireLock(db, { phase: 'daily', mindId: 'm', now: 1000, ttlMs: 5000 }); + const released = releaseLock(db, 'daily', 'dream-daemon:m:0:wrong'); + expect(released).toBe(false); + expect(getLock(db, 'daily')).not.toBeNull(); + + const ok = releaseLock(db, 'daily', a.holder!); + expect(ok).toBe(true); + expect(getLock(db, 'daily')).toBeNull(); + }); + + it('after release, a fresh acquire succeeds with reason=acquired', () => { + const a = acquireLock(db, { phase: 'daily', mindId: 'm', now: 1000, ttlMs: 5000 }); + releaseLock(db, 'daily', a.holder!); + const c = acquireLock(db, { phase: 'daily', mindId: 'm', now: 2000, ttlMs: 5000 }); + expect(c.acquired).toBe(true); + expect(c.reason).toBe('acquired'); + }); +}); diff --git a/packages/services/src/mindMemory/dream-state.ts b/packages/services/src/mindMemory/dream-state.ts new file mode 100644 index 00000000..5bfa9d6b --- /dev/null +++ b/packages/services/src/mindMemory/dream-state.ts @@ -0,0 +1,249 @@ +/** + * dream-state — typed CRUD over the dream-schema tables, plus DB-backed + * lock acquire/release with stale-lock break. + * + * All writes are wrapped in `db.transaction(...)` so a partial failure + * cannot leave the singleton row half-updated. Lock steal is implemented + * inside a single transaction that re-checks `expires_at` so two + * simultaneous stealers cannot both succeed. + * + * Lock holder format: `dream-daemon:::` — the uuid + * component defeats same-process re-acquisition by a stale handle. The + * in-memory mutex layered on top lives in consolidation-scheduler.ts. + */ + +import { randomUUID } from 'node:crypto'; + +import type Database from 'better-sqlite3'; + +import type { DreamPhase, DreamRunStatus } from './dream-schema'; + +export interface DreamState { + readonly turnsSinceLastRun: number; + readonly lastDailyAt: number | null; + readonly lastWeeklyAt: number | null; + readonly lastMonthlyAt: number | null; + readonly lastConsolidatedTurnId: string | null; +} + +export interface RunRecord { + readonly phase: DreamPhase; + readonly startedAt: number; + readonly endedAt: number | null; + readonly status: DreamRunStatus; + readonly reason?: string | null; + readonly fromTurnId?: string | null; + readonly toTurnId?: string | null; +} + +export interface DreamLockRow { + readonly phase: DreamPhase; + readonly holder: string; + readonly acquiredAt: number; + readonly expiresAt: number; +} + +export interface AcquireLockArgs { + readonly phase: DreamPhase; + readonly mindId: string; + readonly pid?: number; + readonly uuid?: string; + readonly now: number; + readonly ttlMs: number; +} + +export type AcquireLockReason = 'acquired' | 'stolen-stale' | 'held'; + +export interface AcquireLockResult { + readonly acquired: boolean; + readonly holder: string | null; + readonly reason: AcquireLockReason; +} + +export interface ListRunsOptions { + readonly phase?: DreamPhase; + readonly limit?: number; +} + +const DEFAULT_STATE: DreamState = { + turnsSinceLastRun: 0, + lastDailyAt: null, + lastWeeklyAt: null, + lastMonthlyAt: null, + lastConsolidatedTurnId: null, +}; + +const PHASE_COLUMN: Record = { + daily: 'last_daily_at', + weekly: 'last_weekly_at', + monthly: 'last_monthly_at', +}; + +interface DreamStateRow { + turns_since_last_run: number; + last_daily_at: number | null; + last_weekly_at: number | null; + last_monthly_at: number | null; + last_consolidated_turn_id: string | null; +} + +export function readState(db: Database.Database): DreamState { + const row = db + .prepare( + `SELECT turns_since_last_run, last_daily_at, last_weekly_at, last_monthly_at, last_consolidated_turn_id + FROM dream_state WHERE id = 1`, + ) + .get() as DreamStateRow | undefined; + + if (!row) return DEFAULT_STATE; + + return { + turnsSinceLastRun: row.turns_since_last_run, + lastDailyAt: row.last_daily_at, + lastWeeklyAt: row.last_weekly_at, + lastMonthlyAt: row.last_monthly_at, + lastConsolidatedTurnId: row.last_consolidated_turn_id, + }; +} + +export function incrementTurnCount(db: Database.Database, n = 1): void { + if (!Number.isInteger(n) || n <= 0) { + throw new Error(`incrementTurnCount: n must be a positive integer (got ${n})`); + } + db.transaction(() => { + db.prepare('INSERT OR IGNORE INTO dream_state (id) VALUES (1)').run(); + db.prepare('UPDATE dream_state SET turns_since_last_run = turns_since_last_run + ? WHERE id = 1').run(n); + })(); +} + +export function resetActivityCounter(db: Database.Database): void { + db.transaction(() => { + db.prepare('INSERT OR IGNORE INTO dream_state (id) VALUES (1)').run(); + db.prepare('UPDATE dream_state SET turns_since_last_run = 0 WHERE id = 1').run(); + })(); +} + +export function markPhaseComplete(db: Database.Database, phase: DreamPhase, ts: number): void { + const col = PHASE_COLUMN[phase]; + db.transaction(() => { + db.prepare('INSERT OR IGNORE INTO dream_state (id) VALUES (1)').run(); + db.prepare(`UPDATE dream_state SET ${col} = ? WHERE id = 1`).run(ts); + })(); +} + +export function setLastConsolidatedTurnId(db: Database.Database, turnId: string | null): void { + db.transaction(() => { + db.prepare('INSERT OR IGNORE INTO dream_state (id) VALUES (1)').run(); + db.prepare('UPDATE dream_state SET last_consolidated_turn_id = ? WHERE id = 1').run(turnId); + })(); +} + +export function recordRun(db: Database.Database, record: RunRecord): number { + const info = db + .prepare( + `INSERT INTO dream_runs (phase, started_at, ended_at, status, reason, from_turn_id, to_turn_id) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + ) + .run( + record.phase, + record.startedAt, + record.endedAt, + record.status, + record.reason ?? null, + record.fromTurnId ?? null, + record.toTurnId ?? null, + ); + return Number(info.lastInsertRowid); +} + +interface DreamRunRow { + phase: DreamPhase; + started_at: number; + ended_at: number | null; + status: DreamRunStatus; + reason: string | null; + from_turn_id: string | null; + to_turn_id: string | null; +} + +export function listRuns(db: Database.Database, opts: ListRunsOptions = {}): RunRecord[] { + const where = opts.phase ? 'WHERE phase = ?' : ''; + const limitClause = opts.limit ? `LIMIT ${Math.max(1, Math.floor(opts.limit))}` : ''; + const sql = `SELECT phase, started_at, ended_at, status, reason, from_turn_id, to_turn_id + FROM dream_runs ${where} + ORDER BY started_at DESC, id DESC ${limitClause}`; + const stmt = db.prepare(sql); + const rows = (opts.phase ? stmt.all(opts.phase) : stmt.all()) as DreamRunRow[]; + return rows.map((r) => ({ + phase: r.phase, + startedAt: r.started_at, + endedAt: r.ended_at, + status: r.status, + reason: r.reason, + fromTurnId: r.from_turn_id, + toTurnId: r.to_turn_id, + })); +} + +export function buildLockHolder(mindId: string, pid: number = process.pid, uuid: string = randomUUID()): string { + return `dream-daemon:${mindId}:${pid}:${uuid}`; +} + +interface LockRowRaw { + phase: DreamPhase; + holder: string; + acquired_at: number; + expires_at: number; +} + +export function getLock(db: Database.Database, phase: DreamPhase): DreamLockRow | null { + const row = db + .prepare('SELECT phase, holder, acquired_at, expires_at FROM dream_locks WHERE phase = ?') + .get(phase) as LockRowRaw | undefined; + if (!row) return null; + return { + phase: row.phase, + holder: row.holder, + acquiredAt: row.acquired_at, + expiresAt: row.expires_at, + }; +} + +export function acquireLock(db: Database.Database, args: AcquireLockArgs): AcquireLockResult { + const holder = buildLockHolder(args.mindId, args.pid, args.uuid); + const expiresAt = args.now + args.ttlMs; + + // The whole acquire is a single transaction: SELECT-then-write under + // BEGIN IMMEDIATE serializes against any other writer attempting the + // same operation, so two would-be stealers cannot both win. + const txn = db.transaction((): AcquireLockResult => { + const existing = db + .prepare('SELECT phase, holder, acquired_at, expires_at FROM dream_locks WHERE phase = ?') + .get(args.phase) as LockRowRaw | undefined; + + if (!existing) { + db.prepare( + 'INSERT INTO dream_locks (phase, holder, acquired_at, expires_at) VALUES (?, ?, ?, ?)', + ).run(args.phase, holder, args.now, expiresAt); + return { acquired: true, holder, reason: 'acquired' }; + } + + if (existing.expires_at <= args.now) { + db.prepare( + 'UPDATE dream_locks SET holder = ?, acquired_at = ?, expires_at = ? WHERE phase = ?', + ).run(holder, args.now, expiresAt, args.phase); + return { acquired: true, holder, reason: 'stolen-stale' }; + } + + return { acquired: false, holder: existing.holder, reason: 'held' }; + }); + + return txn.immediate(); +} + +export function releaseLock(db: Database.Database, phase: DreamPhase, holder: string): boolean { + const info = db + .prepare('DELETE FROM dream_locks WHERE phase = ? AND holder = ?') + .run(phase, holder); + return info.changes > 0; +} diff --git a/packages/services/src/mindMemory/extraction.test.ts b/packages/services/src/mindMemory/extraction.test.ts new file mode 100644 index 00000000..f3018907 --- /dev/null +++ b/packages/services/src/mindMemory/extraction.test.ts @@ -0,0 +1,287 @@ +import { describe, it, expect } from 'vitest'; +import { + parseDailyLog, + classifyEntry, + extractFromLog, + extractFromMultipleLogs, + generateEntryName, + containsSensitive, +} from './extraction'; +import type { DailyLogEntry } from './extraction'; + +const singleSessionLog = `## 2026-04-05 + +### 14:30 — Session abc123 +- Working on SCNS project +- Decided to use Express 5 instead of Fastify +- User prefers TypeScript over JavaScript always +`; + +const multiSessionLog = `## 2026-04-05 + +### 14:30 — Session abc123 +- Working on SCNS project +- Decided to use Express 5 instead of Fastify +- User prefers TypeScript over JavaScript always + +### 16:45 — Session def456 +- Reviewed PR for real estate analysis +- PostgreSQL queries optimized with EXPLAIN ANALYZE +- User wants conventional commits enforced +`; + +const noSessionIdLog = `## 2026-04-05 + +### 09:00 +- Morning standup notes +- Remember to always run lint before commit +`; + +const realisticLog = `## 2026-04-05 + +### 14:30 — Session abc123 +- Working on SCNS project +- Decided to use Express 5 instead of Fastify +- User prefers TypeScript over JavaScript always +- Dashboard at https://grafana.example.com + +### 16:45 — Session def456 +- Reviewed PR for real estate analysis +- User dislikes tabs, prefers spaces +- Remember to always run lint before commit +- Docs at https://docs.example.com/api +`; + +describe('parseDailyLog', () => { + it('parses a single session entry with time and session ID', () => { + const entries = parseDailyLog(singleSessionLog, '2026-04-05'); + expect(entries).toHaveLength(1); + expect(entries[0]).toEqual({ + date: '2026-04-05', + time: '14:30', + sessionId: 'abc123', + lines: [ + 'Working on SCNS project', + 'Decided to use Express 5 instead of Fastify', + 'User prefers TypeScript over JavaScript always', + ], + }); + }); + + it('parses multiple session entries in one log', () => { + const entries = parseDailyLog(multiSessionLog, '2026-04-05'); + expect(entries).toHaveLength(2); + expect(entries[0]!.sessionId).toBe('abc123'); + expect(entries[1]!.sessionId).toBe('def456'); + }); + + it('parses entry without session ID (just time header)', () => { + const entries = parseDailyLog(noSessionIdLog, '2026-04-05'); + expect(entries).toHaveLength(1); + expect(entries[0]!.sessionId).toBeNull(); + expect(entries[0]!.time).toBe('09:00'); + }); + + it('returns empty array for empty content', () => { + expect(parseDailyLog('', '2026-04-05')).toEqual([]); + expect(parseDailyLog(' \n\n ', '2026-04-05')).toEqual([]); + }); + + it('strips bullet "- " prefix from lines', () => { + const entries = parseDailyLog(multiSessionLog, '2026-04-05'); + for (const line of entries[0]!.lines) { + expect(line).not.toMatch(/^- /); + } + }); +}); + +describe('classifyEntry', () => { + it('classifies "User prefers TypeScript" as user type', () => { + const result = classifyEntry('User prefers TypeScript'); + expect(result?.type).toBe('user'); + expect(result!.confidence).toBeGreaterThanOrEqual(0.6); + }); + + it('classifies "Decided to use Express 5" as project type', () => { + expect(classifyEntry('Decided to use Express 5')?.type).toBe('project'); + }); + + it('classifies "Remember to always run lint before commit" as feedback type', () => { + expect(classifyEntry('Remember to always run lint before commit')?.type).toBe('feedback'); + }); + + it('classifies "Dashboard at https://grafana.example.com" as reference type', () => { + expect(classifyEntry('Dashboard at https://grafana.example.com')?.type).toBe('reference'); + }); + + it('returns null for routine action "Fixed a bug in the parser"', () => { + expect(classifyEntry('Fixed a bug in the parser')).toBeNull(); + }); + + it('returns null for status update "Working on SCNS project"', () => { + expect(classifyEntry('Working on SCNS project')).toBeNull(); + }); +}); + +describe('extractFromLog', () => { + it('extracts entries from a realistic daily log', () => { + const entries = extractFromLog(realisticLog, '2026-04-05'); + expect(entries.length).toBeGreaterThan(0); + for (const entry of entries) { + expect(entry.type).toMatch(/^(user|feedback|project|reference|prohibition)$/); + expect(entry.name.length).toBeGreaterThan(0); + } + }); + + it('returns empty array for empty log', () => { + expect(extractFromLog('', '2026-04-05')).toEqual([]); + }); + + it('sets correct source on entries', () => { + const entries = extractFromLog(realisticLog, '2026-04-05'); + for (const entry of entries) { + expect(entry.source).toBe('daily-log:2026-04-05'); + } + }); + + it('sets correct createdAt on entries', () => { + const entries = extractFromLog(realisticLog, '2026-04-05'); + for (const entry of entries) { + expect(entry.createdAt).toMatch(/^2026-04-05T\d{2}:\d{2}:00Z$/); + } + }); +}); + +describe('extractFromMultipleLogs', () => { + it('combines entries from two logs sorted by date', () => { + const log1 = `## 2026-04-04\n\n### 10:00 — Session aaa\n- User prefers dark mode\n`; + const log2 = `## 2026-04-05\n\n### 14:00 — Session bbb\n- Decided to use PostgreSQL\n`; + const entries = extractFromMultipleLogs([ + { content: log1, date: '2026-04-04' }, + { content: log2, date: '2026-04-05' }, + ]); + expect(entries.length).toBe(2); + expect(entries[0]!.createdAt!.startsWith('2026-04-04')).toBe(true); + expect(entries[1]!.createdAt!.startsWith('2026-04-05')).toBe(true); + }); + + it('deduplicates across logs (same fact → keep latest)', () => { + const log1 = `## 2026-04-04\n\n### 10:00 — Session aaa\n- User prefers dark mode\n`; + const log2 = `## 2026-04-05\n\n### 14:00 — Session bbb\n- User prefers dark mode\n`; + const entries = extractFromMultipleLogs([ + { content: log1, date: '2026-04-04' }, + { content: log2, date: '2026-04-05' }, + ]); + expect(entries.length).toBe(1); + expect(entries[0]!.createdAt!.startsWith('2026-04-05')).toBe(true); + }); +}); + +describe('generateEntryName', () => { + it('truncates long content to ~60 chars', () => { + const name = generateEntryName( + 'This is a really long description that goes on and on and should be truncated to fit within sixty characters or so', + ); + expect(name.length).toBeLessThanOrEqual(60); + }); + + it('removes leading bullet markers', () => { + expect(generateEntryName('- User prefers TypeScript')).not.toMatch(/^-/); + }); + + it('title-cases the result', () => { + expect(generateEntryName('user prefers dark mode')).toBe('User Prefers Dark Mode'); + }); + + it('handles empty string', () => { + expect(generateEntryName('')).toBe(''); + }); +}); + +describe('containsSensitive — redaction guard', () => { + it('detects OpenAI-style sk- API keys', () => { + expect(containsSensitive('My API key is sk-abc123def456ghi789jkl012mno345pq')).toBe(true); + }); + + it('detects AWS access keys (AKIA...)', () => { + expect(containsSensitive('Use access key AKIAIOSFODNN7EXAMPLE for S3')).toBe(true); + }); + + it('detects GitHub-style ghp_ tokens', () => { + expect(containsSensitive('token: ghp_abcdefghijklmnopqrstuvwxyz0123456789')).toBe(true); + }); + + it('detects "password is X" patterns', () => { + expect(containsSensitive('the database password is hunter2')).toBe(true); + }); + + it('detects "secret = X" patterns', () => { + expect(containsSensitive('SECRET_TOKEN = abc123xyz789def456')).toBe(true); + }); + + it('does not flag innocent mentions of "key" or "token"', () => { + expect(containsSensitive('We use JWT tokens for auth')).toBe(false); + expect(containsSensitive('The primary key is the user ID')).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// Chamber-flavored fixture pack (5 cases — required by Phase 1 plan) +// 3 positives (memorized) + 2 negatives (transient + sensitive redaction) +// --------------------------------------------------------------------------- +describe('Chamber fixture pack — extractFromLog', () => { + const date = '2026-04-15'; + const log = `## ${date} + +### 10:00 — Session chamber-fixture +- I prefer concise commit messages +- We're using Postgres for the auth service +- Stop saying 'sure thing' +- Run npm test +- My API key is sk-abc123def456ghi789jkl012mno345pq +`; + + it('positive: "I prefer concise commit messages" → user entry', () => { + const entries = extractFromLog(log, date); + const match = entries.find((e) => e.content.toLowerCase().includes('concise commit')); + expect(match).toBeDefined(); + expect(match!.type).toBe('user'); + }); + + it('positive: "We\'re using Postgres for the auth service" → project entry', () => { + const entries = extractFromLog(log, date); + const match = entries.find((e) => e.content.toLowerCase().includes('postgres')); + expect(match).toBeDefined(); + expect(match!.type).toBe('project'); + }); + + it('positive: "Stop saying \'sure thing\'" → prohibition entry', () => { + const entries = extractFromLog(log, date); + const match = entries.find((e) => e.type === 'prohibition'); + expect(match).toBeDefined(); + }); + + it('negative: "Run npm test" → transient, NOT memorized', () => { + const entries = extractFromLog(log, date); + const match = entries.find((e) => e.content.toLowerCase().includes('run npm test')); + expect(match).toBeUndefined(); + }); + + it('negative: "My API key is sk-..." → sensitive, NOT memorized (redaction guard)', () => { + const entries = extractFromLog(log, date); + const leaked = entries.find( + (e) => + e.content.includes('sk-abc123def456ghi789jkl012mno345pq') || + e.description.includes('sk-abc123def456ghi789jkl012mno345pq'), + ); + expect(leaked).toBeUndefined(); + }); + + it('redaction guard also catches sensitive content even when classifier matches', () => { + // "I prefer using sk-..." would normally match the user 'prefer' pattern. + // The redaction guard must override and drop the entry. + const sneaky = `## ${date}\n\n### 11:00 — Session sneaky\n- I prefer using sk-abc123def456ghi789jkl012mno345pq for auth\n`; + const entries = extractFromLog(sneaky, date); + expect(entries.find((e) => e.content.includes('sk-'))).toBeUndefined(); + }); +}); diff --git a/packages/services/src/mindMemory/extraction.ts b/packages/services/src/mindMemory/extraction.ts new file mode 100644 index 00000000..454e9cf0 --- /dev/null +++ b/packages/services/src/mindMemory/extraction.ts @@ -0,0 +1,465 @@ +/** + * Daily log parsing and memory entry extraction. + * + * Pure functions — no file I/O. Content is passed as strings. + * + * Ported from SCNS (`scns/src/dream/extraction.ts`) with a Chamber-specific + * **sensitive-content redaction guard** added (see {@link containsSensitive}): + * before any extracted line is synthesized into a MemoryEntry, the raw text + * is screened for credential-shaped substrings (sk-… , AKIA… , ghp_… , + * "password is …", "secret = …"). Matches are dropped on the floor — they + * are NOT memorized, even if the classifier would otherwise accept them. + */ + +import { randomUUID } from 'node:crypto'; +import type { MemoryEntry } from './memory-entries'; +import { deduplicateEntries } from './memory-entries'; + +export interface DailyLogEntry { + readonly date: string; + readonly time: string; + readonly sessionId: string | null; + readonly lines: ReadonlyArray; +} + +interface Classification { + readonly type: MemoryEntry['type']; + readonly confidence: number; +} + +const STRONG_CONFIDENCE = 0.9; +const WEAK_CONFIDENCE = 0.6; + +type PatternDef = readonly [RegExp, number]; + +const USER_PATTERNS: readonly PatternDef[] = [ + [/\bprefers?\b/i, STRONG_CONFIDENCE], + [/\balways wants?\b/i, STRONG_CONFIDENCE], + [/\blikes?\b/i, WEAK_CONFIDENCE], + [/\bdislikes?\b/i, STRONG_CONFIDENCE], + [/\bstyle\b/i, WEAK_CONFIDENCE], + [/\bconvention\b/i, WEAK_CONFIDENCE], +]; + +const FEEDBACK_PATTERNS: readonly PatternDef[] = [ + [/\bshould\b/i, WEAK_CONFIDENCE], + [/\bdon['']t\b/i, STRONG_CONFIDENCE], + [/\bremember to\b/i, STRONG_CONFIDENCE], + [/\bbetter to\b/i, STRONG_CONFIDENCE], + [/\blesson learned\b/i, STRONG_CONFIDENCE], +]; + +const PROJECT_PATTERNS: readonly PatternDef[] = [ + [/\bdecided\b/i, STRONG_CONFIDENCE], + [/\busing\b/i, WEAK_CONFIDENCE], + [/\barchitecture\b/i, STRONG_CONFIDENCE], + [/\bdatabase\b/i, WEAK_CONFIDENCE], + [/\bdeployed\b/i, STRONG_CONFIDENCE], + [/\bconfigured\b/i, WEAK_CONFIDENCE], +]; + +const REFERENCE_PATTERNS: readonly PatternDef[] = [ + [/\bURL\b/i, STRONG_CONFIDENCE], + [/\blink\b/i, WEAK_CONFIDENCE], + [/\bdashboard at\b/i, STRONG_CONFIDENCE], + [/\bdocs at\b/i, STRONG_CONFIDENCE], + [/\bAPI at\b/i, STRONG_CONFIDENCE], + [/\blocated at\b/i, STRONG_CONFIDENCE], + [/https?:\/\//i, WEAK_CONFIDENCE], +]; + +const PROHIBITION_PATTERNS: readonly PatternDef[] = [ + [/\bnever (do|claim|say|assume|make|lie|skip|write|mark)\b/i, STRONG_CONFIDENCE], + [/\bstop (doing|making|saying|lying|claiming|ignoring|skipping)\b/i, STRONG_CONFIDENCE], + [/\bdo not (ever|again|assume|claim|skip)\b/i, STRONG_CONFIDENCE], + [/\bmust not\b/i, STRONG_CONFIDENCE], + [/\bavoid\b.*\b(always|never|must)\b/i, WEAK_CONFIDENCE], + [/\bnegative feedback\b/i, WEAK_CONFIDENCE], + [/\bprohibit/i, STRONG_CONFIDENCE], +]; + +// --------------------------------------------------------------------------- +// Sensitive-content redaction guard (Chamber addition) +// --------------------------------------------------------------------------- + +const SENSITIVE_PATTERNS: readonly RegExp[] = [ + /\bsk-[A-Za-z0-9_-]{16,}\b/, // OpenAI-style API key + /\bAKIA[0-9A-Z]{16}\b/, // AWS access key id + /\bghp_[A-Za-z0-9]{20,}\b/, // GitHub personal access token + /\bgho_[A-Za-z0-9]{20,}\b/, // GitHub OAuth token + /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/, // Slack token + /\b(?:password|passwd|pwd)\s*(?:is|=|:)\s*\S{3,}/i, + /\b(?:secret|api[_-]?key|access[_-]?token|auth[_-]?token)\w*\s*(?:is|=|:)\s*\S{8,}/i, + /\bBearer\s+[A-Za-z0-9._-]{20,}/, + /-----BEGIN [A-Z ]*PRIVATE KEY-----/, +]; + +/** + * Returns true when the input text contains a credential-shaped substring. + * Used to drop entries that would otherwise be memorized. + */ +export function containsSensitive(text: string): boolean { + for (const re of SENSITIVE_PATTERNS) { + if (re.test(text)) return true; + } + return false; +} + +// --------------------------------------------------------------------------- +// parseDailyLog +// --------------------------------------------------------------------------- + +const OLD_HEADER_RE = /^###\s+(\d{1,2}:\d{2})(?:\s+[—–-]\s+Session\s+(\S+))?$/; +const PROD_HEADER_RE = /^##\s+(\d{1,2}:\d{2}:\d{2})\s*$/; +const TAGS_RE = /^Tags:\s*(.+)$/; +const CONTENT_RE = /^\*\*\[([^\]]+)\]\*\*\s*(.*)$/; + +const NOISE_SOURCES = new Set([ + 'pre-tool-use', + 'post-tool-use', + 'tool-use', + 'session-start', + 'session-end', +]); + +function parseTags(tagsLine: string): Map { + const tags = new Map(); + for (const part of tagsLine.split(',')) { + const trimmed = part.trim(); + const colonIdx = trimmed.indexOf(':'); + if (colonIdx === -1) continue; + const key = trimmed.slice(0, colonIdx).trim(); + const value = trimmed.slice(colonIdx + 1).trim(); + if (key && value) tags.set(key, value); + } + return tags; +} + +function stripContentWrapper(content: string): string { + const wrappers = [ + /^User prompt:\s*/i, + /^Tool used:\s*/i, + /^Pre-tool:\s*/i, + /^Post-tool:\s*/i, + /^Session\s+\S+\s+(started|ended)\s*/i, + ]; + for (const re of wrappers) { + const match = re.exec(content); + if (match) return content.slice(match[0].length).trim(); + } + return content; +} + +export function parseDailyLog(content: string, date: string): DailyLogEntry[] { + if (!content.trim()) return []; + + const results: DailyLogEntry[] = []; + let current: { time: string; sessionId: string | null; lines: string[] } | null = null; + let currentIsNoise = false; + + const rawLines = content.split('\n'); + + for (let i = 0; i < rawLines.length; i++) { + const line = rawLines[i]!.trimEnd(); + + const oldMatch = OLD_HEADER_RE.exec(line); + if (oldMatch) { + if (current && !currentIsNoise) { + results.push({ + date, + time: current.time, + sessionId: current.sessionId, + lines: current.lines, + }); + } + current = { time: oldMatch[1]!, sessionId: oldMatch[2] ?? null, lines: [] }; + currentIsNoise = false; + continue; + } + + const prodMatch = PROD_HEADER_RE.exec(line); + if (prodMatch) { + if (current && !currentIsNoise) { + results.push({ + date, + time: current.time, + sessionId: current.sessionId, + lines: current.lines, + }); + } + current = { time: prodMatch[1]!, sessionId: null, lines: [] }; + currentIsNoise = false; + continue; + } + + if (!current) continue; + + const tagsMatch = TAGS_RE.exec(line); + if (tagsMatch) { + const tags = parseTags(tagsMatch[1]!); + const sessionId = tags.get('session-id'); + if (sessionId && !current.sessionId) { + current.sessionId = sessionId; + } + const source = tags.get('source'); + if (source && NOISE_SOURCES.has(source)) { + currentIsNoise = true; + current.lines = []; + } + continue; + } + + const contentMatch = CONTENT_RE.exec(line); + if (contentMatch) { + const rawContent = contentMatch[2]!.trim(); + if (rawContent) { + const stripped = stripContentWrapper(rawContent); + if (stripped) current.lines.push(stripped); + } + continue; + } + + if (line.startsWith('- ')) { + current.lines.push(line.slice(2)); + continue; + } + + const trimmed = line.trim(); + if (trimmed && !trimmed.startsWith('## ')) { + current.lines.push(trimmed); + } + } + + if (current && !currentIsNoise) { + results.push({ + date, + time: current.time, + sessionId: current.sessionId, + lines: current.lines, + }); + } + + return results.filter((e) => e.lines.some((l) => l.trim().length > 0)); +} + +// --------------------------------------------------------------------------- +// classifyEntry +// --------------------------------------------------------------------------- + +export function classifyEntry(line: string): Classification | null { + const prohibition = matchPatterns(line, PROHIBITION_PATTERNS); + if (prohibition) return { type: 'prohibition', confidence: prohibition }; + + const ref = matchPatterns(line, REFERENCE_PATTERNS); + if (ref) return { type: 'reference', confidence: ref }; + + const user = matchPatterns(line, USER_PATTERNS); + if (user) return { type: 'user', confidence: user }; + + const feedback = matchPatterns(line, FEEDBACK_PATTERNS); + if (feedback) return { type: 'feedback', confidence: feedback }; + + const project = matchPatterns(line, PROJECT_PATTERNS); + if (project) return { type: 'project', confidence: project }; + + return null; +} + +function matchPatterns(line: string, patterns: readonly PatternDef[]): number | null { + let best: number | null = null; + for (const [regex, confidence] of patterns) { + if (regex.test(line)) { + if (best === null || confidence > best) best = confidence; + } + } + return best; +} + +// --------------------------------------------------------------------------- +// generateEntryName +// --------------------------------------------------------------------------- + +const FILLER_WORDS = new Set(['the', 'a', 'an']); + +export function generateEntryName(content: string): string { + if (!content.trim()) return ''; + + let text = content.replace(/^[-*•]\s+/, '').trim(); + + const clauseMatch = /^([^,.\-—–]+)/.exec(text); + if (clauseMatch) text = clauseMatch[1]!.trim(); + + const words = text.split(/\s+/); + while (words.length > 1 && FILLER_WORDS.has(words[0]!.toLowerCase())) { + words.shift(); + } + + const titled = words.map((w) => w.charAt(0).toUpperCase() + w.slice(1)).join(' '); + + if (titled.length <= 60) return titled; + const truncated = titled.slice(0, 60); + const lastSpace = truncated.lastIndexOf(' '); + return lastSpace > 20 ? truncated.slice(0, lastSpace) : truncated; +} + +// --------------------------------------------------------------------------- +// synthesizeEntry +// --------------------------------------------------------------------------- + +const TYPO_MAP: ReadonlyArray = [ + [/\bii\b/gi, 'I'], + [/\bsi\b/gi, 'is'], + [/\bteh\b/gi, 'the'], + [/\bidk\b/gi, "I don't know"], +]; + +const SYNTHESIS_RULES: ReadonlyArray<{ + readonly match: RegExp; + readonly type: MemoryEntry['type']; + readonly transform: (text: string) => { name: string; description: string }; +}> = [ + { + match: /\bfollow TDD\b/i, + type: 'feedback', + transform: () => ({ + name: 'Follow TDD Workflow', + description: + 'Always follow Test-Driven Development: write tests first, implement to pass, then verify with both automated and manual testing before declaring work complete.', + }), + }, + { + match: /\bstop\s+(doing|making|ignoring|skipping|lying)\b/i, + type: 'prohibition', + transform: (text: string) => { + if (/test/i.test(text)) { + return { + name: 'Never Skip Required Testing', + description: 'Testing requirements are non-negotiable. Do not skip manual or automated testing steps.', + }; + } + return { + name: 'Follow All Quality Requirements', + description: 'Follow all stated quality requirements without shortcuts or omissions.', + }; + }, + }, +]; + +export function fixTypos(text: string): string { + let result = text; + for (const [pattern, replacement] of TYPO_MAP) { + result = result.replace(pattern, replacement); + } + return result; +} + +export function synthesizeEntry( + rawText: string, + type: MemoryEntry['type'], +): { name: string; description: string; content: string } | null { + // Sensitive-content redaction guard — drop entries that contain credentials. + if (containsSensitive(rawText)) return null; + + for (const rule of SYNTHESIS_RULES) { + if (rule.match.test(rawText)) { + const { name, description } = rule.transform(rawText); + return { name, description, content: description }; + } + } + + const cleaned = fixTypos(rawText); + + if (cleaned.length < 15) return null; + + if (type === 'prohibition' || type === 'feedback') { + const normalized = cleanProhibitionFeedback(cleaned); + if (normalized) { + return { + name: generateEntryName(normalized), + description: normalized, + content: normalized, + }; + } + } + + return { + name: generateEntryName(cleaned), + description: cleaned, + content: cleaned, + }; +} + +function cleanProhibitionFeedback(text: string): string | null { + let cleaned = text + .replace(/\?\s*$/g, '.') + .replace(/^why\s+would\s+you\s+/i, '') + .replace(/^what\s+gives?\s*/i, '') + .trim(); + + if (cleaned.length > 0) { + cleaned = cleaned.charAt(0).toUpperCase() + cleaned.slice(1); + } + + if (cleaned.length > 0 && !cleaned.endsWith('.')) cleaned += '.'; + + return cleaned.length > 10 ? cleaned : null; +} + +// --------------------------------------------------------------------------- +// extractFromLog +// --------------------------------------------------------------------------- + +export function extractFromLog(content: string, date: string): ReadonlyArray { + const parsed = parseDailyLog(content, date); + const entries: MemoryEntry[] = []; + + for (const session of parsed) { + for (const line of session.lines) { + // Redaction guard fires before classification — sensitive lines never + // get a chance to match a pattern. + if (containsSensitive(line)) continue; + + const classification = classifyEntry(line); + if (!classification) continue; + + const time = session.time.includes(':') + ? session.time.split(':').length === 3 + ? session.time + : `${session.time}:00` + : `${session.time}:00`; + + const synthesized = synthesizeEntry(line, classification.type); + if (!synthesized) continue; + + entries.push({ + id: randomUUID(), + type: classification.type, + name: synthesized.name, + description: synthesized.description, + content: synthesized.content, + source: `daily-log:${date}`, + createdAt: `${date}T${time}Z`, + }); + } + } + + return entries; +} + +// --------------------------------------------------------------------------- +// extractFromMultipleLogs +// --------------------------------------------------------------------------- + +export function extractFromMultipleLogs( + logs: ReadonlyArray<{ content: string; date: string }>, +): ReadonlyArray { + const allEntries: MemoryEntry[] = []; + + for (const log of logs) { + allEntries.push(...extractFromLog(log.content, log.date)); + } + + allEntries.sort((a, b) => (a.createdAt ?? '').localeCompare(b.createdAt ?? '')); + + return deduplicateEntries(allEntries); +} diff --git a/packages/services/src/mindMemory/index.test.ts b/packages/services/src/mindMemory/index.test.ts new file mode 100644 index 00000000..71fae331 --- /dev/null +++ b/packages/services/src/mindMemory/index.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from 'vitest'; + +describe('mindMemory package scaffold', () => { + it('exposes a public surface module that loads without side effects', async () => { + const mod = await import('./index'); + expect(mod).toBeDefined(); + expect(mod).not.toBeNull(); + }); + + it('exposes a sentinel marker identifying the scaffold version', async () => { + const mod = await import('./index'); + expect(mod.MIND_MEMORY_PACKAGE_VERSION).toBe('0.0.0-scaffold'); + }); +}); + +describe('better-sqlite3 native binding (Phase 0 packaging gate)', () => { + it('opens an in-memory database and round-trips a value', async () => { + const { default: Database } = await import('better-sqlite3'); + const db = new Database(':memory:'); + try { + db.exec('CREATE TABLE t (k TEXT PRIMARY KEY, v TEXT)'); + db.prepare('INSERT INTO t (k, v) VALUES (?, ?)').run('hello', 'world'); + const row = db.prepare('SELECT v FROM t WHERE k = ?').get('hello') as { v: string }; + expect(row.v).toBe('world'); + } finally { + db.close(); + } + }); + + it('supports WAL journal mode (used by per-mind dream.db)', async () => { + const { default: Database } = await import('better-sqlite3'); + const db = new Database(':memory:'); + try { + const mode = db.pragma('journal_mode = WAL', { simple: true }); + // In-memory DBs report 'memory' for journal_mode; the call must not throw. + expect(typeof mode).toBe('string'); + } finally { + db.close(); + } + }); +}); diff --git a/packages/services/src/mindMemory/index.ts b/packages/services/src/mindMemory/index.ts new file mode 100644 index 00000000..8e428711 --- /dev/null +++ b/packages/services/src/mindMemory/index.ts @@ -0,0 +1,37 @@ +/** + * Public surface for the per-mind background memory consolidation engine + * (a.k.a. "Dream Daemon"). + * + * Phase 0 scaffold only — no exports yet. Subsequent phases will add: + * - pure modules: memory-limits, date-utils, consolidation-priorities, + * memory-entries, consolidation, extraction + * - I/O: MindMemoryVault, MindArchiveStore, StructuredLogFormat, + * DailyLogWriter + * - state: dream-schema, dream-state, dream-gates, scheduler + * - orchestrator: LLMClient, CopilotLLMClient, DreamDaemon, + * InternalScheduler, MindMemoryService + * + * See plan: feature/dream-daemon-memory-consolidation. + */ + +export const MIND_MEMORY_PACKAGE_VERSION = '0.0.0-scaffold'; + +export * from './memory-limits'; +export * from './date-utils'; +export * from './memory-entries'; +export * from './consolidation-priorities'; +export * from './consolidation'; +export * from './extraction'; +export * from './StructuredLogFormat'; +export * from './MindMemoryVault'; +export * from './MindArchiveStore'; +export * from './DailyLogWriter'; +export * from './dream-schema'; +export * from './dream-state'; +export * from './dream-gates'; +export * from './consolidation-scheduler'; +export * from './LLMClient'; +export * from './CopilotLLMClient'; +export * from './DreamDaemon'; +export * from './InternalScheduler'; +export * from './MindMemoryService'; diff --git a/packages/services/src/mindMemory/memory-entries.test.ts b/packages/services/src/mindMemory/memory-entries.test.ts new file mode 100644 index 00000000..e4916621 --- /dev/null +++ b/packages/services/src/mindMemory/memory-entries.test.ts @@ -0,0 +1,259 @@ +import { describe, expect, it } from 'vitest'; + +import { + type MemoryEntry, + deduplicateEntries, + detectDuplicates, + parseMemoryMd, + resolveContradictions, + resynthesizeMemoryEntries, + serializeMemoryMd, + validateMemoryEntry, +} from './memory-entries'; + +describe('validateMemoryEntry', () => { + it('accepts a minimal valid entry', () => { + const entry: MemoryEntry = { + type: 'user', + name: 'Prefer concise commits', + description: 'User prefers concise commit messages', + content: 'concise commits', + }; + expect(validateMemoryEntry(entry)).toBe(true); + }); + + it('rejects entries with an unknown type', () => { + expect( + validateMemoryEntry({ + type: 'gibberish', + name: 'x', + description: 'y', + content: 'z', + }), + ).toBe(false); + }); + + it('rejects entries with empty required strings', () => { + expect( + validateMemoryEntry({ + type: 'user', + name: '', + description: 'x', + content: 'y', + }), + ).toBe(false); + expect( + validateMemoryEntry({ + type: 'user', + name: 'x', + description: '', + content: 'y', + }), + ).toBe(false); + }); + + it('rejects non-objects', () => { + expect(validateMemoryEntry(null)).toBe(false); + expect(validateMemoryEntry('string')).toBe(false); + expect(validateMemoryEntry(42)).toBe(false); + }); + + it('accepts every type from the enum', () => { + for (const type of ['user', 'feedback', 'project', 'reference', 'prohibition'] as const) { + expect( + validateMemoryEntry({ type, name: 'n', description: 'd', content: 'c' }), + ).toBe(true); + } + }); +}); + +describe('parseMemoryMd', () => { + it('returns [] for empty content', () => { + expect(parseMemoryMd('')).toEqual([]); + expect(parseMemoryMd(' \n ')).toEqual([]); + }); + + it('parses a single entry with frontmatter', () => { + const md = `---\nname: Prefer concise commits\ndescription: User prefers brevity\ntype: user\n---\nconcise commit messages`; + const result = parseMemoryMd(md); + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ + name: 'Prefer concise commits', + description: 'User prefers brevity', + type: 'user', + content: 'concise commit messages', + }); + }); + + it('parses multiple entries separated by ---', () => { + const md = [ + `---\nname: A\ndescription: dA\ntype: user\n---\ncontentA`, + `---\nname: B\ndescription: dB\ntype: project\n---\ncontentB`, + ].join('\n'); + const result = parseMemoryMd(md); + expect(result).toHaveLength(2); + expect(result[0]?.name).toBe('A'); + expect(result[1]?.name).toBe('B'); + }); + + it('preserves optional fields when present', () => { + const md = `---\nid: abc-123\nname: X\ndescription: dx\ntype: user\nsource: daily-log:2026-05-12\ncreatedAt: 2026-05-12T03:00:00Z\n---\nbody`; + const [entry] = parseMemoryMd(md); + expect(entry?.id).toBe('abc-123'); + expect(entry?.source).toBe('daily-log:2026-05-12'); + expect(entry?.createdAt).toBe('2026-05-12T03:00:00Z'); + }); + + it('skips blocks missing required fields', () => { + const md = `---\ntype: user\n---\nno name`; + expect(parseMemoryMd(md)).toEqual([]); + }); +}); + +describe('serializeMemoryMd', () => { + it('round-trips through parseMemoryMd', () => { + const entries: MemoryEntry[] = [ + { type: 'user', name: 'A', description: 'dA', content: 'cA' }, + { type: 'project', name: 'B', description: 'dB', content: 'cB' }, + ]; + const serialized = serializeMemoryMd(entries); + const reparsed = parseMemoryMd(serialized); + expect(reparsed).toHaveLength(2); + expect(reparsed[0]).toMatchObject(entries[0]!); + expect(reparsed[1]).toMatchObject(entries[1]!); + }); + + it('emits id, source, createdAt only when present', () => { + const md = serializeMemoryMd([ + { type: 'user', name: 'A', description: 'dA', content: 'cA' }, + ]); + expect(md).not.toContain('id:'); + expect(md).not.toContain('source:'); + expect(md).not.toContain('createdAt:'); + }); +}); + +describe('detectDuplicates', () => { + it('flags entries with identical names case-insensitively', () => { + const entries: MemoryEntry[] = [ + { type: 'user', name: 'Prefer Concise Commits', description: 'd', content: 'c' }, + { type: 'user', name: 'prefer concise commits', description: 'd', content: 'c' }, + ]; + expect(detectDuplicates(entries)).toEqual([[0, 1]]); + }); + + it('flags entries with very similar content', () => { + const long = 'we are using postgres for the auth service'; + const entries: MemoryEntry[] = [ + { type: 'project', name: 'A', description: 'd', content: long }, + { type: 'project', name: 'B', description: 'd', content: long }, + ]; + expect(detectDuplicates(entries)).toEqual([[0, 1]]); + }); + + it('returns no pairs when entries are distinct', () => { + const entries: MemoryEntry[] = [ + { type: 'user', name: 'A', description: 'dA', content: 'totally different' }, + { type: 'user', name: 'B', description: 'dB', content: 'completely unrelated' }, + ]; + expect(detectDuplicates(entries)).toEqual([]); + }); +}); + +describe('deduplicateEntries', () => { + it('keeps the later entry when names collide', () => { + const entries: MemoryEntry[] = [ + { type: 'user', name: 'A', description: 'old', content: 'old' }, + { type: 'user', name: 'a', description: 'new', content: 'new' }, + ]; + const out = deduplicateEntries(entries); + expect(out).toHaveLength(1); + expect(out[0]?.description).toBe('new'); + }); + + it('returns all entries when no duplicates exist', () => { + const entries: MemoryEntry[] = [ + { type: 'user', name: 'A', description: 'd', content: 'totally different content here' }, + { type: 'project', name: 'B', description: 'd', content: 'completely unrelated material' }, + ]; + expect(deduplicateEntries(entries)).toHaveLength(2); + }); +}); + +describe('resolveContradictions', () => { + it('keeps the later entry when names match exactly', () => { + const entries: MemoryEntry[] = [ + { type: 'user', name: 'X', description: 'old', content: 'old' }, + { type: 'user', name: 'X', description: 'new', content: 'new' }, + ]; + const out = resolveContradictions(entries); + expect(out).toHaveLength(1); + expect(out[0]?.description).toBe('new'); + }); + + it('keeps the later entry when same type and similar names', () => { + const entries: MemoryEntry[] = [ + { type: 'user', name: 'use postgres', description: 'old', content: 'c' }, + { type: 'user', name: 'use postgres!', description: 'new', content: 'c' }, + ]; + const out = resolveContradictions(entries); + expect(out).toHaveLength(1); + expect(out[0]?.description).toBe('new'); + }); + + it('does not collapse entries of different types even with similar names', () => { + const entries: MemoryEntry[] = [ + { type: 'user', name: 'use postgres', description: 'a', content: 'a' }, + { type: 'project', name: 'use postgres', description: 'b', content: 'b' }, + ]; + // Note: identical name (case-insensitive) STILL collapses regardless of type per SCNS rules. + const out = resolveContradictions(entries); + expect(out).toHaveLength(1); + expect(out[0]?.type).toBe('project'); + }); +}); + +describe('resynthesizeMemoryEntries', () => { + it('uses the synthesizer output when it returns a result', () => { + const entries: MemoryEntry[] = [ + { type: 'user', name: 'old', description: 'old', content: 'raw text' }, + ]; + const out = resynthesizeMemoryEntries( + entries, + () => ({ name: 'New', description: 'New desc', content: 'New content' }), + (t) => t, + ); + expect(out[0]).toMatchObject({ + name: 'New', + description: 'New desc', + content: 'New content', + }); + expect(out[0]?.id).toBeDefined(); + }); + + it('falls back to typo-fix when the synthesizer returns null', () => { + const entries: MemoryEntry[] = [ + { type: 'user', name: 'teh name', description: 'teh desc', content: 'teh body' }, + ]; + const out = resynthesizeMemoryEntries( + entries, + () => null, + (t) => t.replace(/teh/g, 'the'), + ); + expect(out[0]?.name).toBe('the name'); + expect(out[0]?.description).toBe('the desc'); + expect(out[0]?.content).toBe('the body'); + }); + + it('preserves an existing id when present', () => { + const entries: MemoryEntry[] = [ + { id: 'preserved', type: 'user', name: 'A', description: 'd', content: 'c' }, + ]; + const out = resynthesizeMemoryEntries( + entries, + () => ({ name: 'A', description: 'd', content: 'c' }), + (t) => t, + ); + expect(out[0]?.id).toBe('preserved'); + }); +}); diff --git a/packages/services/src/mindMemory/memory-entries.ts b/packages/services/src/mindMemory/memory-entries.ts new file mode 100644 index 00000000..6a0ef92c --- /dev/null +++ b/packages/services/src/mindMemory/memory-entries.ts @@ -0,0 +1,238 @@ +/** + * Memory entry types, parsing, deduplication, and validation. + * + * Pure module: no I/O, no logging. + * + * Zod v4 audit: this module uses `z.object`, `z.string()`, `z.string().min(1)`, + * `z.string().optional()`, `z.enum([...] as const)`, and only the `.success` field + * of `safeParse(...)`. All of these are stable across zod v3→v4. The `.error.issues` + * shape changed in v4 but is not consumed here. + */ +import { randomUUID } from 'node:crypto'; +import { z } from 'zod'; + +const MEMORY_ENTRY_TYPES = ['user', 'feedback', 'project', 'reference', 'prohibition'] as const; + +export interface MemoryEntry { + readonly id?: string; + readonly type: (typeof MEMORY_ENTRY_TYPES)[number]; + readonly name: string; + readonly description: string; + readonly content: string; + readonly source?: string; + readonly createdAt?: string; +} + +const MemoryEntrySchema = z.object({ + id: z.string().optional(), + type: z.enum(MEMORY_ENTRY_TYPES), + name: z.string().min(1), + description: z.string().min(1), + content: z.string(), + source: z.string().optional(), + createdAt: z.string().optional(), +}); + +export function validateMemoryEntry(entry: unknown): entry is MemoryEntry { + return MemoryEntrySchema.safeParse(entry).success; +} + +export function parseMemoryMd(content: string): MemoryEntry[] { + if (!content.trim()) return []; + + const entries: MemoryEntry[] = []; + const blocks = splitIntoBlocks(content); + + for (const block of blocks) { + const parsed = parseBlock(block); + if (parsed) entries.push(parsed); + } + + return entries; +} + +function splitIntoBlocks(content: string): string[] { + const blocks: string[] = []; + const regex = /---\n([\s\S]*?)---\n([\s\S]*?)(?=\n---\n|$)/g; + let match: RegExpExecArray | null; + + while ((match = regex.exec(content)) !== null) { + const frontmatter = match[1]!; + const body = match[2]!; + blocks.push(`---\n${frontmatter}---\n${body}`); + } + + return blocks; +} + +function parseBlock(block: string): MemoryEntry | null { + const fmMatch = /^---\n([\s\S]*?)\n?---\n([\s\S]*)$/.exec(block.trim()); + if (!fmMatch) return null; + + const frontmatter = fmMatch[1]!; + const content = fmMatch[2]!.trim(); + + const fields = parseSimpleYaml(frontmatter); + if (!fields.name || !fields.description || !fields.type) return null; + + const entry: Record = { + type: fields.type, + name: fields.name, + description: fields.description, + content, + }; + + if (fields.id) entry['id'] = fields.id; + if (fields.source) entry['source'] = fields.source; + if (fields.createdAt) entry['createdAt'] = fields.createdAt; + + return validateMemoryEntry(entry) ? (entry as unknown as MemoryEntry) : null; +} + +function parseSimpleYaml(text: string): Record { + const result: Record = {}; + for (const line of text.split('\n')) { + const colonIdx = line.indexOf(':'); + if (colonIdx === -1) continue; + const key = line.slice(0, colonIdx).trim(); + const value = line.slice(colonIdx + 1).trim(); + if (key && value) result[key] = value; + } + return result; +} + +export function serializeMemoryMd(entries: ReadonlyArray): string { + return entries.map(serializeEntry).join('\n\n'); +} + +function serializeEntry(entry: MemoryEntry): string { + const fields: string[] = []; + + if (entry.id) fields.push(`id: ${entry.id}`); + fields.push(`name: ${entry.name}`); + fields.push(`description: ${entry.description}`); + fields.push(`type: ${entry.type}`); + + if (entry.source) fields.push(`source: ${entry.source}`); + if (entry.createdAt) fields.push(`createdAt: ${entry.createdAt}`); + + return `---\n${fields.join('\n')}\n---\n${entry.content}`; +} + +export function detectDuplicates( + entries: ReadonlyArray, +): ReadonlyArray { + const pairs: Array = []; + + for (let i = 0; i < entries.length; i++) { + for (let j = i + 1; j < entries.length; j++) { + const a = entries[i]!; + const b = entries[j]!; + + if (a.name.toLowerCase() === b.name.toLowerCase()) { + pairs.push([i, j] as const); + continue; + } + + if (contentSimilarity(a.content, b.content) > 0.8) { + pairs.push([i, j] as const); + } + } + } + + return pairs; +} + +function contentSimilarity(a: string, b: string): number { + const na = normalize(a); + const nb = normalize(b); + if (na.length === 0 && nb.length === 0) return 1; + if (na.length === 0 || nb.length === 0) return 0; + + const longer = na.length >= nb.length ? na : nb; + const shorter = na.length < nb.length ? na : nb; + + let matches = 0; + for (let i = 0; i < shorter.length; i++) { + if (shorter[i] === longer[i]) matches++; + } + + return matches / longer.length; +} + +function normalize(text: string): string { + return text.toLowerCase().replace(/\s+/g, ' ').trim(); +} + +export function deduplicateEntries( + entries: ReadonlyArray, +): ReadonlyArray { + const dupes = detectDuplicates(entries); + const removeSet = new Set(); + + for (const [earlier] of dupes) { + removeSet.add(earlier); + } + + return entries.filter((_, i) => !removeSet.has(i)); +} + +export function resolveContradictions( + entries: ReadonlyArray, +): ReadonlyArray { + const removeSet = new Set(); + + for (let i = 0; i < entries.length; i++) { + for (let j = i + 1; j < entries.length; j++) { + const a = entries[i]!; + const b = entries[j]!; + + if (a.name.toLowerCase() === b.name.toLowerCase()) { + removeSet.add(i); + continue; + } + + if (a.type === b.type && nameSimilarity(a.name, b.name) > 0.8) { + removeSet.add(i); + } + } + } + + return entries.filter((_, i) => !removeSet.has(i)); +} + +function nameSimilarity(a: string, b: string): number { + return contentSimilarity(a, b); +} + +export function resynthesizeMemoryEntries( + entries: ReadonlyArray, + synthesize: ( + content: string, + type: MemoryEntry['type'], + ) => { name: string; description: string; content: string } | null, + fixTyposFn: (text: string) => string, +): MemoryEntry[] { + const results: MemoryEntry[] = []; + for (const entry of entries) { + const synthesized = synthesize(entry.content, entry.type); + if (synthesized) { + results.push({ + ...entry, + id: entry.id ?? randomUUID(), + name: synthesized.name, + description: synthesized.description, + content: synthesized.content, + }); + } else { + results.push({ + ...entry, + id: entry.id ?? randomUUID(), + name: fixTyposFn(entry.name), + description: fixTyposFn(entry.description), + content: fixTyposFn(entry.content), + }); + } + } + return results; +} diff --git a/packages/services/src/mindMemory/memory-limits.test.ts b/packages/services/src/mindMemory/memory-limits.test.ts new file mode 100644 index 00000000..9d89c95e --- /dev/null +++ b/packages/services/src/mindMemory/memory-limits.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from 'vitest'; + +import { + MAX_ENTRYPOINT_BYTES, + MAX_ENTRYPOINT_LINES, + countBytes, + countLines, + truncateEntrypoint, +} from './memory-limits'; + +describe('memory-limits constants', () => { + it('caps the entrypoint at the SCNS spec limits', () => { + expect(MAX_ENTRYPOINT_LINES).toBe(200); + expect(MAX_ENTRYPOINT_BYTES).toBe(25_000); + }); +}); + +describe('countLines', () => { + it('returns 0 for an empty string', () => { + expect(countLines('')).toBe(0); + }); + + it('returns 0 for a string containing only a trailing newline', () => { + expect(countLines('\n')).toBe(0); + }); + + it('returns 1 for a single line with no trailing newline', () => { + expect(countLines('hello')).toBe(1); + }); + + it('treats a trailing newline as terminator, not as an extra line', () => { + expect(countLines('hello\n')).toBe(1); + }); + + it('counts interior newlines as separators', () => { + expect(countLines('a\nb')).toBe(2); + expect(countLines('a\nb\nc\n')).toBe(3); + }); +}); + +describe('countBytes', () => { + it('returns the UTF-8 byte length of an ASCII string', () => { + expect(countBytes('hello')).toBe(5); + }); + + it('counts multi-byte UTF-8 characters by their byte length', () => { + expect(countBytes('héllo')).toBe(6); + expect(countBytes('🦆')).toBe(4); + }); + + it('returns 0 for an empty string', () => { + expect(countBytes('')).toBe(0); + }); +}); + +describe('truncateEntrypoint', () => { + it('passes empty content through unchanged', () => { + const result = truncateEntrypoint(''); + expect(result.content).toBe(''); + expect(result.truncated).toBe(false); + expect(result.warning).toBeNull(); + }); + + it('passes content under both limits through unchanged', () => { + const content = 'one\ntwo\nthree'; + const result = truncateEntrypoint(content); + expect(result.content).toBe(content); + expect(result.truncated).toBe(false); + expect(result.warning).toBeNull(); + }); + + it('truncates content that exceeds the line limit and reports a lines warning', () => { + const content = Array.from({ length: MAX_ENTRYPOINT_LINES + 50 }, (_, i) => `line${i}`).join( + '\n', + ); + const result = truncateEntrypoint(content); + expect(result.truncated).toBe(true); + expect(result.warning).toBe(''); + // Resulting content has MAX_ENTRYPOINT_LINES kept lines + the warning line. + const lines = result.content.split('\n'); + expect(lines).toHaveLength(MAX_ENTRYPOINT_LINES + 1); + expect(lines[MAX_ENTRYPOINT_LINES]).toBe(''); + }); + + it('truncates content that exceeds the byte limit and reports a bytes warning', () => { + const oversized = 'x'.repeat(MAX_ENTRYPOINT_BYTES + 100); + const result = truncateEntrypoint(oversized); + expect(result.truncated).toBe(true); + expect(result.warning).toBe(''); + expect(countBytes(result.content)).toBeLessThanOrEqual(MAX_ENTRYPOINT_BYTES + 100); + }); + + it('reports both lines and bytes when both limits are exceeded', () => { + // Many short lines so we trip BOTH gates. + const content = Array.from({ length: MAX_ENTRYPOINT_LINES + 500 }, () => + 'a'.repeat(200), + ).join('\n'); + const result = truncateEntrypoint(content); + expect(result.truncated).toBe(true); + expect(result.warning).toBe(''); + }); + + it('does not split multi-byte sequences when hard-cutting a single oversized line', () => { + const single = '🦆'.repeat(Math.ceil(MAX_ENTRYPOINT_BYTES / 4) + 50); + const result = truncateEntrypoint(single); + expect(result.truncated).toBe(true); + // The truncated content must be valid UTF-8 (no replacement character at the end). + expect(result.content).not.toMatch(/\uFFFD$/); + }); +}); diff --git a/packages/services/src/mindMemory/memory-limits.ts b/packages/services/src/mindMemory/memory-limits.ts new file mode 100644 index 00000000..d59bfc55 --- /dev/null +++ b/packages/services/src/mindMemory/memory-limits.ts @@ -0,0 +1,80 @@ +/** + * MEMORY.md size enforcement — keeps the entrypoint within SCNS-style spec limits. + * + * Pure module: no I/O, no logging, no global state. + */ + +export const MAX_ENTRYPOINT_LINES = 200; +export const MAX_ENTRYPOINT_BYTES = 25_000; + +export interface TruncateResult { + readonly content: string; + readonly truncated: boolean; + readonly warning: string | null; +} + +export function countLines(content: string): number { + if (content === '') return 0; + const stripped = content.endsWith('\n') ? content.slice(0, -1) : content; + if (stripped === '') return 0; + return stripped.split('\n').length; +} + +export function countBytes(content: string): number { + return Buffer.byteLength(content, 'utf-8'); +} + +export function truncateEntrypoint(content: string): TruncateResult { + if (content === '') { + return { content: '', truncated: false, warning: null }; + } + + const overLines = countLines(content) > MAX_ENTRYPOINT_LINES; + const overBytes = countBytes(content) > MAX_ENTRYPOINT_BYTES; + + if (!overLines && !overBytes) { + return { content, truncated: false, warning: null }; + } + + let result = content; + let hitLines = false; + let hitBytes = false; + + if (overLines) { + const lines = result.split('\n'); + result = lines.slice(0, MAX_ENTRYPOINT_LINES).join('\n'); + hitLines = true; + } + + if (countBytes(result) > MAX_ENTRYPOINT_BYTES) { + result = truncateToByteLimit(result); + hitBytes = true; + } + + const cap = hitLines && hitBytes ? 'lines and bytes' : hitLines ? 'lines' : 'bytes'; + const warning = ``; + + return { + content: `${result}\n${warning}`, + truncated: true, + warning, + }; +} + +function truncateToByteLimit(content: string): string { + const buf = Buffer.from(content, 'utf-8'); + const sliced = buf.subarray(0, MAX_ENTRYPOINT_BYTES); + let str = sliced.toString('utf-8'); + + const lastNl = str.lastIndexOf('\n'); + if (lastNl > 0) { + str = str.slice(0, lastNl); + } else { + str = Buffer.from(content, 'utf-8').subarray(0, MAX_ENTRYPOINT_BYTES).toString('utf-8'); + if (str.endsWith('\uFFFD')) { + str = str.slice(0, -1); + } + } + + return str; +} From ca15812b1cf5daeef26f0381a2c04d47ff66757b Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Tue, 12 May 2026 19:20:48 -0400 Subject: [PATCH 05/23] ADD: TurnCompletionObserver wiring in ChatService --- .../src/chat/ChatService.observers.test.ts | 300 ++++++++++++++++++ packages/services/src/chat/ChatService.ts | 98 +++++- packages/services/src/chat/index.ts | 5 + 3 files changed, 395 insertions(+), 8 deletions(-) create mode 100644 packages/services/src/chat/ChatService.observers.test.ts diff --git a/packages/services/src/chat/ChatService.observers.test.ts b/packages/services/src/chat/ChatService.observers.test.ts new file mode 100644 index 00000000..d7553490 --- /dev/null +++ b/packages/services/src/chat/ChatService.observers.test.ts @@ -0,0 +1,300 @@ +/** + * Phase 6 — TurnCompletionObserver wiring in ChatService. + * + * The success contract is exact: every observer is notified exactly once per + * turn that reached the SDK `done` state, with the full CompletedTurn + * payload. Aborted turns, errored turns, and SDK contract drift all suppress + * notification. One observer throwing — sync or async — must not block any + * other observer and must not leak back into the streaming path. + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { ChatService } from './ChatService'; +import { TurnQueue } from './TurnQueue'; +import type { MindManager } from '../mind'; +import type { CompletedTurn, TurnCompletionObserver } from '@chamber/shared/turn-observer'; + +interface SessionListeners { + idle: Array<() => void>; + error: Array<(event: unknown) => void>; + message: Array<(event: unknown) => void>; + delta: Array<(event: unknown) => void>; +} + +function createMockSession() { + const listeners: SessionListeners = { idle: [], error: [], message: [], delta: [] }; + const session = { + send: vi.fn().mockResolvedValue(undefined), + abort: vi.fn().mockResolvedValue(undefined), + destroy: vi.fn().mockResolvedValue(undefined), + on: vi.fn((event: string, cb: (...args: unknown[]) => void) => { + if (event === 'session.idle') listeners.idle.push(cb as () => void); + else if (event === 'session.error') listeners.error.push(cb as (event: unknown) => void); + else if (event === 'assistant.message') listeners.message.push(cb as (event: unknown) => void); + else if (event === 'assistant.message_delta') listeners.delta.push(cb as (event: unknown) => void); + return vi.fn(); + }), + }; + return { session, listeners }; +} + +function createMockManager(session: unknown) { + return { + getMind: vi.fn(() => ({ + session, + client: { listModels: vi.fn(async () => []) }, + activeSessionId: 'sdk-session-abc', + selectedModel: 'gpt-5.4', + })), + setMindModel: vi.fn(async () => null), + recoverActiveConversationSession: vi.fn(), + markActiveConversationHasMessages: vi.fn(), + listConversationHistory: vi.fn(() => []), + startNewConversation: vi.fn(), + resumeConversation: vi.fn(), + deleteConversation: vi.fn(), + renameConversation: vi.fn(() => []), + recycleClientForMind: vi.fn(), + reloadMind: vi.fn(), + }; +} + +const dateTimeProvider = () => ({ + currentDateTime: '2026-05-12T17:00:00.000Z', + timezone: 'America/New_York', +}); + +describe('ChatService — TurnCompletionObserver wiring (Phase 6)', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('notifies every observer exactly once per successful turn with the full CompletedTurn payload', async () => { + const { session, listeners } = createMockSession(); + const mgr = createMockManager(session); + session.send.mockImplementation(async () => { + // Final assistant message arrives between send and idle. + listeners.message.forEach((cb) => + cb({ data: { messageId: 'sdk-msg-1', content: 'pong' } }), + ); + listeners.idle.forEach((cb) => cb()); + }); + + const captured: CompletedTurn[] = []; + const observerA: TurnCompletionObserver = { onTurnCompleted: (t) => { captured.push(t); } }; + const observerB: TurnCompletionObserver = { onTurnCompleted: (t) => { captured.push(t); } }; + + const svc = new ChatService( + mgr as unknown as MindManager, + new TurnQueue(), + dateTimeProvider, + [observerA, observerB], + ); + + await svc.sendMessage('mind-1', 'ping', 'msg-1', vi.fn()); + + expect(captured).toHaveLength(2); + const [a, b] = captured; + expect(a).toEqual(b); + expect(a.prompt).toBe('ping'); + expect(a.finalAssistantMessage).toBe('pong'); + expect(a.sessionId).toBe('sdk-session-abc'); + expect(a.model).toBe('gpt-5.4'); + expect(a.status).toBe('completed'); + expect(typeof a.turnId).toBe('string'); + expect(a.turnId.length).toBeGreaterThan(0); + expect(typeof a.startedAt).toBe('string'); + expect(typeof a.endedAt).toBe('string'); + expect(Date.parse(a.startedAt)).not.toBeNaN(); + expect(Date.parse(a.endedAt)).not.toBeNaN(); + expect(Date.parse(a.endedAt)).toBeGreaterThanOrEqual(Date.parse(a.startedAt)); + }); + + it('uses the explicitly-requested model from sendMessage when provided', async () => { + const { session, listeners } = createMockSession(); + const mgr = createMockManager(session); + session.send.mockImplementation(async () => { + listeners.message.forEach((cb) => cb({ data: { messageId: 'm', content: 'hi' } })); + listeners.idle.forEach((cb) => cb()); + }); + + const captured: CompletedTurn[] = []; + const svc = new ChatService( + mgr as unknown as MindManager, + new TurnQueue(), + dateTimeProvider, + [{ onTurnCompleted: (t) => { captured.push(t); } }], + ); + + await svc.sendMessage('mind-1', 'hi', 'msg-1', vi.fn(), 'opus-4.7'); + + expect(captured).toHaveLength(1); + expect(captured[0].model).toBe('opus-4.7'); + }); + + it('does NOT notify observers when the user aborts the turn mid-stream', async () => { + const { session, listeners } = createMockSession(); + const mgr = createMockManager(session); + // session.idle never fires; we abort externally. + session.send.mockResolvedValue(undefined); + + const observer = { onTurnCompleted: vi.fn() }; + const svc = new ChatService( + mgr as unknown as MindManager, + new TurnQueue(), + dateTimeProvider, + [observer], + ); + + const pending = svc.sendMessage('mind-1', 'long-running', 'msg-1', vi.fn()); + // Allow the queue/streamTurn to wire up listeners and call send. + for (let i = 0; i < 10; i++) await Promise.resolve(); + // Even if the assistant text was captured before abort, abort suppresses notification. + listeners.message.forEach((cb) => cb({ data: { messageId: 'm', content: 'partial' } })); + await svc.cancelMessage('mind-1', 'msg-1'); + await pending; + + expect(observer.onTurnCompleted).not.toHaveBeenCalled(); + }); + + it('does NOT notify observers when the SDK signals session.error', async () => { + const { session, listeners } = createMockSession(); + const mgr = createMockManager(session); + session.send.mockImplementation(async () => { + listeners.error.forEach((cb) => cb({ data: { message: 'boom' } })); + }); + + const observer = { onTurnCompleted: vi.fn() }; + const svc = new ChatService( + mgr as unknown as MindManager, + new TurnQueue(), + dateTimeProvider, + [observer], + ); + + const emit = vi.fn(); + await svc.sendMessage('mind-1', 'hi', 'msg-1', emit); + + expect(emit).toHaveBeenCalledWith(expect.objectContaining({ type: 'error' })); + expect(observer.onTurnCompleted).not.toHaveBeenCalled(); + }); + + it('does NOT notify observers when sendMessage rejects synchronously (mind missing)', async () => { + const mgr = createMockManager(null); + mgr.getMind.mockReturnValue(undefined as never); + + const observer = { onTurnCompleted: vi.fn() }; + const svc = new ChatService( + mgr as unknown as MindManager, + new TurnQueue(), + dateTimeProvider, + [observer], + ); + + const emit = vi.fn(); + await svc.sendMessage('missing', 'hi', 'msg-1', emit); + + expect(emit).toHaveBeenCalledWith(expect.objectContaining({ type: 'error' })); + expect(observer.onTurnCompleted).not.toHaveBeenCalled(); + }); + + it('one observer throwing synchronously does NOT block subsequent observers and does NOT break streaming', async () => { + const { session, listeners } = createMockSession(); + const mgr = createMockManager(session); + session.send.mockImplementation(async () => { + listeners.message.forEach((cb) => cb({ data: { messageId: 'm', content: 'pong' } })); + listeners.idle.forEach((cb) => cb()); + }); + + const order: string[] = []; + const observerA: TurnCompletionObserver = { + onTurnCompleted: () => { order.push('a'); throw new Error('observer A boom'); }, + }; + const observerB: TurnCompletionObserver = { + onTurnCompleted: () => { order.push('b'); }, + }; + const observerC: TurnCompletionObserver = { + onTurnCompleted: () => { order.push('c'); }, + }; + + const consoleWarn = vi.spyOn(console, 'log').mockImplementation(() => undefined); + try { + const svc = new ChatService( + mgr as unknown as MindManager, + new TurnQueue(), + dateTimeProvider, + [observerA, observerB, observerC], + ); + + const emit = vi.fn(); + // Whole streaming path must still resolve cleanly. + await expect(svc.sendMessage('mind-1', 'ping', 'msg-1', emit)).resolves.toBeUndefined(); + + expect(order).toEqual(['a', 'b', 'c']); + expect(emit).toHaveBeenCalledWith({ type: 'done' }); + expect(emit).not.toHaveBeenCalledWith(expect.objectContaining({ type: 'error' })); + } finally { + consoleWarn.mockRestore(); + } + }); + + it('an observer rejecting asynchronously is logged and does NOT surface back into the streaming path', async () => { + const { session, listeners } = createMockSession(); + const mgr = createMockManager(session); + session.send.mockImplementation(async () => { + listeners.message.forEach((cb) => cb({ data: { messageId: 'm', content: 'pong' } })); + listeners.idle.forEach((cb) => cb()); + }); + + const observerA: TurnCompletionObserver = { + onTurnCompleted: async () => { throw new Error('async boom'); }, + }; + const observerB = { onTurnCompleted: vi.fn() }; + + const consoleWarn = vi.spyOn(console, 'log').mockImplementation(() => undefined); + try { + const svc = new ChatService( + mgr as unknown as MindManager, + new TurnQueue(), + dateTimeProvider, + [observerA, observerB], + ); + + const emit = vi.fn(); + await expect(svc.sendMessage('mind-1', 'ping', 'msg-1', emit)).resolves.toBeUndefined(); + + // observer B was still invoked despite A's async rejection. + expect(observerB.onTurnCompleted).toHaveBeenCalledTimes(1); + // Streaming path stayed clean — no error event leaked from the observer failure. + expect(emit).toHaveBeenCalledWith({ type: 'done' }); + expect(emit).not.toHaveBeenCalledWith(expect.objectContaining({ type: 'error' })); + + // Drain microtasks to give the .catch on the rejected observer promise + // a chance to run before the test ends so we know it was attached + // (otherwise vitest would surface an unhandled rejection). + for (let i = 0; i < 5; i++) await Promise.resolve(); + } finally { + consoleWarn.mockRestore(); + } + }); + + it('default observers list is empty — existing 3-arg constructor callers keep working unchanged', async () => { + const { session, listeners } = createMockSession(); + const mgr = createMockManager(session); + session.send.mockImplementation(async () => { + listeners.idle.forEach((cb) => cb()); + }); + + // Three-arg construction (no observers) is the legacy contract. + const svc = new ChatService( + mgr as unknown as MindManager, + new TurnQueue(), + dateTimeProvider, + ); + + const emit = vi.fn(); + await svc.sendMessage('mind-1', 'hi', 'msg-1', emit); + + expect(emit).toHaveBeenCalledWith({ type: 'done' }); + }); +}); diff --git a/packages/services/src/chat/ChatService.ts b/packages/services/src/chat/ChatService.ts index 7ea112fd..59a36b51 100644 --- a/packages/services/src/chat/ChatService.ts +++ b/packages/services/src/chat/ChatService.ts @@ -1,8 +1,10 @@ // ChatService — thin message streaming layer. // Gets sessions from MindManager, streams SDK events via callback. +import { randomUUID } from 'node:crypto'; import type { MindManager } from '../mind'; import type { ChatEvent, ChatImageAttachment, ConversationResumeResult, ConversationSummary, ModelInfo } from '@chamber/shared/types'; +import type { CompletedTurn, TurnCompletionObserver } from '@chamber/shared/turn-observer'; import type { CopilotSession } from '../mind/types'; import { isStaleSessionError, SEND_TIMEOUT_MS, sendTimeoutError } from '@chamber/shared/sessionErrors'; import { Logger } from '../logger'; @@ -28,12 +30,37 @@ const log = Logger.create('ChatService'); export class ChatService { private abortControllers = new Map(); + private readonly observers: TurnCompletionObserver[]; constructor( private readonly mindManager: MindManager, private readonly turnQueue: TurnQueue, private readonly dateTimeContextProvider: DateTimeContextProvider = getCurrentDateTimeContext, - ) {} + initialObservers: readonly TurnCompletionObserver[] = [], + ) { + // Defensive copy — callers may mutate the array they passed in. + this.observers = [...initialObservers]; + } + + /** + * Register a turn-completion observer (Phase 11 wiring — used by + * MindMemoryService to attach the per-mind DailyLogWriter when a mind is + * activated). Adding an observer mid-flight is safe because + * `notifyTurnCompleted` reads the array at notify time. + */ + addObserver(observer: TurnCompletionObserver): void { + this.observers.push(observer); + } + + /** + * Remove a previously-registered observer (no-op if not present). Used + * by MindMemoryService.releaseMind so a swapped-out mind stops receiving + * turn frames. + */ + removeObserver(observer: TurnCompletionObserver): void { + const i = this.observers.indexOf(observer); + if (i !== -1) this.observers.splice(i, 1); + } async sendMessage( mindId: string, @@ -47,17 +74,21 @@ export class ChatService { const abortController = new AbortController(); this.abortControllers.set(mindId, abortController); + const startedAt = new Date().toISOString(); + const turnId = randomUUID(); + try { const context = this.mindManager.getMind(mindId); if (!context?.session) { throw new Error(`Mind ${mindId} not found or has no session`); } + let finalAssistantMessage: string | null = null; try { const session = model ? await this.mindManager.setMindModel(mindId, model) : null; const currentSession = session ? this.mindManager.getMind(mindId)?.session : context.session; if (!currentSession) throw new Error(`Mind ${mindId} not found or has no session`); - await this.streamTurn(currentSession, prompt, abortController, emit, attachments, () => { + finalAssistantMessage = await this.streamTurn(currentSession, prompt, abortController, emit, attachments, () => { this.mindManager.markActiveConversationHasMessages(mindId, prompt); }); } catch (err) { @@ -69,10 +100,30 @@ export class ChatService { emit({ type: 'reconnecting' }); const recoveredSession = await this.mindManager.recoverActiveConversationSession(mindId); if (abortController.signal.aborted) return; - await this.streamTurn(recoveredSession, prompt, abortController, emit, attachments, () => { + finalAssistantMessage = await this.streamTurn(recoveredSession, prompt, abortController, emit, attachments, () => { this.mindManager.markActiveConversationHasMessages(mindId, prompt); }); } + + // Notify TurnCompletionObservers ONLY on successful completion. The + // streamTurn helper returns null whenever the turn was aborted by + // the user or torn down by an SDK contract failure; both branches + // skip notification. Errors thrown out of streamTurn fall through + // to the outer catch below, which also bypasses notification. + if (finalAssistantMessage !== null && !abortController.signal.aborted) { + const endedAt = new Date().toISOString(); + const refreshed = this.mindManager.getMind(mindId); + this.notifyTurnCompleted({ + turnId, + sessionId: refreshed?.activeSessionId ?? '', + model: model ?? refreshed?.selectedModel ?? '', + status: 'completed', + startedAt, + endedAt, + prompt, + finalAssistantMessage, + }); + } } catch (err) { if (abortController.signal.aborted) return; const message = err instanceof Error ? err.message : String(err); @@ -83,6 +134,28 @@ export class ChatService { }); } + /** + * Notify each observer of a completed turn. One observer throwing (sync + * or async) must NOT block subsequent observers and must NOT propagate + * back into the streaming path. Failures are logged at warn level with + * the offending observer's index for triage. + */ + private notifyTurnCompleted(turn: CompletedTurn): void { + for (let i = 0; i < this.observers.length; i++) { + const observer = this.observers[i]; + try { + const result = observer.onTurnCompleted(turn); + if (result && typeof (result as Promise).then === 'function') { + Promise.resolve(result).catch((err: unknown) => { + log.warn(`TurnCompletionObserver[${i}] failed asynchronously`, err); + }); + } + } catch (err) { + log.warn(`TurnCompletionObserver[${i}] failed`, err); + } + } + } + private async streamTurn( session: CopilotSession, prompt: string, @@ -90,10 +163,11 @@ export class ChatService { emit: (event: ChatEvent) => void, attachments?: ChatImageAttachment[], onSendAccepted?: () => void, - ): Promise{ - if (abortController.signal.aborted) return; + ): Promise{ + if (abortController.signal.aborted) return null; const unsubs: (() => void)[] = []; + let finalAssistantMessage = ''; const guard = (fn: () => void) => { if (!abortController.signal.aborted) fn(); }; let sdkContractFailed = false; const failSdkContract = (error: unknown) => { @@ -120,9 +194,16 @@ export class ChatService { emitMapped(() => mapSdkAssistantMessageDelta(event)); })); - // Final assistant message + // Final assistant message — also captured for TurnCompletionObservers + // so the observer payload carries the same text the renderer sees. + // The SDK can fire `assistant.message` more than once per turn (e.g. + // around tool use); keep the most recent non-null content. unsubs.push(session.on('assistant.message', (event) => { - emitMapped(() => mapSdkAssistantMessage(event)); + emitMapped(() => { + const mapped = mapSdkAssistantMessage(event); + if (mapped) finalAssistantMessage = mapped.content; + return mapped; + }); })); // Reasoning @@ -224,8 +305,9 @@ export class ChatService { // Wait for idle (listeners already active from before send). await turnDone; - if (abortController.signal.aborted) return; + if (abortController.signal.aborted) return null; emit({ type: 'done' }); + return finalAssistantMessage; } finally { for (const unsub of unsubs) unsub(); } diff --git a/packages/services/src/chat/index.ts b/packages/services/src/chat/index.ts index c3571b1f..933dcf35 100644 --- a/packages/services/src/chat/index.ts +++ b/packages/services/src/chat/index.ts @@ -1,3 +1,8 @@ export { ChatService } from './ChatService'; export { IdentityLoader } from './IdentityLoader'; export { TurnQueue } from './TurnQueue'; +export { + createWorkingMemoryComposer, + type WorkingMemoryComposer, + type WorkingMemoryComposerConfig, +} from './WorkingMemoryComposer'; From 40c75ff7f3a386015e4b50c09896beda9fd19964 Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Tue, 12 May 2026 19:20:54 -0400 Subject: [PATCH 06/23] ADD: WorkingMemoryComposer with sentinel-aware log composition --- .../services/src/chat/IdentityLoader.test.ts | 89 ++++++++- packages/services/src/chat/IdentityLoader.ts | 56 ++++-- .../src/chat/WorkingMemoryComposer.test.ts | 185 +++++++++++++++++ .../src/chat/WorkingMemoryComposer.ts | 189 ++++++++++++++++++ 4 files changed, 502 insertions(+), 17 deletions(-) create mode 100644 packages/services/src/chat/WorkingMemoryComposer.test.ts create mode 100644 packages/services/src/chat/WorkingMemoryComposer.ts diff --git a/packages/services/src/chat/IdentityLoader.test.ts b/packages/services/src/chat/IdentityLoader.test.ts index 7935d4d7..11754b87 100644 --- a/packages/services/src/chat/IdentityLoader.test.ts +++ b/packages/services/src/chat/IdentityLoader.test.ts @@ -89,11 +89,27 @@ describe('IdentityLoader', () => { return [] as unknown as ReturnType; }); - const result = loader.load('/tmp/test'); + // Use an injected composer so the test does not depend on `node:fs` + // (the real composer reads via `node:fs`, which the `vi.mock('fs')` setup + // above does not intercept). The composer contract here is only that + // IdentityLoader forwards mindPath and inserts the returned section. + const composer = { + compose: vi.fn(() => 'Curated memory\n\n---\n\nOperational rule'), + }; + const customLoader = new IdentityLoader(() => [], composer); + const result = customLoader.load('/tmp/test'); + expect(composer.compose).toHaveBeenCalledWith('/tmp/test', expect.objectContaining({ + lastKTurns: expect.any(Number), + perTurnMaxBytes: expect.any(Number), + memoryMaxBytes: expect.any(Number), + })); expect(result?.systemMessage).toContain('Curated memory'); expect(result?.systemMessage).toContain('Operational rule'); - expect(result?.systemMessage).toContain('Chronological note'); + // Unstructured log.md is filtered out by the composer; the IdentityLoader + // never includes it directly. The fake composer above returns no log + // section, so the chronological note must NOT appear. + expect(result?.systemMessage).not.toContain('Chronological note'); }); it('does not extract the mind name from working-memory headings', () => { @@ -118,7 +134,9 @@ describe('IdentityLoader', () => { return [] as unknown as ReturnType; }); - const result = loader.load('/tmp/agents/fox'); + const composer = { compose: vi.fn(() => '# Memory\nCurated memory') }; + const customLoader = new IdentityLoader(() => [], composer); + const result = customLoader.load('/tmp/agents/fox'); expect(result?.name).toBe('fox'); expect(result?.systemMessage).toContain('# Memory'); @@ -182,5 +200,70 @@ describe('IdentityLoader', () => { expect(systemMessage.indexOf('## Chamber')).toBeGreaterThan(systemMessage.indexOf('# Q')); expect(systemMessage.indexOf('## Tools')).toBeGreaterThan(systemMessage.indexOf('## Chamber')); }); + + it('uses a default WorkingMemoryComposer when none injected', () => { + vi.mocked(fs.existsSync).mockReturnValue(false); + vi.mocked(fs.readFileSync).mockReturnValue(''); + vi.mocked(fs.readdirSync).mockReturnValue([]); + // Should not throw — proves the default composer is constructed and called. + const defaultLoader = new IdentityLoader(); + expect(() => defaultLoader.load('/tmp/test')).not.toThrow(); + }); + + it('forwards mindPath and resolved config defaults to the composer', () => { + vi.mocked(fs.existsSync).mockImplementation((candidate) => { + const normalized = String(candidate).replace(/\\/g, '/'); + return normalized.endsWith('SOUL.md'); + }); + vi.mocked(fs.readFileSync).mockReturnValue('# Soul'); + vi.mocked(fs.readdirSync).mockReturnValue([]); + const composer = { compose: vi.fn(() => '') }; + const loader2 = new IdentityLoader(() => [], composer); + loader2.load('/tmp/agents/widget'); + + expect(composer.compose).toHaveBeenCalledWith( + '/tmp/agents/widget', + { + // Defaults from chamberMindConfig (Phase 4) when no .chamber.json exists. + lastKTurns: 10, + perTurnMaxBytes: 2048, + memoryMaxBytes: 8192, + }, + ); + }); + + it('backward compat: builds a system prompt when composer returns empty', () => { + vi.mocked(fs.existsSync).mockImplementation((candidate) => { + const normalized = String(candidate).replace(/\\/g, '/'); + return normalized.endsWith('SOUL.md'); + }); + vi.mocked(fs.readFileSync).mockReturnValue('# Mind\nIdentity body'); + vi.mocked(fs.readdirSync).mockReturnValue([]); + const composer = { compose: vi.fn(() => '') }; + const loader2 = new IdentityLoader(() => [], composer); + const result = loader2.load('/tmp/test'); + + expect(result).not.toBeNull(); + expect(result?.systemMessage).toContain('Identity body'); + expect(result?.systemMessage).toContain('## Chamber'); + }); + + it('backward compat: does not crash when composer throws', () => { + vi.mocked(fs.existsSync).mockImplementation((candidate) => { + const normalized = String(candidate).replace(/\\/g, '/'); + return normalized.endsWith('SOUL.md'); + }); + vi.mocked(fs.readFileSync).mockReturnValue('# Soul'); + vi.mocked(fs.readdirSync).mockReturnValue([]); + const composer = { + compose: vi.fn(() => { + throw new Error('composer boom'); + }), + }; + const loader2 = new IdentityLoader(() => [], composer); + expect(() => loader2.load('/tmp/test')).not.toThrow(); + const result = loader2.load('/tmp/test'); + expect(result?.systemMessage).toContain('# Soul'); + }); }); }); diff --git a/packages/services/src/chat/IdentityLoader.ts b/packages/services/src/chat/IdentityLoader.ts index 97b6d59d..4cbff9a3 100644 --- a/packages/services/src/chat/IdentityLoader.ts +++ b/packages/services/src/chat/IdentityLoader.ts @@ -3,15 +3,30 @@ import * as path from 'path'; import type { InstalledTool, MindIdentity } from '@chamber/shared/types'; import { buildToolsSection } from '../tools/toolsSystemMessage'; import { buildChamberSection } from './chamberSystemMessage'; +import { + createWorkingMemoryComposer, + type WorkingMemoryComposer, + type WorkingMemoryComposerConfig, +} from './WorkingMemoryComposer'; +import { + loadChamberMindConfig, + DEFAULT_WORKING_MEMORY_CONSOLIDATION, +} from '../mind/chamberMindConfig'; const FRONTMATTER_RE = /^---\r?\n[\s\S]*?\r?\n---\r?\n?/; const H1_RE = /^#\s+(.+)$/m; -const WORKING_MEMORY_FILES = ['memory.md', 'rules.md', 'log.md']; export type InstalledToolsProvider = () => InstalledTool[]; export class IdentityLoader { - constructor(private readonly getInstalledTools: InstalledToolsProvider = () => []) {} + private readonly composer: WorkingMemoryComposer; + + constructor( + private readonly getInstalledTools: InstalledToolsProvider = () => [], + composer: WorkingMemoryComposer = createWorkingMemoryComposer(), + ) { + this.composer = composer; + } load(mindPath: string | null): MindIdentity | null { if (!mindPath) return null; @@ -39,18 +54,10 @@ export class IdentityLoader { } catch { /* missing */ } try { - const memoryDir = path.join(mindPath, '.working-memory'); - if (!fs.existsSync(memoryDir)) throw new Error('missing working-memory'); - const files = fs.readdirSync(memoryDir) - .map((file) => String(file)) - .filter((file) => WORKING_MEMORY_FILES.includes(file)) - .sort((a, b) => WORKING_MEMORY_FILES.indexOf(a) - WORKING_MEMORY_FILES.indexOf(b)); - for (const file of files) { - const filePath = path.join(memoryDir, file); - const content = fs.readFileSync(filePath, 'utf-8').trim(); - if (content.length > 0) memoryParts.push(content); - } - } catch { /* missing */ } + const composerConfig = this.resolveComposerConfig(mindPath); + const memorySection = this.composer.compose(mindPath, composerConfig); + if (memorySection.length > 0) memoryParts.push(memorySection); + } catch { /* composer is defensive; defense-in-depth */ } const parts = [...identityParts, ...memoryParts]; if (parts.length === 0) return null; @@ -74,4 +81,25 @@ export class IdentityLoader { } return path.basename(mindPath); } + + private resolveComposerConfig(mindPath: string): WorkingMemoryComposerConfig { + // .chamber.json is the source of truth for composer caps. loadChamberMindConfig + // already returns DEFAULT_WORKING_MEMORY_CONSOLIDATION when the file is missing, + // unparseable, or schema-invalid, so this never throws. Defaults are also + // exported here so a composer-only failure path still has a fallback. + try { + const c = loadChamberMindConfig(mindPath).workingMemory.consolidation; + return { + lastKTurns: c.lastKTurns, + perTurnMaxBytes: c.perTurnMaxBytes, + memoryMaxBytes: c.memoryMaxBytes, + }; + } catch { + return { + lastKTurns: DEFAULT_WORKING_MEMORY_CONSOLIDATION.lastKTurns, + perTurnMaxBytes: DEFAULT_WORKING_MEMORY_CONSOLIDATION.perTurnMaxBytes, + memoryMaxBytes: DEFAULT_WORKING_MEMORY_CONSOLIDATION.memoryMaxBytes, + }; + } + } } diff --git a/packages/services/src/chat/WorkingMemoryComposer.test.ts b/packages/services/src/chat/WorkingMemoryComposer.test.ts new file mode 100644 index 00000000..c5a5666d --- /dev/null +++ b/packages/services/src/chat/WorkingMemoryComposer.test.ts @@ -0,0 +1,185 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { createWorkingMemoryComposer, type WorkingMemoryComposerConfig } from './WorkingMemoryComposer'; +import { STRUCTURED_LOG_SENTINEL, serializeTurn, type CompletedTurn } from '../mindMemory/StructuredLogFormat'; + +const DEFAULTS: WorkingMemoryComposerConfig = { + lastKTurns: 10, + perTurnMaxBytes: 2048, + memoryMaxBytes: 8192, +}; + +let mindRoot: string; +let workingMemoryDir: string; + +beforeEach(() => { + mindRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-wmc-')); + workingMemoryDir = path.join(mindRoot, '.working-memory'); + fs.mkdirSync(workingMemoryDir, { recursive: true }); +}); + +afterEach(() => { + fs.rmSync(mindRoot, { recursive: true, force: true }); +}); + +function makeTurn(i: number, overrides: Partial = {}): CompletedTurn { + const ts = `2026-05-12T17:${String(20 + i).padStart(2, '0')}:00Z`; + return { + turnId: `turn-${i}`, + sessionId: `sess-${i}`, + model: 'claude-opus-4.7', + status: 'completed', + startedAt: ts, + endedAt: ts, + prompt: `prompt body ${i}`, + finalAssistantMessage: `assistant body ${i}`, + ...overrides, + }; +} + +function writeStructuredLog(turns: CompletedTurn[]): void { + const body = STRUCTURED_LOG_SENTINEL + '\n\n' + turns.map(serializeTurn).join('\n'); + fs.writeFileSync(path.join(workingMemoryDir, 'log.md'), body, 'utf-8'); +} + +describe('WorkingMemoryComposer.compose', () => { + it('returns empty string when working-memory dir is empty', () => { + const composer = createWorkingMemoryComposer(); + expect(composer.compose(mindRoot, DEFAULTS)).toBe(''); + }); + + it('returns empty string when working-memory dir is missing', () => { + fs.rmSync(workingMemoryDir, { recursive: true, force: true }); + const composer = createWorkingMemoryComposer(); + expect(composer.compose(mindRoot, DEFAULTS)).toBe(''); + }); + + it('includes only rules.md when memory and log are absent', () => { + fs.writeFileSync(path.join(workingMemoryDir, 'rules.md'), 'Operational rule', 'utf-8'); + const composer = createWorkingMemoryComposer(); + const out = composer.compose(mindRoot, DEFAULTS); + expect(out).toContain('Operational rule'); + expect(out).not.toContain('---'); + }); + + it('includes memory.md and rules.md joined by separator', () => { + fs.writeFileSync(path.join(workingMemoryDir, 'memory.md'), 'Curated memory', 'utf-8'); + fs.writeFileSync(path.join(workingMemoryDir, 'rules.md'), 'Operational rule', 'utf-8'); + const composer = createWorkingMemoryComposer(); + const out = composer.compose(mindRoot, DEFAULTS); + expect(out).toBe('Curated memory\n\n---\n\nOperational rule'); + }); + + it('returns only the last K structured turns when log has more than K', () => { + const turns = Array.from({ length: 15 }, (_, i) => makeTurn(i)); + writeStructuredLog(turns); + const composer = createWorkingMemoryComposer(); + const out = composer.compose(mindRoot, { ...DEFAULTS, lastKTurns: 5 }); + expect(out).toContain('turn:turn-14'); + expect(out).toContain('turn:turn-10'); + expect(out).not.toContain('turn:turn-9'); + expect(out).not.toContain('turn:turn-0'); + }); + + it('returns all turns when log has fewer than K', () => { + writeStructuredLog([makeTurn(0), makeTurn(1)]); + const composer = createWorkingMemoryComposer(); + const out = composer.compose(mindRoot, { ...DEFAULTS, lastKTurns: 10 }); + expect(out).toContain('turn:turn-0'); + expect(out).toContain('turn:turn-1'); + }); + + it('truncates a turn whose rendered size exceeds perTurnMaxBytes', () => { + const huge = 'x'.repeat(10_000); + writeStructuredLog([makeTurn(0, { finalAssistantMessage: huge })]); + const composer = createWorkingMemoryComposer(); + const out = composer.compose(mindRoot, { ...DEFAULTS, perTurnMaxBytes: 512 }); + // The rendered turn (or its truncation block) must not exceed the cap by more than a small margin. + // We assert two things: + // 1) The full huge body is NOT present verbatim. + expect(out).not.toContain(huge); + // 2) A truncation marker is present. + expect(out).toMatch(/truncated/); + }); + + it('omits log entirely when log.md is unstructured (no sentinel) and warns', () => { + fs.writeFileSync(path.join(workingMemoryDir, 'log.md'), 'just freeform notes\nnot structured\n', 'utf-8'); + fs.writeFileSync(path.join(workingMemoryDir, 'memory.md'), 'mem', 'utf-8'); + const warn = vi.fn(); + const composer = createWorkingMemoryComposer({ logger: { warn, info: () => {} } }); + const out = composer.compose(mindRoot, DEFAULTS); + expect(out).toBe('mem'); + expect(out).not.toContain('freeform'); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0][0]).toMatch(/unstructured/i); + }); + + it('contributes nothing when log.md is missing', () => { + fs.writeFileSync(path.join(workingMemoryDir, 'memory.md'), 'mem', 'utf-8'); + const composer = createWorkingMemoryComposer(); + expect(composer.compose(mindRoot, DEFAULTS)).toBe('mem'); + }); + + it('never includes log.legacy.md content', () => { + writeStructuredLog([makeTurn(0)]); + fs.writeFileSync( + path.join(workingMemoryDir, 'log.legacy.md'), + 'LEGACY-CONTENT-MUST-NOT-APPEAR', + 'utf-8', + ); + const composer = createWorkingMemoryComposer(); + const out = composer.compose(mindRoot, DEFAULTS); + expect(out).not.toContain('LEGACY-CONTENT-MUST-NOT-APPEAR'); + expect(out).toContain('turn:turn-0'); + }); + + it('truncates memory.md when it exceeds memoryMaxBytes', () => { + const huge = 'm'.repeat(20_000); + fs.writeFileSync(path.join(workingMemoryDir, 'memory.md'), huge, 'utf-8'); + const composer = createWorkingMemoryComposer(); + const out = composer.compose(mindRoot, { ...DEFAULTS, memoryMaxBytes: 1024 }); + expect(Buffer.byteLength(out, 'utf-8')).toBeLessThanOrEqual(1024 + 100); // marker tolerance + expect(out).toMatch(/truncated/); + }); + + it('does not throw when memory.md, rules.md, and log.md are all missing', () => { + const composer = createWorkingMemoryComposer(); + expect(() => composer.compose(mindRoot, DEFAULTS)).not.toThrow(); + expect(composer.compose(mindRoot, DEFAULTS)).toBe(''); + }); + + it('respects custom lastKTurns from config', () => { + const turns = Array.from({ length: 8 }, (_, i) => makeTurn(i)); + writeStructuredLog(turns); + const composer = createWorkingMemoryComposer(); + const out = composer.compose(mindRoot, { ...DEFAULTS, lastKTurns: 3 }); + expect(out).toContain('turn:turn-7'); + expect(out).toContain('turn:turn-6'); + expect(out).toContain('turn:turn-5'); + expect(out).not.toContain('turn:turn-4'); + }); + + it('orders sections memory → rules → log', () => { + fs.writeFileSync(path.join(workingMemoryDir, 'memory.md'), 'MEMSECTION', 'utf-8'); + fs.writeFileSync(path.join(workingMemoryDir, 'rules.md'), 'RULESECTION', 'utf-8'); + writeStructuredLog([makeTurn(0)]); + const composer = createWorkingMemoryComposer(); + const out = composer.compose(mindRoot, DEFAULTS); + const memIdx = out.indexOf('MEMSECTION'); + const ruleIdx = out.indexOf('RULESECTION'); + const logIdx = out.indexOf('turn:turn-0'); + expect(memIdx).toBeGreaterThanOrEqual(0); + expect(ruleIdx).toBeGreaterThan(memIdx); + expect(logIdx).toBeGreaterThan(ruleIdx); + }); + + it('skips empty/whitespace-only files', () => { + fs.writeFileSync(path.join(workingMemoryDir, 'memory.md'), ' \n\n', 'utf-8'); + fs.writeFileSync(path.join(workingMemoryDir, 'rules.md'), 'rules-content', 'utf-8'); + const composer = createWorkingMemoryComposer(); + expect(composer.compose(mindRoot, DEFAULTS)).toBe('rules-content'); + }); +}); diff --git a/packages/services/src/chat/WorkingMemoryComposer.ts b/packages/services/src/chat/WorkingMemoryComposer.ts new file mode 100644 index 00000000..6dc9b4fd --- /dev/null +++ b/packages/services/src/chat/WorkingMemoryComposer.ts @@ -0,0 +1,189 @@ +/** + * WorkingMemoryComposer — assembles the working-memory section of a mind's + * system prompt from `/.working-memory/{memory.md, rules.md, log.md}`. + * + * Phase 12 scope (locked by plan): + * - `memory.md` → full content, hard-capped at `memoryMaxBytes` (defense-in- + * depth; the consolidator already caps at write time). + * - `rules.md` → full content (small file, no cap). + * - `log.md` → only included when the file's first non-blank line is the + * `chamber-structured-log/v1` sentinel. The composer takes the last + * `lastKTurns` parsed turns and renders each, truncating any rendered + * turn that exceeds `perTurnMaxBytes`. Unstructured / missing logs + * contribute NOTHING and emit a warning (sentinel detection is owned by + * this composer, not by DailyLogWriter — a mind that never ran the + * writer must not leak its legacy log into the prompt). + * - `log.legacy.md` → never included. + * + * Section order: memory → rules → log. Sections are joined by the same + * `\n\n---\n\n` separator IdentityLoader uses for its top-level parts so + * the resulting string can be slotted into the system prompt as a single + * element without disturbing the existing layout. + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import { Logger } from '../logger'; +import { + parseLog, + type ParsedTurn, +} from '../mindMemory/StructuredLogFormat'; + +const WORKING_MEMORY_DIRNAME = '.working-memory'; +const SECTION_SEPARATOR = '\n\n---\n\n'; + +export interface WorkingMemoryComposerConfig { + /** Max number of structured turns to include from `log.md`. */ + readonly lastKTurns: number; + /** Max bytes per rendered turn frame; over-budget turns get a truncation marker. */ + readonly perTurnMaxBytes: number; + /** Hard cap on `memory.md` bytes (defense-in-depth over the consolidator). */ + readonly memoryMaxBytes: number; +} + +export interface ComposerLogger { + warn(message: string, ...args: unknown[]): void; + info(message: string, ...args: unknown[]): void; +} + +export interface WorkingMemoryComposerDeps { + readonly logger?: ComposerLogger; +} + +export interface WorkingMemoryComposer { + /** Build the working-memory section. Returns `''` when nothing applies. */ + compose(mindPath: string, config: WorkingMemoryComposerConfig): string; +} + +export function createWorkingMemoryComposer( + deps: WorkingMemoryComposerDeps = {}, +): WorkingMemoryComposer { + const log: ComposerLogger = deps.logger ?? Logger.create('WorkingMemoryComposer'); + + return { + compose(mindPath, config) { + const dir = path.join(mindPath, WORKING_MEMORY_DIRNAME); + if (!safeExists(dir)) return ''; + + const sections: string[] = []; + + const memory = readMemory(dir, config.memoryMaxBytes, log); + if (memory) sections.push(memory); + + const rules = readSimple(dir, 'rules.md'); + if (rules) sections.push(rules); + + const logSection = readLog(dir, config, log); + if (logSection) sections.push(logSection); + + return sections.join(SECTION_SEPARATOR); + }, + }; +} + +function safeExists(p: string): boolean { + try { + return fs.existsSync(p); + } catch { + return false; + } +} + +function readSimple(dir: string, name: string): string { + const filePath = path.join(dir, name); + if (!safeExists(filePath)) return ''; + try { + return fs.readFileSync(filePath, 'utf-8').trim(); + } catch { + return ''; + } +} + +function readMemory(dir: string, maxBytes: number, log: ComposerLogger): string { + const raw = readSimple(dir, 'memory.md'); + if (!raw) return ''; + return truncateToBytes(raw, maxBytes, log, 'memory.md'); +} + +function readLog( + dir: string, + config: WorkingMemoryComposerConfig, + log: ComposerLogger, +): string { + const filePath = path.join(dir, 'log.md'); + if (!safeExists(filePath)) return ''; + + let raw: string; + try { + raw = fs.readFileSync(filePath, 'utf-8'); + } catch (err) { + log.warn(`WorkingMemoryComposer: failed to read log.md; skipping log section`, err); + return ''; + } + + if (raw.trim().length === 0) return ''; + + const parsed = parseLog(raw); + if (!parsed.sentinel) { + log.warn( + `WorkingMemoryComposer: log.md is unstructured (no chamber-structured-log/v1 sentinel); skipping log section`, + ); + return ''; + } + + if (parsed.turns.length === 0) return ''; + + const k = Math.max(0, config.lastKTurns | 0); + if (k === 0) return ''; + + const tail = parsed.turns.slice(-k); + const rendered = tail.map((t) => truncateToBytes(renderTurn(t), config.perTurnMaxBytes, log, `turn ${t.turnId}`)); + return rendered.join('\n\n'); +} + +function renderTurn(turn: ParsedTurn): string { + return [ + `## ${turn.timestamp} turn:${turn.turnId} status:${turn.status}`, + `session: ${turn.sessionId}`, + `model: ${turn.model}`, + '', + '### user', + turn.prompt, + '', + '### assistant', + turn.assistant, + ].join('\n'); +} + +function truncateToBytes( + s: string, + maxBytes: number, + log: ComposerLogger, + label: string, +): string { + const originalBytes = Buffer.byteLength(s, 'utf-8'); + if (originalBytes <= maxBytes) return s; + + const originalKb = Math.max(1, Math.round(originalBytes / 1024)); + const marker = `\n[…truncated, originally ${originalKb} KB]`; + const markerBytes = Buffer.byteLength(marker, 'utf-8'); + + if (markerBytes >= maxBytes) { + log.warn( + `WorkingMemoryComposer: ${label} exceeds ${maxBytes}B and the truncation marker alone (${markerBytes}B) does not fit; emitting marker only`, + ); + return marker.slice(0, maxBytes); + } + + const room = maxBytes - markerBytes; + let truncated = s; + while (Buffer.byteLength(truncated, 'utf-8') > room && truncated.length > 0) { + truncated = truncated.slice(0, -1); + } + + log.info( + `WorkingMemoryComposer: truncated ${label} from ${originalBytes}B to ${Buffer.byteLength(truncated + marker, 'utf-8')}B`, + ); + return truncated + marker; +} From da5fedab1d3ac0575641350dd9a3f49cc53fd62d Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Tue, 12 May 2026 19:20:59 -0400 Subject: [PATCH 07/23] REFACTOR: convert MindManager.createSessionForMind call to named args --- packages/services/src/mind/MindManager.ts | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/packages/services/src/mind/MindManager.ts b/packages/services/src/mind/MindManager.ts index d2969515..10f7daa0 100644 --- a/packages/services/src/mind/MindManager.ts +++ b/packages/services/src/mind/MindManager.ts @@ -274,16 +274,13 @@ export class MindManager extends EventEmitter { context.activeSessionId, context.selectedModel, ) - : await this.createSessionForMind( - newClient, - context.mindPath, - context.identity.systemMessage, - sessionTools, - undefined, - approveForSessionCompat, - false, - context.selectedModel, - ); + : await this.createSessionForMind({ + client: newClient, + mindPath: context.mindPath, + systemMessage: context.identity.systemMessage, + tools: sessionTools, + model: context.selectedModel, + }); context.client = newClient; context.session = newSession; From 3f62fa1608b723dd93e9d3bb82a25378ee247f0e Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Tue, 12 May 2026 19:21:06 -0400 Subject: [PATCH 08/23] ADD: wire MindMemoryService into desktop composition root --- apps/desktop/src/main.ts | 34 ++- .../mindMemory/buildMindMemoryService.ts | 231 ++++++++++++++++++ 2 files changed, 264 insertions(+), 1 deletion(-) create mode 100644 apps/desktop/src/main/services/mindMemory/buildMindMemoryService.ts diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index 0655c744..8f8dc290 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -51,6 +51,8 @@ import { type Notifier, } from '@chamber/services'; import { Logger } from '@chamber/services'; +import type { MindContext } from '@chamber/shared'; +import { buildMindMemoryService } from './main/services/mindMemory/buildMindMemoryService'; import { createAppTray, loadAppIcon } from './main/tray/Tray'; import { installContextMenu } from './main/contextMenu/ContextMenu'; import { installExternalNavigationGuard } from './main/navigationGuard'; @@ -315,6 +317,30 @@ if (configService.load().chamberCopilotEnabled === true) { mindManager.setProviders(mindToolProviders); +// --------------------------------------------------------------------------- +// MindMemory (Dream Daemon) — per-mind background memory consolidation. +// Wires after providers so chatService observers + scheduler are ready before +// any mind:loaded event fires. The lifecycle hooks below own the per-mind +// activate/release dance; the composition root owns close() during quit. +// --------------------------------------------------------------------------- +const mindMemoryComposition = buildMindMemoryService({ + mindManager, + chatService, + isPackaged: app.isPackaged, + resourcesPath: app.isPackaged ? process.resourcesPath : undefined, +}); +const mindMemoryService = mindMemoryComposition.service; +mindManager.on('mind:loaded', (ctx: MindContext) => { + mindMemoryService.activateMind(ctx.mindId, ctx.mindPath).catch((err) => { + log.warn('mindMemory: activateMind failed', { mindId: ctx.mindId, err: String(err) }); + }); +}); +mindManager.on('mind:unloaded', (mindId: string) => { + mindMemoryService.releaseMind(mindId).catch((err) => { + log.warn('mindMemory: releaseMind failed', { mindId, err: String(err) }); + }); +}); + wireLifecycleEvents({ mindManager, agentCardRegistry, taskManager, a2aEventBus }); // Wire Lens refresh to use the mind's session @@ -343,7 +369,13 @@ const requestQuit = () => { if (isQuitting) return; isQuitting = true; - mindManager.shutdown() + // INVARIANT: close MindMemoryService BEFORE MindManager.shutdown so each + // mind's dream.db handle and scheduler entry tear down while the underlying + // Mind / SDK client is still alive — avoids cron ticks racing with mind + // teardown and leaves dream.db files cleanly closed on disk. + mindMemoryComposition.close() + .catch((err) => { log.warn('mindMemory: shutdown close failed', { err: String(err) }); }) + .then(() => mindManager.shutdown()) .then(() => { updaterService.stop(); return stopMvpServer(); diff --git a/apps/desktop/src/main/services/mindMemory/buildMindMemoryService.ts b/apps/desktop/src/main/services/mindMemory/buildMindMemoryService.ts new file mode 100644 index 00000000..cdab57aa --- /dev/null +++ b/apps/desktop/src/main/services/mindMemory/buildMindMemoryService.ts @@ -0,0 +1,231 @@ +/** + * Phase 13 desktop wiring for the per-mind background memory engine + * ("Dream Daemon"). This module is a *thin adapter*: it knows how to + * + * - load better-sqlite3 from either dev `node_modules` or the packaged + * ASAR-unpacked tree, + * - open per-mind `dream.db` files at the canonical path, + * - mint *one-shot* Copilot sessions with tools disabled and a refusing + * permission handler (defense-in-depth — tools=[] should already mean + * no permission requests reach the handler), + * - construct a `DreamDaemon` from a `WorkingMemoryConsolidationConfig` + * by supplying defaults for the wider `DreamDaemonConfig` surface. + * + * The composition root in `apps/desktop/src/main.ts` calls + * `buildMindMemoryService` once and wires the resulting service into + * `MindManager`'s `mind:loaded` / `mind:unloaded` events. + */ +import path from 'node:path'; +import fs from 'node:fs'; +import { createRequire } from 'node:module'; +import { + createMindMemoryService, + createInternalScheduler, + createMindMemoryVault, + createMindArchiveStore, + createDreamDaemon, + createCopilotLLMClient, + defaultConfigReader, + Logger, + migrate as migrateDreamDb, + type MindMemoryService, + type MindManager, + type DaemonFactoryOptions, + type CreateOneShotSessionArgs, + type OneShotSession, +} from '@chamber/services'; +import type { TurnCompletionObserver } from '@chamber/shared'; +import type { PermissionHandler, MessageOptions, SessionConfig } from '@github/copilot-sdk'; + +type BetterSqlite3Module = typeof import('better-sqlite3'); +type BetterSqlite3Database = import('better-sqlite3').Database; + +interface BuildMindMemoryServiceOptions { + readonly mindManager: MindManager; + readonly chatService: { + addObserver(o: TurnCompletionObserver): void; + removeObserver(o: TurnCompletionObserver): void; + }; + readonly isPackaged: boolean; + readonly resourcesPath: string | undefined; + readonly logger?: Logger; +} + +export interface MindMemoryComposition { + readonly service: MindMemoryService; + readonly scheduler: ReturnType; + /** + * Shut everything down cleanly. Releases the service first (which releases + * each mind, closing its observer + dream.db handle), then the scheduler. + * Idempotent. + */ + close(): Promise; +} + +const runtimeRequire = createRequire(__filename); + +function loadBetterSqlite3(isPackaged: boolean, resourcesPath: string | undefined): BetterSqlite3Module { + if (!isPackaged) { + return runtimeRequire('better-sqlite3') as BetterSqlite3Module; + } + // forge.config.ts unpacks `**/node_modules/{...,better-sqlite3,bindings,file-uri-to-path}/**/*` + // out of the ASAR, so `app.asar.unpacked` becomes the canonical resolve root. + // Try the unpacked path first; fall back to a bare require so we still load + // when the packaging layout changes (e.g., hoisted to resources/). + const unpacked = resourcesPath + ? path.join(resourcesPath, 'app.asar.unpacked', 'node_modules', 'better-sqlite3') + : null; + if (unpacked && fs.existsSync(unpacked)) { + return runtimeRequire(unpacked) as BetterSqlite3Module; + } + return runtimeRequire('better-sqlite3') as BetterSqlite3Module; +} + +const refusingPermissionHandler: PermissionHandler = () => ({ + kind: 'reject', + feedback: 'Tool permissions are disabled for memory-consolidation sessions.', +}); + +/** + * Build the createOneShotSession adapter for CopilotLLMClient. Each + * synthesize call mints a fresh CopilotSession scoped to the mind's + * working directory, with NO tools, NO config discovery, and a refusing + * permission handler. The session is closed in the LLMClient's `finally`. + */ +function makeCreateOneShotSession( + mindManager: MindManager, + logger: Logger, +): (args: CreateOneShotSessionArgs) => Promise { + return async ({ mindId, mindPath, signal }) => { + const ctx = mindManager.getMind(mindId); + if (!ctx) { + throw new Error(`MindMemory: cannot create session — mind ${mindId} is not loaded`); + } + const sessionConfig: SessionConfig = { + workingDirectory: mindPath, + enableConfigDiscovery: false, + tools: [], + systemMessage: { mode: 'replace', content: '' }, + onPermissionRequest: refusingPermissionHandler, + }; + const session = await ctx.client.createSession(sessionConfig); + + const onAbort = (): void => { + // Best-effort abort. The CLI may or may not have an in-flight call. + session.abort().catch(() => { /* noop */ }); + }; + if (signal.aborted) { + onAbort(); + } else { + signal.addEventListener('abort', onAbort, { once: true }); + } + + return { + async send(prompt: string): Promise { + const opts: MessageOptions = { prompt }; + const event = await session.sendAndWait(opts); + return event?.data.content ?? ''; + }, + async close(): Promise { + signal.removeEventListener('abort', onAbort); + try { + await session.disconnect(); + } catch (err) { + logger.warn('mindMemory: session disconnect failed', { err: String(err) }); + } + }, + }; + }; +} + +// Defaults for the wider DreamDaemonConfig fields that aren't covered by +// WorkingMemoryConsolidationConfig. Tuned to match Phase 9/10 unit-test +// defaults so production behavior matches the test harness. +const MS_PER_DAY = 24 * 60 * 60 * 1000; +const DEFAULT_LLM_TIMEOUT_MS = 90_000; +const DEFAULT_LOCK_TTL_MS = 5 * 60 * 1000; +const DEFAULT_MIN_TURNS_BETWEEN_RUNS = 1; +const DEFAULT_MIN_DAILY_INTERVAL_MS = 0; +const DEFAULT_WEEKLY_ROLLUP_AFTER_DAILIES = 7; +const DEFAULT_MONTHLY_ROLLUP_AFTER_WEEKLIES = 4; +const DEFAULT_WEEKLY_MIN_INTERVAL_MS = 7 * MS_PER_DAY; +const DEFAULT_MONTHLY_MIN_INTERVAL_MS = 30 * MS_PER_DAY; + +export function buildMindMemoryService(opts: BuildMindMemoryServiceOptions): MindMemoryComposition { + const logger = opts.logger ?? Logger.create('mindMemory'); + const Database = loadBetterSqlite3(opts.isPackaged, opts.resourcesPath); + const scheduler = createInternalScheduler({ logger }); + const createOneShotSession = makeCreateOneShotSession(opts.mindManager, logger); + + const service = createMindMemoryService({ + scheduler, + chatService: opts.chatService, + configReader: defaultConfigReader, + dbFactory: (dbPath: string): BetterSqlite3Database => { + // INVARIANT: must apply the dream.db schema before returning. Without + // `migrate(db)`, the daemon's first call to `readState` / `acquireLock` + // would throw `no such table: dream_state`. Mirrors `openDreamDb` in + // dream-schema.ts, but uses the dynamically-loaded better-sqlite3 + // module so packaged builds resolve the unpacked native binding. + fs.mkdirSync(path.dirname(dbPath), { recursive: true }); + const db = new Database(dbPath); + db.pragma('journal_mode = WAL'); + db.pragma('foreign_keys = ON'); + migrateDreamDb(db); + return db; + }, + vaultFactory: createMindMemoryVault, + archiveFactory: createMindArchiveStore, + daemonFactory: ({ mindId, mindPath, vault, archive, db, config }: DaemonFactoryOptions) => { + const llmClient = createCopilotLLMClient({ + mindId, + mindPath, + deps: { createOneShotSession }, + }); + return createDreamDaemon({ + mindId, + mindPath, + llmClient, + vault, + archiveStore: archive, + db, + config: { + memoryMaxBytes: config.memoryMaxBytes, + llmTimeoutMs: DEFAULT_LLM_TIMEOUT_MS, + lockTtlMs: DEFAULT_LOCK_TTL_MS, + minTurnsBetweenRuns: DEFAULT_MIN_TURNS_BETWEEN_RUNS, + minDailyIntervalMs: DEFAULT_MIN_DAILY_INTERVAL_MS, + weeklyRollupAfterDailies: DEFAULT_WEEKLY_ROLLUP_AFTER_DAILIES, + monthlyRollupAfterWeeklies: DEFAULT_MONTHLY_ROLLUP_AFTER_WEEKLIES, + weeklyMinIntervalMs: DEFAULT_WEEKLY_MIN_INTERVAL_MS, + monthlyMinIntervalMs: DEFAULT_MONTHLY_MIN_INTERVAL_MS, + }, + logger, + }); + }, + logger, + }); + + let closed = false; + return { + service, + scheduler, + async close(): Promise { + if (closed) return; + closed = true; + // INVARIANT: release the service BEFORE closing the scheduler so the + // scheduler can still observe `unregister` calls coming from each + // mind release. Otherwise close() on a closed scheduler would throw. + try { + await service.close(); + } catch (err) { + logger.warn('mindMemory: service close failed', { err: String(err) }); + } + try { + scheduler.close(); + } catch (err) { + logger.warn('mindMemory: scheduler close failed', { err: String(err) }); + } + }, + }; +} From 50fd7cc6df0cfc1c10cd1f902330ac460aa3e0bf Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Tue, 12 May 2026 19:21:11 -0400 Subject: [PATCH 09/23] TEST: integration coverage for dream daemon composition and multi-day cycle --- config/vitest.config.ts | 1 + .../mindMemory.composition.test.ts | 257 ++++++++++ .../mindMemory.integration.test.ts | 477 ++++++++++++++++++ 3 files changed, 735 insertions(+) create mode 100644 tests/integration/mindMemory.composition.test.ts create mode 100644 tests/integration/mindMemory.integration.test.ts diff --git a/config/vitest.config.ts b/config/vitest.config.ts index 9fe62dc5..5b6c2ed7 100644 --- a/config/vitest.config.ts +++ b/config/vitest.config.ts @@ -10,6 +10,7 @@ export default defineConfig({ 'apps/**/*.{test,spec}.{ts,tsx}', 'packages/**/*.{test,spec}.{ts,tsx}', 'tests/regression/**/*.{test,spec}.{ts,tsx}', + 'tests/integration/**/*.{test,spec}.{ts,tsx}', ], exclude: ['node_modules', 'dist', 'out', '.vite', 'apps/*/dist', 'packages/*/dist'], testTimeout: 10_000, diff --git a/tests/integration/mindMemory.composition.test.ts b/tests/integration/mindMemory.composition.test.ts new file mode 100644 index 00000000..ba4d0d8c --- /dev/null +++ b/tests/integration/mindMemory.composition.test.ts @@ -0,0 +1,257 @@ +/** + * Phase 13 composition smoke for MindMemoryService. + * + * Goal: catch native-module / DI failures cheaply, without launching + * Electron. Builds the full service graph against a tmpdir mindPath using: + * - real `createInternalScheduler` + * - real `createMindMemoryVault` / `createMindArchiveStore` + * - real `defaultConfigReader` (.chamber.json on disk) + * - real better-sqlite3 dbFactory at /.working-memory/.state/dream.db + * - a fake daemon factory + chat-observer registry (the real DreamDaemon + * needs a CopilotClient, which needs Electron — out of scope for a + * non-Electron smoke). + * + * The expensive integration that exercises the real CopilotLLMClient lives + * in Phase 14. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync, existsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { + createMindMemoryService, + createInternalScheduler, + createMindMemoryVault, + createMindArchiveStore, + defaultConfigReader, + dreamDbPath, + type DreamDaemon, + type DreamRunResult, + type MindMemoryService, + type ChatObserverRegistry, + type DaemonFactoryOptions, +} from '@chamber/services'; +import type { TurnCompletionObserver } from '@chamber/shared'; + +const runtimeRequire = createRequire(__filename); + +function makeFakeDaemon(): DreamDaemon & { closeCalls: number } { + let closeCalls = 0; + const daemon: DreamDaemon = { + async run(): Promise { + return { status: 'skipped', reason: 'no-turns' }; + }, + async forceRun(): Promise { + return { status: 'skipped', reason: 'no-turns' }; + }, + getStatus() { + return { phase: 'idle', locked: false, lastRunAt: null, lastResult: null }; + }, + notifyTurnCompleted() { + /* no-op */ + }, + async close(): Promise { + closeCalls += 1; + }, + }; + return Object.defineProperty(daemon, 'closeCalls', { + get: () => closeCalls, + enumerable: true, + }) as DreamDaemon & { closeCalls: number }; +} + +function makeFakeChatRegistry(): ChatObserverRegistry & { + readonly observers: TurnCompletionObserver[]; +} { + const observers: TurnCompletionObserver[] = []; + return { + addObserver(o: TurnCompletionObserver): void { + observers.push(o); + }, + removeObserver(o: TurnCompletionObserver): void { + const i = observers.indexOf(o); + if (i !== -1) observers.splice(i, 1); + }, + get observers() { + return observers; + }, + }; +} + +function writeChamberConfig(mindPath: string, enabled: boolean): void { + const cfg = { + workingMemory: { + consolidation: { + enabled, + cron: '0 3 * * *', + lastKTurns: 10, + perTurnMaxBytes: 2048, + memoryMaxBytes: 8192, + }, + }, + }; + writeFileSync(path.join(mindPath, '.chamber.json'), JSON.stringify(cfg, null, 2), 'utf-8'); +} + +interface Harness { + readonly mindPath: string; + readonly mindId: string; + readonly service: MindMemoryService; + readonly scheduler: ReturnType; + readonly chat: ReturnType; + readonly daemonFactoryCalls: { count: number }; + readonly openDbs: import('better-sqlite3').Database[]; + cleanup(): Promise; +} + +function buildHarness(mindRoot: string, mindId: string): Harness { + const mindPath = path.join(mindRoot, mindId); + mkdirSync(mindPath, { recursive: true }); + + const scheduler = createInternalScheduler(); + const chat = makeFakeChatRegistry(); + const daemonFactoryCalls = { count: 0 }; + const openDbs: import('better-sqlite3').Database[] = []; + + const Database = runtimeRequire('better-sqlite3') as typeof import('better-sqlite3'); + + const service = createMindMemoryService({ + scheduler, + chatService: chat, + configReader: defaultConfigReader, + dbFactory: (dbPath: string) => { + // Mirror the production wiring contract: ensure the parent dir exists + // before opening. dreamDbPath is /.working-memory/.state/dream.db + mkdirSync(path.dirname(dbPath), { recursive: true }); + const db = new Database(dbPath); + openDbs.push(db); + return db; + }, + vaultFactory: createMindMemoryVault, + archiveFactory: createMindArchiveStore, + daemonFactory: (_opts: DaemonFactoryOptions) => { + void _opts; + daemonFactoryCalls.count += 1; + return makeFakeDaemon(); + }, + }); + + return { + mindPath, + mindId, + service, + scheduler, + chat, + daemonFactoryCalls, + openDbs, + async cleanup() { + try { + await service.close(); + } catch { + /* noop */ + } + try { + scheduler.close(); + } catch { + /* noop */ + } + // Final guard — anything still open from a half-failed activation. + for (const db of openDbs) { + try { + if (db.open) db.close(); + } catch { + /* noop */ + } + } + }, + }; +} + +describe('MindMemoryService composition (Phase 13 smoke)', () => { + let mindRoot: string; + let harness: Harness | null = null; + + beforeEach(() => { + mindRoot = mkdtempSync(path.join(tmpdir(), 'chamber-mindmem-smoke-')); + }); + + afterEach(async () => { + if (harness) { + await harness.cleanup(); + harness = null; + } + rmSync(mindRoot, { recursive: true, force: true }); + }); + + it('loads better-sqlite3 native module without throwing', () => { + expect(() => runtimeRequire('better-sqlite3')).not.toThrow(); + }); + + it('activateMind on a disabled config is a no-op (no db, no observer, no scheduler entry)', async () => { + harness = buildHarness(mindRoot, 'mind-disabled'); + writeChamberConfig(harness.mindPath, false); + + await harness.service.activateMind(harness.mindId, harness.mindPath); + + expect(existsSync(dreamDbPath(harness.mindPath))).toBe(false); + expect(harness.chat.observers).toHaveLength(0); + expect(harness.scheduler.list().size).toBe(0); + expect(harness.daemonFactoryCalls.count).toBe(0); + }); + + it('activateMind on an enabled config wires db, observer, and scheduler entry; releaseMind tears them all down', async () => { + harness = buildHarness(mindRoot, 'mind-enabled'); + writeChamberConfig(harness.mindPath, true); + + await harness.service.activateMind(harness.mindId, harness.mindPath); + + const dbFile = dreamDbPath(harness.mindPath); + expect(existsSync(dbFile)).toBe(true); + expect(harness.chat.observers).toHaveLength(1); + const entries = harness.scheduler.list(); + expect(entries.size).toBe(1); + expect(entries.get(harness.mindId)).toBe('0 3 * * *'); + expect(harness.daemonFactoryCalls.count).toBe(1); + + await harness.service.releaseMind(harness.mindId); + + expect(harness.chat.observers).toHaveLength(0); + expect(harness.scheduler.list().size).toBe(0); + // dream.db file persists on disk (state survives mind release), but the + // handle should be closed — `db.open` flips to false on close. + expect(harness.openDbs).toHaveLength(1); + expect(harness.openDbs[0]?.open).toBe(false); + }); + + it('activateMind is idempotent and close() is safe when called multiple times', async () => { + harness = buildHarness(mindRoot, 'mind-idem'); + writeChamberConfig(harness.mindPath, true); + + await harness.service.activateMind(harness.mindId, harness.mindPath); + await harness.service.activateMind(harness.mindId, harness.mindPath); + + expect(harness.scheduler.list().size).toBe(1); + expect(harness.daemonFactoryCalls.count).toBe(1); + + await harness.service.close(); + await expect(harness.service.close()).resolves.toBeUndefined(); + expect(harness.scheduler.list().size).toBe(0); + expect(harness.openDbs[0]?.open).toBe(false); + }); + + it('uses the production dream.db path under /.working-memory/.state/', async () => { + harness = buildHarness(mindRoot, 'mind-path'); + writeChamberConfig(harness.mindPath, true); + + await harness.service.activateMind(harness.mindId, harness.mindPath); + + const expected = path.join(harness.mindPath, '.working-memory', '.state', 'dream.db'); + expect(dreamDbPath(harness.mindPath)).toBe(expected); + expect(existsSync(expected)).toBe(true); + }); +}); + +// Reference vi to keep the import-set stable across future edits even if all +// existing tests stop using it directly. +void vi; diff --git a/tests/integration/mindMemory.integration.test.ts b/tests/integration/mindMemory.integration.test.ts new file mode 100644 index 00000000..3ae39982 --- /dev/null +++ b/tests/integration/mindMemory.integration.test.ts @@ -0,0 +1,477 @@ +/** + * Phase 14 — Dream Daemon end-to-end integration. + * + * Builds the full per-mind consolidation graph against a real on-disk + * mind directory and a real better-sqlite3 dream.db, then drives a + * 7-day simulated run with a deterministic in-test LLM client. The + * test substitutes the LLM client (the only seam that would otherwise + * require Electron / network) — every other collaborator is the real + * production implementation. + * + * Properties verified (matches the Phase 14 deliverables in plan.md): + * + * 1. memory.md exists at the end and stays under `memoryMaxBytes` (8192). + * 2. weekly/.md rollup is materialised after 7 daily ticks. + * 3. log.md is pruned (sentinel preserved + only post-cutoff turns survive). + * 4. archive/consolidated/ accumulates one file per source turn. + * 5. Re-running the daemon when no new turns have arrived is skipped + * with a non-success result (no double-processing). + * 6. Two parallel `daemon.run()` calls produce exactly one cycle — + * the second one short-circuits with a `locked` skip. + * 7. dream_state.last_consolidated_turn_id advances monotonically. + * 8. A turn appended between snapshot and prune survives in log.md. + */ + +import { + describe, + it, + expect, + beforeEach, + afterEach, +} from 'vitest'; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync, existsSync, readFileSync } from 'node:fs'; +import { readdir } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { createRequire } from 'node:module'; + +import { + createMindMemoryService, + createInternalScheduler, + createMindMemoryVault, + createMindArchiveStore, + createDreamDaemon, + defaultConfigReader, + migrate as migrateDreamDb, + readState, + listRuns, + type DaemonFactoryOptions, + type DreamDaemon, + type DreamDaemonConfig, + type LLMClient, + type SynthesizeRequest, + type MindMemoryService, + type ChatObserverRegistry, +} from '@chamber/services'; +import type { TurnCompletionObserver, CompletedTurn } from '@chamber/shared'; + +const runtimeRequire = createRequire(__filename); + +// --------------------------------------------------------------------------- +// Test-local LLM client +// --------------------------------------------------------------------------- + +/** + * In-test LLMClient. The default response is the canonical daily-log + * vault delta `extractFromLog` accepts (header `## HH:MM:SS`, content + * lines `**[type]** body`). `pauseNext()` lets a single test arrange + * for the LLM call to suspend so a turn can be appended mid-cycle. + */ +interface TestLLMController { + readonly client: LLMClient; + readonly calls: SynthesizeRequest[]; + pauseNext(): () => void; +} + +function makeTestLLMController(canned: string): TestLLMController { + const calls: SynthesizeRequest[] = []; + let pendingPause: { release: () => void; promise: Promise } | null = null; + + const client: LLMClient = { + async synthesize(req: SynthesizeRequest): Promise { + calls.push(req); + if (pendingPause) { + const p = pendingPause; + pendingPause = null; + await p.promise; + } + return canned; + }, + }; + + return { + client, + calls, + pauseNext(): () => void { + let release: () => void = () => { + /* placeholder */ + }; + const promise = new Promise((res) => { + release = res; + }); + pendingPause = { release, promise }; + return release; + }, + }; +} + +const CANNED_VAULT_DELTA = [ + '## 12:00:00', + '**[user-prompt]** I prefer kebab-case file names.', + '**[user-prompt]** Always follow TDD when implementing features.', + '**[user-prompt]** Never skip required testing steps before claiming done.', + '', +].join('\n'); + +// --------------------------------------------------------------------------- +// Fakes (only ChatService — everything else is real) +// --------------------------------------------------------------------------- + +function makeChatRegistry(): ChatObserverRegistry & { + readonly observers: TurnCompletionObserver[]; +} { + const observers: TurnCompletionObserver[] = []; + return { + addObserver(o: TurnCompletionObserver): void { + observers.push(o); + }, + removeObserver(o: TurnCompletionObserver): void { + const i = observers.indexOf(o); + if (i !== -1) observers.splice(i, 1); + }, + get observers() { + return observers; + }, + }; +} + +// --------------------------------------------------------------------------- +// Harness +// --------------------------------------------------------------------------- + +const MIND_ID = 'mind-integration'; + +interface Harness { + readonly mindRoot: string; + readonly mindPath: string; + readonly service: MindMemoryService; + readonly scheduler: ReturnType; + readonly chat: ReturnType; + readonly llm: TestLLMController; + readonly openDbs: import('better-sqlite3').Database[]; + daemon(): DreamDaemon; + db(): import('better-sqlite3').Database; + cleanup(): Promise; +} + +const MS_PER_DAY = 24 * 60 * 60 * 1000; + +const INTEGRATION_CONFIG: DreamDaemonConfig = { + memoryMaxBytes: 8192, + llmTimeoutMs: 60_000, + lockTtlMs: 300_000, + minTurnsBetweenRuns: 1, + minDailyIntervalMs: 0, + weeklyRollupAfterDailies: 7, + monthlyRollupAfterWeeklies: 4, + weeklyMinIntervalMs: 7 * MS_PER_DAY, + monthlyMinIntervalMs: 30 * MS_PER_DAY, +}; + +function writeChamberConfig(mindPath: string): void { + const cfg = { + workingMemory: { + consolidation: { + enabled: true, + cron: '0 3 * * *', + lastKTurns: 10, + perTurnMaxBytes: 2048, + memoryMaxBytes: 8192, + }, + }, + }; + writeFileSync(path.join(mindPath, '.chamber.json'), JSON.stringify(cfg, null, 2), 'utf-8'); +} + +function buildHarness(): Harness { + const mindRoot = mkdtempSync(path.join(tmpdir(), 'chamber-mindmem-int-')); + const mindPath = path.join(mindRoot, MIND_ID); + mkdirSync(mindPath, { recursive: true }); + writeChamberConfig(mindPath); + + const scheduler = createInternalScheduler(); + const chat = makeChatRegistry(); + const llm = makeTestLLMController(CANNED_VAULT_DELTA); + const openDbs: import('better-sqlite3').Database[] = []; + const Database = runtimeRequire('better-sqlite3') as typeof import('better-sqlite3'); + + let capturedDaemon: DreamDaemon | null = null; + let capturedDb: import('better-sqlite3').Database | null = null; + + const service = createMindMemoryService({ + scheduler, + chatService: chat, + configReader: defaultConfigReader, + dbFactory: (dbPath: string) => { + mkdirSync(path.dirname(dbPath), { recursive: true }); + const db = new Database(dbPath); + db.pragma('journal_mode = WAL'); + db.pragma('foreign_keys = ON'); + migrateDreamDb(db); + openDbs.push(db); + capturedDb = db; + return db; + }, + vaultFactory: createMindMemoryVault, + archiveFactory: createMindArchiveStore, + daemonFactory: (opts: DaemonFactoryOptions) => { + // Real daemon, real vault/archive/db — only the LLM is substituted. + const daemon = createDreamDaemon({ + mindId: opts.mindId, + mindPath: opts.mindPath, + vault: opts.vault, + archiveStore: opts.archive, + db: opts.db, + llmClient: llm.client, + config: INTEGRATION_CONFIG, + }); + capturedDaemon = daemon; + return daemon; + }, + }); + + return { + mindRoot, + mindPath, + service, + scheduler, + chat, + llm, + openDbs, + daemon(): DreamDaemon { + if (!capturedDaemon) throw new Error('daemon not yet constructed (call activateMind first)'); + return capturedDaemon; + }, + db(): import('better-sqlite3').Database { + if (!capturedDb) throw new Error('db not yet opened (call activateMind first)'); + return capturedDb; + }, + async cleanup() { + try { + await service.close(); + } catch { + /* noop */ + } + try { + scheduler.close(); + } catch { + /* noop */ + } + for (const db of openDbs) { + try { + if (db.open) db.close(); + } catch { + /* noop */ + } + } + rmSync(mindRoot, { recursive: true, force: true }); + }, + }; +} + +function makeTurn( + dayIndex: number, + withinDay: number, + prompt: string, + assistant: string, +): CompletedTurn { + const turnId = `t-d${String(dayIndex).padStart(2, '0')}-${String(withinDay).padStart(2, '0')}`; + const startedAt = new Date(Date.now()).toISOString(); + return { + turnId, + sessionId: `s-day-${dayIndex}`, + model: 'gpt-test', + status: 'completed', + startedAt, + endedAt: startedAt, + prompt, + finalAssistantMessage: assistant, + }; +} + +const DAILY_PROMPTS: ReadonlyArray = [ + ['I prefer kebab-case file names.', 'Got it — I will use kebab-case.'], + ['Always follow TDD when implementing features.', 'Acknowledged — TDD by default.'], + ['Never skip required testing steps before claiming done.', 'Understood.'], + ['Use Tailwind for styling.', 'OK — Tailwind it is.'], +]; + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe('Dream Daemon — multi-day integration', () => { + let harness: Harness | null = null; + + beforeEach(() => { + harness = null; + }); + + afterEach(async () => { + if (harness) { + await harness.cleanup(); + harness = null; + } + }); + + it('consolidates 7 simulated days end-to-end with bounded memory.md, weekly rollup, pruned log, and monotonic state', async () => { + harness = buildHarness(); + await harness.service.activateMind(MIND_ID, harness.mindPath); + + const observer = harness.chat.observers[0]; + expect(observer).toBeDefined(); + const db = harness.db(); + + let prevLastTurnId: string | null = null; + const lastTurnIdsByDay: string[] = []; + + // Simulate 7 days. Each day: append several turns, then trigger one + // consolidation cycle via the scheduler entry (the same fn croner + // would invoke at 03:00). + for (let day = 1; day <= 7; day++) { + for (let i = 0; i < DAILY_PROMPTS.length; i++) { + const [p, a] = DAILY_PROMPTS[i]!; + await observer!.onTurnCompleted(makeTurn(day, i, p, a)); + } + + // Activity counter must have advanced (proves the onTurnRecorded + // hook is wired into DailyLogWriter). + expect(readState(db).turnsSinceLastRun).toBe(DAILY_PROMPTS.length); + + await harness.scheduler.runNow(MIND_ID); + + const state = readState(db); + expect(state.lastConsolidatedTurnId).not.toBeNull(); + lastTurnIdsByDay.push(state.lastConsolidatedTurnId!); + + if (prevLastTurnId !== null) { + // Monotonic advance: the new id must be a turn from this day. + expect(state.lastConsolidatedTurnId!.startsWith(`t-d${String(day).padStart(2, '0')}-`)).toBe(true); + } + prevLastTurnId = state.lastConsolidatedTurnId; + + // Activity counter is reset after each successful run. + expect(readState(db).turnsSinceLastRun).toBe(0); + } + + // -- Property 1: memory.md exists and is bounded ----------------- + const memoryPath = path.join(harness.mindPath, '.working-memory', 'memory.md'); + expect(existsSync(memoryPath)).toBe(true); + const memoryBuf = readFileSync(memoryPath); + expect(memoryBuf.byteLength).toBeGreaterThan(0); + expect(memoryBuf.byteLength).toBeLessThanOrEqual(INTEGRATION_CONFIG.memoryMaxBytes); + const memoryText = memoryBuf.toString('utf-8'); + // Curated content from the canned vault delta should be visible. + expect(memoryText.toLowerCase()).toContain('kebab-case'); + + // -- Property 2: weekly rollup materialised ---------------------- + const weeklyDir = path.join(harness.mindPath, '.working-memory', 'archive', 'weekly'); + expect(existsSync(weeklyDir)).toBe(true); + const weeklyFiles = await readdir(weeklyDir); + expect(weeklyFiles.length).toBeGreaterThanOrEqual(1); + expect(weeklyFiles.some((f) => /^\d{4}-W\d{2}\.md$/.test(f))).toBe(true); + + // -- Property 3: log.md is pruned (sentinel preserved) ----------- + const logPath = path.join(harness.mindPath, '.working-memory', 'log.md'); + expect(existsSync(logPath)).toBe(true); + const logText = readFileSync(logPath, 'utf-8'); + expect(logText).toContain('chamber-structured-log/v1'); + // After the final cycle, no in-scope turn ids remain (everything + // already archived). Survivors would be turns appended *after* the + // last snapshot — none in this test, so no `turn:t-dXX-YY` headers. + expect(/turn:t-d\d{2}-\d{2}/.test(logText)).toBe(false); + + // -- Property 4: archive/consolidated/ accumulates --------------- + const consolidatedDir = path.join(harness.mindPath, '.working-memory', 'archive', 'consolidated'); + expect(existsSync(consolidatedDir)).toBe(true); + const consolidatedFiles = await readdir(consolidatedDir); + expect(consolidatedFiles.length).toBe(7 * DAILY_PROMPTS.length); + + // -- Property 5: re-run with no new turns is skipped -------------- + const llmCallsBefore = harness.llm.calls.length; + await harness.scheduler.runNow(MIND_ID); + expect(harness.llm.calls.length).toBe(llmCallsBefore); // synthesize NOT invoked + const stateAfterIdleRun = readState(db); + expect(stateAfterIdleRun.lastConsolidatedTurnId).toBe(prevLastTurnId); + + // -- Property 7: monotonic last_consolidated_turn_id -------------- + for (let i = 1; i < lastTurnIdsByDay.length; i++) { + // Day-prefixed ids — newer-day prefix lexicographically greater. + expect(lastTurnIdsByDay[i]!.localeCompare(lastTurnIdsByDay[i - 1]!)).toBeGreaterThan(0); + } + + // -- Run history shows the success cycles plus the trailing skip -- + const runs = listRuns(db, { limit: 100 }); + const successes = runs.filter((r) => r.status === 'success'); + expect(successes.length).toBe(7); + }); + + it('parallel daemon.run() calls — only one cycle executes; the loser short-circuits with a skip', async () => { + harness = buildHarness(); + await harness.service.activateMind(MIND_ID, harness.mindPath); + + const observer = harness.chat.observers[0]!; + await observer.onTurnCompleted(makeTurn(1, 1, 'I prefer kebab-case file names.', 'ok')); + + const release = harness.llm.pauseNext(); + + const daemon = harness.daemon(); + const a = daemon.run(); + // Yield once so the first run() acquires the in-process mutex / DB lock. + await Promise.resolve(); + await Promise.resolve(); + const b = daemon.run(); + + // Release the LLM call so the first run can complete. + release(); + + const [ra, rb] = await Promise.all([a, b]); + const outcomes = [ra.status, rb.status].sort(); + // Exactly one success, one skip. + expect(outcomes).toEqual(['skipped', 'success']); + + // Synthesize was called exactly once across both runs. + expect(harness.llm.calls.length).toBe(1); + }); + + it('a turn appended between snapshot and prune survives in log.md', async () => { + harness = buildHarness(); + await harness.service.activateMind(MIND_ID, harness.mindPath); + + const observer = harness.chat.observers[0]!; + // Pre-cycle turns. + await observer.onTurnCompleted(makeTurn(1, 1, 'I prefer kebab-case file names.', 'ok')); + await observer.onTurnCompleted(makeTurn(1, 2, 'Always follow TDD when implementing features.', 'ok')); + + const release = harness.llm.pauseNext(); + const daemon = harness.daemon(); + + const runPromise = daemon.run(); + + // Wait until synthesize has been called — that proves snapshot has + // already been taken (snapshot precedes synthesize in runCycleLocked). + while (harness.llm.calls.length === 0) { + await new Promise((r) => setImmediate(r)); + } + + // Append a tail turn AFTER snapshot but BEFORE prune. + const tail = makeTurn(1, 9, 'tail prompt that arrives mid-cycle', 'tail assistant'); + await observer.onTurnCompleted(tail); + + release(); + const result = await runPromise; + expect(result.status).toBe('success'); + + // The tail turn must still be in log.md (not archived, not pruned). + const logPath = path.join(harness.mindPath, '.working-memory', 'log.md'); + const logText = readFileSync(logPath, 'utf-8'); + expect(logText).toContain('chamber-structured-log/v1'); + expect(logText).toContain(tail.turnId); + + // And it should NOT have been archived (only snapshot turns are). + const consolidatedDir = path.join(harness.mindPath, '.working-memory', 'archive', 'consolidated'); + const consolidatedFiles = await readdir(consolidatedDir); + expect(consolidatedFiles.length).toBe(2); + expect(consolidatedFiles.some((f) => f.includes(tail.turnId))).toBe(false); + }); +}); From ec6b45ce37f5b22e19f202748461a032bed94bc3 Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Tue, 12 May 2026 19:21:18 -0400 Subject: [PATCH 10/23] CONFIG: gitignore .orchestrator/ and resources/copilot-runtime.new/ --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index b5586abc..7ee29feb 100644 --- a/.gitignore +++ b/.gitignore @@ -102,6 +102,10 @@ resources/node/ # Packaged Copilot runtime (generated by scripts/prepare-copilot-runtime.js) resources/copilot-runtime/ +resources/copilot-runtime.new/ + +# Orchestrator session forensics (winorch / tmux-orchestrator-win) +.orchestrator/ # Packaged Sharp runtime (generated by scripts/prepare-sharp-runtime.js) resources/sharp-runtime/ From 0330a58b44dd846ff141a8411d5eef3958e25d83 Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Tue, 12 May 2026 19:42:31 -0400 Subject: [PATCH 11/23] ADD: MindMemoryService.__debugGet for E2E access to per-mind daemon + dbPath Test-only accessor on the MindMemoryService interface, matching the existing `__resetMindMutexForTesting` pattern in consolidation-scheduler. Returns `{ daemon, dbPath } | null` so a Playwright driver attached via `electronApp.evaluate()` can drive forceRun()/getStatus() and read the per-mind dream.db from disk without us building a renderer-facing IPC bridge. --- .../src/mindMemory/MindMemoryService.test.ts | 45 +++++++++++++++++++ .../src/mindMemory/MindMemoryService.ts | 28 ++++++++++-- 2 files changed, 70 insertions(+), 3 deletions(-) diff --git a/packages/services/src/mindMemory/MindMemoryService.test.ts b/packages/services/src/mindMemory/MindMemoryService.test.ts index ce185aef..484b403f 100644 --- a/packages/services/src/mindMemory/MindMemoryService.test.ts +++ b/packages/services/src/mindMemory/MindMemoryService.test.ts @@ -661,3 +661,48 @@ describe('MindMemoryService — multi-mind isolation', () => { expect(chat.observers).toHaveLength(1); }); }); + +describe('MindMemoryService — __debugGet (E2E accessor)', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('returns null for an unknown mind id', () => { + const { factories } = makeFactories({ chamberConfig: ENABLED_CONFIG }); + const svc = createMindMemoryService(factories); + + expect(svc.__debugGet('does-not-exist')).toBeNull(); + }); + + it('returns null for a disabled mind (activate was a no-op)', async () => { + const { factories } = makeFactories({ chamberConfig: DISABLED_CONFIG }); + const svc = createMindMemoryService(factories); + + await svc.activateMind(MIND_ID, MIND_PATH); + + expect(svc.__debugGet(MIND_ID)).toBeNull(); + }); + + it('returns the live daemon + dbPath for an activated mind', async () => { + const { factories, daemon } = makeFactories({ chamberConfig: ENABLED_CONFIG }); + const svc = createMindMemoryService(factories); + + await svc.activateMind(MIND_ID, MIND_PATH); + + const entry = svc.__debugGet(MIND_ID); + expect(entry).not.toBeNull(); + expect(entry!.daemon).toBe(daemon); + expect(entry!.dbPath).toBe(dreamDbPath(MIND_PATH)); + }); + + it('returns null again after releaseMind', async () => { + const { factories } = makeFactories({ chamberConfig: ENABLED_CONFIG }); + const svc = createMindMemoryService(factories); + + await svc.activateMind(MIND_ID, MIND_PATH); + expect(svc.__debugGet(MIND_ID)).not.toBeNull(); + + await svc.releaseMind(MIND_ID); + expect(svc.__debugGet(MIND_ID)).toBeNull(); + }); +}); diff --git a/packages/services/src/mindMemory/MindMemoryService.ts b/packages/services/src/mindMemory/MindMemoryService.ts index 3fbda336..279c5ae7 100644 --- a/packages/services/src/mindMemory/MindMemoryService.ts +++ b/packages/services/src/mindMemory/MindMemoryService.ts @@ -101,6 +101,20 @@ export interface MindMemoryService { activateMind(mindId: string, mindPath: string): Promise; releaseMind(mindId: string): Promise; close(): Promise; + /** + * Test/E2E-only accessor. Returns the live `DreamDaemon` plus the + * `dream.db` path for an active mind, or `null` if the mind is not + * currently activated. Production code must NOT depend on this surface; + * callers are expected to gate on `process.env.CHAMBER_E2E === '1'`. + * + * Lifecycle: + * - returns `null` for unknown / disabled mind ids + * - returns the same `daemon` reference handed back by `daemonFactory` + * during `activateMind` + * - returns `null` again after `releaseMind` (entry is removed from + * the internal active map) + */ + __debugGet(mindId: string): { readonly daemon: DreamDaemon; readonly dbPath: string } | null; } // --------------------------------------------------------------------------- @@ -109,6 +123,7 @@ export interface MindMemoryService { interface ActiveEntry { readonly mindPath: string; + readonly dbPath: string; readonly db: Database.Database; readonly daemon: DreamDaemon; readonly writer: DailyLogWriter; @@ -151,7 +166,8 @@ export function createMindMemoryService( try { const vault = factories.vaultFactory(mindPath); const archive = factories.archiveFactory(mindPath); - db = factories.dbFactory(dreamDbPath(mindPath)); + const dbPath = dreamDbPath(mindPath); + db = factories.dbFactory(dbPath); daemon = factories.daemonFactory({ mindId, @@ -196,7 +212,7 @@ export function createMindMemoryService( factories.chatService.addObserver(observer); - active.set(mindId, { mindPath, db, daemon, writer, observer }); + active.set(mindId, { mindPath, dbPath, db, daemon, writer, observer }); } catch (err) { // Unwind in reverse order — only what we successfully built. Each // step is wrapped to keep the original error as the surfaced one. @@ -269,7 +285,13 @@ export function createMindMemoryService( } } - return { activateMind, releaseMind, close }; + return { activateMind, releaseMind, close, __debugGet }; + + function __debugGet(mindId: string): { readonly daemon: DreamDaemon; readonly dbPath: string } | null { + const entry = active.get(mindId); + if (!entry) return null; + return { daemon: entry.daemon, dbPath: entry.dbPath }; + } } // Re-export the default loader so the composition root can pass it as From d254ceca40bbc7bd13981a011dbac01099317996 Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Tue, 12 May 2026 19:42:42 -0400 Subject: [PATCH 12/23] ADD: dream daemon E2E hooks (mid-cycle sleep shim + globalThis service handle) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two narrow, env-gated hooks so the chamber-ui-tester Playwright driver can exercise the consolidation cycle deterministically without a renderer bridge: 1. DreamDaemon: `maybeTestSleep()` between the memory.md write and the log.md prune, gated on `CHAMBER_E2E=1 && CHAMBER_DREAM_TEST_SLEEP_MS > 0`. Lets M7 reproduce the mid-cycle append race (turn arrives after snapshot but before prune). Production builds never see CHAMBER_E2E=1, so this is a no-op by construction. 2. main.ts: when CHAMBER_E2E=1, stash mindMemoryService on globalThis.__chamberMindMemoryService. `electronApp.evaluate(...)` can then call `__debugGet(mindId).daemon.forceRun()` and read `dbPath` directly — no IPC channel, no preload bridge, no shared-package type contract. --- apps/desktop/src/main.ts | 9 +++ .../src/mindMemory/DreamDaemon.test.ts | 60 ++++++++++++++++++- .../services/src/mindMemory/DreamDaemon.ts | 26 ++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index 8f8dc290..8afaaed5 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -668,6 +668,15 @@ app.on('ready', async () => { setupChatroomIPC(chatroomService); setupUpdaterIPC(updaterService); + // Test-only hook: expose the MindMemoryService on globalThis so a Playwright + // driver attached via `electronApp.evaluate(...)` can drive the Dream Daemon + // (forceRun, getStatus, dbPath) without us building a renderer-facing bridge + // and the production type contract that comes with it. Gated by CHAMBER_E2E. + if (process.env.CHAMBER_E2E === '1') { + (globalThis as { __chamberMindMemoryService?: typeof mindMemoryService }).__chamberMindMemoryService = + mindMemoryService; + } + // Fire-and-forget tool reconciliation: install any new marketplace tools. // Errors are logged in ToolsService and surface via tools:list later. reconcileMarketplaceTools(); diff --git a/packages/services/src/mindMemory/DreamDaemon.test.ts b/packages/services/src/mindMemory/DreamDaemon.test.ts index 72f08488..c5dbf94e 100644 --- a/packages/services/src/mindMemory/DreamDaemon.test.ts +++ b/packages/services/src/mindMemory/DreamDaemon.test.ts @@ -14,7 +14,7 @@ * - injected clock for deterministic tiered-rollup gates */ -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import Database from 'better-sqlite3'; import fs from 'node:fs'; @@ -447,6 +447,64 @@ describe('DreamDaemon — mid-run append', () => { }); }); +// --------------------------------------------------------------------------- +// E2E mid-cycle sleep shim (test-only) +// --------------------------------------------------------------------------- + +describe('DreamDaemon — E2E mid-cycle sleep shim', () => { + it('is a no-op when CHAMBER_E2E is unset, even if CHAMBER_DREAM_TEST_SLEEP_MS is set', async () => { + vi.stubEnv('CHAMBER_E2E', ''); + vi.stubEnv('CHAMBER_DREAM_TEST_SLEEP_MS', '5000'); + try { + await seedLog([makeTurn({ turnId: 'turn-noop' })]); + incrementTurnCount(db, 1); + const daemon = makeDaemon(); + const started = Date.now(); + const result = await daemon.run(); + const elapsed = Date.now() - started; + expect(result.status).toBe('success'); + // Without the env gate, the sleep is bypassed entirely. + expect(elapsed).toBeLessThan(2000); + } finally { + vi.unstubAllEnvs(); + } + }); + + it('pauses between the snapshot and the prune when CHAMBER_E2E=1 AND CHAMBER_DREAM_TEST_SLEEP_MS is positive', async () => { + vi.stubEnv('CHAMBER_E2E', '1'); + vi.stubEnv('CHAMBER_DREAM_TEST_SLEEP_MS', '120'); + try { + await seedLog([makeTurn({ turnId: 'turn-sleep' })]); + incrementTurnCount(db, 1); + const daemon = makeDaemon(); + const started = Date.now(); + const result = await daemon.run(); + const elapsed = Date.now() - started; + expect(result.status).toBe('success'); + expect(elapsed).toBeGreaterThanOrEqual(100); + } finally { + vi.unstubAllEnvs(); + } + }); + + it('ignores a non-finite or non-positive CHAMBER_DREAM_TEST_SLEEP_MS value', async () => { + vi.stubEnv('CHAMBER_E2E', '1'); + vi.stubEnv('CHAMBER_DREAM_TEST_SLEEP_MS', 'banana'); + try { + await seedLog([makeTurn({ turnId: 'turn-bad-env' })]); + incrementTurnCount(db, 1); + const daemon = makeDaemon(); + const started = Date.now(); + const result = await daemon.run(); + const elapsed = Date.now() - started; + expect(result.status).toBe('success'); + expect(elapsed).toBeLessThan(2000); + } finally { + vi.unstubAllEnvs(); + } + }); +}); + // --------------------------------------------------------------------------- // LLM failures // --------------------------------------------------------------------------- diff --git a/packages/services/src/mindMemory/DreamDaemon.ts b/packages/services/src/mindMemory/DreamDaemon.ts index d9457e9d..740f1989 100644 --- a/packages/services/src/mindMemory/DreamDaemon.ts +++ b/packages/services/src/mindMemory/DreamDaemon.ts @@ -353,6 +353,13 @@ export function createDreamDaemon(opts: DreamDaemonOptions): DreamDaemon { phase = 'writing'; await vault.write(MEMORY_REL_PATH, memoryMd); + // Test-only mid-cycle sleep used by M7 (mid-run append race). + // Honored ONLY when CHAMBER_E2E=1 so production builds never read the + // env var. Inserted between the memory.md write and the prune phase + // so a test can append a turn AFTER the snapshot was taken and assert + // the tail entry survives the prune. + await maybeTestSleep(); + // Step 7 — re-read log.md and prune only the snapshot turn ids. Tail // entries appended during the LLM call MUST survive. phase = 'pruning'; @@ -579,3 +586,22 @@ function isoMonthKey(date: Date): string { const mm = String(date.getUTCMonth() + 1).padStart(2, '0'); return `${yyyy}-${mm}`; } + +/** + * Test-only mid-cycle pause. Returns immediately unless BOTH: + * - `CHAMBER_E2E=1` (the global test-surface gate the rest of the app + * uses to expose `IPC.E2E.*` handlers and `electronAPI.e2e`) + * - `CHAMBER_DREAM_TEST_SLEEP_MS` is a positive finite number + * + * Production builds never see CHAMBER_E2E=1, so this function is a no-op + * by construction. Used by M7 to inject an append between the snapshot + * and the prune. + */ +async function maybeTestSleep(): Promise { + if (process.env.CHAMBER_E2E !== '1') return; + const raw = process.env.CHAMBER_DREAM_TEST_SLEEP_MS; + if (!raw) return; + const ms = Number(raw); + if (!Number.isFinite(ms) || ms <= 0) return; + await new Promise((resolve) => setTimeout(resolve, ms)); +} From e8a8d009bddbd336a989e8172d1f4f26d625b9ca Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Tue, 12 May 2026 21:24:23 -0400 Subject: [PATCH 13/23] ADD: expose DailyLogWriter on MindMemoryService.__debugGet for E2E rotation/sentinel scenarios --- .../src/mindMemory/MindMemoryService.test.ts | 2 ++ .../services/src/mindMemory/MindMemoryService.ts | 14 +++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/services/src/mindMemory/MindMemoryService.test.ts b/packages/services/src/mindMemory/MindMemoryService.test.ts index 484b403f..237de754 100644 --- a/packages/services/src/mindMemory/MindMemoryService.test.ts +++ b/packages/services/src/mindMemory/MindMemoryService.test.ts @@ -693,6 +693,8 @@ describe('MindMemoryService — __debugGet (E2E accessor)', () => { expect(entry).not.toBeNull(); expect(entry!.daemon).toBe(daemon); expect(entry!.dbPath).toBe(dreamDbPath(MIND_PATH)); + expect(entry!.writer).toBeDefined(); + expect(typeof entry!.writer.write).toBe('function'); }); it('returns null again after releaseMind', async () => { diff --git a/packages/services/src/mindMemory/MindMemoryService.ts b/packages/services/src/mindMemory/MindMemoryService.ts index 279c5ae7..b06e4f72 100644 --- a/packages/services/src/mindMemory/MindMemoryService.ts +++ b/packages/services/src/mindMemory/MindMemoryService.ts @@ -114,7 +114,11 @@ export interface MindMemoryService { * - returns `null` again after `releaseMind` (entry is removed from * the internal active map) */ - __debugGet(mindId: string): { readonly daemon: DreamDaemon; readonly dbPath: string } | null; + __debugGet(mindId: string): { + readonly daemon: DreamDaemon; + readonly dbPath: string; + readonly writer: DailyLogWriter; + } | null; } // --------------------------------------------------------------------------- @@ -287,10 +291,14 @@ export function createMindMemoryService( return { activateMind, releaseMind, close, __debugGet }; - function __debugGet(mindId: string): { readonly daemon: DreamDaemon; readonly dbPath: string } | null { + function __debugGet(mindId: string): { + readonly daemon: DreamDaemon; + readonly dbPath: string; + readonly writer: DailyLogWriter; + } | null { const entry = active.get(mindId); if (!entry) return null; - return { daemon: entry.daemon, dbPath: entry.dbPath }; + return { daemon: entry.daemon, dbPath: entry.dbPath, writer: entry.writer }; } } From 93cb04bcff75d68e12960f00d7949c55fcbf4d21 Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Tue, 12 May 2026 23:59:03 -0400 Subject: [PATCH 14/23] ADD: ensure-native-abi guard for better-sqlite3 across vitest and Playwright Electron --- package.json | 2 + scripts/ensure-native-abi.cjs | 44 ++++++ scripts/lib/ensure-native-abi.cjs | 79 ++++++++++ tests/regression/ensure-native-abi.test.ts | 163 +++++++++++++++++++++ 4 files changed, 288 insertions(+) create mode 100644 scripts/ensure-native-abi.cjs create mode 100644 scripts/lib/ensure-native-abi.cjs create mode 100644 tests/regression/ensure-native-abi.test.ts diff --git a/package.json b/package.json index 37b47240..379114e7 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "deps:check": "depcruise apps packages --config config/dependency-cruiser.cjs", "typecheck": "tsc --noEmit", "test": "vitest run --config config/vitest.config.ts --no-file-parallelism --maxWorkers=1", + "pretest": "node scripts/ensure-native-abi.cjs node", "test:coverage": "vitest run --config config/vitest.config.ts --coverage --no-file-parallelism --maxWorkers=1", "clean": "node scripts/clean-artifacts.js", "capture:hero": "node scripts/capture-readme-hero.js", @@ -35,6 +36,7 @@ "smoke:server-sdk": "npm --workspace @chamber/server run build && node scripts/run-server-sdk-smoke-test.js", "smoke:web": "npm run playwright:install && playwright test --config config/playwright.config.ts --project=web", "smoke:desktop": "npm run playwright:install && playwright test --config config/playwright.config.ts --project=electron --workers=1", + "presmoke:desktop": "node scripts/ensure-native-abi.cjs electron", "smoke:packaged-runtime": "npm --workspace @chamber/server run build && node scripts/packaged-smoke.js" }, "keywords": [], diff --git a/scripts/ensure-native-abi.cjs b/scripts/ensure-native-abi.cjs new file mode 100644 index 00000000..ee64050a --- /dev/null +++ b/scripts/ensure-native-abi.cjs @@ -0,0 +1,44 @@ +#!/usr/bin/env node +'use strict'; + +// Thin CLI wrapper around scripts/lib/ensure-native-abi.cjs. +// Usage: node scripts/ensure-native-abi.cjs +// Wired into npm `pretest` and `presmoke:desktop` lifecycle hooks. + +const { + readSentinel, + decideAction, + writeSentinel, + rebuild, + TARGETS, +} = require('./lib/ensure-native-abi.cjs'); + +const target = process.argv[2]; + +if (!target || !TARGETS.includes(target)) { + console.error( + `[ensure-native-abi] usage: node scripts/ensure-native-abi.cjs <${TARGETS.join('|')}>`, + ); + process.exit(2); +} + +const current = readSentinel(); +const action = decideAction({ target, current }); + +if (action === 'noop') { + console.log(`[ensure-native-abi] better-sqlite3 already built for ${target} — skipping rebuild`); + process.exit(0); +} + +console.log( + `[ensure-native-abi] better-sqlite3 ABI target=${target}, current=${current ?? 'unknown'} — rebuilding...`, +); + +try { + rebuild(target); + writeSentinel(target); + console.log(`[ensure-native-abi] better-sqlite3 now built for ${target}`); +} catch (err) { + console.error(`[ensure-native-abi] rebuild failed: ${err && err.message ? err.message : err}`); + process.exit(1); +} diff --git a/scripts/lib/ensure-native-abi.cjs b/scripts/lib/ensure-native-abi.cjs new file mode 100644 index 00000000..788369c0 --- /dev/null +++ b/scripts/lib/ensure-native-abi.cjs @@ -0,0 +1,79 @@ +'use strict'; + +// Pure-logic core for the ensure-native-abi guard. +// +// Why this exists: +// better-sqlite3 is a native N-API module. Node 24 and Electron 41 ship +// different V8 ABIs (137 vs 145). electron-forge silently rebuilds the +// binary against the Electron ABI on `npm start` / `npm run package`, +// but vitest (Node) and Playwright `_electron.launch` (Electron) have no +// such hook — so a developer who switches between `npm test` and +// `npm run smoke:desktop` hits "Cannot read properties of undefined" +// crashes from the wrong-ABI .node file. +// +// This module records the last-built ABI target in a sentinel file under +// `node_modules/better-sqlite3/build/Release/.abi-target` and exposes a +// pure `decideAction` so callers can short-circuit when the binary already +// matches. The CLI wrapper (`scripts/ensure-native-abi.cjs`) drives the +// actual rebuild via `npm rebuild` or `electron-rebuild`. + +const fs = require('node:fs'); +const path = require('node:path'); +const { execSync } = require('node:child_process'); + +const TARGETS = Object.freeze(['node', 'electron']); + +const DEFAULT_SENTINEL_PATH = path.join( + 'node_modules', + 'better-sqlite3', + 'build', + 'Release', + '.abi-target', +); + +function readSentinel(sentinelPath = DEFAULT_SENTINEL_PATH) { + try { + return fs.readFileSync(sentinelPath, 'utf8').trim(); + } catch { + return null; + } +} + +function decideAction({ target, current }) { + if (!TARGETS.includes(target)) { + throw new Error( + `ensure-native-abi: unknown target "${target}". Expected one of: ${TARGETS.join(', ')}`, + ); + } + return current === target ? 'noop' : 'rebuild'; +} + +function writeSentinel(target, sentinelPath = DEFAULT_SENTINEL_PATH) { + if (!TARGETS.includes(target)) { + throw new Error( + `ensure-native-abi: refusing to write sentinel with unknown target "${target}"`, + ); + } + fs.mkdirSync(path.dirname(sentinelPath), { recursive: true }); + fs.writeFileSync(sentinelPath, `${target}\n`); +} + +function rebuildCommand(target) { + if (target === 'node') return 'npm rebuild better-sqlite3'; + if (target === 'electron') return 'npx --no-install electron-rebuild -f -w better-sqlite3'; + throw new Error(`ensure-native-abi: unknown target "${target}"`); +} + +function rebuild(target, runner = (cmd) => execSync(cmd, { stdio: 'inherit' })) { + runner(rebuildCommand(target)); +} + +module.exports = { + TARGETS, + DEFAULT_SENTINEL_PATH, + readSentinel, + decideAction, + writeSentinel, + rebuildCommand, + rebuild, +}; diff --git a/tests/regression/ensure-native-abi.test.ts b/tests/regression/ensure-native-abi.test.ts new file mode 100644 index 00000000..cf8d6ff8 --- /dev/null +++ b/tests/regression/ensure-native-abi.test.ts @@ -0,0 +1,163 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +// eslint-disable-next-line @typescript-eslint/no-require-imports +const lib = require('../../scripts/lib/ensure-native-abi.cjs') as { + TARGETS: readonly string[]; + DEFAULT_SENTINEL_PATH: string; + readSentinel: (p?: string) => string | null; + decideAction: (input: { target: string; current: string | null }) => 'noop' | 'rebuild'; + writeSentinel: (target: string, p?: string) => void; + rebuildCommand: (target: string) => string; + rebuild: (target: string, runner?: (cmd: string) => void) => void; +}; + +describe('ensure-native-abi guard', () => { + let tmpDir: string; + let sentinelPath: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-abi-test-')); + sentinelPath = path.join(tmpDir, 'build', 'Release', '.abi-target'); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + describe('TARGETS', () => { + it('is locked to node and electron only', () => { + expect([...lib.TARGETS]).toEqual(['node', 'electron']); + }); + }); + + describe('decideAction', () => { + it('returns "noop" when current matches target', () => { + expect(lib.decideAction({ target: 'node', current: 'node' })).toBe('noop'); + expect(lib.decideAction({ target: 'electron', current: 'electron' })).toBe('noop'); + }); + + it('returns "rebuild" when current differs from target', () => { + expect(lib.decideAction({ target: 'node', current: 'electron' })).toBe('rebuild'); + expect(lib.decideAction({ target: 'electron', current: 'node' })).toBe('rebuild'); + }); + + it('returns "rebuild" when no sentinel exists yet (current is null)', () => { + expect(lib.decideAction({ target: 'node', current: null })).toBe('rebuild'); + expect(lib.decideAction({ target: 'electron', current: null })).toBe('rebuild'); + }); + + it('throws on unknown target rather than silently skipping', () => { + expect(() => lib.decideAction({ target: 'wasm', current: 'node' })).toThrow(/unknown target/); + expect(() => lib.decideAction({ target: '', current: 'node' })).toThrow(/unknown target/); + }); + }); + + describe('readSentinel', () => { + it('returns null when the sentinel file does not exist', () => { + expect(lib.readSentinel(sentinelPath)).toBeNull(); + }); + + it('returns the trimmed sentinel contents when present', () => { + fs.mkdirSync(path.dirname(sentinelPath), { recursive: true }); + fs.writeFileSync(sentinelPath, 'electron\n'); + expect(lib.readSentinel(sentinelPath)).toBe('electron'); + }); + + it('returns null when the sentinel directory is unreadable / missing parents', () => { + const deepMissing = path.join(tmpDir, 'never', 'made', '.abi-target'); + expect(lib.readSentinel(deepMissing)).toBeNull(); + }); + }); + + describe('writeSentinel', () => { + it('creates parent directories and writes the target with a trailing newline', () => { + lib.writeSentinel('node', sentinelPath); + const raw = fs.readFileSync(sentinelPath, 'utf8'); + expect(raw).toBe('node\n'); + }); + + it('round-trips with readSentinel', () => { + lib.writeSentinel('electron', sentinelPath); + expect(lib.readSentinel(sentinelPath)).toBe('electron'); + }); + + it('refuses to write an unknown target', () => { + expect(() => lib.writeSentinel('wasm', sentinelPath)).toThrow(/unknown target/); + expect(fs.existsSync(sentinelPath)).toBe(false); + }); + + it('overwrites an existing sentinel rather than appending', () => { + lib.writeSentinel('node', sentinelPath); + lib.writeSentinel('electron', sentinelPath); + expect(lib.readSentinel(sentinelPath)).toBe('electron'); + }); + }); + + describe('rebuildCommand', () => { + it('uses `npm rebuild better-sqlite3` for the node target', () => { + expect(lib.rebuildCommand('node')).toBe('npm rebuild better-sqlite3'); + }); + + it('uses electron-rebuild scoped to better-sqlite3 for the electron target', () => { + const cmd = lib.rebuildCommand('electron'); + expect(cmd).toContain('electron-rebuild'); + expect(cmd).toContain('better-sqlite3'); + expect(cmd).toContain('-f'); + expect(cmd).toContain('-w better-sqlite3'); + }); + + it('throws on unknown target', () => { + expect(() => lib.rebuildCommand('wasm')).toThrow(/unknown target/); + }); + }); + + describe('rebuild', () => { + it('invokes the runner with the resolved command for node', () => { + const calls: string[] = []; + lib.rebuild('node', (cmd: string) => calls.push(cmd)); + expect(calls).toEqual([lib.rebuildCommand('node')]); + }); + + it('invokes the runner with the resolved command for electron', () => { + const calls: string[] = []; + lib.rebuild('electron', (cmd: string) => calls.push(cmd)); + expect(calls).toEqual([lib.rebuildCommand('electron')]); + }); + + it('propagates runner failures so the CLI can exit non-zero', () => { + expect(() => + lib.rebuild('node', () => { + throw new Error('toolchain missing'); + }), + ).toThrow(/toolchain missing/); + }); + }); + + describe('integration: full guard sequence', () => { + it('rebuilds on first run, then noops on the second run with the same target', () => { + // First run: no sentinel yet → rebuild path + const first = lib.decideAction({ target: 'node', current: lib.readSentinel(sentinelPath) }); + expect(first).toBe('rebuild'); + const calls: string[] = []; + lib.rebuild('node', (cmd) => calls.push(cmd)); + lib.writeSentinel('node', sentinelPath); + + // Second run: sentinel matches → noop, no rebuild invoked + const second = lib.decideAction({ target: 'node', current: lib.readSentinel(sentinelPath) }); + expect(second).toBe('noop'); + expect(calls).toHaveLength(1); + }); + + it('rebuilds when switching target from node → electron', () => { + lib.writeSentinel('node', sentinelPath); + const action = lib.decideAction({ + target: 'electron', + current: lib.readSentinel(sentinelPath), + }); + expect(action).toBe('rebuild'); + }); + }); +}); From 72e646355058c6e816c8211c7671779cb0427b3c Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Wed, 13 May 2026 01:21:31 -0400 Subject: [PATCH 15/23] FIX: prevent better-sqlite3 finalizer race in vitest by switching to forks pool --- config/vitest.config.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/config/vitest.config.ts b/config/vitest.config.ts index 5b6c2ed7..8075871c 100644 --- a/config/vitest.config.ts +++ b/config/vitest.config.ts @@ -6,6 +6,15 @@ export default defineConfig({ test: { globals: true, environment: 'node', + // INVARIANT: use the `forks` pool, not the default `threads` pool. + // better-sqlite3 native finalizers race the worker-thread teardown when + // running on a thread pool; the result is a benign "Worker exited + // unexpectedly" error after all tests pass, which makes vitest exit 1 + // and silently drops ~25 tests partway through the run. Forks isolate + // each test file in a real subprocess, so native modules clean up at + // process exit. Pair with --no-file-parallelism --maxWorkers=1 in the + // npm script for serial execution. + pool: 'forks', include: [ 'apps/**/*.{test,spec}.{ts,tsx}', 'packages/**/*.{test,spec}.{ts,tsx}', From ec6cb564838ef66c517d839b6a1570870e7f2de7 Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Wed, 13 May 2026 01:21:39 -0400 Subject: [PATCH 16/23] REFACTOR: extract buildOneShotSession helper and add live-SDK integration test --- .../mindMemory/buildMindMemoryService.ts | 49 +---- .../CopilotLLMClient.integration.test.ts | 112 +++++++++-- packages/services/src/mindMemory/index.ts | 1 + .../src/mindMemory/oneShotSession.test.ts | 186 ++++++++++++++++++ .../services/src/mindMemory/oneShotSession.ts | 84 ++++++++ 5 files changed, 372 insertions(+), 60 deletions(-) create mode 100644 packages/services/src/mindMemory/oneShotSession.test.ts create mode 100644 packages/services/src/mindMemory/oneShotSession.ts diff --git a/apps/desktop/src/main/services/mindMemory/buildMindMemoryService.ts b/apps/desktop/src/main/services/mindMemory/buildMindMemoryService.ts index cdab57aa..d80a79ef 100644 --- a/apps/desktop/src/main/services/mindMemory/buildMindMemoryService.ts +++ b/apps/desktop/src/main/services/mindMemory/buildMindMemoryService.ts @@ -19,6 +19,7 @@ import path from 'node:path'; import fs from 'node:fs'; import { createRequire } from 'node:module'; import { + buildOneShotSession, createMindMemoryService, createInternalScheduler, createMindMemoryVault, @@ -35,7 +36,6 @@ import { type OneShotSession, } from '@chamber/services'; import type { TurnCompletionObserver } from '@chamber/shared'; -import type { PermissionHandler, MessageOptions, SessionConfig } from '@github/copilot-sdk'; type BetterSqlite3Module = typeof import('better-sqlite3'); type BetterSqlite3Database = import('better-sqlite3').Database; @@ -81,16 +81,14 @@ function loadBetterSqlite3(isPackaged: boolean, resourcesPath: string | undefine return runtimeRequire('better-sqlite3') as BetterSqlite3Module; } -const refusingPermissionHandler: PermissionHandler = () => ({ - kind: 'reject', - feedback: 'Tool permissions are disabled for memory-consolidation sessions.', -}); - /** * Build the createOneShotSession adapter for CopilotLLMClient. Each * synthesize call mints a fresh CopilotSession scoped to the mind's * working directory, with NO tools, NO config discovery, and a refusing * permission handler. The session is closed in the LLMClient's `finally`. + * + * The SDK-touching plumbing lives in `@chamber/services` `buildOneShotSession` + * so the same contract is exercised by the live-SDK integration test. */ function makeCreateOneShotSession( mindManager: MindManager, @@ -101,40 +99,13 @@ function makeCreateOneShotSession( if (!ctx) { throw new Error(`MindMemory: cannot create session — mind ${mindId} is not loaded`); } - const sessionConfig: SessionConfig = { + return buildOneShotSession({ + client: ctx.client, workingDirectory: mindPath, - enableConfigDiscovery: false, - tools: [], - systemMessage: { mode: 'replace', content: '' }, - onPermissionRequest: refusingPermissionHandler, - }; - const session = await ctx.client.createSession(sessionConfig); - - const onAbort = (): void => { - // Best-effort abort. The CLI may or may not have an in-flight call. - session.abort().catch(() => { /* noop */ }); - }; - if (signal.aborted) { - onAbort(); - } else { - signal.addEventListener('abort', onAbort, { once: true }); - } - - return { - async send(prompt: string): Promise { - const opts: MessageOptions = { prompt }; - const event = await session.sendAndWait(opts); - return event?.data.content ?? ''; - }, - async close(): Promise { - signal.removeEventListener('abort', onAbort); - try { - await session.disconnect(); - } catch (err) { - logger.warn('mindMemory: session disconnect failed', { err: String(err) }); - } - }, - }; + signal, + onDisconnectError: (err) => + logger.warn('mindMemory: session disconnect failed', { err: String(err) }), + }); }; } diff --git a/packages/services/src/mindMemory/CopilotLLMClient.integration.test.ts b/packages/services/src/mindMemory/CopilotLLMClient.integration.test.ts index 1dd84533..0c620960 100644 --- a/packages/services/src/mindMemory/CopilotLLMClient.integration.test.ts +++ b/packages/services/src/mindMemory/CopilotLLMClient.integration.test.ts @@ -1,35 +1,105 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { CopilotClient } from '@github/copilot-sdk'; + +import { createCopilotLLMClient } from './CopilotLLMClient'; +import { buildOneShotSession } from './oneShotSession'; +import { + getPlatformCopilotBinaryPath, + resolveNodeModulesDir, +} from '../sdk'; + /** - * Phase 8 — CopilotLLMClient integration test stub. + * Phase 8 / 13 — CopilotLLMClient × Copilot SDK contract test. * * Hermetic by default. Set `CHAMBER_LIVE_SDK=1` (and ensure the Copilot * CLI runtime + valid keychain credentials are present) to exercise the * adapter against a real one-shot session. * - * Wiring lives in Phase 13 (composition root); this test documents the - * contract the desktop wiring must satisfy: - * - `createOneShotSession` builds a session with NO tools and NO - * permission handler. - * - The session's `send` resolves with the final assistant text. - * - The session is torn down by `close()` so no CLI process leaks. + * The unit-test contract for `buildOneShotSession` lives in + * `oneShotSession.test.ts` (hermetic, fakes only). This file proves the + * production adapter actually drives the SDK end-to-end with the + * documented contract: + * + * - NO tools registered (tools = []) + * - NO config discovery (`enableConfigDiscovery: false`) + * - PermissionHandler refuses any request that leaks through + * - `synthesize` returns the final assistant text + * - the underlying CLI process is torn down by `close()` */ -import { describe, it } from 'vitest'; - const liveSdk = process.env.CHAMBER_LIVE_SDK === '1'; -describe('CopilotLLMClient — live SDK', () => { - it.skipIf(!liveSdk)( - 'returns a string containing "pong" for the canonical smoke prompt', +describe.skipIf(!liveSdk)('CopilotLLMClient — live SDK', () => { + const mindId = 'chamber-llm-integration-mind'; + let mindPath: string; + let logDir: string; + let client: CopilotClient; + + beforeAll(async () => { + const modulesDir = resolveNodeModulesDir(); + const cliPath = getPlatformCopilotBinaryPath(modulesDir); + + mindPath = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-llm-int-')); + // SDK --log-dir requires the directory to pre-exist; we don't read logDir again. + logDir = path.join(os.homedir(), '.chamber', 'logs'); + fs.mkdirSync(logDir, { recursive: true }); + + client = new CopilotClient({ + cliPath, + cwd: mindPath, + logLevel: 'all', + cliArgs: [ + '--log-dir', logDir, + '--allow-all-tools', + '--allow-all-paths', + '--allow-all-urls', + ], + }); + await client.start(); + }, 60_000); + + afterAll(async () => { + if (client) { + await client.stop().catch(() => undefined); + } + if (mindPath) { + for (let attempt = 0; attempt < 5; attempt += 1) { + try { + fs.rmSync(mindPath, { recursive: true, force: true }); + break; + } catch { + await new Promise((resolve) => setTimeout(resolve, 250)); + } + } + } + }, 30_000); + + it( + 'returns assistant text containing "pong" for the canonical smoke prompt', async () => { - // Intentionally not implemented in Phase 8. The composition root - // (Phase 13) supplies the real `createOneShotSession` factory; this - // stub will be filled in then. Keeping the file in place so the - // contract is visible at review time. - throw new Error( - 'TODO(phase-13): wire CopilotLLMClient to the real Copilot SDK and ' + - 'assert synthesize({prompt: "Reply with the word \'pong\' and nothing else.", timeoutMs: 30_000}) ' + - 'returns a string containing "pong".', - ); + const llm = createCopilotLLMClient({ + mindId, + mindPath, + deps: { + createOneShotSession: ({ signal }) => + buildOneShotSession({ + client, + workingDirectory: mindPath, + signal, + }), + }, + }); + + const response = await llm.synthesize({ + prompt: "Reply with the word 'pong' and nothing else.", + timeoutMs: 60_000, + }); + + expect(response.toLowerCase()).toContain('pong'); }, + 120_000, ); }); diff --git a/packages/services/src/mindMemory/index.ts b/packages/services/src/mindMemory/index.ts index 8e428711..6266b611 100644 --- a/packages/services/src/mindMemory/index.ts +++ b/packages/services/src/mindMemory/index.ts @@ -32,6 +32,7 @@ export * from './dream-gates'; export * from './consolidation-scheduler'; export * from './LLMClient'; export * from './CopilotLLMClient'; +export * from './oneShotSession'; export * from './DreamDaemon'; export * from './InternalScheduler'; export * from './MindMemoryService'; diff --git a/packages/services/src/mindMemory/oneShotSession.test.ts b/packages/services/src/mindMemory/oneShotSession.test.ts new file mode 100644 index 00000000..e1a3da51 --- /dev/null +++ b/packages/services/src/mindMemory/oneShotSession.test.ts @@ -0,0 +1,186 @@ +import { describe, it, expect, vi } from 'vitest'; +import type { + CopilotClient, + CopilotSession, + SessionConfig, +} from '@github/copilot-sdk'; + +import { buildOneShotSession } from './oneShotSession'; + +interface FakeSession { + sendAndWait: ReturnType; + abort: ReturnType; + disconnect: ReturnType; +} + +interface FakeWorld { + client: CopilotClient; + capturedConfig: SessionConfig | undefined; + session: FakeSession; +} + +function makeWorld(overrides: Partial = {}): FakeWorld { + const session: FakeSession = { + sendAndWait: vi.fn().mockResolvedValue({ data: { content: 'pong' } }), + abort: vi.fn().mockResolvedValue(undefined), + disconnect: vi.fn().mockResolvedValue(undefined), + ...overrides, + }; + let capturedConfig: SessionConfig | undefined; + const client = { + createSession: vi.fn(async (cfg: SessionConfig) => { + capturedConfig = cfg; + return session as unknown as CopilotSession; + }), + } as unknown as CopilotClient; + return { + client, + get capturedConfig() { + return capturedConfig; + }, + session, + }; +} + +describe('buildOneShotSession', () => { + it('creates a session with the locked-down memory-consolidation contract', async () => { + const world = makeWorld(); + const controller = new AbortController(); + + await buildOneShotSession({ + client: world.client, + workingDirectory: '/tmp/mind-x', + signal: controller.signal, + }); + + const cfg = world.capturedConfig; + expect(cfg?.workingDirectory).toBe('/tmp/mind-x'); + expect(cfg?.enableConfigDiscovery).toBe(false); + expect(cfg?.tools).toEqual([]); + expect(cfg?.systemMessage).toEqual({ mode: 'replace', content: '' }); + const result = cfg?.onPermissionRequest?.({} as never, {} as never); + expect(result).toEqual({ + kind: 'reject', + feedback: 'Tool permissions are disabled for memory-consolidation sessions.', + }); + }); + + it('returns the assistant content from sendAndWait', async () => { + const world = makeWorld(); + const oneShot = await buildOneShotSession({ + client: world.client, + workingDirectory: '/tmp/mind-x', + signal: new AbortController().signal, + }); + + const text = await oneShot.send('hello'); + + expect(text).toBe('pong'); + expect(world.session.sendAndWait).toHaveBeenCalledWith({ prompt: 'hello' }); + }); + + it('returns empty string when the SDK reports no assistant event', async () => { + const world = makeWorld({ + sendAndWait: vi.fn().mockResolvedValue(undefined), + }); + const oneShot = await buildOneShotSession({ + client: world.client, + workingDirectory: '/tmp/mind-x', + signal: new AbortController().signal, + }); + + expect(await oneShot.send('hello')).toBe(''); + }); + + it('aborts the live session when the caller signal fires', async () => { + const world = makeWorld(); + const controller = new AbortController(); + await buildOneShotSession({ + client: world.client, + workingDirectory: '/tmp/mind-x', + signal: controller.signal, + }); + + expect(world.session.abort).not.toHaveBeenCalled(); + + controller.abort(); + + expect(world.session.abort).toHaveBeenCalledTimes(1); + }); + + it('aborts immediately when the signal is already aborted', async () => { + const world = makeWorld(); + const controller = new AbortController(); + controller.abort(); + + await buildOneShotSession({ + client: world.client, + workingDirectory: '/tmp/mind-x', + signal: controller.signal, + }); + + expect(world.session.abort).toHaveBeenCalledTimes(1); + }); + + it('swallows abort errors so the abort handler cannot crash the daemon', async () => { + const world = makeWorld({ + abort: vi.fn().mockRejectedValue(new Error('already aborted')), + }); + const controller = new AbortController(); + await buildOneShotSession({ + client: world.client, + workingDirectory: '/tmp/mind-x', + signal: controller.signal, + }); + + expect(() => controller.abort()).not.toThrow(); + await Promise.resolve(); + }); + + it('close disconnects the session and removes the abort listener', async () => { + const world = makeWorld(); + const controller = new AbortController(); + const oneShot = await buildOneShotSession({ + client: world.client, + workingDirectory: '/tmp/mind-x', + signal: controller.signal, + }); + + await oneShot.close(); + expect(world.session.disconnect).toHaveBeenCalledTimes(1); + + // Aborting after close must not call session.abort again. + controller.abort(); + expect(world.session.abort).not.toHaveBeenCalled(); + }); + + it('reports disconnect errors via onDisconnectError instead of throwing', async () => { + const world = makeWorld({ + disconnect: vi.fn().mockRejectedValue(new Error('boom')), + }); + const reported: unknown[] = []; + const oneShot = await buildOneShotSession({ + client: world.client, + workingDirectory: '/tmp/mind-x', + signal: new AbortController().signal, + onDisconnectError: (err) => reported.push(err), + }); + + await expect(oneShot.close()).resolves.toBeUndefined(); + expect(reported).toHaveLength(1); + expect((reported[0] as Error).message).toBe('boom'); + }); + + it('swallows disconnect errors silently when no reporter is supplied', async () => { + const world = makeWorld({ + disconnect: vi.fn().mockRejectedValue(new Error('boom')), + }); + const oneShot = await buildOneShotSession({ + client: world.client, + workingDirectory: '/tmp/mind-x', + signal: new AbortController().signal, + }); + + await expect(oneShot.close()).resolves.toBeUndefined(); + }); +}); diff --git a/packages/services/src/mindMemory/oneShotSession.ts b/packages/services/src/mindMemory/oneShotSession.ts new file mode 100644 index 00000000..979bbf8b --- /dev/null +++ b/packages/services/src/mindMemory/oneShotSession.ts @@ -0,0 +1,84 @@ +/** + * One-shot Copilot SDK session adapter for the Dream Daemon. + * + * Encapsulates the *contract* a memory-consolidation session must satisfy + * so callers cannot accidentally weaken it: + * + * - tools = [] (empty surface) + * - enableConfigDiscovery = false (no project config leakage) + * - systemMessage replaced (the mind's own SOUL.md is not loaded) + * - PermissionHandler refuses every request (defense-in-depth — an empty + * tool surface should never produce a permission request, but if one + * ever leaks through it is denied loudly) + * - the caller's AbortSignal aborts the in-flight CLI call + * - close() removes the abort listener and disconnects the session + * + * The SDK-typed plumbing lives here so both the desktop adapter + * (apps/desktop/.../buildMindMemoryService.ts) and the live-SDK + * integration test bind to the *same* session contract. + */ +import type { + CopilotClient, + PermissionHandler, + SessionConfig, +} from '@github/copilot-sdk'; + +import type { OneShotSession } from './CopilotLLMClient'; + +const refusingPermissionHandler: PermissionHandler = () => ({ + kind: 'reject', + feedback: 'Tool permissions are disabled for memory-consolidation sessions.', +}); + +export interface BuildOneShotSessionArgs { + readonly client: CopilotClient; + readonly workingDirectory: string; + readonly signal: AbortSignal; + /** + * Invoked when `close()` swallows a disconnect error. Optional so + * tests don't have to plumb a logger; production wiring passes a + * structured-log callback. + */ + readonly onDisconnectError?: (err: unknown) => void; +} + +export async function buildOneShotSession( + args: BuildOneShotSessionArgs, +): Promise { + const { client, workingDirectory, signal, onDisconnectError } = args; + + const sessionConfig: SessionConfig = { + workingDirectory, + enableConfigDiscovery: false, + tools: [], + systemMessage: { mode: 'replace', content: '' }, + onPermissionRequest: refusingPermissionHandler, + }; + + const session = await client.createSession(sessionConfig); + + const onAbort = (): void => { + session.abort().catch(() => { /* best-effort: abort can race with natural completion */ }); + }; + + if (signal.aborted) { + onAbort(); + } else { + signal.addEventListener('abort', onAbort, { once: true }); + } + + return { + async send(prompt: string): Promise { + const event = await session.sendAndWait({ prompt }); + return event?.data.content ?? ''; + }, + async close(): Promise { + signal.removeEventListener('abort', onAbort); + try { + await session.disconnect(); + } catch (err) { + onDisconnectError?.(err); + } + }, + }; +} From 9e76aaeff798c4900a15d3afd381315717a15f81 Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Wed, 13 May 2026 12:37:20 -0400 Subject: [PATCH 17/23] FIX: stop genesis-time errors that bricked fresh-mind console hygiene (v0.59.7) Two production-blocking defects surfaced during manual testing of feature/dream-daemon-memory-consolidation: every fresh mind logged "[MindScaffold] Capability bootstrap failed (non-fatal): Error: Upgrade skill not found in genesis repo", and "[WorkingMemoryComposer] log.md is unstructured" repeated on every system-prompt rebuild. Root causes: 1. Epic #67 moved the upgrade skill from ianphil/genesis to ianphil/genesis-frontier in commit 17580f5, but MindScaffold.GENESIS_SOURCE was never updated. 2. MindScaffold.createStructure() wrote a zero-byte log.md placeholder which violated the chamber-structured-log/v1 sentinel contract, so WorkingMemoryComposer.readLog warned and skipped on every prompt rebuild until the first DailyLogWriter rotation fired. Fixes: - MindScaffold now points at ianphil/genesis-frontier@main; the upgrade-skill not-found error names the repo and branch. - createStructure() pre-seeds log.md with the sentinel + double newline (matching DailyLogWriter.seedFreshLog byte-for-byte) BEFORE the WORKING_MEMORY_FILES placeholder loop runs. - Genesis prompt no longer instructs the LLM to write to log.md (the file is reserved for DailyLogWriter frames). SOUL.md "Continuity" rewritten and the Mind Index drops the log.md bullet. - WorkingMemoryComposer.readLog downgrades the unstructured-log message from warn to info for the migration window. Validation: - New tests/integration/mindScaffold.integration.test.ts (7 tests) drives the full MindScaffold.create() against mkdtempSync with fake registry + SDK clients. Locks the on-disk contract for new minds AND the cross-cutting migration story for existing pre-fix minds (composer info -> DailyLogWriter rotates -> composer silent). - 49/49 targeted unit tests across the 4 affected files. - Full suite: 2035/2035 pass; lint clean. - Live Electron smoke (chamber-ui-tester) confirmed both error patterns are gone end-to-end. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 6 + package-lock.json | 4 +- package.json | 2 +- .../src/chat/WorkingMemoryComposer.test.ts | 45 ++- .../src/chat/WorkingMemoryComposer.ts | 5 +- .../services/src/genesis/MindScaffold.test.ts | 202 ++++++++++ packages/services/src/genesis/MindScaffold.ts | 22 +- .../src/genesis/genesisPrompt.test.ts | 26 +- .../services/src/genesis/genesisPrompt.ts | 21 +- .../mindScaffold.integration.test.ts | 362 ++++++++++++++++++ 10 files changed, 669 insertions(+), 26 deletions(-) create mode 100644 tests/integration/mindScaffold.integration.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 25e05ddb..d9634897 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## v0.59.7 (2026-05-13) + +### Fixes + +- **Stop the two genesis-time errors that bricked fresh-mind console hygiene** — Two production-blocking defects surfaced during manual testing of `feature/dream-daemon-memory-consolidation`: every fresh mind logged `[MindScaffold] Capability bootstrap failed (non-fatal): Error: Upgrade skill not found in genesis repo`, and `[WorkingMemoryComposer] log.md is unstructured` repeated on every system-prompt rebuild. Root causes: (1) Epic #67 moved the upgrade skill from `ianphil/genesis` to `ianphil/genesis-frontier` in commit `17580f5` but `MindScaffold.GENESIS_SOURCE` was never updated; (2) `MindScaffold.createStructure()` wrote a zero-byte `log.md` placeholder which violated the chamber-structured-log/v1 sentinel contract, so `WorkingMemoryComposer.readLog` warned and skipped on every prompt rebuild until the first `DailyLogWriter` rotation fired. `MindScaffold` now points at `ianphil/genesis-frontier@main`, the upgrade-skill error message names the repo and branch (`Upgrade skill not found in ianphil/genesis-frontier@main`) so operators can immediately see which coordinate was searched, and `createStructure()` pre-seeds `log.md` with `\n\n` (matching the byte-level format `DailyLogWriter.seedFreshLog` emits) before the `WORKING_MEMORY_FILES` placeholder loop runs — the loop's `existsSync` guard then skips the seeded file. The genesis prompt is also rewritten to drop log.md as a write target (the file is reserved for `DailyLogWriter` frames) and the SOUL.md "Continuity" section now reads "Your turn-by-turn history is preserved automatically; you do not write to it." `WorkingMemoryComposer.readLog` downgrades the unstructured-log message from `warn` to `info` so SRE dashboards don't alert on pre-existing minds whose `log.md` rotates lazily on the first turn. Validated end-to-end with a new tmpdir integration test (`tests/integration/mindScaffold.integration.test.ts`, 7 tests) that drives the full `MindScaffold.create()` against `mkdtempSync` with a fake `GitHubRegistryClient` and a fake `CopilotClientFactory`, asserting the sentinel-prefixed `log.md`, `registry.json.source === 'ianphil/genesis-frontier'`, the on-disk upgrade.js, no `log.legacy.md` after a `DailyLogWriter.write()`, and zero `WorkingMemoryComposer` warnings for a fresh mind. The 7th test locks the cross-cutting migration story for users upgrading into this release: an existing pre-fix mind (unstructured `log.md` on disk) → composer fires `info` (not `warn`) on the next system-prompt rebuild → `DailyLogWriter` rotates the legacy content to `log.legacy.md` and seeds a fresh sentinel-prefixed log on the first chat turn → composer is silent on subsequent rebuilds. Live Electron smoke (chamber-ui-tester driving the genesis wizard against an isolated `CHAMBER_E2E_GENESIS_BASE_PATH` tmpdir) confirmed both error patterns are gone and the on-disk contract holds end-to-end. + ## v0.59.6 (2026-05-11) ### Refactoring diff --git a/package-lock.json b/package-lock.json index 8642c38f..66ac24ed 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "chamber", - "version": "0.59.6", + "version": "0.59.7", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "chamber", - "version": "0.59.6", + "version": "0.59.7", "license": "MIT", "workspaces": [ "apps/*", diff --git a/package.json b/package.json index 379114e7..1264f962 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "chamber", "productName": "Chamber", - "version": "0.59.6", + "version": "0.59.7", "description": "Genesis Mind Interface — desktop chat UI for Genesis agents", "main": ".vite/build/main.js", "private": true, diff --git a/packages/services/src/chat/WorkingMemoryComposer.test.ts b/packages/services/src/chat/WorkingMemoryComposer.test.ts index c5a5666d..9e2fb917 100644 --- a/packages/services/src/chat/WorkingMemoryComposer.test.ts +++ b/packages/services/src/chat/WorkingMemoryComposer.test.ts @@ -105,16 +105,53 @@ describe('WorkingMemoryComposer.compose', () => { expect(out).toMatch(/truncated/); }); - it('omits log entirely when log.md is unstructured (no sentinel) and warns', () => { + it('logs at info-level (not warn) when log.md is unstructured, and contributes nothing', () => { + // Migration window: pre-existing minds may still have an unstructured + // log.md. DailyLogWriter will rotate it on first turn. Until then, the + // composer must skip the section without elevating to a [warn] (which + // misleads SREs into thinking something failed). Uncle Bob (plan review) + // rejected the Set dedupe — this is the lightweight alternative. + // TODO: remove the info-level fallback after the migration window closes. fs.writeFileSync(path.join(workingMemoryDir, 'log.md'), 'just freeform notes\nnot structured\n', 'utf-8'); fs.writeFileSync(path.join(workingMemoryDir, 'memory.md'), 'mem', 'utf-8'); const warn = vi.fn(); - const composer = createWorkingMemoryComposer({ logger: { warn, info: () => {} } }); + const info = vi.fn(); + const composer = createWorkingMemoryComposer({ logger: { warn, info } }); + const out = composer.compose(mindRoot, DEFAULTS); + expect(out).toBe('mem'); + expect(out).not.toContain('freeform'); + expect(warn).not.toHaveBeenCalled(); + expect(info).toHaveBeenCalledTimes(1); + expect(info.mock.calls[0][0]).toMatch(/unstructured/i); + }); + + it('emits neither warn nor info when log.md is sentinel-only with zero turns (the new-mind default)', () => { + // After Fix 2, MindScaffold.createStructure seeds log.md with the + // chamber-structured-log/v1 sentinel and no turn frames. The composer must + // recognise this as a structured-but-empty log, not as unstructured noise. + fs.writeFileSync( + path.join(workingMemoryDir, 'log.md'), + STRUCTURED_LOG_SENTINEL + '\n', + 'utf-8', + ); + const warn = vi.fn(); + const info = vi.fn(); + const composer = createWorkingMemoryComposer({ logger: { warn, info } }); + const out = composer.compose(mindRoot, DEFAULTS); + expect(out).toBe(''); + expect(warn).not.toHaveBeenCalled(); + expect(info).not.toHaveBeenCalled(); + }); + + it('omits log entirely when log.md is unstructured (no sentinel) — historical assertion superseded by warn/info split above', () => { + // Kept as a regression check that the section is omitted, regardless of + // log level. The level itself is locked by the test above. + fs.writeFileSync(path.join(workingMemoryDir, 'log.md'), 'just freeform notes\nnot structured\n', 'utf-8'); + fs.writeFileSync(path.join(workingMemoryDir, 'memory.md'), 'mem', 'utf-8'); + const composer = createWorkingMemoryComposer(); const out = composer.compose(mindRoot, DEFAULTS); expect(out).toBe('mem'); expect(out).not.toContain('freeform'); - expect(warn).toHaveBeenCalledTimes(1); - expect(warn.mock.calls[0][0]).toMatch(/unstructured/i); }); it('contributes nothing when log.md is missing', () => { diff --git a/packages/services/src/chat/WorkingMemoryComposer.ts b/packages/services/src/chat/WorkingMemoryComposer.ts index 6dc9b4fd..17f7ccf3 100644 --- a/packages/services/src/chat/WorkingMemoryComposer.ts +++ b/packages/services/src/chat/WorkingMemoryComposer.ts @@ -126,7 +126,10 @@ function readLog( const parsed = parseLog(raw); if (!parsed.sentinel) { - log.warn( + // Migration-window log level: pre-existing minds may still hold an + // unstructured log.md until DailyLogWriter rotates it on the first turn. + // Use info (not warn) so SRE dashboards don't flag this benign state. + log.info( `WorkingMemoryComposer: log.md is unstructured (no chamber-structured-log/v1 sentinel); skipping log section`, ); return ''; diff --git a/packages/services/src/genesis/MindScaffold.test.ts b/packages/services/src/genesis/MindScaffold.test.ts index 6929da75..387608c9 100644 --- a/packages/services/src/genesis/MindScaffold.test.ts +++ b/packages/services/src/genesis/MindScaffold.test.ts @@ -6,6 +6,7 @@ import * as path from 'path'; import * as os from 'os'; import type { CopilotClientFactory } from '../sdk/CopilotClientFactory'; import type { GitHubRegistryClient } from './GitHubRegistryClient'; +import { STRUCTURED_LOG_SENTINEL } from '../mindMemory/StructuredLogFormat'; describe('MindScaffold.slugify', () => { it('lowercases and replaces spaces with hyphens', () => { @@ -163,3 +164,204 @@ describe('MindScaffold constructor', () => { expect(scaffold).toBeDefined(); }); }); + +// The upgrade skill lives in ianphil/genesis-frontier@main as of 2026-04-24 +// (Epic #67). Calling against the legacy ianphil/genesis repo silently throws +// "Upgrade skill not found in genesis repo" and leaves new minds without the +// bootloader. The tests below lock the source coordinate so a future rename or +// typo is caught at the unit level. +describe('MindScaffold.bootstrapCapabilities — registry source', () => { + function makeFakeRegistryClient() { + const calls: { fetchTree: Array<[string, string, string]>; fetchJsonContent: Array<[string, string, string, string]> } = { + fetchTree: [], + fetchJsonContent: [], + }; + const tree = [ + { path: '.github/skills/upgrade/upgrade.js', type: 'blob', sha: 'sha-upgrade-js' }, + { path: '.github/skills/upgrade/skill.json', type: 'blob', sha: 'sha-upgrade-json' }, + ]; + const client = { + fetchTree: vi.fn(async (owner: string, repo: string, branch: string) => { + calls.fetchTree.push([owner, repo, branch]); + return tree; + }), + fetchBlob: vi.fn(async () => Buffer.from('// stub upgrade content', 'utf8')), + fetchJsonContent: vi.fn(async (owner: string, repo: string, file: string, ref: string) => { + calls.fetchJsonContent.push([owner, repo, file, ref]); + return { skills: { upgrade: { version: '1.0.0', description: 'stub' } } }; + }), + } as unknown as GitHubRegistryClient; + return { client, calls }; + } + + it('pullUpgradeSkill fetches the tree from ianphil/genesis-frontier@main', async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-mindscaffold-source-')); + try { + const mindPath = path.join(tmpDir, 'mind'); + fs.mkdirSync(path.join(mindPath, '.github'), { recursive: true }); + fs.writeFileSync( + path.join(mindPath, '.github', 'registry.json'), + JSON.stringify({ version: '0.0.0', source: 'placeholder', channel: 'main', extensions: {}, skills: {}, prompts: {}, packages: [] }, null, 2), + ); + + const { client, calls } = makeFakeRegistryClient(); + const scaffold = new MindScaffold(client, {} as unknown as CopilotClientFactory); + + const internal = scaffold as unknown as { pullUpgradeSkill(mp: string): Promise }; + await internal.pullUpgradeSkill(mindPath); + + expect(calls.fetchTree).toHaveLength(1); + expect(calls.fetchTree[0]).toEqual(['ianphil', 'genesis-frontier', 'main']); + expect(calls.fetchJsonContent[0]?.slice(0, 3)).toEqual(['ianphil', 'genesis-frontier', '.github/registry.json']); + expect(calls.fetchJsonContent[0]?.[3]).toBe('main'); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it('seedRegistry writes source: ianphil/genesis-frontier', async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-mindscaffold-source-')); + try { + const mindPath = path.join(tmpDir, 'mind'); + fs.mkdirSync(path.join(mindPath, '.github'), { recursive: true }); + + const { client } = makeFakeRegistryClient(); + const scaffold = new MindScaffold(client, {} as unknown as CopilotClientFactory); + + const internal = scaffold as unknown as { bootstrapCapabilities(mp: string): Promise }; + // bootstrapCapabilities will fail at the execSync step (upgrade.js exec), but + // by then seedRegistry has already written the registry.json. We only care + // about that file's contents here. + await internal.bootstrapCapabilities(mindPath).catch(() => { /* expected */ }); + + const reg = JSON.parse(fs.readFileSync(path.join(mindPath, '.github', 'registry.json'), 'utf8')); + expect(reg.source).toBe('ianphil/genesis-frontier'); + expect(reg.channel).toBe('main'); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it('pullUpgradeSkill error message names the searched owner/repo/branch when the skill is missing', async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-mindscaffold-source-')); + try { + const mindPath = path.join(tmpDir, 'mind'); + fs.mkdirSync(path.join(mindPath, '.github'), { recursive: true }); + fs.writeFileSync( + path.join(mindPath, '.github', 'registry.json'), + JSON.stringify({ version: '0.0.0', source: 'placeholder', channel: 'main', extensions: {}, skills: {}, prompts: {}, packages: [] }, null, 2), + ); + + const emptyTreeClient = { + fetchTree: vi.fn(async () => [ + { path: '.github/skills/commit/commit.js', type: 'blob', sha: 'sha-commit' }, + ]), + fetchBlob: vi.fn(async () => Buffer.from('')), + fetchJsonContent: vi.fn(async () => ({})), + } as unknown as GitHubRegistryClient; + + const scaffold = new MindScaffold(emptyTreeClient, {} as unknown as CopilotClientFactory); + const internal = scaffold as unknown as { pullUpgradeSkill(mp: string): Promise }; + + await expect(internal.pullUpgradeSkill(mindPath)).rejects.toThrow(/ianphil\/genesis-frontier@main/); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); +}); + +// Fix 2: a new mind's .working-memory/log.md must be sentinel-prefixed from the +// moment createStructure() runs. Otherwise WorkingMemoryComposer emits the +// "log.md is unstructured" warning on every system-prompt rebuild for the +// lifetime of the mind, and DailyLogWriter has to rotate the legacy line on +// first turn. Seeding from MindScaffold owns the on-disk shape of a mind. +describe('MindScaffold.createStructure — log.md sentinel seed', () => { + it('seeds log.md with the chamber-structured-log/v1 sentinel as its first non-blank line', () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-mindscaffold-sentinel-')); + try { + const mindPath = path.join(tmpDir, 'mind'); + const scaffold = new MindScaffold( + {} as unknown as GitHubRegistryClient, + {} as unknown as CopilotClientFactory, + ); + + const internal = scaffold as unknown as { createStructure(mp: string): void }; + internal.createStructure(mindPath); + + const logPath = path.join(mindPath, '.working-memory', 'log.md'); + expect(fs.existsSync(logPath)).toBe(true); + const content = fs.readFileSync(logPath, 'utf-8'); + const firstNonBlank = content.split('\n').find((l) => l.trim() !== ''); + expect(firstNonBlank).toBe(STRUCTURED_LOG_SENTINEL); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it('leaves the seeded sentinel intact even though WORKING_MEMORY_FILES still iterates log.md', () => { + // Guard against accidental regression: if the WORKING_MEMORY_FILES loop + // ran AFTER the seed without the existsSync guard, log.md would be blanked. + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-mindscaffold-sentinel-')); + try { + const mindPath = path.join(tmpDir, 'mind'); + const scaffold = new MindScaffold( + {} as unknown as GitHubRegistryClient, + {} as unknown as CopilotClientFactory, + ); + + const internal = scaffold as unknown as { createStructure(mp: string): void }; + internal.createStructure(mindPath); + + const logPath = path.join(mindPath, '.working-memory', 'log.md'); + const content = fs.readFileSync(logPath, 'utf-8'); + expect(content.length).toBeGreaterThan(0); + expect(content).toContain(STRUCTURED_LOG_SENTINEL); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); +}); + +// Fix 2: the genesis prompt must not instruct the LLM to write to log.md any +// more — that file is reserved for structured CompletedTurn frames produced by +// DailyLogWriter. The "I am born" observation in genesis is deliberately +// dropped (recorded by the "Genesis" git commit and SOUL.md instead). +describe('MindScaffold.generateSoul — genesis prompt no longer references log.md', () => { + it('does not pass log.md as a write target to buildGenesisPrompt', async () => { + const session = { + send: vi.fn<(_: { prompt: string }) => Promise>(async () => undefined), + destroy: vi.fn(async () => undefined), + on: vi.fn((event: string, callback: () => void) => { + if (event === 'session.idle') setTimeout(callback, 0); + return vi.fn(); + }), + rpc: { permissions: { setApproveAll: vi.fn(async () => ({ success: true })) } }, + }; + const client = { createSession: vi.fn(async () => session) }; + const clientFactory = { + createClient: vi.fn(async () => client), + destroyClient: vi.fn(async () => undefined), + } as unknown as CopilotClientFactory; + const scaffold = new MindScaffold( + {} as unknown as GitHubRegistryClient, + clientFactory, + ); + + const generateSoul = scaffold as unknown as { + generateSoul(mindPath: string, config: Parameters[0], slug: string): Promise; + }; + await generateSoul.generateSoul('/tmp/minds/seed', { + name: 'Seed', + role: 'reviewer', + voice: 'plain', + voiceDescription: 'plain', + basePath: '/tmp/minds', + }, 'seed'); + + const sentPrompt = session.send.mock.calls[0]?.[0]?.prompt ?? ''; + expect(sentPrompt).not.toContain('log.md'); + expect(sentPrompt).toContain('SOUL.md'); + expect(sentPrompt).toContain('memory.md'); + expect(sentPrompt).toContain('rules.md'); + }); +}); diff --git a/packages/services/src/genesis/MindScaffold.ts b/packages/services/src/genesis/MindScaffold.ts index a56b4fcb..66cd08ca 100644 --- a/packages/services/src/genesis/MindScaffold.ts +++ b/packages/services/src/genesis/MindScaffold.ts @@ -10,13 +10,14 @@ import { approveForSessionCompat } from '../sdk/approveForSessionCompat'; import { getCurrentDateTimeContext, injectCurrentDateTimeContext } from '../chat/currentDateTimeContext'; import { buildGenesisPrompt } from './genesisPrompt'; import { GitHubRegistryClient } from './GitHubRegistryClient'; +import { STRUCTURED_LOG_SENTINEL } from '../mindMemory/StructuredLogFormat'; const log = Logger.create('MindScaffold'); const IDEA_FOLDERS = ['inbox', 'domains', 'expertise', 'initiatives', 'Archive']; const WORKING_MEMORY_FILES = ['memory.md', 'rules.md', 'log.md']; -const GENESIS_SOURCE = 'ianphil/genesis'; +const GENESIS_SOURCE = 'ianphil/genesis-frontier'; const GENESIS_CHANNEL = 'main'; export interface GenesisConfig { @@ -125,6 +126,17 @@ export class MindScaffold { const wmDir = path.join(mindPath, '.working-memory'); fs.mkdirSync(wmDir, { recursive: true }); + // Seed log.md with the chamber-structured-log/v1 sentinel BEFORE the + // WORKING_MEMORY_FILES placeholder loop runs. The loop's `existsSync` guard + // then skips it, preserving the sentinel. This keeps `validate()` happy + // (sentinel-only content trims to non-empty) while honoring the + // structured-log contract from creation. Trailing `\n\n` matches the + // byte-level format that DailyLogWriter.seedFreshLog emits for a fresh + // mind so on-disk content stays consistent regardless of which path + // seeded the sentinel. See WorkingMemoryComposer.readLog and + // DailyLogWriter.doWrite for the consumer side of this contract. + fs.writeFileSync(path.join(wmDir, 'log.md'), STRUCTURED_LOG_SENTINEL + '\n\n'); + // Create placeholder files so the agent has targets for (const file of WORKING_MEMORY_FILES) { const filePath = path.join(wmDir, file); @@ -141,14 +153,16 @@ export class MindScaffold { const agentPath = path.join(mindPath, '.github', 'agents', `${slug}.agent.md`); const memoryPath = path.join(mindPath, '.working-memory', 'memory.md'); const rulesPath = path.join(mindPath, '.working-memory', 'rules.md'); - const logPath = path.join(mindPath, '.working-memory', 'log.md'); const indexPath = path.join(mindPath, 'mind-index.md'); + // log.md is intentionally NOT a prompt target. It is pre-seeded with the + // chamber-structured-log/v1 sentinel by createStructure() and reserved for + // structured CompletedTurn frames written by DailyLogWriter. const prompt = buildGenesisPrompt({ name: config.name, role: config.role, voiceDescription: config.voiceDescription, - paths: { soul: soulPath, agent: agentPath, memory: memoryPath, rules: rulesPath, log: logPath, index: indexPath }, + paths: { soul: soulPath, agent: agentPath, memory: memoryPath, rules: rulesPath, index: indexPath }, }); const sessionConfig: Record = { @@ -276,7 +290,7 @@ export class MindScaffold { } if (upgradeFiles.length === 0) { - throw new Error('Upgrade skill not found in genesis repo'); + throw new Error(`Upgrade skill not found in ${GENESIS_SOURCE}@${GENESIS_CHANNEL}`); } // Download and write each file diff --git a/packages/services/src/genesis/genesisPrompt.test.ts b/packages/services/src/genesis/genesisPrompt.test.ts index 18dc3f48..6183afab 100644 --- a/packages/services/src/genesis/genesisPrompt.test.ts +++ b/packages/services/src/genesis/genesisPrompt.test.ts @@ -11,7 +11,6 @@ describe('buildGenesisPrompt', () => { agent: '/test/.github/agents/test.agent.md', memory: '/test/.working-memory/memory.md', rules: '/test/.working-memory/rules.md', - log: '/test/.working-memory/log.md', index: '/test/mind-index.md', }, }; @@ -26,13 +25,34 @@ describe('buildGenesisPrompt', () => { expect(prompt).toContain('calm and precise'); }); - it('includes all six file paths', () => { + it('includes the five user-visible identity file paths', () => { const prompt = buildGenesisPrompt(input); expect(prompt).toContain('SOUL.md'); expect(prompt).toContain('memory.md'); expect(prompt).toContain('rules.md'); - expect(prompt).toContain('log.md'); expect(prompt).toContain('mind-index.md'); expect(prompt).toContain('.agent.md'); }); + + // log.md is reserved for structured CompletedTurn frames written by + // DailyLogWriter. The genesis prompt must not instruct the LLM to write + // there or it poisons the chamber-structured-log/v1 contract before the + // first turn ever runs. + it('does not instruct the LLM to write to log.md', () => { + expect(buildGenesisPrompt(input)).not.toContain('log.md'); + }); + + it('still accepts an input with paths.log set for backward compatibility', () => { + const withLog = { + ...input, + paths: { ...input.paths, log: '/test/.working-memory/log.md' }, + }; + expect(() => buildGenesisPrompt(withLog)).not.toThrow(); + expect(buildGenesisPrompt(withLog)).not.toContain('log.md'); + }); + + it('accepts an input that omits paths.log entirely', () => { + expect(() => buildGenesisPrompt(input)).not.toThrow(); + expect(buildGenesisPrompt(input)).not.toContain('log.md'); + }); }); diff --git a/packages/services/src/genesis/genesisPrompt.ts b/packages/services/src/genesis/genesisPrompt.ts index 79d7b072..3aef19f4 100644 --- a/packages/services/src/genesis/genesisPrompt.ts +++ b/packages/services/src/genesis/genesisPrompt.ts @@ -7,7 +7,12 @@ export interface GenesisPromptInput { agent: string; memory: string; rules: string; - log: string; + /** + * @deprecated log.md is reserved for structured CompletedTurn frames + * written by DailyLogWriter. The genesis prompt no longer references this + * path. Kept optional for backward-compatible call-site shape only. + */ + log?: string; index: string; }; } @@ -45,10 +50,11 @@ Write to: ${paths.soul} [One paragraph on how you communicate. Your tone, your style, your energy.] ## Continuity -You maintain memory across sessions through three files: +You maintain memory across sessions through these working-memory files: - \`.working-memory/memory.md\` — curated long-term reference - \`.working-memory/rules.md\` — operational rules learned from experience -- \`.working-memory/log.md\` — raw chronological observations + +Your turn-by-turn history is preserved automatically; you do not write to it. --- Write to: ${paths.agent} @@ -76,12 +82,6 @@ Write to: ${paths.rules} [One starter rule that fits your character voice.] --- -Write to: ${paths.log} ---- -# Log -- ${new Date().toISOString()}: Genesis. I am ${name}. My purpose is ${role}. Let's begin. ---- - Write to: ${paths.index} --- # Mind Index @@ -93,8 +93,7 @@ Write to: ${paths.index} ## Working Memory - \`.working-memory/memory.md\` — curated long-term reference - \`.working-memory/rules.md\` — operational rules -- \`.working-memory/log.md\` — chronological observations --- -Write all six files now.`; +Write all five files now.`; } diff --git a/tests/integration/mindScaffold.integration.test.ts b/tests/integration/mindScaffold.integration.test.ts new file mode 100644 index 00000000..4ca34b6f --- /dev/null +++ b/tests/integration/mindScaffold.integration.test.ts @@ -0,0 +1,362 @@ +/** + * MindScaffold bootstrap integration smoke. + * + * Goal: lock in the dream-daemon contract that a freshly-scaffolded mind is + * born with a structured `log.md` (chamber-structured-log/v1 sentinel) and a + * registry pointing at `ianphil/genesis-frontier`. Exercises the full + * `MindScaffold.create()` path against a tmpdir with the network and SDK + * mocked, then verifies the on-disk state plus downstream consumer behaviour + * (DailyLogWriter, WorkingMemoryComposer). + * + * Why integration? The MindScaffold unit tests mock `fs` and can't see the + * actual byte-level contract on disk. This test runs real `fs` against a + * tmpdir so a future regression (e.g. someone "optimizing" the seed write + * away, or the WORKING_MEMORY_FILES loop re-blanking log.md) gets caught + * here. + * + * Filesystem hygiene: every assertion group runs against a fresh tmpdir, + * cleaned up in `afterEach`. ZERO writes to `~/agents` or any user-visible + * location. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { + MindScaffold, + STRUCTURED_LOG_SENTINEL, + createDailyLogWriter, + createWorkingMemoryComposer, + type CompletedTurn, + type CopilotClientFactory, + type GitHubRegistryClient, +} from '@chamber/services'; + +let tmpRoot: string; + +beforeEach(() => { + tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-mindscaffold-int-')); +}); + +afterEach(() => { + fs.rmSync(tmpRoot, { recursive: true, force: true }); +}); + +interface RegistryCallLog { + fetchTree: Array<[string, string, string]>; + fetchBlob: Array<[string, string, string]>; + fetchJsonContent: Array<[string, string, string, string]>; +} + +function makeFakeRegistryClient(callLog: RegistryCallLog): GitHubRegistryClient { + // A minimal valid upgrade.js so initGit's commit doesn't fail and the + // bootstrapCapabilities exec path can either run or be skipped cleanly. + // Returning {skills:{}} from fetchJsonContent means skillNames is empty and + // bootstrapCapabilities exits before invoking execSync on upgrade.js. + const tree = [ + { path: '.github/skills/upgrade/upgrade.js', type: 'blob', sha: 'sha-upgrade-js' }, + { path: '.github/skills/upgrade/skill.json', type: 'blob', sha: 'sha-upgrade-json' }, + ]; + return { + fetchTree: vi.fn(async (owner: string, repo: string, branch: string) => { + callLog.fetchTree.push([owner, repo, branch]); + return tree; + }), + fetchBlob: vi.fn(async (owner: string, repo: string, sha: string) => { + callLog.fetchBlob.push([owner, repo, sha]); + if (sha === 'sha-upgrade-js') { + return Buffer.from('// stub upgrade bootloader\nprocess.exit(0);\n', 'utf8'); + } + return Buffer.from('{"name":"upgrade","version":"1.0.0"}\n', 'utf8'); + }), + fetchJsonContent: vi.fn(async (owner: string, repo: string, filePath: string, ref: string) => { + callLog.fetchJsonContent.push([owner, repo, filePath, ref]); + // No remote skills besides the bootloader itself → bootstrapCapabilities + // early-returns without execSync. + return { skills: {} }; + }), + } as unknown as GitHubRegistryClient; +} + +interface SoulPaths { + soul: string; + agent: string; + memory: string; + rules: string; + index: string; +} + +// A fake session that, on `send()`, synchronously writes the minimal set of +// files genesis-prompt would otherwise drive the LLM to write. This keeps +// `validate()` happy so the test exercises the full create() path including +// initGit and bootstrapCapabilities. +function makeFakeClientFactory(seedFiles: (paths: SoulPaths) => void): CopilotClientFactory { + return { + createClient: vi.fn(async (mindPath: string) => { + const slug = path.basename(mindPath); + const paths: SoulPaths = { + soul: path.join(mindPath, 'SOUL.md'), + agent: path.join(mindPath, '.github', 'agents', `${slug}.agent.md`), + memory: path.join(mindPath, '.working-memory', 'memory.md'), + rules: path.join(mindPath, '.working-memory', 'rules.md'), + index: path.join(mindPath, 'mind-index.md'), + }; + const session = { + send: vi.fn(async () => { + seedFiles(paths); + }), + destroy: vi.fn(async () => undefined), + on: vi.fn((event: string, callback: () => void) => { + if (event === 'session.idle') setTimeout(callback, 0); + return vi.fn(); + }), + rpc: { permissions: { setApproveAll: vi.fn(async () => ({ success: true })) } }, + }; + return { createSession: vi.fn(async () => session) }; + }), + destroyClient: vi.fn(async () => undefined), + } as unknown as CopilotClientFactory; +} + +function defaultSeedFiles(paths: SoulPaths): void { + fs.writeFileSync(paths.soul, '# Test Soul\n\nA mind for tests.\n'); + fs.writeFileSync(paths.agent, '---\nname: test\ndescription: test\n---\n'); + fs.writeFileSync(paths.memory, '# Memory\n'); + fs.writeFileSync(paths.rules, '# Rules\n'); + fs.writeFileSync(paths.index, '# Mind Index\n'); +} + +describe('MindScaffold.create — bootstrap integration', () => { + it('produces a sentinel-prefixed log.md on disk', async () => { + const callLog: RegistryCallLog = { fetchTree: [], fetchBlob: [], fetchJsonContent: [] }; + const scaffold = new MindScaffold( + makeFakeRegistryClient(callLog), + makeFakeClientFactory(defaultSeedFiles), + ); + + const mindPath = await scaffold.create({ + name: 'Sentinel Mind', + role: 'integration tester', + voice: 'plain', + voiceDescription: 'plain', + basePath: tmpRoot, + }); + + const logPath = path.join(mindPath, '.working-memory', 'log.md'); + expect(fs.existsSync(logPath)).toBe(true); + const content = fs.readFileSync(logPath, 'utf-8'); + const firstNonBlank = content.split('\n').find((l) => l.trim() !== ''); + expect(firstNonBlank).toBe(STRUCTURED_LOG_SENTINEL); + }); + + it('records source: ianphil/genesis-frontier in registry.json', async () => { + const callLog: RegistryCallLog = { fetchTree: [], fetchBlob: [], fetchJsonContent: [] }; + const scaffold = new MindScaffold( + makeFakeRegistryClient(callLog), + makeFakeClientFactory(defaultSeedFiles), + ); + + const mindPath = await scaffold.create({ + name: 'Frontier Mind', + role: 'integration tester', + voice: 'plain', + voiceDescription: 'plain', + basePath: tmpRoot, + }); + + const registry = JSON.parse( + fs.readFileSync(path.join(mindPath, '.github', 'registry.json'), 'utf-8'), + ); + expect(registry.source).toBe('ianphil/genesis-frontier'); + expect(registry.channel).toBe('main'); + }); + + it('pulls the upgrade skill from ianphil/genesis-frontier and writes upgrade.js on disk', async () => { + const callLog: RegistryCallLog = { fetchTree: [], fetchBlob: [], fetchJsonContent: [] }; + const scaffold = new MindScaffold( + makeFakeRegistryClient(callLog), + makeFakeClientFactory(defaultSeedFiles), + ); + + const mindPath = await scaffold.create({ + name: 'Upgrade Mind', + role: 'integration tester', + voice: 'plain', + voiceDescription: 'plain', + basePath: tmpRoot, + }); + + // Network coordinates + expect(callLog.fetchTree).toEqual([['ianphil', 'genesis-frontier', 'main']]); + // On-disk artifact + const upgradeJs = path.join(mindPath, '.github', 'skills', 'upgrade', 'upgrade.js'); + expect(fs.existsSync(upgradeJs)).toBe(true); + expect(fs.readFileSync(upgradeJs, 'utf-8')).toContain('stub upgrade bootloader'); + }); + + it('lays down the full IDEA + .github + working-memory structure', async () => { + const callLog: RegistryCallLog = { fetchTree: [], fetchBlob: [], fetchJsonContent: [] }; + const scaffold = new MindScaffold( + makeFakeRegistryClient(callLog), + makeFakeClientFactory(defaultSeedFiles), + ); + + const mindPath = await scaffold.create({ + name: 'Structure Mind', + role: 'integration tester', + voice: 'plain', + voiceDescription: 'plain', + basePath: tmpRoot, + }); + + for (const folder of ['inbox', 'domains', 'expertise', 'initiatives', 'Archive']) { + expect(fs.existsSync(path.join(mindPath, folder))).toBe(true); + } + expect(fs.existsSync(path.join(mindPath, '.github', 'agents'))).toBe(true); + expect(fs.existsSync(path.join(mindPath, '.github', 'skills'))).toBe(true); + expect(fs.existsSync(path.join(mindPath, '.working-memory', 'memory.md'))).toBe(true); + expect(fs.existsSync(path.join(mindPath, '.working-memory', 'rules.md'))).toBe(true); + expect(fs.existsSync(path.join(mindPath, '.working-memory', 'log.md'))).toBe(true); + }); + + it('lets DailyLogWriter append a structured frame without producing log.legacy.md', async () => { + const callLog: RegistryCallLog = { fetchTree: [], fetchBlob: [], fetchJsonContent: [] }; + const scaffold = new MindScaffold( + makeFakeRegistryClient(callLog), + makeFakeClientFactory(defaultSeedFiles), + ); + + const mindPath = await scaffold.create({ + name: 'Writer Mind', + role: 'integration tester', + voice: 'plain', + voiceDescription: 'plain', + basePath: tmpRoot, + }); + + const writer = createDailyLogWriter({ mindId: 'writer-mind', mindPath }); + const turn: CompletedTurn = { + turnId: '00000000-0000-4000-8000-000000000001', + sessionId: 'sess-int-1', + model: 'claude-opus-4.7', + status: 'completed', + startedAt: '2026-05-13T14:00:00Z', + endedAt: '2026-05-13T14:00:05Z', + prompt: 'hello', + finalAssistantMessage: 'hi back', + }; + await writer.write(turn); + + const logContent = fs.readFileSync( + path.join(mindPath, '.working-memory', 'log.md'), + 'utf-8', + ); + expect(logContent).toContain(STRUCTURED_LOG_SENTINEL); + expect(logContent).toContain('turn:00000000-0000-4000-8000-000000000001'); + expect(logContent).toContain('### user'); + expect(logContent).toContain('### assistant'); + // The sentinel pre-seed means no rotation happens — log.legacy.md must not exist. + expect(fs.existsSync(path.join(mindPath, '.working-memory', 'log.legacy.md'))).toBe(false); + }); + + it('WorkingMemoryComposer treats the fresh mind as structured (no info or warn fired)', async () => { + const callLog: RegistryCallLog = { fetchTree: [], fetchBlob: [], fetchJsonContent: [] }; + const scaffold = new MindScaffold( + makeFakeRegistryClient(callLog), + makeFakeClientFactory(defaultSeedFiles), + ); + + const mindPath = await scaffold.create({ + name: 'Composer Mind', + role: 'integration tester', + voice: 'plain', + voiceDescription: 'plain', + basePath: tmpRoot, + }); + + const warn = vi.fn(); + const info = vi.fn(); + const composer = createWorkingMemoryComposer({ logger: { warn, info } }); + composer.compose(mindPath, { lastKTurns: 10, perTurnMaxBytes: 2048, memoryMaxBytes: 8192 }); + + expect(warn).not.toHaveBeenCalled(); + // info may be called for benign reasons (e.g. memory.md truncation) but + // must NEVER be called with the "unstructured" message for a fresh mind. + for (const call of info.mock.calls) { + expect(call[0]).not.toMatch(/unstructured/i); + } + }); + + // Cross-cutting migration story for minds that already exist on disk in the + // pre-fix shape: an unstructured `log.md` (no sentinel). Locks in three + // promises end-to-end: + // 1. Composer reads the unstructured file at `info` level — never `warn` + // (the migration-window contract). + // 2. DailyLogWriter on the first chat turn rotates the legacy content out + // to `log.legacy.md` and seeds a fresh sentinel-prefixed log.md. + // 3. After rotation, the composer is silent (no further unstructured + // messages on subsequent prompt rebuilds). + it('migrates an existing pre-fix mind: composer info → first turn rotates → composer silent', async () => { + const callLog: RegistryCallLog = { fetchTree: [], fetchBlob: [], fetchJsonContent: [] }; + const scaffold = new MindScaffold( + makeFakeRegistryClient(callLog), + makeFakeClientFactory(defaultSeedFiles), + ); + + const mindPath = await scaffold.create({ + name: 'Legacy Mind', + role: 'integration tester', + voice: 'plain', + voiceDescription: 'plain', + basePath: tmpRoot, + }); + + // Simulate a mind that was scaffolded BEFORE the fix shipped: unstructured + // log.md with no sentinel. This is the on-disk shape for every existing + // user upgrading into this release. + const logPath = path.join(mindPath, '.working-memory', 'log.md'); + const legacyPath = path.join(mindPath, '.working-memory', 'log.legacy.md'); + const legacyContent = '# pre-fix freeform notes\n\nthis is what existing minds have.\n'; + fs.writeFileSync(logPath, legacyContent); + expect(fs.existsSync(legacyPath)).toBe(false); + + const warn = vi.fn(); + const info = vi.fn(); + const composer = createWorkingMemoryComposer({ logger: { warn, info } }); + + // Step 1: opening the mind triggers a system-prompt rebuild → info, never warn. + composer.compose(mindPath, { lastKTurns: 10, perTurnMaxBytes: 2048, memoryMaxBytes: 8192 }); + expect(warn).not.toHaveBeenCalled(); + const unstructuredInfoCalls = info.mock.calls.filter((c) => /unstructured/i.test(c[0])); + expect(unstructuredInfoCalls.length).toBe(1); + + // Step 2: first chat turn → DailyLogWriter rotates and seeds the sentinel. + const writer = createDailyLogWriter({ mindId: 'legacy-mind', mindPath }); + const turn: CompletedTurn = { + turnId: '00000000-0000-4000-8000-000000000099', + sessionId: 'sess-legacy-1', + model: 'claude-opus-4.7', + status: 'completed', + startedAt: '2026-05-13T15:00:00Z', + endedAt: '2026-05-13T15:00:05Z', + prompt: 'first turn after upgrade', + finalAssistantMessage: 'welcome back', + }; + await writer.write(turn); + + expect(fs.existsSync(legacyPath)).toBe(true); + expect(fs.readFileSync(legacyPath, 'utf-8')).toBe(legacyContent); + const rotatedLog = fs.readFileSync(logPath, 'utf-8'); + expect(rotatedLog).toContain(STRUCTURED_LOG_SENTINEL); + expect(rotatedLog).toContain('turn:00000000-0000-4000-8000-000000000099'); + + // Step 3: subsequent prompt rebuild is silent — migration is complete. + info.mockClear(); + warn.mockClear(); + composer.compose(mindPath, { lastKTurns: 10, perTurnMaxBytes: 2048, memoryMaxBytes: 8192 }); + expect(warn).not.toHaveBeenCalled(); + for (const call of info.mock.calls) { + expect(call[0]).not.toMatch(/unstructured/i); + } + }); +}); From 3ce101a9f6265df4380278474e63bd59ff0b7025 Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Thu, 14 May 2026 11:18:30 -0400 Subject: [PATCH 18/23] FIX: pin better-sqlite3 ABI sentinel by NODE_MODULE_VERSION to catch Node-major upgrades --- scripts/ensure-native-abi.cjs | 17 +- scripts/lib/ensure-native-abi.cjs | 41 +++-- tests/regression/ensure-native-abi.test.ts | 178 +++++++++++++++++---- 3 files changed, 191 insertions(+), 45 deletions(-) diff --git a/scripts/ensure-native-abi.cjs b/scripts/ensure-native-abi.cjs index ee64050a..383046ec 100644 --- a/scripts/ensure-native-abi.cjs +++ b/scripts/ensure-native-abi.cjs @@ -22,22 +22,29 @@ if (!target || !TARGETS.includes(target)) { process.exit(2); } +// process.versions.modules is the V8 ABI version (NODE_MODULE_VERSION). It's +// what a native addon must be compiled against to load in the current runtime. +// Pinning the sentinel to {target, moduleVersion} catches Node-major upgrades +// that keep target=='node' but flip the ABI. +const moduleVersion = process.versions.modules; const current = readSentinel(); -const action = decideAction({ target, current }); +const action = decideAction({ target, current, moduleVersion }); if (action === 'noop') { - console.log(`[ensure-native-abi] better-sqlite3 already built for ${target} — skipping rebuild`); + console.log( + `[ensure-native-abi] better-sqlite3 already built for ${target}:${moduleVersion} — skipping rebuild`, + ); process.exit(0); } console.log( - `[ensure-native-abi] better-sqlite3 ABI target=${target}, current=${current ?? 'unknown'} — rebuilding...`, + `[ensure-native-abi] better-sqlite3 ABI target=${target}:${moduleVersion}, current=${current ?? 'unknown'} — rebuilding...`, ); try { rebuild(target); - writeSentinel(target); - console.log(`[ensure-native-abi] better-sqlite3 now built for ${target}`); + writeSentinel({ target, moduleVersion }); + console.log(`[ensure-native-abi] better-sqlite3 now built for ${target}:${moduleVersion}`); } catch (err) { console.error(`[ensure-native-abi] rebuild failed: ${err && err.message ? err.message : err}`); process.exit(1); diff --git a/scripts/lib/ensure-native-abi.cjs b/scripts/lib/ensure-native-abi.cjs index 788369c0..6c5b4507 100644 --- a/scripts/lib/ensure-native-abi.cjs +++ b/scripts/lib/ensure-native-abi.cjs @@ -11,11 +11,14 @@ // `npm run smoke:desktop` hits "Cannot read properties of undefined" // crashes from the wrong-ABI .node file. // -// This module records the last-built ABI target in a sentinel file under -// `node_modules/better-sqlite3/build/Release/.abi-target` and exposes a -// pure `decideAction` so callers can short-circuit when the binary already -// matches. The CLI wrapper (`scripts/ensure-native-abi.cjs`) drives the -// actual rebuild via `npm rebuild` or `electron-rebuild`. +// What the sentinel records: +// `${target}:${moduleVersion}` — e.g. `node:137`, `electron:125`. Both +// axes must match the current runtime for the guard to short-circuit. +// Recording only the framework (`node` vs `electron`) is not enough: +// Node 23 and Node 24 share target=='node' but differ in MODULE_VERSION +// (145 vs 137), and a developer who upgrades Node would otherwise sail +// past the guard with a stale binary. (Caveat C-1 from the v0.60.0 +// ship review — this is the fix.) const fs = require('node:fs'); const path = require('node:path'); @@ -39,23 +42,39 @@ function readSentinel(sentinelPath = DEFAULT_SENTINEL_PATH) { } } -function decideAction({ target, current }) { +function assertTarget(target) { if (!TARGETS.includes(target)) { throw new Error( `ensure-native-abi: unknown target "${target}". Expected one of: ${TARGETS.join(', ')}`, ); } - return current === target ? 'noop' : 'rebuild'; } -function writeSentinel(target, sentinelPath = DEFAULT_SENTINEL_PATH) { - if (!TARGETS.includes(target)) { +function assertModuleVersion(moduleVersion) { + // process.versions.modules is always a numeric string (e.g. "137"). A bad + // value here would corrupt the sentinel — fail loudly rather than write garbage. + if (typeof moduleVersion !== 'string' || !/^[0-9]+$/.test(moduleVersion)) { throw new Error( - `ensure-native-abi: refusing to write sentinel with unknown target "${target}"`, + `ensure-native-abi: invalid moduleVersion ${JSON.stringify(moduleVersion)} — expected a numeric string from process.versions.modules`, ); } +} + +function sentinelValue(target, moduleVersion) { + return `${target}:${moduleVersion}`; +} + +function decideAction({ target, current, moduleVersion }) { + assertTarget(target); + assertModuleVersion(moduleVersion); + return current === sentinelValue(target, moduleVersion) ? 'noop' : 'rebuild'; +} + +function writeSentinel({ target, moduleVersion }, sentinelPath = DEFAULT_SENTINEL_PATH) { + assertTarget(target); + assertModuleVersion(moduleVersion); fs.mkdirSync(path.dirname(sentinelPath), { recursive: true }); - fs.writeFileSync(sentinelPath, `${target}\n`); + fs.writeFileSync(sentinelPath, `${sentinelValue(target, moduleVersion)}\n`); } function rebuildCommand(target) { diff --git a/tests/regression/ensure-native-abi.test.ts b/tests/regression/ensure-native-abi.test.ts index cf8d6ff8..fcb07529 100644 --- a/tests/regression/ensure-native-abi.test.ts +++ b/tests/regression/ensure-native-abi.test.ts @@ -8,8 +8,12 @@ const lib = require('../../scripts/lib/ensure-native-abi.cjs') as { TARGETS: readonly string[]; DEFAULT_SENTINEL_PATH: string; readSentinel: (p?: string) => string | null; - decideAction: (input: { target: string; current: string | null }) => 'noop' | 'rebuild'; - writeSentinel: (target: string, p?: string) => void; + decideAction: (input: { + target: string; + current: string | null; + moduleVersion: string; + }) => 'noop' | 'rebuild'; + writeSentinel: (input: { target: string; moduleVersion: string }, p?: string) => void; rebuildCommand: (target: string) => string; rebuild: (target: string, runner?: (cmd: string) => void) => void; }; @@ -34,24 +38,87 @@ describe('ensure-native-abi guard', () => { }); describe('decideAction', () => { - it('returns "noop" when current matches target', () => { - expect(lib.decideAction({ target: 'node', current: 'node' })).toBe('noop'); - expect(lib.decideAction({ target: 'electron', current: 'electron' })).toBe('noop'); + it('returns "noop" when current matches target AND module ABI version', () => { + expect( + lib.decideAction({ target: 'node', current: 'node:137', moduleVersion: '137' }), + ).toBe('noop'); + expect( + lib.decideAction({ target: 'electron', current: 'electron:125', moduleVersion: '125' }), + ).toBe('noop'); }); - it('returns "rebuild" when current differs from target', () => { - expect(lib.decideAction({ target: 'node', current: 'electron' })).toBe('rebuild'); - expect(lib.decideAction({ target: 'electron', current: 'node' })).toBe('rebuild'); + it('returns "rebuild" when target matches but module ABI version differs', () => { + // The exact bug C-1 caught: Node 22→24 keeps target='node' but flips MODULE_VERSION. + expect( + lib.decideAction({ target: 'node', current: 'node:145', moduleVersion: '137' }), + ).toBe('rebuild'); + expect( + lib.decideAction({ target: 'electron', current: 'electron:125', moduleVersion: '127' }), + ).toBe('rebuild'); + }); + + it('returns "rebuild" on a legacy single-token sentinel (pre-ABI format)', () => { + // A sentinel written by an older version of this script has no `:NNN` suffix. + // We must rebuild rather than trust it — we cannot prove the ABI matches. + expect( + lib.decideAction({ target: 'node', current: 'node', moduleVersion: '137' }), + ).toBe('rebuild'); + expect( + lib.decideAction({ target: 'electron', current: 'electron', moduleVersion: '125' }), + ).toBe('rebuild'); + }); + + it('returns "rebuild" when the framework target differs (even with matching ABI)', () => { + expect( + lib.decideAction({ target: 'node', current: 'electron:137', moduleVersion: '137' }), + ).toBe('rebuild'); + expect( + lib.decideAction({ target: 'electron', current: 'node:125', moduleVersion: '125' }), + ).toBe('rebuild'); }); it('returns "rebuild" when no sentinel exists yet (current is null)', () => { - expect(lib.decideAction({ target: 'node', current: null })).toBe('rebuild'); - expect(lib.decideAction({ target: 'electron', current: null })).toBe('rebuild'); + expect( + lib.decideAction({ target: 'node', current: null, moduleVersion: '137' }), + ).toBe('rebuild'); + expect( + lib.decideAction({ target: 'electron', current: null, moduleVersion: '125' }), + ).toBe('rebuild'); }); it('throws on unknown target rather than silently skipping', () => { - expect(() => lib.decideAction({ target: 'wasm', current: 'node' })).toThrow(/unknown target/); - expect(() => lib.decideAction({ target: '', current: 'node' })).toThrow(/unknown target/); + expect(() => + lib.decideAction({ target: 'wasm', current: 'node:137', moduleVersion: '137' }), + ).toThrow(/unknown target/); + expect(() => + lib.decideAction({ target: '', current: 'node:137', moduleVersion: '137' }), + ).toThrow(/unknown target/); + }); + + it('throws on missing or malformed moduleVersion', () => { + // A bad moduleVersion would corrupt the sentinel — fail loud rather than write garbage. + expect(() => + lib.decideAction({ + target: 'node', + current: 'node:137', + moduleVersion: '', + }), + ).toThrow(/moduleVersion/); + expect(() => + lib.decideAction({ + target: 'node', + current: 'node:137', + moduleVersion: 'undefined', + }), + ).toThrow(/moduleVersion/); + expect(() => + lib.decideAction({ + target: 'node', + current: 'node:137', + // @ts-expect-error: deliberately wrong type to test runtime guard + moduleVersion: 137, + }), + ).toThrow(/moduleVersion/); }); }); @@ -62,8 +129,15 @@ describe('ensure-native-abi guard', () => { it('returns the trimmed sentinel contents when present', () => { fs.mkdirSync(path.dirname(sentinelPath), { recursive: true }); - fs.writeFileSync(sentinelPath, 'electron\n'); - expect(lib.readSentinel(sentinelPath)).toBe('electron'); + fs.writeFileSync(sentinelPath, 'electron:125\n'); + expect(lib.readSentinel(sentinelPath)).toBe('electron:125'); + }); + + it('returns a legacy single-token sentinel verbatim (decideAction handles rejection)', () => { + // readSentinel stays a dumb file reader; semantic interpretation belongs to decideAction. + fs.mkdirSync(path.dirname(sentinelPath), { recursive: true }); + fs.writeFileSync(sentinelPath, 'node\n'); + expect(lib.readSentinel(sentinelPath)).toBe('node'); }); it('returns null when the sentinel directory is unreadable / missing parents', () => { @@ -73,26 +147,38 @@ describe('ensure-native-abi guard', () => { }); describe('writeSentinel', () => { - it('creates parent directories and writes the target with a trailing newline', () => { - lib.writeSentinel('node', sentinelPath); + it('writes ${target}:${moduleVersion} with a trailing newline, creating parents', () => { + lib.writeSentinel({ target: 'node', moduleVersion: '137' }, sentinelPath); const raw = fs.readFileSync(sentinelPath, 'utf8'); - expect(raw).toBe('node\n'); + expect(raw).toBe('node:137\n'); }); - it('round-trips with readSentinel', () => { - lib.writeSentinel('electron', sentinelPath); - expect(lib.readSentinel(sentinelPath)).toBe('electron'); + it('round-trips with readSentinel for both targets', () => { + lib.writeSentinel({ target: 'electron', moduleVersion: '125' }, sentinelPath); + expect(lib.readSentinel(sentinelPath)).toBe('electron:125'); }); it('refuses to write an unknown target', () => { - expect(() => lib.writeSentinel('wasm', sentinelPath)).toThrow(/unknown target/); + expect(() => + lib.writeSentinel({ target: 'wasm', moduleVersion: '137' }, sentinelPath), + ).toThrow(/unknown target/); + expect(fs.existsSync(sentinelPath)).toBe(false); + }); + + it('refuses to write a missing or malformed moduleVersion', () => { + expect(() => + lib.writeSentinel({ target: 'node', moduleVersion: '' }, sentinelPath), + ).toThrow(/moduleVersion/); + expect(() => + lib.writeSentinel({ target: 'node', moduleVersion: 'NaN' }, sentinelPath), + ).toThrow(/moduleVersion/); expect(fs.existsSync(sentinelPath)).toBe(false); }); it('overwrites an existing sentinel rather than appending', () => { - lib.writeSentinel('node', sentinelPath); - lib.writeSentinel('electron', sentinelPath); - expect(lib.readSentinel(sentinelPath)).toBe('electron'); + lib.writeSentinel({ target: 'node', moduleVersion: '137' }, sentinelPath); + lib.writeSentinel({ target: 'electron', moduleVersion: '125' }, sentinelPath); + expect(lib.readSentinel(sentinelPath)).toBe('electron:125'); }); }); @@ -137,25 +223,59 @@ describe('ensure-native-abi guard', () => { }); describe('integration: full guard sequence', () => { - it('rebuilds on first run, then noops on the second run with the same target', () => { + it('rebuilds on first run, then noops on the second run with the same target+ABI', () => { // First run: no sentinel yet → rebuild path - const first = lib.decideAction({ target: 'node', current: lib.readSentinel(sentinelPath) }); + const first = lib.decideAction({ + target: 'node', + current: lib.readSentinel(sentinelPath), + moduleVersion: '137', + }); expect(first).toBe('rebuild'); const calls: string[] = []; lib.rebuild('node', (cmd) => calls.push(cmd)); - lib.writeSentinel('node', sentinelPath); + lib.writeSentinel({ target: 'node', moduleVersion: '137' }, sentinelPath); // Second run: sentinel matches → noop, no rebuild invoked - const second = lib.decideAction({ target: 'node', current: lib.readSentinel(sentinelPath) }); + const second = lib.decideAction({ + target: 'node', + current: lib.readSentinel(sentinelPath), + moduleVersion: '137', + }); expect(second).toBe('noop'); expect(calls).toHaveLength(1); }); it('rebuilds when switching target from node → electron', () => { - lib.writeSentinel('node', sentinelPath); + lib.writeSentinel({ target: 'node', moduleVersion: '137' }, sentinelPath); const action = lib.decideAction({ target: 'electron', current: lib.readSentinel(sentinelPath), + moduleVersion: '125', + }); + expect(action).toBe('rebuild'); + }); + + it('rebuilds when Node ABI shifts under the same target (the C-1 bug)', () => { + // Simulates: binary was built on Node 23 (MODULE_VERSION 145), developer upgraded + // to Node 24 (MODULE_VERSION 137). Old guard silently said noop. New guard rebuilds. + lib.writeSentinel({ target: 'node', moduleVersion: '145' }, sentinelPath); + const action = lib.decideAction({ + target: 'node', + current: lib.readSentinel(sentinelPath), + moduleVersion: '137', + }); + expect(action).toBe('rebuild'); + }); + + it('treats a legacy single-token sentinel as a rebuild signal', () => { + // Older sentinel left by pre-fix versions of the guard. Force a one-time rebuild + // so the new format is written and future runs can short-circuit. + fs.mkdirSync(path.dirname(sentinelPath), { recursive: true }); + fs.writeFileSync(sentinelPath, 'node\n'); + const action = lib.decideAction({ + target: 'node', + current: lib.readSentinel(sentinelPath), + moduleVersion: '137', }); expect(action).toBe('rebuild'); }); From 5fc9cdf9195a7ff8b1ddaa7dc06b774332f1cc5a Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Thu, 14 May 2026 11:18:56 -0400 Subject: [PATCH 19/23] ADD: dream daemon opt-in UX with bidirectional log migration (v0.60.0) --- AGENTS.md | 6 + CHANGELOG.md | 6 + apps/desktop/src/main/ipc/mind.ts | 6 + apps/desktop/src/preload.ts | 1 + apps/web/src/browserApi.ts | 1 + .../components/genesis/GenesisFlow.test.tsx | 40 +- .../components/genesis/GenesisFlow.tsx | 3 +- .../components/genesis/RoleScreen.test.tsx | 98 +++ .../components/genesis/RoleScreen.tsx | 54 +- .../profile/AgentProfileModal.test.tsx | 51 ++ .../components/profile/AgentProfileModal.tsx | 58 ++ apps/web/src/test/helpers.ts | 3 + package-lock.json | 4 +- package.json | 2 +- packages/services/src/chat/ChatService.ts | 7 +- .../services/src/chat/IdentityLoader.test.ts | 2 + packages/services/src/chat/IdentityLoader.ts | 2 + .../src/chat/WorkingMemoryComposer.test.ts | 127 ++++ .../src/chat/WorkingMemoryComposer.ts | 48 +- .../services/src/genesis/MindScaffold.test.ts | 140 +++- packages/services/src/genesis/MindScaffold.ts | 55 +- .../services/src/mind/MindManager.test.ts | 145 +++- packages/services/src/mind/MindManager.ts | 68 +- .../src/mind/chamberMindConfig.test.ts | 96 +++ .../services/src/mind/chamberMindConfig.ts | 73 ++ .../src/mindMemory/DailyLogWriter.test.ts | 215 +++++- .../services/src/mindMemory/DailyLogWriter.ts | 83 ++- .../src/mindMemory/MindMemoryService.test.ts | 200 ++++++ .../src/mindMemory/MindMemoryService.ts | 51 +- .../mindMemory/StructuredLogFormat.test.ts | 12 + .../src/mindMemory/StructuredLogFormat.ts | 6 +- .../services/src/mindMemory/rollback.test.ts | 234 +++++++ packages/services/src/mindMemory/rollback.ts | 199 ++++++ .../mindProfile/MindProfileService.test.ts | 24 + .../src/mindProfile/MindProfileService.ts | 3 + packages/shared/src/electron-types.ts | 3 +- packages/shared/src/ipc-channels.ts | 1 + packages/shared/src/types.ts | 7 + tests/e2e/electron/dream-daemon-bidir.spec.ts | 647 ++++++++++++++++++ .../mindScaffold.integration.test.ts | 44 +- 40 files changed, 2769 insertions(+), 56 deletions(-) create mode 100644 apps/web/src/renderer/components/genesis/RoleScreen.test.tsx create mode 100644 packages/services/src/mindMemory/rollback.test.ts create mode 100644 packages/services/src/mindMemory/rollback.ts create mode 100644 tests/e2e/electron/dream-daemon-bidir.spec.ts diff --git a/AGENTS.md b/AGENTS.md index 5dd694d2..cd2b102b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,6 +21,12 @@ Chamber is a desktop application where AI agents ("minds") operate as a Chief of - **Handoff**: Agent-to-agent delegation - **Magentic**: Manager-driven task ledger +### Memory Consolidation (Dream Daemon — experimental, opt-in) +- **Per-mind opt-in**: `workingMemory.consolidation.enabled` in each mind's `.chamber.json`. Default OFF. +- **Toggle surfaces**: Genesis wizard role screen (pre-genesis) and agent profile modal (post-genesis); never silent. +- **Bidirectional**: Disabling rolls the structured `log.md` (and any `log.legacy.md`) back to unstructured turn-by-turn markdown via `rollbackToUnstructured`. Single source of truth after rollback. +- **Failure semantics**: Toggle resolves even if rollback fails; config flip is the contract. + ## Security Boundaries ### Credential Storage diff --git a/CHANGELOG.md b/CHANGELOG.md index d9634897..b656bbb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## v0.60.0 (2026-05-13) + +### Features + +- **Dream daemon ships as an opt-in toggle with bidirectional log migration** — The background memory consolidation daemon (introduced as scaffolding in 0.59.x) is now a first-class user-controlled feature. The genesis wizard exposes a single "Enable dream daemon (experimental)" switch on the role screen; the agent profile modal shows the same switch so existing minds can flip it post-genesis without touching `.chamber.json` by hand. Default is **OFF** — minds opted-in pre-genesis ship with `workingMemory.consolidation.enabled: true` written by `MindScaffold.create()`; minds opted-in post-genesis go through `MindManager.enableDreamDaemon(mindId)` which atomically patches the on-disk config (tmp+fsync+rename via the new `patchChamberMindConfig` helper that preserves passthrough fields and deep-merges only the `workingMemory.consolidation` subtree) and reloads the mind context so the new `MindMemoryService` daemon spins up against fresh providers. **Opt-out runs the migration in reverse**: `MindManager.disableDreamDaemon(mindId)` patches config → reloads (tearing down `DailyLogWriter` so the chat observer detaches and the daemon closes against a quiescent log) → calls the new `rollbackToUnstructured(mindPath)` which reads the structured `log.md`, parses every `chamber-structured-log/v1` frame, renders each as `## {ISO} — turn {turnId} ({model})\n\n**User**: …\n\n**Assistant**: …`, folds in any pre-existing `log.legacy.md` ahead of the rendered turns, atomically rewrites `log.md`, and removes `log.legacy.md` so the rolled-back mind has a single source of truth. Rollback is non-fatal (try/catch + `console.warn`): the user-visible toggle resolves successfully even if the rewrite fails — config is already flipped and the daemon is gone, so the worst case is a structured log that needs manual cleanup. Concurrent toggles for the same mindId are serialized through a `daemonToggling: Map>` in MindManager (same pattern as the existing `loading` map); rapid clicks return the same in-flight promise rather than racing the patch+reload+rollback pipeline. Three data-loss safety properties are now locked in by tests after the chamber-ui-tester E2E surfaced a real Flow 4 bug: (1) the structured-log parser at `StructuredLogFormat.ts::parseBlock` accepts an empty `model:` line (`/^model: (.*)$/` not `(.+)$`), so frames written by `ChatService` when no model was selected at turn time still round-trip cleanly through `parseLog`; (2) `ChatService.notifyTurnCompleted` coerces an empty `model` to the sentinel string `'unknown'` before serializing, so on-disk frames are always semantically complete; (3) `rollbackToUnstructured` adds a new `'no-op-malformed'` outcome — when the file has the sentinel and non-empty content but the parser produces zero turns AND `parsed.malformed > 0`, the rewrite is refused (file preserved byte-identical, warn logged) so unparseable history is never silently overwritten with an empty file. Tool surface for the renderer: `mind:setDreamDaemon` IPC channel (desktop adapter routes to `enable/disableDreamDaemon`, browser shim returns `unavailable`), `dreamDaemonEnabled: boolean` added to `AgentProfile` (populated from `loadChamberMindConfig`), `Switch` component in `AgentProfileModal` mirrors the role-screen ARIA pattern (`role="switch"` + `aria-checked` + sky-500/slate-700 colors). Validated by 11 `rollback.test.ts` scenarios (convert N frames, fold legacy + remove file, no-op variants for missing/empty/no-sentinel logs, idempotency, atomicity under synthetic rename failure, zero-frames sentinel-only + legacy → legacy preserved, zero-frames sentinel-only no-legacy → empty file, **Flow 4 regression: empty-model frame round-trips through rollback without data loss**, **all-malformed frames → preserved byte-identical**), 8 `MindManager` `enableDreamDaemon`/`disableDreamDaemon` tests (patch+reload happy paths, throw-on-missing, MindContext return, per-mindId serialization with promise identity, rollback ordering `['patch','reload','rollback']`, no-rollback-on-enable, disable-resolves-even-when-rollback-throws), and a live chamber-ui-tester Electron E2E driving the real Copilot SDK through all four flows (genesis OFF, genesis ON, post-genesis OFF→ON, post-genesis ON→OFF with rollback). Refs #288. + ## v0.59.7 (2026-05-13) ### Fixes diff --git a/apps/desktop/src/main/ipc/mind.ts b/apps/desktop/src/main/ipc/mind.ts index b1a15c74..a2a0e335 100644 --- a/apps/desktop/src/main/ipc/mind.ts +++ b/apps/desktop/src/main/ipc/mind.ts @@ -47,6 +47,12 @@ export function setupMindIPC(mindManager: MindManager, chatService: ChatService, return chatService.setMindModel(mindId, model); }); + ipcMain.handle(IPC.MIND.SET_DREAM_DAEMON, async (_event, mindId: string, enabled: boolean) => { + return enabled + ? mindManager.enableDreamDaemon(mindId) + : mindManager.disableDreamDaemon(mindId); + }); + ipcMain.handle(IPC.MIND.SELECT_DIRECTORY, async (event) => { const win = BrowserWindow.fromWebContents(event.sender); if (!win) return null; diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index cf34ce3f..564f0dbf 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -28,6 +28,7 @@ const electronAPI: ElectronAPI = { list: () => ipcRenderer.invoke(IPC.MIND.LIST), setActive: (mindId) => ipcRenderer.invoke(IPC.MIND.SET_ACTIVE, mindId), setModel: (mindId, model) => ipcRenderer.invoke(IPC.MIND.SET_MODEL, mindId, model), + setDreamDaemon: (mindId, enabled) => ipcRenderer.invoke(IPC.MIND.SET_DREAM_DAEMON, mindId, enabled), selectDirectory: () => ipcRenderer.invoke(IPC.MIND.SELECT_DIRECTORY), openWindow: (mindId) => ipcRenderer.invoke(IPC.MIND.OPEN_WINDOW, mindId), onMindChanged: (callback) => createIpcListener(ipcRenderer, IPC.MIND.CHANGED, callback), diff --git a/apps/web/src/browserApi.ts b/apps/web/src/browserApi.ts index be16c4c1..f222574c 100644 --- a/apps/web/src/browserApi.ts +++ b/apps/web/src/browserApi.ts @@ -164,6 +164,7 @@ export function installBrowserApi(): void { list: () => client.listMinds() as Promise, setActive: async () => unavailable('active mind changes'), setModel: async () => null, + setDreamDaemon: async () => unavailable('dream daemon toggling'), selectDirectory: async () => window.prompt('Enter a local agent folder path on this computer:')?.trim() || null, openWindow: async (mindId) => { window.open(`/?mindId=${encodeURIComponent(mindId)}`, '_blank', 'noopener,noreferrer'); diff --git a/apps/web/src/renderer/components/genesis/GenesisFlow.test.tsx b/apps/web/src/renderer/components/genesis/GenesisFlow.test.tsx index 5b55f973..4e3d3d52 100644 --- a/apps/web/src/renderer/components/genesis/GenesisFlow.test.tsx +++ b/apps/web/src/renderer/components/genesis/GenesisFlow.test.tsx @@ -47,8 +47,17 @@ vi.mock('./VoiceScreen', () => ({ })); vi.mock('./RoleScreen', () => ({ - RoleScreen: ({ onSelect }: { onSelect: (role: string) => void }) => ( - + RoleScreen: ({ + onSelect, + }: { + onSelect: (role: string, enableDreamDaemon: boolean) => void; + }) => ( + <> + + + ), })); @@ -213,11 +222,38 @@ describe('GenesisFlow', () => { voice: 'Test Agent', voiceDescription: 'Test voice', basePath: 'C:\\Users\\test\\agents', + enableDreamDaemon: false, }); }); expect(api.genesis.createFromTemplate).not.toHaveBeenCalled(); }); + it('forwards enableDreamDaemon=true into the genesis.create IPC payload', async () => { + // RoleScreen owns the Switch; GenesisFlow.handleRole must thread the + // captured opt-in into the IPC call so MindScaffold sees it. Without + // this the user toggles the Switch and nothing reaches the main process. + render( + + + , + ); + + fireEvent.click(screen.getByText('Begin')); + fireEvent.click(screen.getByText('Choose voice')); + fireEvent.click(await screen.findByText('Choose role with daemon')); + + await waitFor(() => { + expect(api.genesis.create).toHaveBeenCalledWith({ + name: 'Test Agent', + role: 'Engineering Partner', + voice: 'Test Agent', + voiceDescription: 'Test voice', + basePath: 'C:\\Users\\test\\agents', + enableDreamDaemon: true, + }); + }); + }); + it('adds a marketplace from the landing page and refreshes templates', async () => { render( diff --git a/apps/web/src/renderer/components/genesis/GenesisFlow.tsx b/apps/web/src/renderer/components/genesis/GenesisFlow.tsx index e68fb123..3cae2ce2 100644 --- a/apps/web/src/renderer/components/genesis/GenesisFlow.tsx +++ b/apps/web/src/renderer/components/genesis/GenesisFlow.tsx @@ -52,7 +52,7 @@ export function GenesisFlow({ onComplete }: Props) { return { success: true, message: `Added ${result.registry.label}. It will appear in New Agent templates.` }; }, [loadTemplates]); - const handleRole= useCallback(async (r: string) => { + const handleRole= useCallback(async (r: string, enableDreamDaemon: boolean) => { setRole(r); setStage('boot'); setCreationError(null); @@ -64,6 +64,7 @@ export function GenesisFlow({ onComplete }: Props) { voice: name, voiceDescription: voiceDesc, basePath: defaultPath, + enableDreamDaemon, }).catch((error: unknown) => ({ success: false, error: error instanceof Error ? error.message : String(error), diff --git a/apps/web/src/renderer/components/genesis/RoleScreen.test.tsx b/apps/web/src/renderer/components/genesis/RoleScreen.test.tsx new file mode 100644 index 00000000..c2acf294 --- /dev/null +++ b/apps/web/src/renderer/components/genesis/RoleScreen.test.tsx @@ -0,0 +1,98 @@ +/** + * @vitest-environment jsdom + * + * v0.60.0 Phase 2: the dream-daemon Switch lives at the bottom of RoleScreen + * because Role is the LAST input the user makes before `genesis.create` fires. + * Capturing the Switch state here means GenesisFlow can forward it into the + * IPC payload without an extra screen + extra reload of state. + * + * Contract: + * - Switch defaults to OFF (strict opt-in). + * - `onSelect` signature is `(role: string, enableDreamDaemon: boolean)`. + * - The Switch is purely a captured field — toggling it does NOT submit. + * Clicking a role card (or pressing "That's my purpose" in the custom + * branch) is what fires `onSelect` with the captured Switch state. + */ +import React from 'react'; +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { render, screen, fireEvent, cleanup, waitFor } from '@testing-library/react'; + +// Mock TypeWriter to fire onComplete immediately — the real implementation +// uses a 35ms-per-char setInterval that takes ~1.5s to finish, which would +// blow past Vitest's default `findBy*` 1000ms wait. The cards/Switch render +// after onComplete + a 500ms delay; bypassing the typewriter is the standard +// pattern in this codebase (see GenesisFlow.test.tsx — same approach). +vi.mock('./TypeWriter', () => ({ + TypeWriter: ({ text, onComplete }: { text: string; onComplete?: () => void }) => { + React.useEffect(() => { + onComplete?.(); + }, [onComplete]); + return {text}; + }, +})); + +import { RoleScreen } from './RoleScreen'; + +afterEach(() => { + cleanup(); +}); + +describe('RoleScreen — dream-daemon opt-in switch', () => { + it('renders the dream-daemon Switch in the OFF position by default', async () => { + render(); + const toggle = await screen.findByRole('switch', { name: /dream daemon/i }); + expect(toggle).not.toBeNull(); + expect(toggle.getAttribute('aria-checked')).toBe('false'); + }); + + it('toggling the Switch updates aria-checked to true', async () => { + render(); + const toggle = await screen.findByRole('switch', { name: /dream daemon/i }); + fireEvent.click(toggle); + await waitFor(() => { + expect(toggle.getAttribute('aria-checked')).toBe('true'); + }); + }); + + it('opt-out (default): clicking a role card calls onSelect with enableDreamDaemon=false', async () => { + const onSelect = vi.fn(); + render(); + const card = await screen.findByRole('button', { name: /Chief of Staff/i }); + fireEvent.click(card); + await waitFor(() => { + expect(onSelect).toHaveBeenCalledTimes(1); + }); + expect(onSelect).toHaveBeenCalledWith('Chief of Staff', false); + }); + + it('opt-in: toggling the Switch ON, then clicking a card, calls onSelect with enableDreamDaemon=true', async () => { + const onSelect = vi.fn(); + render(); + const toggle = await screen.findByRole('switch', { name: /dream daemon/i }); + fireEvent.click(toggle); + const card = await screen.findByRole('button', { name: /Engineering Partner/i }); + fireEvent.click(card); + await waitFor(() => { + expect(onSelect).toHaveBeenCalledTimes(1); + }); + expect(onSelect).toHaveBeenCalledWith('Engineering Partner', true); + }); + + it('custom-role branch: opt-in propagates through "That\'s my purpose"', async () => { + const onSelect = vi.fn(); + render(); + const toggle = await screen.findByRole('switch', { name: /dream daemon/i }); + fireEvent.click(toggle); + + const customCard = await screen.findByRole('button', { name: /Something else/i }); + fireEvent.click(customCard); + + const input = await screen.findByPlaceholderText(/Creative Director/i); + fireEvent.change(input, { target: { value: 'Debate Coach' } }); + const submit = await screen.findByRole('button', { name: /That's my purpose/i }); + fireEvent.click(submit); + + expect(onSelect).toHaveBeenCalledTimes(1); + expect(onSelect).toHaveBeenCalledWith('Debate Coach', true); + }); +}); diff --git a/apps/web/src/renderer/components/genesis/RoleScreen.tsx b/apps/web/src/renderer/components/genesis/RoleScreen.tsx index 4c310551..2ad27aca 100644 --- a/apps/web/src/renderer/components/genesis/RoleScreen.tsx +++ b/apps/web/src/renderer/components/genesis/RoleScreen.tsx @@ -4,7 +4,15 @@ import { cn } from '../../lib/utils'; interface Props { name: string; - onSelect: (role: string) => void; + /** + * v0.60.0 Phase 2: signature changed from `(role: string)` to + * `(role: string, enableDreamDaemon: boolean)`. The boolean is captured + * from the dream-daemon Switch at the bottom of this screen — Role is the + * last input the user makes before `genesis.create` fires, so colocating + * the Switch here means GenesisFlow can forward the choice into the IPC + * payload without an extra screen or extra state hop. + */ + onSelect: (role: string, enableDreamDaemon: boolean) => void; } const ROLES = [ @@ -19,6 +27,10 @@ export function RoleScreen({ name, onSelect }: Props) { const [selected, setSelected] = useState(null); const [customRole, setCustomRole] = useState(''); const [showCustomInput, setShowCustomInput] = useState(false); + // Strict opt-in. Defaults to OFF so a user who never touches the Switch + // ends up with a quiet mind. The dream daemon never starts, log.md stays + // empty, and `.chamber.json` is never written — see MindScaffold.createStructure. + const [enableDreamDaemon, setEnableDreamDaemon] = useState(false); const inputRef = useRef(null); useEffect(() => { @@ -36,14 +48,14 @@ export function RoleScreen({ name, onSelect }: Props) { setSelected(roleId); setTimeout(() => { const role = ROLES.find(r => r.id === roleId); - onSelect(role?.label ?? roleId); + onSelect(role?.label ?? roleId, enableDreamDaemon); }, 300); }; const handleCustomSubmit = () => { const role = customRole.trim(); if (!role) return; - onSelect(role); + onSelect(role, enableDreamDaemon); }; return ( @@ -102,6 +114,42 @@ export function RoleScreen({ name, onSelect }: Props) { )} )} + + {/* + Dream-daemon opt-in. Sits at the bottom because it's a + secondary, optional choice — the role cards are the primary + decision. ARIA `switch` role + `aria-checked` is the WCAG- + recommended shape for an on/off toggle (better than a raw + checkbox here because the binary state is the whole UI). + */} +
+
+
Enable dream daemon
+
+ Background memory consolidation. Off by default — you can change this later. +
+
+ +
)} diff --git a/apps/web/src/renderer/components/profile/AgentProfileModal.test.tsx b/apps/web/src/renderer/components/profile/AgentProfileModal.test.tsx index e909af5d..774fab71 100644 --- a/apps/web/src/renderer/components/profile/AgentProfileModal.test.tsx +++ b/apps/web/src/renderer/components/profile/AgentProfileModal.test.tsx @@ -84,6 +84,56 @@ describe('AgentProfileModal', () => { crop: expect.objectContaining({ width: 600, height: 600 }), })); }); + + describe('dream-daemon switch', () => { + it('renders the switch in the OFF position when dreamDaemonEnabled is false', async () => { + (api.mindProfile.get as ReturnType).mockResolvedValue(makeProfile({ dreamDaemonEnabled: false })); + renderProfileModal(); + + const toggle = await screen.findByRole('switch', { name: /dream daemon/i }); + expect(toggle.getAttribute('aria-checked')).toBe('false'); + }); + + it('renders the switch in the ON position when dreamDaemonEnabled is true', async () => { + (api.mindProfile.get as ReturnType).mockResolvedValue(makeProfile({ dreamDaemonEnabled: true })); + renderProfileModal(); + + const toggle = await screen.findByRole('switch', { name: /dream daemon/i }); + expect(toggle.getAttribute('aria-checked')).toBe('true'); + }); + + it('flipping the switch from OFF to ON calls mind.setDreamDaemon(mindId, true) then refreshes the profile', async () => { + const offProfile = makeProfile({ dreamDaemonEnabled: false }); + const onProfile = makeProfile({ dreamDaemonEnabled: true }); + const getMock = api.mindProfile.get as ReturnType; + getMock.mockResolvedValueOnce(offProfile); + getMock.mockResolvedValueOnce(onProfile); + (api.mind.setDreamDaemon as ReturnType).mockResolvedValue({ ...mind }); + + renderProfileModal(); + const toggle = await screen.findByRole('switch', { name: /dream daemon/i }); + fireEvent.click(toggle); + + await waitFor(() => expect(api.mind.setDreamDaemon).toHaveBeenCalledWith('mind-1', true)); + await waitFor(() => expect(getMock).toHaveBeenCalledTimes(2)); + }); + + it('flipping the switch from ON to OFF calls mind.setDreamDaemon(mindId, false)', async () => { + const onProfile = makeProfile({ dreamDaemonEnabled: true }); + const offProfile = makeProfile({ dreamDaemonEnabled: false }); + const getMock = api.mindProfile.get as ReturnType; + getMock.mockResolvedValueOnce(onProfile); + getMock.mockResolvedValueOnce(offProfile); + (api.mind.setDreamDaemon as ReturnType).mockResolvedValue({ ...mind }); + + renderProfileModal(); + const toggle = await screen.findByRole('switch', { name: /dream daemon/i }); + expect(toggle.getAttribute('aria-checked')).toBe('true'); + fireEvent.click(toggle); + + await waitFor(() => expect(api.mind.setDreamDaemon).toHaveBeenCalledWith('mind-1', false)); + }); + }); }); function renderProfileModal() { @@ -111,6 +161,7 @@ function makeProfile(overrides?: Partial): AgentProfile { }, agentFiles: [makeAgentFile('moneypenny.agent.md')], needsRestart: false, + dreamDaemonEnabled: false, ...overrides, }; } diff --git a/apps/web/src/renderer/components/profile/AgentProfileModal.tsx b/apps/web/src/renderer/components/profile/AgentProfileModal.tsx index 3182d6d4..0be95158 100644 --- a/apps/web/src/renderer/components/profile/AgentProfileModal.tsx +++ b/apps/web/src/renderer/components/profile/AgentProfileModal.tsx @@ -37,6 +37,7 @@ export function AgentProfileModal({ mind, open, onOpenChange, onProfileChanged } const [editingFile, setEditingFile] = useState(null); const [avatarSource, setAvatarSource] = useState(null); const [restarting, setRestarting] = useState(false); + const [togglingDreamDaemon, setTogglingDreamDaemon] = useState(false); useEffect(() => { if (!open || !mind) { @@ -105,6 +106,23 @@ export function AgentProfileModal({ mind, open, onOpenChange, onProfileChanged } } }; + const handleToggleDreamDaemon = async () => { + if (!profile || togglingDreamDaemon) return; + const next = !profile.dreamDaemonEnabled; + setTogglingDreamDaemon(true); + setError(null); + try { + await window.electronAPI.mind.setDreamDaemon(profile.mindId, next); + const updatedProfile = await window.electronAPI.mindProfile.get(profile.mindId); + setProfile(updatedProfile); + onProfileChanged?.(updatedProfile); + } catch (toggleError) { + setError(toggleError instanceof Error ? toggleError.message : String(toggleError)); + } finally { + setTogglingDreamDaemon(false); + } + }; + return ( <> @@ -178,6 +196,46 @@ export function AgentProfileModal({ mind, open, onOpenChange, onProfileChanged } /> ))} + + {/* + Dream-daemon opt-in toggle. Mirrors the genesis-time + switch in RoleScreen so an existing mind can be opted + in/out post-creation. Flipping it triggers a mind reload + (see MindManager.enableDreamDaemon / disableDreamDaemon) + so the new opt-in state takes effect immediately — + composer reads the gate, DailyLogWriter migrates legacy + `log.md` if needed. + */} +
+
+
Enable dream daemon
+
+ Background memory consolidation. When enabled, this agent's chat history is structured and summarized over time. +
+
+ +
) : null} diff --git a/apps/web/src/test/helpers.ts b/apps/web/src/test/helpers.ts index 75dd6602..ef0c2422 100644 --- a/apps/web/src/test/helpers.ts +++ b/apps/web/src/test/helpers.ts @@ -132,6 +132,7 @@ export function mockElectronAPI(): ElectronAPI { list: vi.fn().mockResolvedValue([]), setActive: vi.fn().mockResolvedValue(undefined), setModel: vi.fn().mockResolvedValue(null), + setDreamDaemon: vi.fn().mockResolvedValue({ mindId: 'test-1234', mindPath: 'C:\\test', identity: { name: 'Test', systemMessage: '' }, status: 'ready' }), selectDirectory: vi.fn().mockResolvedValue(null), openWindow: vi.fn().mockResolvedValue(undefined), onMindChanged: vi.fn().mockReturnValue(vi.fn()), @@ -146,6 +147,7 @@ export function mockElectronAPI(): ElectronAPI { soul: { kind: 'soul', label: 'SOUL.md', relativePath: 'SOUL.md', content: '# Test\n', exists: true, mtimeMs: 1 }, agentFiles: [{ kind: 'agent', label: 'test.agent.md', relativePath: '.github\\agents\\test.agent.md', content: '# Test agent\n', exists: true, mtimeMs: 2 }], needsRestart: false, + dreamDaemonEnabled: false, })), saveFile: vi.fn().mockResolvedValue({ success: true, needsRestart: true, profile: { mindId: 'test-1234', @@ -156,6 +158,7 @@ export function mockElectronAPI(): ElectronAPI { soul: { kind: 'soul', label: 'SOUL.md', relativePath: 'SOUL.md', content: '# Test\n', exists: true, mtimeMs: 3 }, agentFiles: [{ kind: 'agent', label: 'test.agent.md', relativePath: '.github\\agents\\test.agent.md', content: '# Test agent\n', exists: true, mtimeMs: 2 }], needsRestart: true, + dreamDaemonEnabled: false, } }), pickAvatarImage: vi.fn().mockResolvedValue({ success: false, error: 'not stubbed' }), saveAvatar: vi.fn().mockResolvedValue({ success: false, error: 'not stubbed' }), diff --git a/package-lock.json b/package-lock.json index 66ac24ed..4d5f029b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "chamber", - "version": "0.59.7", + "version": "0.60.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "chamber", - "version": "0.59.7", + "version": "0.60.0", "license": "MIT", "workspaces": [ "apps/*", diff --git a/package.json b/package.json index 1264f962..fc476c1c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "chamber", "productName": "Chamber", - "version": "0.59.7", + "version": "0.60.0", "description": "Genesis Mind Interface — desktop chat UI for Genesis agents", "main": ".vite/build/main.js", "private": true, diff --git a/packages/services/src/chat/ChatService.ts b/packages/services/src/chat/ChatService.ts index 59a36b51..63047c7e 100644 --- a/packages/services/src/chat/ChatService.ts +++ b/packages/services/src/chat/ChatService.ts @@ -113,10 +113,15 @@ export class ChatService { if (finalAssistantMessage !== null && !abortController.signal.aborted) { const endedAt = new Date().toISOString(); const refreshed = this.mindManager.getMind(mindId); + // Coerce empty model to a sentinel so the structured-log frame is + // semantically meaningful. The parser accepts empty values, but + // 'unknown' is more useful in rendered rollback markdown than a + // bare `(`. + const turnModel = model ?? refreshed?.selectedModel ?? ''; this.notifyTurnCompleted({ turnId, sessionId: refreshed?.activeSessionId ?? '', - model: model ?? refreshed?.selectedModel ?? '', + model: turnModel.length > 0 ? turnModel : 'unknown', status: 'completed', startedAt, endedAt, diff --git a/packages/services/src/chat/IdentityLoader.test.ts b/packages/services/src/chat/IdentityLoader.test.ts index 11754b87..9e485b98 100644 --- a/packages/services/src/chat/IdentityLoader.test.ts +++ b/packages/services/src/chat/IdentityLoader.test.ts @@ -225,6 +225,8 @@ describe('IdentityLoader', () => { '/tmp/agents/widget', { // Defaults from chamberMindConfig (Phase 4) when no .chamber.json exists. + // Phase 1 of v0.60.0 added `enabled` (strict opt-in for the dream daemon). + enabled: false, lastKTurns: 10, perTurnMaxBytes: 2048, memoryMaxBytes: 8192, diff --git a/packages/services/src/chat/IdentityLoader.ts b/packages/services/src/chat/IdentityLoader.ts index 4cbff9a3..a3be7f9e 100644 --- a/packages/services/src/chat/IdentityLoader.ts +++ b/packages/services/src/chat/IdentityLoader.ts @@ -90,12 +90,14 @@ export class IdentityLoader { try { const c = loadChamberMindConfig(mindPath).workingMemory.consolidation; return { + enabled: c.enabled, lastKTurns: c.lastKTurns, perTurnMaxBytes: c.perTurnMaxBytes, memoryMaxBytes: c.memoryMaxBytes, }; } catch { return { + enabled: DEFAULT_WORKING_MEMORY_CONSOLIDATION.enabled, lastKTurns: DEFAULT_WORKING_MEMORY_CONSOLIDATION.lastKTurns, perTurnMaxBytes: DEFAULT_WORKING_MEMORY_CONSOLIDATION.perTurnMaxBytes, memoryMaxBytes: DEFAULT_WORKING_MEMORY_CONSOLIDATION.memoryMaxBytes, diff --git a/packages/services/src/chat/WorkingMemoryComposer.test.ts b/packages/services/src/chat/WorkingMemoryComposer.test.ts index 9e2fb917..1bc44069 100644 --- a/packages/services/src/chat/WorkingMemoryComposer.test.ts +++ b/packages/services/src/chat/WorkingMemoryComposer.test.ts @@ -6,7 +6,12 @@ import path from 'node:path'; import { createWorkingMemoryComposer, type WorkingMemoryComposerConfig } from './WorkingMemoryComposer'; import { STRUCTURED_LOG_SENTINEL, serializeTurn, type CompletedTurn } from '../mindMemory/StructuredLogFormat'; +// Most existing tests assert opted-in behaviour (sentinel logs, truncation, +// info-on-unstructured). Default `enabled: true` keeps those tests unchanged. +// New tests below pass `enabled: false` to exercise the opt-out gate added +// in v0.60.0. See "Dream Daemon Opt-In UX" (issue tracked in plan.md). const DEFAULTS: WorkingMemoryComposerConfig = { + enabled: true, lastKTurns: 10, perTurnMaxBytes: 2048, memoryMaxBytes: 8192, @@ -123,6 +128,11 @@ describe('WorkingMemoryComposer.compose', () => { expect(warn).not.toHaveBeenCalled(); expect(info).toHaveBeenCalledTimes(1); expect(info.mock.calls[0][0]).toMatch(/unstructured/i); + // Cosmetic (Uncle Bob plan-review finding 6): the message text must NOT + // start with `WorkingMemoryComposer:` — the Logger already prepends the + // tag, so duplicating it produces noisy `[WorkingMemoryComposer] WorkingMemoryComposer: ...` + // lines in tray logs. Locked here to prevent regression. + expect(info.mock.calls[0][0]).not.toMatch(/^WorkingMemoryComposer:/); }); it('emits neither warn nor info when log.md is sentinel-only with zero turns (the new-mind default)', () => { @@ -220,3 +230,120 @@ describe('WorkingMemoryComposer.compose', () => { expect(composer.compose(mindRoot, DEFAULTS)).toBe('rules-content'); }); }); + +// --------------------------------------------------------------------------- +// v0.60.0 — Dream Daemon opt-in gate (Phase 1) +// +// The composer must NOT include the structured-log section when the mind has +// opted out of dream-daemon consolidation (the default). For opted-out minds +// the composer also must NOT log info or warn — silence is the contract; +// otherwise tray logs would scream "log.md is unstructured" for every brand- +// new mind that hasn't enabled the feature. +// +// Per Uncle Bob plan-review (finding 6) the info-once dedupe Set lives on the +// COMPOSER INSTANCE, not module scope, so each test gets fresh state without +// resetModules() acrobatics. The test below proves the dedupe path on a +// single instance shared across two compose() calls. +// --------------------------------------------------------------------------- + +describe('WorkingMemoryComposer.compose — dream-daemon opt-in gate', () => { + it('opted-out + structured log → log section omitted, no info, no warn', () => { + writeStructuredLog([makeTurn(0)]); + fs.writeFileSync(path.join(workingMemoryDir, 'memory.md'), 'mem', 'utf-8'); + const warn = vi.fn(); + const info = vi.fn(); + const composer = createWorkingMemoryComposer({ logger: { warn, info } }); + const out = composer.compose(mindRoot, { ...DEFAULTS, enabled: false }); + expect(out).toBe('mem'); + expect(out).not.toContain('turn:turn-0'); + expect(warn).not.toHaveBeenCalled(); + expect(info).not.toHaveBeenCalled(); + }); + + it('opted-out + unstructured log → log section omitted, no info, no warn (silence is the contract)', () => { + fs.writeFileSync(path.join(workingMemoryDir, 'log.md'), 'just freeform notes\nnot structured\n', 'utf-8'); + fs.writeFileSync(path.join(workingMemoryDir, 'memory.md'), 'mem', 'utf-8'); + const warn = vi.fn(); + const info = vi.fn(); + const composer = createWorkingMemoryComposer({ logger: { warn, info } }); + const out = composer.compose(mindRoot, { ...DEFAULTS, enabled: false }); + expect(out).toBe('mem'); + expect(out).not.toContain('freeform'); + expect(warn).not.toHaveBeenCalled(); + expect(info).not.toHaveBeenCalled(); + }); + + it('opted-out + sentinel-only log → log section omitted, no info, no warn', () => { + fs.writeFileSync( + path.join(workingMemoryDir, 'log.md'), + STRUCTURED_LOG_SENTINEL + '\n\n', + 'utf-8', + ); + fs.writeFileSync(path.join(workingMemoryDir, 'memory.md'), 'mem', 'utf-8'); + const warn = vi.fn(); + const info = vi.fn(); + const composer = createWorkingMemoryComposer({ logger: { warn, info } }); + const out = composer.compose(mindRoot, { ...DEFAULTS, enabled: false }); + expect(out).toBe('mem'); + expect(warn).not.toHaveBeenCalled(); + expect(info).not.toHaveBeenCalled(); + }); + + it('opted-in + unstructured log → info fires AT MOST ONCE per composer instance, even across many compose() calls', () => { + fs.writeFileSync(path.join(workingMemoryDir, 'log.md'), 'unstructured\n', 'utf-8'); + fs.writeFileSync(path.join(workingMemoryDir, 'memory.md'), 'mem', 'utf-8'); + const warn = vi.fn(); + const info = vi.fn(); + const composer = createWorkingMemoryComposer({ logger: { warn, info } }); + + composer.compose(mindRoot, DEFAULTS); + composer.compose(mindRoot, DEFAULTS); + composer.compose(mindRoot, DEFAULTS); + + expect(warn).not.toHaveBeenCalled(); + // Three reads of the same unstructured log.md must produce ONE info line. + // The composer keeps a per-instance Set of already-warned paths. + expect(info).toHaveBeenCalledTimes(1); + expect(info.mock.calls[0][0]).not.toMatch(/^WorkingMemoryComposer:/); + }); + + it('opted-in + unstructured logs across DIFFERENT mind paths each fire info once on the same composer', () => { + // The dedupe is keyed by mindPath, not "any mind". Two different opted-in + // minds with unstructured logs must each get their own info line. + const otherMind = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-wmc-other-')); + const otherWmDir = path.join(otherMind, '.working-memory'); + fs.mkdirSync(otherWmDir, { recursive: true }); + try { + fs.writeFileSync(path.join(workingMemoryDir, 'log.md'), 'unstructured-a\n', 'utf-8'); + fs.writeFileSync(path.join(otherWmDir, 'log.md'), 'unstructured-b\n', 'utf-8'); + const warn = vi.fn(); + const info = vi.fn(); + const composer = createWorkingMemoryComposer({ logger: { warn, info } }); + + composer.compose(mindRoot, DEFAULTS); + composer.compose(otherMind, DEFAULTS); + composer.compose(mindRoot, DEFAULTS); + composer.compose(otherMind, DEFAULTS); + + expect(warn).not.toHaveBeenCalled(); + expect(info).toHaveBeenCalledTimes(2); + } finally { + fs.rmSync(otherMind, { recursive: true, force: true }); + } + }); + + it('opted-out is the default for missing/incomplete config (defensive)', () => { + // If a caller forgets to thread enabled through, composer must not leak + // log section content. Pass a config object missing `enabled`. + fs.writeFileSync(path.join(workingMemoryDir, 'log.md'), 'unstructured\n', 'utf-8'); + fs.writeFileSync(path.join(workingMemoryDir, 'memory.md'), 'mem', 'utf-8'); + const warn = vi.fn(); + const info = vi.fn(); + const composer = createWorkingMemoryComposer({ logger: { warn, info } }); + const noFlag = { lastKTurns: 10, perTurnMaxBytes: 2048, memoryMaxBytes: 8192 } as unknown as WorkingMemoryComposerConfig; + const out = composer.compose(mindRoot, noFlag); + expect(out).toBe('mem'); + expect(warn).not.toHaveBeenCalled(); + expect(info).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/services/src/chat/WorkingMemoryComposer.ts b/packages/services/src/chat/WorkingMemoryComposer.ts index 17f7ccf3..8454df02 100644 --- a/packages/services/src/chat/WorkingMemoryComposer.ts +++ b/packages/services/src/chat/WorkingMemoryComposer.ts @@ -34,6 +34,19 @@ const WORKING_MEMORY_DIRNAME = '.working-memory'; const SECTION_SEPARATOR = '\n\n---\n\n'; export interface WorkingMemoryComposerConfig { + /** + * Strict opt-in for the dream-daemon log section. When `true` (the user + * enabled consolidation in `.chamber.json` or via the agent profile UI), + * the composer reads `log.md`, validates the sentinel, and includes the + * last-K turns. When `false` (the default for new minds), the log section + * is omitted entirely — no read, no info, no warn. Silence is the + * contract: a freshly-genesis'd mind that hasn't opted in must not yield + * "log.md is unstructured" tray noise. + * + * Threaded through from `IdentityLoader.resolveComposerConfig`, which + * sources it from `loadChamberMindConfig(mindPath).workingMemory.consolidation.enabled`. + */ + readonly enabled: boolean; /** Max number of structured turns to include from `log.md`. */ readonly lastKTurns: number; /** Max bytes per rendered turn frame; over-budget turns get a truncation marker. */ @@ -61,6 +74,14 @@ export function createWorkingMemoryComposer( ): WorkingMemoryComposer { const log: ComposerLogger = deps.logger ?? Logger.create('WorkingMemoryComposer'); + // Per-instance dedupe of the unstructured-log info line, keyed by mindPath. + // Lives on the closure so each composer instance gets fresh state — Uncle + // Bob's plan-review (finding 6) rejected module-scope state because tests + // would leak between cases. Two opted-in minds with unstructured logs will + // each get one info line; calling compose() three times for the same mind + // produces ONE info line. + const unstructuredWarned = new Set(); + return { compose(mindPath, config) { const dir = path.join(mindPath, WORKING_MEMORY_DIRNAME); @@ -74,8 +95,13 @@ export function createWorkingMemoryComposer( const rules = readSimple(dir, 'rules.md'); if (rules) sections.push(rules); - const logSection = readLog(dir, config, log); - if (logSection) sections.push(logSection); + // Strict opt-in gate. The log section is omitted entirely when the mind + // has not enabled dream-daemon consolidation. No read, no info, no warn + // — see the field doc on WorkingMemoryComposerConfig.enabled. + if (config.enabled === true) { + const logSection = readLog(mindPath, dir, config, log, unstructuredWarned); + if (logSection) sections.push(logSection); + } return sections.join(SECTION_SEPARATOR); }, @@ -107,9 +133,11 @@ function readMemory(dir: string, maxBytes: number, log: ComposerLogger): string } function readLog( + mindPath: string, dir: string, config: WorkingMemoryComposerConfig, log: ComposerLogger, + unstructuredWarned: Set, ): string { const filePath = path.join(dir, 'log.md'); if (!safeExists(filePath)) return ''; @@ -118,7 +146,7 @@ function readLog( try { raw = fs.readFileSync(filePath, 'utf-8'); } catch (err) { - log.warn(`WorkingMemoryComposer: failed to read log.md; skipping log section`, err); + log.warn(`failed to read log.md; skipping log section`, err); return ''; } @@ -129,9 +157,13 @@ function readLog( // Migration-window log level: pre-existing minds may still hold an // unstructured log.md until DailyLogWriter rotates it on the first turn. // Use info (not warn) so SRE dashboards don't flag this benign state. - log.info( - `WorkingMemoryComposer: log.md is unstructured (no chamber-structured-log/v1 sentinel); skipping log section`, - ); + // Dedupe per-mindPath so we emit at most one line per process per mind. + if (!unstructuredWarned.has(mindPath)) { + unstructuredWarned.add(mindPath); + log.info( + `log.md is unstructured (no chamber-structured-log/v1 sentinel); skipping log section`, + ); + } return ''; } @@ -174,7 +206,7 @@ function truncateToBytes( if (markerBytes >= maxBytes) { log.warn( - `WorkingMemoryComposer: ${label} exceeds ${maxBytes}B and the truncation marker alone (${markerBytes}B) does not fit; emitting marker only`, + `${label} exceeds ${maxBytes}B and the truncation marker alone (${markerBytes}B) does not fit; emitting marker only`, ); return marker.slice(0, maxBytes); } @@ -186,7 +218,7 @@ function truncateToBytes( } log.info( - `WorkingMemoryComposer: truncated ${label} from ${originalBytes}B to ${Buffer.byteLength(truncated + marker, 'utf-8')}B`, + `truncated ${label} from ${originalBytes}B to ${Buffer.byteLength(truncated + marker, 'utf-8')}B`, ); return truncated + marker; } diff --git a/packages/services/src/genesis/MindScaffold.test.ts b/packages/services/src/genesis/MindScaffold.test.ts index 387608c9..0ca3fd1f 100644 --- a/packages/services/src/genesis/MindScaffold.test.ts +++ b/packages/services/src/genesis/MindScaffold.test.ts @@ -270,13 +270,15 @@ describe('MindScaffold.bootstrapCapabilities — registry source', () => { }); }); -// Fix 2: a new mind's .working-memory/log.md must be sentinel-prefixed from the -// moment createStructure() runs. Otherwise WorkingMemoryComposer emits the -// "log.md is unstructured" warning on every system-prompt rebuild for the -// lifetime of the mind, and DailyLogWriter has to rotate the legacy line on -// first turn. Seeding from MindScaffold owns the on-disk shape of a mind. -describe('MindScaffold.createStructure — log.md sentinel seed', () => { - it('seeds log.md with the chamber-structured-log/v1 sentinel as its first non-blank line', () => { +// v0.60.0 Phase 2: sentinel-seed becomes strict opt-in. The dream-daemon Switch +// in the Genesis wizard threads `enableDreamDaemon` through GenesisConfig → +// MindScaffold.createStructure. Opt-in seeds the sentinel exactly as before; +// opt-out leaves log.md as an empty placeholder so the WorkingMemoryComposer +// short-circuits cleanly (no read, no warn, no info — see Phase 1 enabled +// gate). The on-disk shape of a mind is owned by createStructure regardless +// of which Genesis path created the mind. +describe('MindScaffold.createStructure — log.md sentinel seed (opt-in)', () => { + it('opt-in (enableDreamDaemon=true): seeds log.md with the chamber-structured-log/v1 sentinel as its first non-blank line', () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-mindscaffold-sentinel-')); try { const mindPath = path.join(tmpDir, 'mind'); @@ -285,8 +287,10 @@ describe('MindScaffold.createStructure — log.md sentinel seed', () => { {} as unknown as CopilotClientFactory, ); - const internal = scaffold as unknown as { createStructure(mp: string): void }; - internal.createStructure(mindPath); + const internal = scaffold as unknown as { + createStructure(mp: string, opts: { enableDreamDaemon: boolean }): void; + }; + internal.createStructure(mindPath, { enableDreamDaemon: true }); const logPath = path.join(mindPath, '.working-memory', 'log.md'); expect(fs.existsSync(logPath)).toBe(true); @@ -298,7 +302,7 @@ describe('MindScaffold.createStructure — log.md sentinel seed', () => { } }); - it('leaves the seeded sentinel intact even though WORKING_MEMORY_FILES still iterates log.md', () => { + it('opt-in: leaves the seeded sentinel intact even though WORKING_MEMORY_FILES still iterates log.md', () => { // Guard against accidental regression: if the WORKING_MEMORY_FILES loop // ran AFTER the seed without the existsSync guard, log.md would be blanked. const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-mindscaffold-sentinel-')); @@ -309,8 +313,10 @@ describe('MindScaffold.createStructure — log.md sentinel seed', () => { {} as unknown as CopilotClientFactory, ); - const internal = scaffold as unknown as { createStructure(mp: string): void }; - internal.createStructure(mindPath); + const internal = scaffold as unknown as { + createStructure(mp: string, opts: { enableDreamDaemon: boolean }): void; + }; + internal.createStructure(mindPath, { enableDreamDaemon: true }); const logPath = path.join(mindPath, '.working-memory', 'log.md'); const content = fs.readFileSync(logPath, 'utf-8'); @@ -320,6 +326,116 @@ describe('MindScaffold.createStructure — log.md sentinel seed', () => { fs.rmSync(tmpDir, { recursive: true, force: true }); } }); + + it('opt-out (enableDreamDaemon=false): log.md exists but is empty (no sentinel)', () => { + // When the user does NOT opt in, the structured-log sentinel must NOT be + // written. Otherwise a never-opted-in mind would have a sentinel byte + // sitting on disk that would activate DailyLogWriter's structured path on + // the first turn — defeating the opt-in. The placeholder loop still + // creates log.md (so paths are valid) but its content is exactly empty. + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-mindscaffold-sentinel-')); + try { + const mindPath = path.join(tmpDir, 'mind'); + const scaffold = new MindScaffold( + {} as unknown as GitHubRegistryClient, + {} as unknown as CopilotClientFactory, + ); + + const internal = scaffold as unknown as { + createStructure(mp: string, opts: { enableDreamDaemon: boolean }): void; + }; + internal.createStructure(mindPath, { enableDreamDaemon: false }); + + const logPath = path.join(mindPath, '.working-memory', 'log.md'); + expect(fs.existsSync(logPath)).toBe(true); + const content = fs.readFileSync(logPath, 'utf-8'); + expect(content).toBe(''); + expect(content).not.toContain(STRUCTURED_LOG_SENTINEL); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it('opt-out: omitting `enableDreamDaemon` defaults to OFF (defense-in-depth)', () => { + // The Genesis IPC schema forwards the field explicitly, but if a future + // refactor or a programmatic call drops the flag we must default to the + // safer (off) state. Strict opt-in: anything other than `true` means OFF. + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-mindscaffold-sentinel-')); + try { + const mindPath = path.join(tmpDir, 'mind'); + const scaffold = new MindScaffold( + {} as unknown as GitHubRegistryClient, + {} as unknown as CopilotClientFactory, + ); + + const internal = scaffold as unknown as { + createStructure(mp: string, opts?: { enableDreamDaemon?: boolean }): void; + }; + internal.createStructure(mindPath); + + const logPath = path.join(mindPath, '.working-memory', 'log.md'); + const content = fs.readFileSync(logPath, 'utf-8'); + expect(content).toBe(''); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); +}); + +// v0.60.0 Phase 2: opt-in must persist the choice into `.chamber.json` so +// MindMemoryService.activateMind can read it back on the very next mind +// load. Opt-out is the default (no consolidation block) so existing minds +// upgrading into this release stay opted-out without a migration. We deep- +// merge in case future Genesis features write other fields. +describe('MindScaffold.create — `.chamber.json` consolidation block (opt-in)', () => { + it('opt-in: writes .chamber.json with workingMemory.consolidation.enabled=true', () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-mindscaffold-chamberjson-')); + try { + const mindPath = path.join(tmpDir, 'mind'); + const scaffold = new MindScaffold( + {} as unknown as GitHubRegistryClient, + {} as unknown as CopilotClientFactory, + ); + + const internal = scaffold as unknown as { + createStructure(mp: string, opts: { enableDreamDaemon: boolean }): void; + }; + internal.createStructure(mindPath, { enableDreamDaemon: true }); + + const chamberJsonPath = path.join(mindPath, '.chamber.json'); + expect(fs.existsSync(chamberJsonPath)).toBe(true); + const parsed = JSON.parse(fs.readFileSync(chamberJsonPath, 'utf-8')) as { + workingMemory?: { consolidation?: { enabled?: unknown } }; + }; + expect(parsed.workingMemory?.consolidation?.enabled).toBe(true); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it('opt-out: does NOT write .chamber.json (defaults are off → file is absent)', () => { + // No file = chamberMindConfig.loadChamberMindConfig returns the default + // shape with `consolidation.enabled: false`. Writing an empty marker file + // would be wasted I/O AND signal intent the user never expressed. + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-mindscaffold-chamberjson-')); + try { + const mindPath = path.join(tmpDir, 'mind'); + const scaffold = new MindScaffold( + {} as unknown as GitHubRegistryClient, + {} as unknown as CopilotClientFactory, + ); + + const internal = scaffold as unknown as { + createStructure(mp: string, opts: { enableDreamDaemon: boolean }): void; + }; + internal.createStructure(mindPath, { enableDreamDaemon: false }); + + const chamberJsonPath = path.join(mindPath, '.chamber.json'); + expect(fs.existsSync(chamberJsonPath)).toBe(false); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); }); // Fix 2: the genesis prompt must not instruct the LLM to write to log.md any diff --git a/packages/services/src/genesis/MindScaffold.ts b/packages/services/src/genesis/MindScaffold.ts index 66cd08ca..07d5468d 100644 --- a/packages/services/src/genesis/MindScaffold.ts +++ b/packages/services/src/genesis/MindScaffold.ts @@ -26,6 +26,16 @@ export interface GenesisConfig { voice: string; voiceDescription: string; basePath: string; + /** + * Strict opt-in for the dream daemon (v0.60.0). When `true`, MindScaffold + * seeds the chamber-structured-log/v1 sentinel into log.md AND writes + * `.chamber.json` with `workingMemory.consolidation.enabled: true` so + * MindMemoryService.activateMind starts the daemon on the first mind load. + * When `false` or omitted, log.md is left empty and `.chamber.json` is + * not written — the mind operates without consolidation, matching the + * default for every existing user upgrading into this release. + */ + enableDreamDaemon?: boolean; } export interface GenesisProgress { @@ -82,7 +92,7 @@ export class MindScaffold { // 1. Create deterministic structure this.emit('structure', 'Creating mind structure...'); - this.createStructure(mindPath); + this.createStructure(mindPath, { enableDreamDaemon: config.enableDreamDaemon === true }); // 2. Generate soul via agent this.emit('soul', `Writing SOUL.md...`); @@ -112,7 +122,10 @@ export class MindScaffold { return mindPath; } - private createStructure(mindPath: string): void { + private createStructure( + mindPath: string, + opts: { enableDreamDaemon?: boolean } = {}, + ): void { // IDEA folders for (const folder of IDEA_FOLDERS) { fs.mkdirSync(path.join(mindPath, folder), { recursive: true }); @@ -126,16 +139,18 @@ export class MindScaffold { const wmDir = path.join(mindPath, '.working-memory'); fs.mkdirSync(wmDir, { recursive: true }); - // Seed log.md with the chamber-structured-log/v1 sentinel BEFORE the - // WORKING_MEMORY_FILES placeholder loop runs. The loop's `existsSync` guard - // then skips it, preserving the sentinel. This keeps `validate()` happy - // (sentinel-only content trims to non-empty) while honoring the - // structured-log contract from creation. Trailing `\n\n` matches the - // byte-level format that DailyLogWriter.seedFreshLog emits for a fresh - // mind so on-disk content stays consistent regardless of which path - // seeded the sentinel. See WorkingMemoryComposer.readLog and - // DailyLogWriter.doWrite for the consumer side of this contract. - fs.writeFileSync(path.join(wmDir, 'log.md'), STRUCTURED_LOG_SENTINEL + '\n\n'); + const enableDreamDaemon = opts.enableDreamDaemon === true; + + // v0.60.0 Phase 2: log.md sentinel seed is now strict opt-in. When the + // user toggles the dream-daemon Switch in Genesis we seed the sentinel + // exactly as before (byte-for-byte parity with DailyLogWriter.seedFreshLog + // — `SENTINEL + '\n\n'`). When the user opts out (default), log.md is + // left empty by the WORKING_MEMORY_FILES placeholder loop below — no + // sentinel byte means DailyLogWriter never engages even if a future bug + // somehow registers a writer for this mind. + if (enableDreamDaemon) { + fs.writeFileSync(path.join(wmDir, 'log.md'), STRUCTURED_LOG_SENTINEL + '\n\n'); + } // Create placeholder files so the agent has targets for (const file of WORKING_MEMORY_FILES) { @@ -144,6 +159,22 @@ export class MindScaffold { fs.writeFileSync(filePath, ''); } } + + // v0.60.0 Phase 2: persist the opt-in choice into `.chamber.json` so + // MindMemoryService.activateMind reads it back on the very next mind + // load. We only WRITE this file on opt-in — opt-out is the default + // shape returned by chamberMindConfig when the file is absent, so an + // empty marker file would be wasted I/O AND signal intent the user + // never expressed. + if (enableDreamDaemon) { + const chamberJsonPath = path.join(mindPath, '.chamber.json'); + const chamberConfig = { + workingMemory: { + consolidation: { enabled: true }, + }, + }; + fs.writeFileSync(chamberJsonPath, JSON.stringify(chamberConfig, null, 2) + '\n'); + } } private async generateSoul(mindPath: string, config: GenesisConfig, slug: string): Promise { diff --git a/packages/services/src/mind/MindManager.test.ts b/packages/services/src/mind/MindManager.test.ts index f63dbd6c..cc50e1f3 100644 --- a/packages/services/src/mind/MindManager.test.ts +++ b/packages/services/src/mind/MindManager.test.ts @@ -6,7 +6,7 @@ import type { IdentityLoader } from '../chat/IdentityLoader'; import type { ChamberToolProvider } from '../chamberTools'; import type { ConfigService } from '../config/ConfigService'; import type { ViewDiscovery } from '../lens/ViewDiscovery'; -import type { AppConfig, LensViewManifest } from '@chamber/shared/types'; +import type { AppConfig, LensViewManifest, MindContext } from '@chamber/shared/types'; // --- Mocks --- @@ -14,6 +14,10 @@ vi.mock('fs', () => ({ existsSync: vi.fn(), readdirSync: vi.fn(() => []), readFileSync: vi.fn(), + writeFileSync: vi.fn(), + renameSync: vi.fn(), + mkdirSync: vi.fn(), + rmSync: vi.fn(), realpathSync: Object.assign(vi.fn((candidate: string) => candidate), { native: vi.fn((candidate: string) => candidate), }), @@ -23,8 +27,26 @@ vi.mock('../lens/MindBootstrap', () => ({ bootstrapMindCapabilities: vi.fn(), })); +vi.mock('./chamberMindConfig', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + patchChamberMindConfig: vi.fn(), + }; +}); + +vi.mock('../mindMemory/rollback', () => ({ + rollbackToUnstructured: vi.fn().mockResolvedValue({ + framesConverted: 0, + legacyExisted: false, + outcome: 'no-op-missing', + }), +})); + import * as fs from 'fs'; import { bootstrapMindCapabilities } from '../lens/MindBootstrap'; +import { patchChamberMindConfig } from './chamberMindConfig'; +import { rollbackToUnstructured } from '../mindMemory/rollback'; const mockStart = vi.fn(); const mockStop = vi.fn(); @@ -1506,4 +1528,125 @@ describe('MindManager', () => { consoleSpy.mockRestore(); }); }); + + describe('enableDreamDaemon / disableDreamDaemon', () => { + beforeEach(() => { + vi.mocked(patchChamberMindConfig).mockReset(); + vi.mocked(rollbackToUnstructured).mockReset(); + vi.mocked(rollbackToUnstructured).mockResolvedValue({ + framesConverted: 0, + legacyExisted: false, + outcome: 'no-op-missing', + }); + }); + + it('enableDreamDaemon patches .chamber.json with consolidation.enabled=true and reloads the mind', async () => { + const mind = await manager.loadMind('/tmp/agents/q'); + const reloadSpy = vi.spyOn(manager, 'reloadMind'); + + await manager.enableDreamDaemon(mind.mindId); + + expect(patchChamberMindConfig).toHaveBeenCalledWith('/tmp/agents/q', { + workingMemory: { consolidation: { enabled: true } }, + }); + expect(reloadSpy).toHaveBeenCalledWith(mind.mindId); + }); + + it('disableDreamDaemon patches .chamber.json with consolidation.enabled=false and reloads the mind', async () => { + const mind = await manager.loadMind('/tmp/agents/q'); + const reloadSpy = vi.spyOn(manager, 'reloadMind'); + + await manager.disableDreamDaemon(mind.mindId); + + expect(patchChamberMindConfig).toHaveBeenCalledWith('/tmp/agents/q', { + workingMemory: { consolidation: { enabled: false } }, + }); + expect(reloadSpy).toHaveBeenCalledWith(mind.mindId); + }); + + it('enableDreamDaemon throws when the mind is not loaded', async () => { + await expect(manager.enableDreamDaemon('does-not-exist')).rejects.toThrow(/not found/); + expect(patchChamberMindConfig).not.toHaveBeenCalled(); + }); + + it('disableDreamDaemon throws when the mind is not loaded', async () => { + await expect(manager.disableDreamDaemon('does-not-exist')).rejects.toThrow(/not found/); + expect(patchChamberMindConfig).not.toHaveBeenCalled(); + }); + + it('enableDreamDaemon resolves to the reloaded MindContext', async () => { + const mind = await manager.loadMind('/tmp/agents/q'); + const result = await manager.enableDreamDaemon(mind.mindId); + expect(result.mindId).toBe(mind.mindId); + expect(result.mindPath).toBe('/tmp/agents/q'); + }); + + it('serializes concurrent toggle calls for the same mindId', async () => { + const mind = await manager.loadMind('/tmp/agents/q'); + + // Hold reloadMind open so the second call observes an in-flight toggle. + let releaseReload: (() => void) | undefined; + const reloadGate = new Promise((resolve) => { releaseReload = resolve; }); + const reloadSpy = vi.spyOn(manager, 'reloadMind').mockImplementation(async () => { + await reloadGate; + // Return a synthetic context that mirrors the mind we created so + // assertions on the resolved value remain meaningful. + return { ...mind } as MindContext; + }); + + const first = manager.enableDreamDaemon(mind.mindId); + const second = manager.enableDreamDaemon(mind.mindId); + + // Same in-flight promise must be returned to the second caller, so + // patchChamberMindConfig fires exactly once for the pair. + expect(second).toBe(first); + + releaseReload!(); + const [a, b] = await Promise.all([first, second]); + expect(a).toBe(b); + expect(reloadSpy).toHaveBeenCalledTimes(1); + expect(patchChamberMindConfig).toHaveBeenCalledTimes(1); + }); + + it('disableDreamDaemon calls rollbackToUnstructured AFTER patch + reload', async () => { + const mind = await manager.loadMind('/tmp/agents/q'); + const callOrder: string[] = []; + vi.mocked(patchChamberMindConfig).mockImplementation(() => { callOrder.push('patch'); }); + const reloadSpy = vi.spyOn(manager, 'reloadMind').mockImplementation(async () => { + callOrder.push('reload'); + return { ...mind } as MindContext; + }); + vi.mocked(rollbackToUnstructured).mockImplementation(async () => { + callOrder.push('rollback'); + return { framesConverted: 2, legacyExisted: false, outcome: 'rolled-back' }; + }); + + await manager.disableDreamDaemon(mind.mindId); + + expect(callOrder).toEqual(['patch', 'reload', 'rollback']); + expect(rollbackToUnstructured).toHaveBeenCalledWith('/tmp/agents/q'); + expect(reloadSpy).toHaveBeenCalledWith(mind.mindId); + }); + + it('enableDreamDaemon does NOT call rollbackToUnstructured', async () => { + const mind = await manager.loadMind('/tmp/agents/q'); + await manager.enableDreamDaemon(mind.mindId); + expect(rollbackToUnstructured).not.toHaveBeenCalled(); + }); + + it('disableDreamDaemon resolves to the reloaded MindContext even when rollback fails', async () => { + const mind = await manager.loadMind('/tmp/agents/q'); + vi.mocked(rollbackToUnstructured).mockRejectedValueOnce(new Error('rollback boom')); + + const result = await manager.disableDreamDaemon(mind.mindId); + + // Toggle is non-fatal: config has been flipped + reload completed. + // A failed rollback only logs a warning; the user-visible toggle + // succeeds so the UI state matches the on-disk config. + expect(result.mindId).toBe(mind.mindId); + expect(patchChamberMindConfig).toHaveBeenCalledWith('/tmp/agents/q', { + workingMemory: { consolidation: { enabled: false } }, + }); + }); + }); }); diff --git a/packages/services/src/mind/MindManager.ts b/packages/services/src/mind/MindManager.ts index 10f7daa0..49479f91 100644 --- a/packages/services/src/mind/MindManager.ts +++ b/packages/services/src/mind/MindManager.ts @@ -12,7 +12,8 @@ import { Logger } from '../logger'; import type { InternalMindContext, CopilotClient, CopilotSession, Tool, UserInputHandler } from './types'; import { generateMindId } from './generateMindId'; import { loadMcpServersFromMindPath } from './mcpConfig'; -import { loadChamberMindConfig } from './chamberMindConfig'; +import { loadChamberMindConfig, patchChamberMindConfig } from './chamberMindConfig'; +import { rollbackToUnstructured } from '../mindMemory/rollback'; import type { CopilotClientFactory } from '../sdk/CopilotClientFactory'; import { approveForSessionCompat } from '../sdk/approveForSessionCompat'; import type { IdentityLoader } from '../chat/IdentityLoader'; @@ -35,6 +36,13 @@ export class MindManager extends EventEmitter { private reloading = false; private providers: ChamberToolProvider[] = []; private modelUpdates = new Map>(); + // Per-mindId serialization for dream-daemon toggle. Two concurrent + // calls (typical: rapid clicks bypassing the UI's `togglingDreamDaemon` + // guard, or a programmatic IPC caller) must not race `reloadMind` — + // the second `this.minds.get(mindId)` would return undefined while the + // first call is mid-reload (delete-then-loadMind). Same shape as the + // `loading` Map: in-flight promises are returned to subsequent callers. + private daemonToggling = new Map>(); constructor( private readonly clientFactory: CopilotClientFactory, @@ -239,6 +247,64 @@ export class MindManager extends EventEmitter { return reloaded; } + // Flip the dream-daemon opt-in for an existing mind. Patches `.chamber.json` + // to set `workingMemory.consolidation.enabled = true`, then reloads the + // mind so providers (notably MindMemoryService) re-read the opt-in gate + // and `migrateIfNeeded` runs against any pre-existing unstructured `log.md`. + // Per-mindId serialization via `daemonToggling` ensures concurrent calls + // for the same mind return the same in-flight promise rather than racing + // (the second call would otherwise hit a stale `this.minds.get` mid-reload). + enableDreamDaemon(mindId: string): Promise { + return this.toggleDreamDaemon(mindId, true); + } + + // Counterpart to `enableDreamDaemon`. In Phase 3 this only flips the flag + // and reloads — it does NOT roll back structured frames in `log.md` back + // to unstructured markdown. That rollback path is the subject of Phase 4. + disableDreamDaemon(mindId: string): Promise { + return this.toggleDreamDaemon(mindId, false); + } + + private toggleDreamDaemon(mindId: string, enabled: boolean): Promise { + const inflight = this.daemonToggling.get(mindId); + if (inflight) return inflight; + const promise = this.doToggleDreamDaemon(mindId, enabled); + this.daemonToggling.set(mindId, promise); + promise.finally(() => { + // Only clear if still the same promise — guards against a race where + // the entry was somehow replaced (defensive; not currently possible). + if (this.daemonToggling.get(mindId) === promise) this.daemonToggling.delete(mindId); + }).catch(() => { /* swallowed — caller still receives the rejection */ }); + return promise; + } + + private async doToggleDreamDaemon(mindId: string, enabled: boolean): Promise { + const context = this.minds.get(mindId); + if (!context) throw new Error(`Mind ${mindId} not found`); + const mindPath = context.mindPath; + patchChamberMindConfig(mindPath, { + workingMemory: { consolidation: { enabled } }, + }); + const reloaded = await this.reloadMind(mindId); + + if (!enabled) { + // Phase 4 — at this point the mind has been reloaded with the + // opted-out config: MindMemoryService skipped activation, so the + // DailyLogWriter is gone and no observer is attached. Safe to + // rewrite log.md without racing in-flight structured writes. + // Failure is non-fatal to the toggle: the config is already + // flipped, the next app launch (or another rollback attempt) + // can retry. Surfacing as a warning keeps the user-visible + // toggle from breaking on a transient fs error. + try { + await rollbackToUnstructured(mindPath); + } catch (err) { + log.warn(`disableDreamDaemon: rollback failed for ${mindId}`, err); + } + } + return reloaded; + } + /** * Recycle the SDK client and active session for a mind in place. This is * the narrow primitive behind issue #90's "Refresh models" affordance: diff --git a/packages/services/src/mind/chamberMindConfig.test.ts b/packages/services/src/mind/chamberMindConfig.test.ts index 09ca179a..86785db0 100644 --- a/packages/services/src/mind/chamberMindConfig.test.ts +++ b/packages/services/src/mind/chamberMindConfig.test.ts @@ -4,6 +4,7 @@ import * as os from 'node:os'; import * as path from 'node:path'; import { loadChamberMindConfig, + patchChamberMindConfig, CHAMBER_MIND_CONFIG_FILENAME, DEFAULT_WORKING_MEMORY_CONSOLIDATION, } from './chamberMindConfig'; @@ -297,3 +298,98 @@ describe('chamberMindConfig — workingMemory.consolidation', () => { }); }); }); + +describe('patchChamberMindConfig', () => { + let tmpDir: string; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-mind-config-patch-')); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('creates the .chamber.json file when it does not yet exist and applies the patch', () => { + patchChamberMindConfig(tmpDir, { + workingMemory: { consolidation: { enabled: true } }, + }); + const onDisk = JSON.parse(fs.readFileSync(path.join(tmpDir, CHAMBER_MIND_CONFIG_FILENAME), 'utf-8')); + expect(onDisk).toEqual({ + workingMemory: { consolidation: { enabled: true } }, + }); + }); + + it('deep-merges into an existing workingMemory.consolidation block', () => { + fs.writeFileSync( + path.join(tmpDir, CHAMBER_MIND_CONFIG_FILENAME), + JSON.stringify({ + workingMemory: { + consolidation: { enabled: false, cron: '*/5 * * * *', lastKTurns: 5 }, + }, + }, null, 2) + '\n', + ); + + patchChamberMindConfig(tmpDir, { + workingMemory: { consolidation: { enabled: true } }, + }); + + const onDisk = JSON.parse(fs.readFileSync(path.join(tmpDir, CHAMBER_MIND_CONFIG_FILENAME), 'utf-8')); + expect(onDisk.workingMemory.consolidation).toEqual({ + enabled: true, + cron: '*/5 * * * *', + lastKTurns: 5, + }); + }); + + it('preserves existing top-level passthrough fields like excludedTools', () => { + fs.writeFileSync( + path.join(tmpDir, CHAMBER_MIND_CONFIG_FILENAME), + JSON.stringify({ + excludedTools: ['shell', 'str_replace'], + somethingFromAFutureVersion: { keepMe: true }, + }, null, 2) + '\n', + ); + + patchChamberMindConfig(tmpDir, { + workingMemory: { consolidation: { enabled: true } }, + }); + + const onDisk = JSON.parse(fs.readFileSync(path.join(tmpDir, CHAMBER_MIND_CONFIG_FILENAME), 'utf-8')); + expect(onDisk.excludedTools).toEqual(['shell', 'str_replace']); + expect(onDisk.somethingFromAFutureVersion).toEqual({ keepMe: true }); + expect(onDisk.workingMemory.consolidation.enabled).toBe(true); + }); + + it('writes pretty-printed JSON ending with a newline', () => { + patchChamberMindConfig(tmpDir, { + workingMemory: { consolidation: { enabled: true } }, + }); + const raw = fs.readFileSync(path.join(tmpDir, CHAMBER_MIND_CONFIG_FILENAME), 'utf-8'); + expect(raw.endsWith('\n')).toBe(true); + expect(raw).toContain('\n '); + }); + + it('leaves the original .chamber.json byte-identical when the rename step fails', () => { + // Trigger a real OS-level rename failure without monkey-patching fs + // (which is impossible for ESM-imported `node:fs.renameSync`). We make + // `.chamber.json` a non-empty directory so: + // * readRawChamberConfig() sees existsSync=true but readFileSync throws + // (EISDIR) → caught → returns {} + // * writeFileSync(tmpPath) succeeds (tmp lives next to the dir) + // * renameSync(tmp, dir) fails → caught → tmp removed → rethrow + const filePath = path.join(tmpDir, CHAMBER_MIND_CONFIG_FILENAME); + fs.mkdirSync(filePath); + const sentinelPath = path.join(filePath, 'marker.txt'); + fs.writeFileSync(sentinelPath, 'untouched'); + + expect(() => patchChamberMindConfig(tmpDir, { + workingMemory: { consolidation: { enabled: true } }, + })).toThrow(); + + expect(fs.statSync(filePath).isDirectory()).toBe(true); + expect(fs.readFileSync(sentinelPath, 'utf-8')).toBe('untouched'); + const lingering = fs.readdirSync(tmpDir).filter((entry) => entry.endsWith('.tmp')); + expect(lingering).toEqual([]); + }); +}); diff --git a/packages/services/src/mind/chamberMindConfig.ts b/packages/services/src/mind/chamberMindConfig.ts index d2139231..c68d31b6 100644 --- a/packages/services/src/mind/chamberMindConfig.ts +++ b/packages/services/src/mind/chamberMindConfig.ts @@ -156,3 +156,76 @@ export function loadChamberMindConfig(mindPath: string): ChamberMindConfig { } return out; } + +// Atomically merge a partial patch into the mind's `.chamber.json`. Reads +// the current raw JSON (preserving unknown top-level passthrough fields per +// `chamberMindConfigSchema.passthrough()`), deep-merges the patch into +// `workingMemory.consolidation`, then writes the result via tmp-file + +// rename. If any step fails, the original file is left untouched and the +// tmp file is removed. Used by `MindManager.enableDreamDaemon` / +// `disableDreamDaemon` so flipping the toggle never half-writes the file. +export interface ChamberMindConfigPatch { + workingMemory?: { + consolidation?: Partial; + }; +} + +function readRawChamberConfig(filePath: string): Record { + if (!fs.existsSync(filePath)) return {}; + let raw: string; + try { + raw = fs.readFileSync(filePath, 'utf-8'); + } catch (err) { + log.warn(`Failed to read ${filePath} during patch; treating as empty:`, err); + return {}; + } + try { + const parsed = JSON.parse(raw); + if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) { + log.warn(`Existing ${filePath} is not a JSON object during patch; treating as empty.`); + return {}; + } + return parsed as Record; + } catch (err) { + log.warn(`Invalid JSON in ${filePath} during patch; treating as empty:`, err); + return {}; + } +} + +export function patchChamberMindConfig(mindPath: string, patch: ChamberMindConfigPatch): void { + const filePath = path.join(mindPath, CHAMBER_MIND_CONFIG_FILENAME); + const current = readRawChamberConfig(filePath); + + const currentWorkingMemory = (current.workingMemory && typeof current.workingMemory === 'object' && !Array.isArray(current.workingMemory)) + ? current.workingMemory as Record + : {}; + const currentConsolidation = (currentWorkingMemory.consolidation && typeof currentWorkingMemory.consolidation === 'object' && !Array.isArray(currentWorkingMemory.consolidation)) + ? currentWorkingMemory.consolidation as Record + : {}; + + const merged: Record = { ...current }; + if (patch.workingMemory) { + // Only `workingMemory.consolidation` is deep-merged; sibling subkeys + // (e.g. future `workingMemory.archival`) survive via the spread of + // `currentWorkingMemory`. Extend this branch when introducing new + // subkeys that themselves need a deep-merge rather than a clobber. + merged.workingMemory = { + ...currentWorkingMemory, + ...(patch.workingMemory.consolidation + ? { consolidation: { ...currentConsolidation, ...patch.workingMemory.consolidation } } + : {}), + }; + } + + const serialized = `${JSON.stringify(merged, null, 2)}\n`; + const tmpPath = `${filePath}.${process.pid}.${Date.now()}.tmp`; + fs.writeFileSync(tmpPath, serialized, 'utf-8'); + try { + fs.renameSync(tmpPath, filePath); + } catch (err) { + if (fs.existsSync(tmpPath)) { + try { fs.rmSync(tmpPath, { force: true }); } catch { /* best-effort cleanup */ } + } + throw err; + } +} diff --git a/packages/services/src/mindMemory/DailyLogWriter.test.ts b/packages/services/src/mindMemory/DailyLogWriter.test.ts index 6f3bbd43..085badcd 100644 --- a/packages/services/src/mindMemory/DailyLogWriter.test.ts +++ b/packages/services/src/mindMemory/DailyLogWriter.test.ts @@ -5,7 +5,7 @@ import os from 'node:os'; import path from 'node:path'; import { createDailyLogWriter } from './DailyLogWriter'; -import { STRUCTURED_LOG_SENTINEL, parseLog, type CompletedTurn } from './StructuredLogFormat'; +import { STRUCTURED_LOG_SENTINEL, parseLog, serializeTurn, type CompletedTurn } from './StructuredLogFormat'; let mindRoot: string; let logPath: string; @@ -328,3 +328,216 @@ describe('DailyLogWriter — error path safety', () => { expect(parsed.turns[0].turnId).toBe('turn-2'); }); }); + +// --------------------------------------------------------------------------- +// v0.60.0 — migrateIfNeeded + flush (Phase 1) +// +// `migrateIfNeeded(mindPath)` is the eager-migration hook MindMemoryService +// invokes when a previously-opted-out mind flips to opted-in. It reproduces +// the rotation-and-seed half of doWrite() WITHOUT requiring a turn to seed. +// The seed bytes MUST match MindScaffold.createStructure byte-for-byte +// (`SENTINEL + '\n\n'`) so on-disk content is uniform regardless of which +// path created it. +// +// `flush()` returns the writer's chain promise so callers (e.g. Phase 4 +// rollbackToUnstructured) can wait for in-flight writes to settle before +// reading log.md back. Without this, an observer-removal-then-read race +// could miss the last frame. +// --------------------------------------------------------------------------- + +describe('DailyLogWriter — migrateIfNeeded', () => { + it('no log.md → no-op (no rotation, no log.legacy.md, no seed)', async () => { + const info = vi.fn(); + const writer = makeWriter({ logger: { info } }); + + await writer.migrateIfNeeded(); + + expect(fs.existsSync(logPath)).toBe(false); + expect(fs.existsSync(legacyPath)).toBe(false); + expect(info).not.toHaveBeenCalled(); + }); + + it('empty log.md (0 bytes) → no rotation (treated as already-structured)', async () => { + fs.mkdirSync(path.dirname(logPath), { recursive: true }); + fs.writeFileSync(logPath, ''); + + const info = vi.fn(); + const writer = makeWriter({ logger: { info } }); + + await writer.migrateIfNeeded(); + + expect(fs.existsSync(legacyPath)).toBe(false); + expect(info).not.toHaveBeenCalled(); + // Empty → seed sentinel-only so subsequent reads are valid structured. + const content = fs.readFileSync(logPath, 'utf-8'); + expect(content).toBe(STRUCTURED_LOG_SENTINEL + '\n\n'); + }); + + it('sentinel-prefixed log.md → no-op (idempotent, no rotation)', async () => { + fs.mkdirSync(path.dirname(logPath), { recursive: true }); + const original = STRUCTURED_LOG_SENTINEL + '\n\n'; + fs.writeFileSync(logPath, original); + + const info = vi.fn(); + const writer = makeWriter({ logger: { info } }); + + await writer.migrateIfNeeded(); + + expect(fs.existsSync(legacyPath)).toBe(false); + expect(info).not.toHaveBeenCalled(); + expect(fs.readFileSync(logPath, 'utf-8')).toBe(original); + }); + + it('sentinel-prefixed log.md WITH frames → no-op (idempotent, frames preserved)', async () => { + fs.mkdirSync(path.dirname(logPath), { recursive: true }); + const original = STRUCTURED_LOG_SENTINEL + '\n\n' + makeWriterRaw([makeTurn(1), makeTurn(2)]); + fs.writeFileSync(logPath, original); + + const info = vi.fn(); + const writer = makeWriter({ logger: { info } }); + + await writer.migrateIfNeeded(); + + expect(fs.existsSync(legacyPath)).toBe(false); + expect(info).not.toHaveBeenCalled(); + expect(fs.readFileSync(logPath, 'utf-8')).toBe(original); + }); + + it('unstructured log.md → rotates to log.legacy.md and seeds sentinel-only log.md (NO frame)', async () => { + fs.mkdirSync(path.dirname(logPath), { recursive: true }); + const legacyContent = '# notes\n\nrandom freeform content\n'; + fs.writeFileSync(logPath, legacyContent); + + const info = vi.fn(); + const writer = makeWriter({ logger: { info } }); + + await writer.migrateIfNeeded(); + + expect(fs.readFileSync(legacyPath, 'utf-8')).toBe(legacyContent); + + // Byte-for-byte parity with MindScaffold.createStructure. + const fresh = fs.readFileSync(logPath, 'utf-8'); + expect(fresh).toBe(STRUCTURED_LOG_SENTINEL + '\n\n'); + + expect(info).toHaveBeenCalledWith( + 'Rotated unstructured log.md to log.legacy.md for mind mind-x', + ); + }); + + it('collision: log.legacy.md exists → rotates to log.legacy..md', async () => { + fs.mkdirSync(path.dirname(logPath), { recursive: true }); + const priorLegacy = '# previous legacy\n'; + const todayBad = '# today is bad\n'; + fs.writeFileSync(legacyPath, priorLegacy); + fs.writeFileSync(logPath, todayBad); + + const info = vi.fn(); + const writer = makeWriter({ logger: { info } }); + + await writer.migrateIfNeeded(); + + expect(fs.readFileSync(legacyPath, 'utf-8')).toBe(priorLegacy); + + const all = fs.readdirSync(path.dirname(logPath)); + const stamped = all.filter( + (n) => /^log\.legacy\.\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}Z\.md$/.test(n), + ); + expect(stamped).toHaveLength(1); + expect(fs.readFileSync(path.join(path.dirname(logPath), stamped[0]), 'utf-8')).toBe(todayBad); + + expect(fs.readFileSync(logPath, 'utf-8')).toBe(STRUCTURED_LOG_SENTINEL + '\n\n'); + expect(info).toHaveBeenCalledWith( + `Rotated unstructured log.md to ${stamped[0]} for mind mind-x`, + ); + }); + + it('migrateIfNeeded then write(turn) → first frame appends to the seeded sentinel without re-rotating', async () => { + fs.mkdirSync(path.dirname(logPath), { recursive: true }); + fs.writeFileSync(logPath, '# legacy\n'); + + const info = vi.fn(); + const writer = makeWriter({ logger: { info } }); + + await writer.migrateIfNeeded(); + await writer.write(makeTurn(1)); + + // Exactly one rotation event. + const rotates = info.mock.calls.filter((c) => /Rotated unstructured/.test(String(c[0]))); + expect(rotates).toHaveLength(1); + + const parsed = parseLog(fs.readFileSync(logPath, 'utf-8')); + expect(parsed.sentinel).toBe(true); + expect(parsed.turns).toHaveLength(1); + }); + + it('serializes through the same per-instance chain as write()', async () => { + // Concurrent write() and migrateIfNeeded() must serialize: never produces + // a doubled sentinel or a half-rotated state. + fs.mkdirSync(path.dirname(logPath), { recursive: true }); + fs.writeFileSync(logPath, '# legacy\n'); + + const writer = makeWriter(); + + // Kick both off "simultaneously" — they must serialize through the chain. + await Promise.all([ + writer.migrateIfNeeded(), + writer.write(makeTurn(1)), + writer.write(makeTurn(2)), + ]); + + const content = fs.readFileSync(logPath, 'utf-8'); + expect(content.split(STRUCTURED_LOG_SENTINEL).length - 1).toBe(1); + const parsed = parseLog(content); + expect(parsed.sentinel).toBe(true); + expect(parsed.malformed).toBe(0); + expect(parsed.turns).toHaveLength(2); + }); +}); + +describe('DailyLogWriter — flush', () => { + it('flush() resolves after all queued writes settle', async () => { + const writer = makeWriter(); + + // Fire many writes without awaiting. + const pending = Array.from({ length: 5 }, (_, i) => writer.write(makeTurn(i + 1))); + + // flush() must not resolve before the chain. + await writer.flush(); + + // After flush, every write is observable on disk. + const parsed = parseLog(fs.readFileSync(logPath, 'utf-8')); + expect(parsed.turns).toHaveLength(5); + + // The original promises also settle. + await Promise.all(pending); + }); + + it('flush() is safe to call when no writes are pending (no-op resolves)', async () => { + const writer = makeWriter(); + await expect(writer.flush()).resolves.toBeUndefined(); + }); + + it('flush() does not throw even if a queued write rejects', async () => { + fs.mkdirSync(path.dirname(logPath), { recursive: true }); + fs.writeFileSync(logPath, '# legacy\n'); + + const failingRename = vi.fn(async () => { + throw new Error('synthetic rename failure'); + }); + const writer = makeWriter({ rename: failingRename }); + + const pending = writer.write(makeTurn(1)); + + // The chain swallows rejections internally so flush stays clean. + await expect(writer.flush()).resolves.toBeUndefined(); + + // The original write() still rejects — error propagation contract preserved. + await expect(pending).rejects.toThrow(/synthetic rename failure/); + }); +}); + +// Helper: produce on-disk frame bytes the same way DailyLogWriter does, so +// "preserved" assertions don't depend on the writer's own behaviour. +function makeWriterRaw(turns: ReturnType[]): string { + return turns.map(serializeTurn).join(''); +} diff --git a/packages/services/src/mindMemory/DailyLogWriter.ts b/packages/services/src/mindMemory/DailyLogWriter.ts index 5fb496b5..c42f480a 100644 --- a/packages/services/src/mindMemory/DailyLogWriter.ts +++ b/packages/services/src/mindMemory/DailyLogWriter.ts @@ -66,6 +66,33 @@ export interface DailyLogWriterOptions { export interface DailyLogWriter { write(turn: CompletedTurn): Promise; + /** + * Eager-migration entry point. Performs the rotation-and-seed half of + * `write()` WITHOUT requiring a turn payload. Behaviour: + * + * - log.md absent → no-op (no rotation, no seed). + * - log.md empty (0 bytes) → seed sentinel-only. + * - log.md sentinel-prefixed → no-op (already structured). + * - log.md unstructured → rotate to log.legacy.md (or timestamped on + * collision) and seed `SENTINEL + '\n\n'` byte-for-byte matching + * `MindScaffold.createStructure`. + * + * Serializes through the same per-instance chain as `write()` so a + * concurrent `migrateIfNeeded` + `write` pair never produces a doubled + * sentinel or half-rotated state. + */ + migrateIfNeeded(): Promise; + /** + * Drain the per-instance write chain. Resolves only after every queued + * `write()` / `migrateIfNeeded()` settles (success OR failure). Phase 4 + * `rollbackToUnstructured` calls this after detaching the observer so it + * can read back log.md without missing in-flight frames. + * + * Errors from queued operations are surfaced through their original + * promises (write returns a rejecting Promise). `flush()` itself never + * rejects — its job is purely to wait for quiescence. + */ + flush(): Promise; } export function createDailyLogWriter(opts: DailyLogWriterOptions): DailyLogWriter { @@ -123,8 +150,16 @@ export function createDailyLogWriter(opts: DailyLogWriterOptions): DailyLogWrite void currentContent; } - async function seedFreshLog(turn: CompletedTurn): Promise { - const content = `${STRUCTURED_LOG_SENTINEL}\n\n${serializeTurn(turn)}`; + async function seedFreshLog(turn: CompletedTurn | null): Promise { + // INVARIANT: when turn is null (eager migration via migrateIfNeeded), + // the seed bytes MUST exactly match MindScaffold.createStructure + // (`SENTINEL + '\n\n'`) so on-disk content is uniform regardless of + // which path created it. Composer + parseLog tolerate trailing blank + // lines, but the byte-for-byte parity is what makes the migration + // path reversible. + const content = turn === null + ? `${STRUCTURED_LOG_SENTINEL}\n\n` + : `${STRUCTURED_LOG_SENTINEL}\n\n${serializeTurn(turn)}`; // Atomic write so a partial seed never lands on disk. const tmp = `${logPath}.tmp.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2)}`; const handle = await fsp.open(tmp, 'wx'); @@ -186,6 +221,34 @@ export function createDailyLogWriter(opts: DailyLogWriterOptions): DailyLogWrite await seedFreshLog(turn); } + async function doMigrateIfNeeded(): Promise { + // Eager-migration variant of doWrite. Same rotation rules, but the seed + // is sentinel-only (no turn frame). Idempotent: missing or already- + // structured logs are no-ops. Reused by MindMemoryService.activateMind + // so a user who flips the dream-daemon switch sees their pre-existing + // unstructured log preserved as log.legacy.md immediately, without + // waiting for the next chat turn. + await fsp.mkdir(workingMemoryDir, { recursive: true }); + + const existing = await readOrNull(logPath); + + // No log.md → no-op. The first write() will seed. + if (existing === null) return; + + // Empty file → seed sentinel-only so subsequent reads see a valid log. + if (existing.length === 0) { + await seedFreshLog(null); + return; + } + + // Already structured → idempotent no-op. + if (detectSentinel(existing)) return; + + // Unstructured → rotate, then seed sentinel-only. + await rotate(existing); + await seedFreshLog(null); + } + function write(turn: CompletedTurn): Promise { const next = chain.then(async () => { await doWrite(turn); @@ -200,7 +263,21 @@ export function createDailyLogWriter(opts: DailyLogWriterOptions): DailyLogWrite return next; } - return { write }; + function migrateIfNeeded(): Promise { + const next = chain.then(() => doMigrateIfNeeded()); + chain = next.catch(() => undefined); + return next; + } + + function flush(): Promise { + // Wait for the current chain tail to settle. The chain itself swallows + // rejections, so awaiting it never throws — flush is purely a quiescence + // barrier. Callers who care about per-write errors observe them via the + // promises returned by `write()` / `migrateIfNeeded()`. + return chain.then(() => undefined); + } + + return { write, migrateIfNeeded, flush }; } function isoStamp(): string { diff --git a/packages/services/src/mindMemory/MindMemoryService.test.ts b/packages/services/src/mindMemory/MindMemoryService.test.ts index 237de754..c776f3dc 100644 --- a/packages/services/src/mindMemory/MindMemoryService.test.ts +++ b/packages/services/src/mindMemory/MindMemoryService.test.ts @@ -708,3 +708,203 @@ describe('MindMemoryService — __debugGet (E2E accessor)', () => { expect(svc.__debugGet(MIND_ID)).toBeNull(); }); }); + +// --------------------------------------------------------------------------- +// v0.60.0 — Eager migration on activate (Phase 1) +// +// When a mind that was previously opted-out flips to opted-in, the user +// experience is "I clicked the switch and now my old freeform log is +// preserved as log.legacy.md and a fresh structured log was seeded". This +// must happen WITHOUT requiring a turn to land — otherwise the user sees no +// effect until they next chat with the mind, and the "what happens to my +// log" question stays scary. +// +// Implementation contract: `activateMind` for an opted-in mind invokes +// `writer.migrateIfNeeded()` BEFORE returning. Opted-out mind: never called +// (no writer is constructed at all per the strict-opt-in contract). +// +// Tests use a real filesystem because the writer is built inline inside +// `activateMind` (no writerFactory injection). The observable contract is +// log.md state after activate resolves. +// --------------------------------------------------------------------------- + +describe('MindMemoryService — activateMind: eager migration (Phase 1)', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('opted-in mind with pre-existing unstructured log.md → after activate, log.md is sentinel-only and log.legacy.md preserves the original', async () => { + const { mkdtempSync, rmSync, mkdirSync, writeFileSync, readFileSync, existsSync } = await import('node:fs'); + const { tmpdir } = await import('node:os'); + const { STRUCTURED_LOG_SENTINEL } = await import('./StructuredLogFormat'); + + const root = mkdtempSync(path.join(tmpdir(), 'chamber-mindmem-migrate-')); + const mindPath = path.join(root, 'mind-real'); + const wmDir = path.join(mindPath, '.working-memory'); + mkdirSync(wmDir, { recursive: true }); + + const original = '# legacy freeform notes\nrandom content\n'; + writeFileSync(path.join(wmDir, 'log.md'), original); + + const { factories } = makeFactories({ chamberConfig: ENABLED_CONFIG }); + const svc = createMindMemoryService(factories); + try { + await svc.activateMind(MIND_ID, mindPath); + + // Original content rotated out of the way. + expect(existsSync(path.join(wmDir, 'log.legacy.md'))).toBe(true); + expect(readFileSync(path.join(wmDir, 'log.legacy.md'), 'utf-8')).toBe(original); + + // log.md is sentinel-only (NO turn frame — migration ran before any turns). + expect(readFileSync(path.join(wmDir, 'log.md'), 'utf-8')).toBe( + STRUCTURED_LOG_SENTINEL + '\n\n', + ); + + await svc.releaseMind(MIND_ID); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('opted-in mind with sentinel log.md → activate is a no-op for migration (idempotent)', async () => { + const { mkdtempSync, rmSync, mkdirSync, writeFileSync, readFileSync, existsSync } = await import('node:fs'); + const { tmpdir } = await import('node:os'); + const { STRUCTURED_LOG_SENTINEL } = await import('./StructuredLogFormat'); + + const root = mkdtempSync(path.join(tmpdir(), 'chamber-mindmem-migrate-noop-')); + const mindPath = path.join(root, 'mind-real'); + const wmDir = path.join(mindPath, '.working-memory'); + mkdirSync(wmDir, { recursive: true }); + + const sentinelOnly = STRUCTURED_LOG_SENTINEL + '\n\n'; + writeFileSync(path.join(wmDir, 'log.md'), sentinelOnly); + + const { factories } = makeFactories({ chamberConfig: ENABLED_CONFIG }); + const svc = createMindMemoryService(factories); + try { + await svc.activateMind(MIND_ID, mindPath); + + expect(existsSync(path.join(wmDir, 'log.legacy.md'))).toBe(false); + expect(readFileSync(path.join(wmDir, 'log.md'), 'utf-8')).toBe(sentinelOnly); + + await svc.releaseMind(MIND_ID); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('opted-out mind with pre-existing unstructured log.md → activate does NOT touch log.md (no migration, no rotation)', async () => { + const { mkdtempSync, rmSync, mkdirSync, writeFileSync, readFileSync, existsSync } = await import('node:fs'); + const { tmpdir } = await import('node:os'); + + const root = mkdtempSync(path.join(tmpdir(), 'chamber-mindmem-migrate-disabled-')); + const mindPath = path.join(root, 'mind-real'); + const wmDir = path.join(mindPath, '.working-memory'); + mkdirSync(wmDir, { recursive: true }); + + const original = '# legacy freeform notes\nrandom content\n'; + writeFileSync(path.join(wmDir, 'log.md'), original); + + const { factories } = makeFactories({ chamberConfig: DISABLED_CONFIG }); + const svc = createMindMemoryService(factories); + try { + await svc.activateMind(MIND_ID, mindPath); + + // Untouched. + expect(existsSync(path.join(wmDir, 'log.legacy.md'))).toBe(false); + expect(readFileSync(path.join(wmDir, 'log.md'), 'utf-8')).toBe(original); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('opted-in mind with no log.md → activate creates the directory but does NOT seed log.md (migrateIfNeeded is a no-op for missing files)', async () => { + const { mkdtempSync, rmSync, mkdirSync, existsSync } = await import('node:fs'); + const { tmpdir } = await import('node:os'); + + const root = mkdtempSync(path.join(tmpdir(), 'chamber-mindmem-migrate-empty-')); + const mindPath = path.join(root, 'mind-real'); + mkdirSync(mindPath, { recursive: true }); + + const { factories } = makeFactories({ chamberConfig: ENABLED_CONFIG }); + const svc = createMindMemoryService(factories); + try { + await svc.activateMind(MIND_ID, mindPath); + + // migrateIfNeeded is a no-op when log.md does not exist. The first + // write() will seed the sentinel — until then, log.md stays absent. + expect(existsSync(path.join(mindPath, '.working-memory', 'log.md'))).toBe(false); + + await svc.releaseMind(MIND_ID); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); + +// --------------------------------------------------------------------------- +// v0.60.0 — Per-mindId activate/release serialization (Uncle Bob finding 2) +// +// `main.ts` wires MindManager events to MindMemoryService via fire-and-forget +// `.catch()` chains. A user who rapid-toggles the daemon switch (ON → OFF → +// ON within a few hundred ms) generates back-to-back activate/release calls +// that may interleave: activate#1 → release while activate#1 still running → +// activate#2 sees `active.has(mindId)` and no-ops. +// +// The fix: serialize per-mindId inside MindMemoryService so the second +// activate genuinely runs after the release completes. The composition root +// stays simple (still fire-and-forget); the service owns the invariant. +// --------------------------------------------------------------------------- + +describe('MindMemoryService — per-mindId activate/release serialization', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('rapid activate → release → activate for the same mindId all complete and end with the mind activated', async () => { + const { factories, chat, scheduler } = makeFactories({ chamberConfig: ENABLED_CONFIG }); + const svc = createMindMemoryService(factories); + + // Fire all three without awaiting between them — exactly the pattern + // main.ts's fire-and-forget event handlers produce on rapid toggle. + const p1 = svc.activateMind(MIND_ID, MIND_PATH); + const p2 = svc.releaseMind(MIND_ID); + const p3 = svc.activateMind(MIND_ID, MIND_PATH); + + await Promise.all([p1, p2, p3]); + + // End state: mind is activated exactly once. + expect(scheduler.registered.size).toBe(1); + expect(chat.observers).toHaveLength(1); + // dbFactory called twice (once per activate); daemonFactory called twice. + expect((factories.dbFactory as ReturnType).mock.calls.length).toBe(2); + expect((factories.daemonFactory as ReturnType).mock.calls.length).toBe(2); + }); + + it('rapid release → activate (when not active) for the same mindId end with the mind activated', async () => { + const { factories, chat, scheduler } = makeFactories({ chamberConfig: ENABLED_CONFIG }); + const svc = createMindMemoryService(factories); + + const p1 = svc.releaseMind(MIND_ID); // no-op (not active yet) + const p2 = svc.activateMind(MIND_ID, MIND_PATH); + + await Promise.all([p1, p2]); + + expect(scheduler.registered.size).toBe(1); + expect(chat.observers).toHaveLength(1); + }); + + it('serialization is per-mindId — independent minds run in parallel', async () => { + const { factories, chat, scheduler } = makeFactories({ chamberConfig: ENABLED_CONFIG }); + const svc = createMindMemoryService(factories); + + await Promise.all([ + svc.activateMind('mind-a', '/tmp/mind-a'), + svc.activateMind('mind-b', '/tmp/mind-b'), + svc.activateMind('mind-c', '/tmp/mind-c'), + ]); + + expect(scheduler.registered.size).toBe(3); + expect(chat.observers).toHaveLength(3); + }); +}); diff --git a/packages/services/src/mindMemory/MindMemoryService.ts b/packages/services/src/mindMemory/MindMemoryService.ts index b06e4f72..3611a639 100644 --- a/packages/services/src/mindMemory/MindMemoryService.ts +++ b/packages/services/src/mindMemory/MindMemoryService.ts @@ -142,7 +142,46 @@ export function createMindMemoryService( const active = new Map(); let closed = false; - async function activateMind(mindId: string, mindPath: string): Promise { + // Per-mindId serialization (Uncle Bob plan-review finding 2). The + // composition root wires `mindManager.on('mind:loaded', ctx => + // mindMemoryService.activateMind(...).catch(...))` — fire-and-forget. A + // user who rapid-toggles the dream-daemon switch (ON → OFF → ON) generates + // back-to-back activate/release events. With the eager-migration await + // added below, activate yields BEFORE calling `active.set`, opening a + // race window where release no-ops (mind not yet active) and the next + // activate's idempotency check no-ops too — leaving the mind in a stale + // state. Serializing here keeps the contract intact at the service + // layer, so the composition root can stay simple. + const lifecycleQueues = new Map>(); + + function enqueueLifecycle(mindId: string, fn: () => Promise): Promise { + const prior = lifecycleQueues.get(mindId) ?? Promise.resolve(); + const next = prior.then(fn, fn); + // Tail tracking — we keep the chain as a Promise that swallows + // rejections so a failed activate/release does not poison the queue. + const tail: Promise = next.then( + () => undefined, + () => undefined, + ); + lifecycleQueues.set(mindId, tail); + // Best-effort cleanup once the queue is fully drained for this mind. + void tail.then(() => { + if (lifecycleQueues.get(mindId) === tail) { + lifecycleQueues.delete(mindId); + } + }); + return next; + } + + function activateMind(mindId: string, mindPath: string): Promise { + return enqueueLifecycle(mindId, () => activateMindInner(mindId, mindPath)); + } + + function releaseMind(mindId: string): Promise { + return enqueueLifecycle(mindId, () => releaseMindInner(mindId)); + } + + async function activateMindInner(mindId: string, mindPath: string): Promise { if (closed) { throw new Error('MindMemoryService is closed'); } @@ -202,6 +241,14 @@ export function createMindMemoryService( }, }, }); + + // Eager migration (v0.60.0 Phase 1). When a mind that previously + // opted out flips ON, the user expects their freeform log.md to be + // preserved as log.legacy.md and a fresh sentinel-only log to be + // seeded — without waiting for the next chat turn. Idempotent for + // already-structured logs; no-op for missing log.md. + await writer.migrateIfNeeded(); + observer = { onTurnCompleted: (turn: CompletedTurn) => writer.write(turn), }; @@ -247,7 +294,7 @@ export function createMindMemoryService( } } - async function releaseMind(mindId: string): Promise { + async function releaseMindInner(mindId: string): Promise { const entry = active.get(mindId); if (!entry) return; // Drop the map entry FIRST so a teardown failure doesn't leave a diff --git a/packages/services/src/mindMemory/StructuredLogFormat.test.ts b/packages/services/src/mindMemory/StructuredLogFormat.test.ts index 71447873..0585d4f6 100644 --- a/packages/services/src/mindMemory/StructuredLogFormat.test.ts +++ b/packages/services/src/mindMemory/StructuredLogFormat.test.ts @@ -223,6 +223,18 @@ describe('parseLog — turn parsing', () => { expect(out.turns).toHaveLength(0); }); + it('accepts a frame with an empty model: line (model selection may not be set at turn time)', () => { + const frame = + '## 2026-05-12T15:00:00Z turn:11111111-1111-4111-8111-111111111111 status:completed\n' + + 'session: sess-abc\n' + + 'model: \n' + + '\n### user\nq\n\n### assistant\na\n'; + const out = parseLog(SENTINEL + '\n' + frame); + expect(out.malformed).toBe(0); + expect(out.turns).toHaveLength(1); + expect(out.turns[0].model).toBe(''); + }); + it('drops a block with malformed timestamp', () => { const bad = '## not-a-date turn:11111111-1111-4111-8111-111111111111 status:completed\n' + diff --git a/packages/services/src/mindMemory/StructuredLogFormat.ts b/packages/services/src/mindMemory/StructuredLogFormat.ts index 9e3416d6..e1265f60 100644 --- a/packages/services/src/mindMemory/StructuredLogFormat.ts +++ b/packages/services/src/mindMemory/StructuredLogFormat.ts @@ -130,7 +130,11 @@ function parseBlock(blockLines: readonly string[]): ParsedTurn | null { if (blockLines.length < 3) return null; const sessionMatch = blockLines[1].match(/^session: (.+)$/); - const modelMatch = blockLines[2].match(/^model: (.+)$/); + // `model:` accepts an empty value because the writer may not have a model + // selected at turn time (ChatService falls back to '' when both the + // turn-time model and the mind's selectedModel are unset). A frame with + // an empty model is still valid; we just record an empty string. + const modelMatch = blockLines[2].match(/^model: (.*)$/); if (!sessionMatch || !modelMatch) return null; const sessionId = sessionMatch[1]; const model = modelMatch[1]; diff --git a/packages/services/src/mindMemory/rollback.test.ts b/packages/services/src/mindMemory/rollback.test.ts new file mode 100644 index 00000000..e7b148bf --- /dev/null +++ b/packages/services/src/mindMemory/rollback.test.ts @@ -0,0 +1,234 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { + STRUCTURED_LOG_SENTINEL, + serializeTurn, + type CompletedTurn, +} from './StructuredLogFormat'; +import { rollbackToUnstructured } from './rollback'; + +const WORKING_MEMORY = '.working-memory'; +const LOG_FILE = 'log.md'; +const LEGACY_FILE = 'log.legacy.md'; + +function turnAt(seq: number): CompletedTurn { + return { + turnId: `turn-${seq}`, + sessionId: `session-${seq}`, + model: 'gpt-test', + status: 'completed', + startedAt: `2026-04-${String(seq).padStart(2, '0')}T11:59:00.000Z`, + endedAt: `2026-04-${String(seq).padStart(2, '0')}T12:00:00.000Z`, + prompt: `User prompt #${seq}`, + finalAssistantMessage: `Assistant reply #${seq}`, + }; +} + +describe('rollbackToUnstructured', () => { + let tmpDir: string; + let workingMemoryDir: string; + let logPath: string; + let legacyPath: string; + const fixedNow = new Date('2026-04-15T08:00:00.000Z'); + const captured: string[] = []; + const testLogger = { + info: (m: string) => captured.push(`INFO ${m}`), + warn: (m: string) => captured.push(`WARN ${m}`), + }; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-rollback-')); + workingMemoryDir = path.join(tmpDir, WORKING_MEMORY); + fs.mkdirSync(workingMemoryDir, { recursive: true }); + logPath = path.join(workingMemoryDir, LOG_FILE); + legacyPath = path.join(workingMemoryDir, LEGACY_FILE); + captured.length = 0; + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + function seedStructuredLog(turns: readonly CompletedTurn[]): void { + const body = turns.map(serializeTurn).join(''); + fs.writeFileSync(logPath, `${STRUCTURED_LOG_SENTINEL}\n\n${body}`, 'utf-8'); + } + + it('converts N structured frames into rendered markdown and removes the sentinel', async () => { + seedStructuredLog([turnAt(1), turnAt(2), turnAt(3)]); + + const result = await rollbackToUnstructured(tmpDir, { now: () => fixedNow }); + + expect(result).toEqual({ + framesConverted: 3, + legacyExisted: false, + outcome: 'rolled-back', + }); + + const after = fs.readFileSync(logPath, 'utf-8'); + expect(after).not.toContain(STRUCTURED_LOG_SENTINEL); + expect(after).toContain('## Resumed unstructured logging — 2026-04-15T08:00:00.000Z'); + expect(after).toContain('## 2026-04-01T12:00:00.000Z — turn turn-1 (gpt-test)'); + expect(after).toContain('**User**: User prompt #1'); + expect(after).toContain('**Assistant**: Assistant reply #1'); + expect(after).toContain('**User**: User prompt #3'); + expect(after).toContain('**Assistant**: Assistant reply #3'); + }); + + it('folds existing log.legacy.md content into the merged output and removes the legacy file', async () => { + fs.writeFileSync(legacyPath, '# Legacy notes\n\nFirst-ever line.\n', 'utf-8'); + seedStructuredLog([turnAt(7)]); + + const result = await rollbackToUnstructured(tmpDir, { now: () => fixedNow, logger: testLogger }); + + expect(result).toEqual({ + framesConverted: 1, + legacyExisted: true, + outcome: 'rolled-back', + }); + + const after = fs.readFileSync(logPath, 'utf-8'); + expect(after.startsWith('# Legacy notes\n\nFirst-ever line.')).toBe(true); + expect(after).toContain('---'); + expect(after).toContain('## Resumed unstructured logging'); + expect(after).toContain('**User**: User prompt #7'); + expect(fs.existsSync(legacyPath)).toBe(false); + }); + + it('is a no-op when log.md is missing', async () => { + const result = await rollbackToUnstructured(tmpDir); + expect(result).toEqual({ + framesConverted: 0, + legacyExisted: false, + outcome: 'no-op-missing', + }); + }); + + it('is a no-op when log.md is present but empty', async () => { + fs.writeFileSync(logPath, '', 'utf-8'); + const result = await rollbackToUnstructured(tmpDir); + expect(result).toEqual({ + framesConverted: 0, + legacyExisted: false, + outcome: 'no-op-empty', + }); + expect(fs.readFileSync(logPath, 'utf-8')).toBe(''); + }); + + it('warns and leaves the file untouched when log.md has no sentinel (already unstructured)', async () => { + const original = '# Already unstructured\n\nUser said something.\n'; + fs.writeFileSync(logPath, original, 'utf-8'); + + const result = await rollbackToUnstructured(tmpDir, { logger: testLogger }); + + expect(result).toEqual({ + framesConverted: 0, + legacyExisted: false, + outcome: 'no-op-no-sentinel', + }); + expect(fs.readFileSync(logPath, 'utf-8')).toBe(original); + expect(captured.some((m) => m.startsWith('WARN') && /no sentinel/.test(m))).toBe(true); + }); + + it('is idempotent — calling rollback twice yields the same content (second call is no-op-no-sentinel)', async () => { + seedStructuredLog([turnAt(1), turnAt(2)]); + + await rollbackToUnstructured(tmpDir, { now: () => fixedNow }); + const afterFirst = fs.readFileSync(logPath, 'utf-8'); + + const second = await rollbackToUnstructured(tmpDir, { now: () => fixedNow, logger: testLogger }); + + expect(second.outcome).toBe('no-op-no-sentinel'); + expect(fs.readFileSync(logPath, 'utf-8')).toBe(afterFirst); + }); + + it('Flow 4 regression: empty-model frame round-trips through rollback without data loss', async () => { + const turn: CompletedTurn = { + turnId: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', + sessionId: 'sess-empty-model', + model: '', + status: 'completed', + startedAt: '2026-04-01T11:59:00.000Z', + endedAt: '2026-04-01T12:00:00.000Z', + prompt: 'Important user content', + finalAssistantMessage: 'Important assistant content', + }; + const raw = `${STRUCTURED_LOG_SENTINEL}\n\n${serializeTurn(turn)}`; + fs.writeFileSync(logPath, raw, 'utf-8'); + + const result = await rollbackToUnstructured(tmpDir, { now: () => fixedNow }); + + expect(result.outcome).toBe('rolled-back'); + expect(result.framesConverted).toBe(1); + const after = fs.readFileSync(logPath, 'utf-8'); + expect(after).toContain('Important user content'); + expect(after).toContain('Important assistant content'); + expect(after).not.toContain(STRUCTURED_LOG_SENTINEL); + }); + + it('preserves log.md unchanged when sentinel is present but all frames are malformed (no-op-malformed)', async () => { + const malformedFrame = + '## not-a-date turn:11111111-1111-4111-8111-111111111111 status:completed\n' + + 'session: s\nmodel: m\n\n### user\nq\n\n### assistant\na\n'; + const original = `${STRUCTURED_LOG_SENTINEL}\n\n${malformedFrame}`; + fs.writeFileSync(logPath, original, 'utf-8'); + + const result = await rollbackToUnstructured(tmpDir, { now: () => fixedNow, logger: testLogger }); + + expect(result.outcome).toBe('no-op-malformed'); + expect(result.framesConverted).toBe(0); + expect(fs.readFileSync(logPath, 'utf-8')).toBe(original); + expect(captured.some((m) => m.startsWith('WARN') && /malformed/.test(m))).toBe(true); + }); + + it('atomicity: simulated rename failure leaves log.md and log.legacy.md byte-equal to prior state', async () => { + fs.writeFileSync(legacyPath, 'legacy bytes', 'utf-8'); + seedStructuredLog([turnAt(5)]); + const beforeLog = fs.readFileSync(logPath, 'utf-8'); + const beforeLegacy = fs.readFileSync(legacyPath, 'utf-8'); + + const failingRename = async () => { throw new Error('synthetic rename failure'); }; + + await expect( + rollbackToUnstructured(tmpDir, { now: () => fixedNow, rename: failingRename }), + ).rejects.toThrow(/synthetic rename failure/); + + expect(fs.readFileSync(logPath, 'utf-8')).toBe(beforeLog); + expect(fs.readFileSync(legacyPath, 'utf-8')).toBe(beforeLegacy); + const lingering = fs.readdirSync(workingMemoryDir).filter((entry) => entry.endsWith('.tmp')); + expect(lingering).toEqual([]); + }); + + it('zero-frames sentinel-only log rolls back to legacy content alone (no spurious "Resumed" header)', async () => { + fs.writeFileSync(legacyPath, 'pre-existing legacy notes', 'utf-8'); + fs.writeFileSync(logPath, `${STRUCTURED_LOG_SENTINEL}\n\n`, 'utf-8'); + + const result = await rollbackToUnstructured(tmpDir, { now: () => fixedNow }); + + expect(result).toEqual({ + framesConverted: 0, + legacyExisted: true, + outcome: 'rolled-back', + }); + + const after = fs.readFileSync(logPath, 'utf-8'); + expect(after).not.toContain('Resumed unstructured logging'); + expect(after).toContain('pre-existing legacy notes'); + expect(fs.existsSync(legacyPath)).toBe(false); + }); + + it('zero-frames sentinel-only log with no legacy yields an empty log.md (no spurious header)', async () => { + fs.writeFileSync(logPath, `${STRUCTURED_LOG_SENTINEL}\n\n`, 'utf-8'); + + const result = await rollbackToUnstructured(tmpDir, { now: () => fixedNow }); + + expect(result).toEqual({ + framesConverted: 0, + legacyExisted: false, + outcome: 'rolled-back', + }); + expect(fs.readFileSync(logPath, 'utf-8')).toBe(''); + }); +}); diff --git a/packages/services/src/mindMemory/rollback.ts b/packages/services/src/mindMemory/rollback.ts new file mode 100644 index 00000000..f80801d1 --- /dev/null +++ b/packages/services/src/mindMemory/rollback.ts @@ -0,0 +1,199 @@ +// Phase 4 — opt-out rollback for the dream daemon. Converts a structured +// (sentinel-prefixed) `log.md` back into freeform markdown and folds in any +// pre-existing `log.legacy.md` content so the user is left with a single +// human-readable file. Designed to run AFTER `MindManager.reloadMind` has +// torn down the writer/observer for the mind, so there's no concurrent +// writer racing the rewrite. + +import * as fs from 'fs'; +import * as fsp from 'fs/promises'; +import * as path from 'path'; +import { Logger } from '../logger'; +import { + parseLog, + type ParsedTurn, +} from './StructuredLogFormat'; + +const WORKING_MEMORY_DIRNAME = '.working-memory'; +const LOG_FILENAME = 'log.md'; +const LEGACY_FILENAME = 'log.legacy.md'; + +export interface RollbackResult { + /** Number of structured frames successfully converted to unstructured markdown. */ + framesConverted: number; + /** + * True if `log.legacy.md` was present (and thus folded into the merged log.md). + * **Only meaningful when `outcome === 'rolled-back'`** — no-op outcomes always report `false` + * without checking the filesystem. + */ + legacyExisted: boolean; + /** + * One of: + * - `'no-op-missing'` — log.md absent. + * - `'no-op-empty'` — log.md present but zero bytes. + * - `'no-op-no-sentinel'` — log.md present but not structured (already unstructured). + * - `'no-op-malformed'` — log.md has sentinel + non-empty body but parser produced + * zero turns (all frames malformed). File is preserved byte-identical to avoid + * data loss; toggle is still successful at the config level. + * - `'rolled-back'` — log.md was structured and was rewritten. + */ + outcome: 'no-op-missing' | 'no-op-empty' | 'no-op-no-sentinel' | 'no-op-malformed' | 'rolled-back'; +} + +export interface RollbackLogger { + info(message: string): void; + warn(message: string, ...rest: unknown[]): void; +} + +export interface RollbackDeps { + logger?: RollbackLogger; + /** Override for tests that need to simulate a rename failure. */ + rename?: (from: string, to: string) => Promise; + /** Override clock for deterministic merged-section header timestamps in tests. */ + now?: () => Date; +} + +function isErrnoCode(err: unknown, code: string): boolean { + return typeof err === 'object' && err !== null && (err as { code?: string }).code === code; +} + +async function readOrNull(absPath: string): Promise { + try { + return await fsp.readFile(absPath, 'utf-8'); + } catch (err) { + if (isErrnoCode(err, 'ENOENT')) return null; + throw err; + } +} + +function renderTurnAsMarkdown(turn: ParsedTurn): string { + // Format approved per plan.md Phase 4 spec: + // ## {ISO} — turn {turnId} ({model}) + // **User**: {prompt} + // + // **Assistant**: {finalAssistantMessage} + return [ + `## ${turn.timestamp} — turn ${turn.turnId} (${turn.model})`, + '', + `**User**: ${turn.prompt}`, + '', + `**Assistant**: ${turn.assistant}`, + '', + ].join('\n'); +} + +function composeMergedContent( + legacyContent: string | null, + turns: readonly ParsedTurn[], + resumedAt: string, +): string { + // Zero-frames sentinel-only rollback: don't emit a "Resumed" header that + // claims content was resumed when nothing was. Just preserve legacy (or + // empty file) and let the caller move on. + if (turns.length === 0) { + if (legacyContent && legacyContent.length > 0) { + return legacyContent.endsWith('\n') ? legacyContent : `${legacyContent}\n`; + } + return ''; + } + + const renderedFrames = turns.map(renderTurnAsMarkdown).join('\n'); + const resumedSection = `## Resumed unstructured logging — ${resumedAt}\n\n${renderedFrames}`; + + if (legacyContent && legacyContent.length > 0) { + const legacyTrimmed = legacyContent.endsWith('\n') ? legacyContent.slice(0, -1) : legacyContent; + return `${legacyTrimmed}\n\n---\n\n${resumedSection}`; + } + + return resumedSection; +} + +export async function rollbackToUnstructured( + mindPath: string, + deps: RollbackDeps = {}, +): Promise { + const log: RollbackLogger = deps.logger ?? Logger.create('rollbackToUnstructured'); + const rename = deps.rename ?? fsp.rename; + const now = deps.now ?? (() => new Date()); + + const workingMemoryDir = path.resolve(mindPath, WORKING_MEMORY_DIRNAME); + const logPath = path.join(workingMemoryDir, LOG_FILENAME); + const legacyPath = path.join(workingMemoryDir, LEGACY_FILENAME); + + const currentContent = await readOrNull(logPath); + if (currentContent === null) { + return { framesConverted: 0, legacyExisted: false, outcome: 'no-op-missing' }; + } + if (currentContent.length === 0) { + return { framesConverted: 0, legacyExisted: false, outcome: 'no-op-empty' }; + } + + const parsed = parseLog(currentContent); + if (!parsed.sentinel) { + log.warn( + `rollbackToUnstructured: log.md at ${logPath} has no sentinel — already unstructured. Leaving file untouched.`, + ); + return { framesConverted: 0, legacyExisted: false, outcome: 'no-op-no-sentinel' }; + } + + // Sentinel-with-content-but-no-parseable-frames: preserve the raw file + // to avoid data loss. The user's chat history is in there (just unparseable); + // overwriting with an empty file would destroy it. The toggle still + // succeeds at the config level — this branch only refuses the rewrite. + if (parsed.turns.length === 0 && parsed.malformed > 0) { + log.warn( + `rollbackToUnstructured: log.md at ${logPath} has ${parsed.malformed} malformed frame(s) and no parseable turns. Preserving file as-is to avoid data loss.`, + ); + return { framesConverted: 0, legacyExisted: false, outcome: 'no-op-malformed' }; + } + + const legacyContent = await readOrNull(legacyPath); + const legacyExisted = legacyContent !== null; + + const merged = composeMergedContent(legacyContent, parsed.turns, now().toISOString()); + + // Atomic rewrite: write to tmp, fsync, rename. If anything fails, the + // original log.md and log.legacy.md remain byte-identical. + const tmpPath = `${logPath}.rollback.${process.pid}.${Date.now()}.tmp`; + const handle = await fsp.open(tmpPath, 'wx'); + try { + await handle.writeFile(merged, 'utf-8'); + await handle.sync(); + } finally { + // SF-3: post-sync close is virtually infallible (data is on disk), but a + // throw here would propagate and skip the rename. Swallow defensively so + // a phantom close failure doesn't poison a successful write. + await handle.close().catch(() => { /* fd will be released by GC */ }); + } + + try { + await rename(tmpPath, logPath); + } catch (err) { + if (fs.existsSync(tmpPath)) { + try { fs.rmSync(tmpPath, { force: true }); } catch { /* best-effort */ } + } + throw err; + } + + if (legacyExisted) { + try { + await fsp.unlink(legacyPath); + } catch (err) { + // Non-fatal — the merged log.md already contains the legacy content. + // We log so the operator knows the file is orphaned, but we don't + // re-raise: rollback succeeded from the user's perspective. + log.warn(`rollbackToUnstructured: failed to remove ${legacyPath} after merge:`, err); + } + } + + log.info( + `rollbackToUnstructured: converted ${parsed.turns.length} frame(s) for ${mindPath}` + + (legacyExisted ? ' (legacy log folded in)' : ''), + ); + + return { + framesConverted: parsed.turns.length, + legacyExisted, + outcome: 'rolled-back', + }; +} diff --git a/packages/services/src/mindProfile/MindProfileService.test.ts b/packages/services/src/mindProfile/MindProfileService.test.ts index 09156284..5542ef68 100644 --- a/packages/services/src/mindProfile/MindProfileService.test.ts +++ b/packages/services/src/mindProfile/MindProfileService.test.ts @@ -105,6 +105,30 @@ describe('MindProfileService', () => { fs.rmSync(root, { recursive: true, force: true }); } }); + + it('exposes dreamDaemonEnabled=false when no .chamber.json is present', () => { + const { root, service } = createProfileFixture(); + try { + const profile = service.getProfile('mind-1'); + expect(profile.dreamDaemonEnabled).toBe(false); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it('exposes dreamDaemonEnabled=true when .chamber.json opts in to consolidation', () => { + const { root, service } = createProfileFixture(); + try { + fs.writeFileSync( + path.join(root, '.chamber.json'), + JSON.stringify({ workingMemory: { consolidation: { enabled: true } } }), + ); + const profile = service.getProfile('mind-1'); + expect(profile.dreamDaemonEnabled).toBe(true); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); }); function createProfileFixture() { diff --git a/packages/services/src/mindProfile/MindProfileService.ts b/packages/services/src/mindProfile/MindProfileService.ts index 16a1fdf7..c21fd7b8 100644 --- a/packages/services/src/mindProfile/MindProfileService.ts +++ b/packages/services/src/mindProfile/MindProfileService.ts @@ -10,6 +10,7 @@ import type { AgentProfileSaveResult, } from '@chamber/shared/types'; import type { IdentityLoader } from '../chat/IdentityLoader'; +import { loadChamberMindConfig } from '../mind/chamberMindConfig'; import type { AvatarNormalizer, MindProfileMindProvider } from './types'; const AVATAR_RELATIVE_PATH = path.join('.chamber', 'avatar.png'); @@ -27,6 +28,7 @@ export class MindProfileService { const identity = this.identityLoader.load(mindPath); const displayName = identity?.name ?? path.basename(mindPath); const avatarPath = path.join(mindPath, AVATAR_RELATIVE_PATH); + const chamberConfig = loadChamberMindConfig(mindPath); return { mindId, @@ -37,6 +39,7 @@ export class MindProfileService { soul: this.readProfileFile(mindPath, 'soul', 'SOUL.md'), agentFiles: this.listAgentFiles(mindPath), needsRestart, + dreamDaemonEnabled: chamberConfig.workingMemory.consolidation.enabled, }; } diff --git a/packages/shared/src/electron-types.ts b/packages/shared/src/electron-types.ts index c0b061ce..e75db637 100644 --- a/packages/shared/src/electron-types.ts +++ b/packages/shared/src/electron-types.ts @@ -64,6 +64,7 @@ export interface ElectronAPI { list: () => Promise; setActive: (mindId: string) => Promise; setModel: (mindId: string, model: string | null) => Promise; + setDreamDaemon: (mindId: string, enabled: boolean) => Promise; selectDirectory: () => Promise; openWindow: (mindId: string) => Promise; onMindChanged: (callback: (minds: MindContext[]) => void) => () => void; @@ -100,7 +101,7 @@ export interface ElectronAPI { getDefaultPath: () => Promise; pickPath: () => Promise; listTemplates: () => Promise; - create: (config: { name: string; role: string; voice: string; voiceDescription: string; basePath: string }) => Promise<{ success: boolean; mindId?: string; mindPath?: string; error?: string }>; + create: (config: { name: string; role: string; voice: string; voiceDescription: string; basePath: string; enableDreamDaemon?: boolean }) => Promise<{ success: boolean; mindId?: string; mindPath?: string; error?: string }>; createFromTemplate: (request: { templateId: string; marketplaceId?: string; basePath: string }) => Promise<{ success: boolean; mindId?: string; mindPath?: string; error?: string }>; onProgress: (callback: (progress: { step: string; detail: string }) => void) => () => void; }; diff --git a/packages/shared/src/ipc-channels.ts b/packages/shared/src/ipc-channels.ts index 25b3f56d..bf5f1f3a 100644 --- a/packages/shared/src/ipc-channels.ts +++ b/packages/shared/src/ipc-channels.ts @@ -32,6 +32,7 @@ export const IPC = { LIST: 'mind:list', SET_ACTIVE: 'mind:setActive', SET_MODEL: 'mind:setModel', + SET_DREAM_DAEMON: 'mind:setDreamDaemon', SELECT_DIRECTORY: 'mind:selectDirectory', OPEN_WINDOW: 'mind:openWindow', CHANGED: 'mind:changed', diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 8abebb49..454c4b62 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -156,6 +156,13 @@ export interface AgentProfile { soul: AgentProfileFile; agentFiles: AgentProfileFile[]; needsRestart: boolean; + /** + * True when this mind has opted in to the dream-daemon working-memory + * consolidation pipeline (`.chamber.json` → + * `workingMemory.consolidation.enabled`). Defaults to false when no + * `.chamber.json` is present, matching `loadChamberMindConfig`. + */ + dreamDaemonEnabled: boolean; } export interface AgentProfileSaveRequest { diff --git a/tests/e2e/electron/dream-daemon-bidir.spec.ts b/tests/e2e/electron/dream-daemon-bidir.spec.ts new file mode 100644 index 00000000..7b7fea3a --- /dev/null +++ b/tests/e2e/electron/dream-daemon-bidir.spec.ts @@ -0,0 +1,647 @@ +import { expect, test } from '@playwright/test'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { findRendererPage, launchElectronApp, type LaunchedElectronApp } from './electronApp'; + +// v0.60.0 dream-daemon opt-in UX + bidirectional migration validation. +// +// One spec, four flows. Mirrors the spec in the validation request: +// Flow 1 — Genesis OFF (default Switch left untouched) +// Flow 2 — Genesis ON (Switch toggled before "That's my purpose") +// Flow 3 — Post-genesis OFF→ON via profile modal +// Flow 4 — Post-genesis ON→OFF via profile modal (rollback path) +// +// Real Copilot SDK is required for Genesis (SOUL.md generation) and chat +// turns. Skipped unless CHAMBER_E2E_LIVE_GENESIS=1, exactly like the +// existing genesis-ernest-chat smoke. CHAMBER_LOG_LEVEL=debug is set so +// MindMemoryService activate/release no-op debug lines are visible. + +const cdpPort = Number(process.env.CHAMBER_E2E_DREAM_DAEMON_CDP_PORT ?? 9351); +const liveGenesisEnabled = process.env.CHAMBER_E2E_LIVE_GENESIS === '1'; + +const offMindName = 'OffMind'; +const onMindName = 'OnMind'; +const offSlug = 'offmind'; +const onSlug = 'onmind'; + +interface FlowEvidence { + flow: string; + passed: boolean; + failures: string[]; + notes: string[]; +} + +test.describe('electron dream-daemon bidirectional toggle smoke', () => { + test.skip(!liveGenesisEnabled, 'Set CHAMBER_E2E_LIVE_GENESIS=1 to run the live dream-daemon smoke.'); + test.setTimeout(45 * 60_000); + + let app: LaunchedElectronApp | undefined; + let userDataPath = ''; + let genesisBasePath = ''; + let offMindPath = ''; + let onMindPath = ''; + const tempRoots: string[] = []; + + test.beforeAll(async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-dream-daemon-bidir-')); + userDataPath = path.join(root, 'user-data'); + genesisBasePath = path.join(root, 'agents'); + offMindPath = path.join(genesisBasePath, offSlug); + onMindPath = path.join(genesisBasePath, onSlug); + tempRoots.push(root); + + app = await launchElectronApp({ + cdpPort, + env: { + CHAMBER_E2E_USER_DATA: userDataPath, + CHAMBER_E2E_GENESIS_BASE_PATH: genesisBasePath, + CHAMBER_LOG_LEVEL: 'debug', + }, + }); + }); + + test.afterAll(async () => { + await app?.close(); + for (const root of tempRoots) { + await removeTempRoot(root); + } + }); + + test('OFF/ON Switch on Genesis + post-genesis bidirectional toggle with rollback', async () => { + const page = await findRendererPage(app?.browser, app?.logs ?? []); + await page.waitForLoadState('domcontentloaded'); + + const consoleMessages: Array<{ type: string; text: string }> = []; + page.on('console', (message) => { + consoleMessages.push({ type: message.type(), text: message.text() }); + }); + const renderedConsoleErrors = (): string[] => consoleMessages.filter((m) => m.type === 'error').map((m) => m.text); + + const evidence: FlowEvidence[] = []; + + // --------------------------------------------------------------------- + // Flow 1 — Genesis with daemon Switch left OFF (default) + // --------------------------------------------------------------------- + const flow1Snapshot = snapshotLogs(app); + const flow1: FlowEvidence = { flow: 'Flow 1 (Genesis OFF)', passed: true, failures: [], notes: [] }; + evidence.push(flow1); + + try { + await driveGenesisCustom(page, { name: offMindName, voiceDescription: 'a calm, methodical operator', purpose: 'Operator', toggleDaemon: false }); + await waitForMindByName(page, offMindName); + + // Disk: log.md exists and is empty (sentinel never seeded). .chamber.json + // is absent — MindScaffold writes it ONLY on opt-in. + const offLogPath = path.join(offMindPath, '.working-memory', 'log.md'); + const offChamberJson = path.join(offMindPath, '.chamber.json'); + const offLogContent = readFileOrNull(offLogPath); + flow1.notes.push(`log.md exists: ${offLogContent !== null}, length: ${offLogContent?.length ?? 'n/a'}`); + flow1.notes.push(`.chamber.json exists: ${fs.existsSync(offChamberJson)}`); + if (offLogContent === null) flow1.failures.push('log.md missing for OFF mind'); + if (offLogContent && offLogContent.length !== 0) { + flow1.failures.push(`log.md is not empty (${offLogContent.length} bytes) — sentinel may have been seeded`); + } + // chamber.json is allowed to be present with enabled=false OR absent. Per + // current implementation it is absent (and loadChamberMindConfig defaults + // to enabled=false). + if (fs.existsSync(offChamberJson)) { + const cfg = JSON.parse(fs.readFileSync(offChamberJson, 'utf-8')); + const enabled = cfg?.workingMemory?.consolidation?.enabled; + flow1.notes.push(`.chamber.json content: enabled=${enabled}`); + if (enabled === true) flow1.failures.push('.chamber.json says enabled=true after Genesis OFF'); + } + + // Dream db must NOT exist + const offDreamDb = path.join(offMindPath, '.working-memory', '.state', 'dream.db'); + flow1.notes.push(`dream.db exists: ${fs.existsSync(offDreamDb)}`); + if (fs.existsSync(offDreamDb)) flow1.failures.push('dream.db should not exist for OFF mind'); + + // Console: NO MindMemoryService activation logs reference the off mind path. + // Activate success path is silent so absence is the assertion. + const flow1Logs = logsSince(app, flow1Snapshot); + const offActivationLines = flow1Logs.filter((l) => /\[MindMemoryService\]/i.test(l) && l.includes(offMindPath)); + flow1.notes.push(`MindMemoryService log lines mentioning OFF mind path: ${offActivationLines.length}`); + if (offActivationLines.some((l) => /already activated|activate/i.test(l))) { + flow1.failures.push(`Unexpected MindMemoryService activate trace for OFF mind: ${offActivationLines.join(' | ')}`); + } + + // ARIA: open profile modal, switch should be OFF + const offMindContext = await getMindContext(page, offMindName); + flow1.notes.push(`OFF mindId: ${offMindContext.mindId}`); + await openProfileModal(page, offMindName); + const offSwitchInitial = await readSwitchAria(page); + flow1.notes.push(`Profile Switch initial aria-checked for OFF mind: ${offSwitchInitial}`); + if (offSwitchInitial !== 'false') { + flow1.failures.push(`Profile Switch aria-checked expected "false" but was "${offSwitchInitial}"`); + } + await closeProfileModal(page); + } catch (err) { + flow1.failures.push(`Exception: ${err instanceof Error ? err.stack ?? err.message : String(err)}`); + } finally { + flow1.passed = flow1.failures.length === 0; + reportFlow(flow1); + } + + // --------------------------------------------------------------------- + // Flow 2 — Genesis with daemon Switch toggled ON + // --------------------------------------------------------------------- + const flow2Snapshot = snapshotLogs(app); + const flow2: FlowEvidence = { flow: 'Flow 2 (Genesis ON)', passed: true, failures: [], notes: [] }; + evidence.push(flow2); + + try { + await driveGenesisCustom(page, { name: onMindName, voiceDescription: 'a precise, observant analyst', purpose: 'Analyst', toggleDaemon: true }); + await waitForMindByName(page, onMindName); + + // Disk: log.md must start with sentinel and .chamber.json must say enabled=true + const onLogPath = path.join(onMindPath, '.working-memory', 'log.md'); + const onChamberJson = path.join(onMindPath, '.chamber.json'); + const onLogContent = readFileOrNull(onLogPath); + flow2.notes.push(`log.md exists: ${onLogContent !== null}, starts with sentinel: ${(onLogContent ?? '').startsWith('')}`); + if (!(onLogContent ?? '').startsWith('')) { + flow2.failures.push('log.md missing sentinel after Genesis ON'); + } + if (!fs.existsSync(onChamberJson)) flow2.failures.push('.chamber.json missing after Genesis ON'); + else { + const cfg = JSON.parse(fs.readFileSync(onChamberJson, 'utf-8')); + flow2.notes.push(`.chamber.json: ${JSON.stringify(cfg)}`); + if (cfg?.workingMemory?.consolidation?.enabled !== true) { + flow2.failures.push(`.chamber.json consolidation.enabled expected true but was ${JSON.stringify(cfg?.workingMemory?.consolidation?.enabled)}`); + } + } + + // dream.db should be created on activation + const onDreamDb = path.join(onMindPath, '.working-memory', '.state', 'dream.db'); + flow2.notes.push(`dream.db exists after Genesis ON: ${fs.existsSync(onDreamDb)}`); + if (!fs.existsSync(onDreamDb)) flow2.failures.push('dream.db not created after Genesis ON activation'); + + // ARIA: profile Switch should be ON + const onMindContext = await getMindContext(page, onMindName); + flow2.notes.push(`ON mindId: ${onMindContext.mindId}`); + await openProfileModal(page, onMindName); + const onSwitchInitial = await readSwitchAria(page); + flow2.notes.push(`Profile Switch initial aria-checked for ON mind: ${onSwitchInitial}`); + if (onSwitchInitial !== 'true') flow2.failures.push(`Profile Switch aria-checked expected "true" but was "${onSwitchInitial}"`); + await closeProfileModal(page); + + // Send a chat turn via the IPC bridge — exercises real SDK + DailyLogWriter + const chatResult = await sendOneShotTurn(page, onMindContext.mindId, 'Reply with the single word: ACK'); + flow2.notes.push(`chat assistantText length=${chatResult.assistantText.length}, error=${chatResult.errorMessage || ''}, doneCount=${chatResult.doneCount}`); + if (chatResult.errorMessage) flow2.failures.push(`Chat turn failed: ${chatResult.errorMessage}`); + + // Allow DailyLogWriter to flush (write happens async after done event). + await delay(1500); + const onLogAfterChat = readFileOrNull(onLogPath) ?? ''; + flow2.notes.push(`log.md size after chat: ${onLogAfterChat.length}`); + flow2.notes.push(`log.md preview: ${preview(onLogAfterChat)}`); + if (!onLogAfterChat.startsWith('')) { + flow2.failures.push('log.md no longer starts with sentinel after chat turn (corrupted?)'); + } + if (!/\n### user\n/.test(onLogAfterChat)) flow2.failures.push('log.md missing "### user" frame marker'); + if (!/\n### assistant\n/.test(onLogAfterChat)) flow2.failures.push('log.md missing "### assistant" frame marker'); + + // No console errors emerged from the renderer during chat + const errs = renderedConsoleErrors(); + if (errs.length > 0) flow2.notes.push(`renderer console errors: ${errs.length} (first: ${errs[0].slice(0, 200)})`); + + // Capture relevant main-process log lines for the report. + const flow2Logs = logsSince(app, flow2Snapshot); + flow2.notes.push(`relevant main logs (sample): ${sampleRelevant(flow2Logs).join(' | ')}`); + } catch (err) { + flow2.failures.push(`Exception: ${err instanceof Error ? err.stack ?? err.message : String(err)}`); + } finally { + flow2.passed = flow2.failures.length === 0; + reportFlow(flow2); + } + + // --------------------------------------------------------------------- + // Flow 3 — toggle the OFF mind ON via profile modal + // --------------------------------------------------------------------- + const flow3Snapshot = snapshotLogs(app); + const flow3: FlowEvidence = { flow: 'Flow 3 (Post-genesis OFF→ON)', passed: true, failures: [], notes: [] }; + evidence.push(flow3); + + try { + const offMindContext = await getMindContext(page, offMindName); + // Set up a renderer-side mind:changed counter so we can attribute the reload sequence. + await page.evaluate((mindId) => { + const w = window as unknown as { + __chamberDreamTest?: { + unsubscribe?: () => void; + mindId?: string; + events: Array<{ ts: number; presentMindIds: string[] }>; + }; + }; + if (w.__chamberDreamTest?.unsubscribe) w.__chamberDreamTest.unsubscribe(); + const events: Array<{ ts: number; presentMindIds: string[] }> = []; + const unsubscribe = window.electronAPI.mind.onMindChanged((minds: { mindId: string }[]) => { + events.push({ ts: Date.now(), presentMindIds: minds.map((m) => m.mindId) }); + }); + w.__chamberDreamTest = { unsubscribe, mindId, events }; + }, offMindContext.mindId); + + await openProfileModal(page, offMindName); + const ariaBefore = await readSwitchAria(page); + flow3.notes.push(`Switch aria-checked before toggle: ${ariaBefore}`); + if (ariaBefore !== 'false') flow3.failures.push(`Pre-toggle aria-checked expected "false" but was "${ariaBefore}"`); + + const switchLocator = page.getByRole('switch', { name: 'Enable dream daemon' }); + await switchLocator.click(); + + // Wait for ARIA to flip — mind reload + Copilot client cold-start can + // take 30+ seconds, so give it 90s. + await expect(switchLocator).toHaveAttribute('aria-checked', 'true', { timeout: 90_000 }); + const ariaAfter = await switchLocator.getAttribute('aria-checked'); + flow3.notes.push(`Switch aria-checked after toggle: ${ariaAfter}`); + + // Disk: .chamber.json should now exist with enabled=true + const offChamberJson = path.join(offMindPath, '.chamber.json'); + const cfgRaw = readFileOrNull(offChamberJson); + flow3.notes.push(`.chamber.json after toggle: ${cfgRaw ?? ''}`); + if (!cfgRaw) flow3.failures.push('.chamber.json missing after toggle ON'); + else { + const cfg = JSON.parse(cfgRaw); + if (cfg?.workingMemory?.consolidation?.enabled !== true) { + flow3.failures.push(`.chamber.json enabled expected true but was ${JSON.stringify(cfg?.workingMemory?.consolidation?.enabled)}`); + } + } + + // mind reload sequence: 2 onMindChanged events (unloaded + loaded) + // First event should not contain the mindId; second event should. + const reloadEvents = await page.evaluate(() => { + const w = window as unknown as { __chamberDreamTest?: { events: Array<{ ts: number; presentMindIds: string[] }>; mindId: string } }; + return w.__chamberDreamTest ?? { events: [], mindId: '' }; + }); + flow3.notes.push(`onMindChanged event count: ${reloadEvents.events.length}`); + const sawUnloaded = reloadEvents.events.some((e) => !e.presentMindIds.includes(offMindContext.mindId)); + const sawLoaded = reloadEvents.events.some((e) => e.presentMindIds.includes(offMindContext.mindId)); + flow3.notes.push(`saw unloaded event: ${sawUnloaded}, saw loaded event: ${sawLoaded}`); + if (!sawUnloaded) flow3.failures.push('No mind:unloaded event observed (mind never absent from list during reload)'); + if (!sawLoaded) flow3.failures.push('No mind:loaded event observed after reload'); + + // Disk: dream.db should now exist + const offDreamDb = path.join(offMindPath, '.working-memory', '.state', 'dream.db'); + flow3.notes.push(`dream.db after toggle ON: ${fs.existsSync(offDreamDb)}`); + if (!fs.existsSync(offDreamDb)) flow3.failures.push('dream.db missing after toggle ON (MindMemoryService failed to activate?)'); + + await closeProfileModal(page); + + // Send a chat turn — must produce structured frames. + const chatResult = await sendOneShotTurn(page, offMindContext.mindId, 'Reply with the single word: GO'); + flow3.notes.push(`chat assistantText length=${chatResult.assistantText.length}, error=${chatResult.errorMessage || ''}`); + if (chatResult.errorMessage) flow3.failures.push(`Chat turn after toggle ON failed: ${chatResult.errorMessage}`); + + await delay(1500); + const offLogPath = path.join(offMindPath, '.working-memory', 'log.md'); + const offLogAfter = readFileOrNull(offLogPath) ?? ''; + flow3.notes.push(`log.md after chat (size=${offLogAfter.length}): ${preview(offLogAfter)}`); + if (!offLogAfter.startsWith('')) { + flow3.failures.push('log.md missing sentinel after toggle ON + chat (eager migrateIfNeeded should have seeded it)'); + } + if (!/\n### user\n/.test(offLogAfter)) flow3.failures.push('log.md missing "### user" frame after toggle ON + chat'); + if (!/\n### assistant\n/.test(offLogAfter)) flow3.failures.push('log.md missing "### assistant" frame after toggle ON + chat'); + + const flow3Logs = logsSince(app, flow3Snapshot); + flow3.notes.push(`relevant main logs (sample): ${sampleRelevant(flow3Logs).join(' | ')}`); + } catch (err) { + flow3.failures.push(`Exception: ${err instanceof Error ? err.stack ?? err.message : String(err)}`); + } finally { + flow3.passed = flow3.failures.length === 0; + reportFlow(flow3); + } + + // --------------------------------------------------------------------- + // Flow 4 — toggle the ON mind OFF via profile modal (rollback path) + // --------------------------------------------------------------------- + const flow4Snapshot = snapshotLogs(app); + const flow4: FlowEvidence = { flow: 'Flow 4 (Post-genesis ON→OFF rollback)', passed: true, failures: [], notes: [] }; + evidence.push(flow4); + + try { + const onMindContext = await getMindContext(page, onMindName); + + // Reset onMindChanged counter for this mind. + await page.evaluate((mindId) => { + const w = window as unknown as { + __chamberDreamTest?: { + unsubscribe?: () => void; + mindId?: string; + events: Array<{ ts: number; presentMindIds: string[] }>; + }; + }; + if (w.__chamberDreamTest?.unsubscribe) w.__chamberDreamTest.unsubscribe(); + const events: Array<{ ts: number; presentMindIds: string[] }> = []; + const unsubscribe = window.electronAPI.mind.onMindChanged((minds: { mindId: string }[]) => { + events.push({ ts: Date.now(), presentMindIds: minds.map((m) => m.mindId) }); + }); + w.__chamberDreamTest = { unsubscribe, mindId, events }; + }, onMindContext.mindId); + + await openProfileModal(page, onMindName); + const ariaBefore = await readSwitchAria(page); + flow4.notes.push(`Switch aria-checked before toggle: ${ariaBefore}`); + if (ariaBefore !== 'true') flow4.failures.push(`Pre-toggle aria-checked expected "true" but was "${ariaBefore}"`); + + const switchLocator = page.getByRole('switch', { name: 'Enable dream daemon' }); + await switchLocator.click(); + await expect(switchLocator).toHaveAttribute('aria-checked', 'false', { timeout: 90_000 }); + const ariaAfter = await switchLocator.getAttribute('aria-checked'); + flow4.notes.push(`Switch aria-checked after toggle: ${ariaAfter}`); + + // Disk: .chamber.json now enabled=false + const onChamberJson = path.join(onMindPath, '.chamber.json'); + const cfgRaw = readFileOrNull(onChamberJson); + flow4.notes.push(`.chamber.json after rollback: ${cfgRaw ?? ''}`); + if (!cfgRaw) flow4.failures.push('.chamber.json removed unexpectedly'); + else { + const cfg = JSON.parse(cfgRaw); + if (cfg?.workingMemory?.consolidation?.enabled !== false) { + flow4.failures.push(`.chamber.json enabled expected false but was ${JSON.stringify(cfg?.workingMemory?.consolidation?.enabled)}`); + } + } + + // Console: rollback log line + const flow4Logs = logsSince(app, flow4Snapshot); + const rollbackLogLines = flow4Logs.filter((l) => /\[rollbackToUnstructured\]/.test(l)); + flow4.notes.push(`rollback log lines: ${rollbackLogLines.length}`); + flow4.notes.push(`rollback log preview: ${rollbackLogLines.map(preview).join(' | ')}`); + if (!rollbackLogLines.some((l) => /converted \d+ frame\(s\)/.test(l))) { + flow4.failures.push('Expected "[rollbackToUnstructured] converted N frame(s)" log line not found'); + } + + // log.md no longer has sentinel; should have rendered turn markdown + const onLogPath = path.join(onMindPath, '.working-memory', 'log.md'); + const onLogAfterRollback = readFileOrNull(onLogPath) ?? ''; + flow4.notes.push(`log.md after rollback (size=${onLogAfterRollback.length}): ${preview(onLogAfterRollback)}`); + if (onLogAfterRollback.includes('')) { + flow4.failures.push('log.md still contains sentinel after rollback'); + } + if (!/^## \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.*— turn .*\(/m.test(onLogAfterRollback)) { + flow4.failures.push('log.md missing rendered turn header (## — turn ())'); + } + if (!/\*\*User\*\*:/.test(onLogAfterRollback)) flow4.failures.push('log.md missing **User**: block'); + if (!/\*\*Assistant\*\*:/.test(onLogAfterRollback)) flow4.failures.push('log.md missing **Assistant**: block'); + + // legacy file should not exist (Genesis-time seed never had legacy content) + const legacyPath = path.join(onMindPath, '.working-memory', 'log.legacy.md'); + flow4.notes.push(`log.legacy.md exists after rollback: ${fs.existsSync(legacyPath)}`); + if (fs.existsSync(legacyPath)) flow4.failures.push('log.legacy.md should be removed (or absent) after rollback'); + + // mind reload events + const reloadEvents = await page.evaluate(() => { + const w = window as unknown as { __chamberDreamTest?: { events: Array<{ ts: number; presentMindIds: string[] }>; mindId: string } }; + return w.__chamberDreamTest ?? { events: [], mindId: '' }; + }); + flow4.notes.push(`onMindChanged event count: ${reloadEvents.events.length}`); + const sawUnloaded = reloadEvents.events.some((e) => !e.presentMindIds.includes(onMindContext.mindId)); + const sawLoaded = reloadEvents.events.some((e) => e.presentMindIds.includes(onMindContext.mindId)); + flow4.notes.push(`saw unloaded event: ${sawUnloaded}, saw loaded event: ${sawLoaded}`); + if (!sawUnloaded) flow4.failures.push('No mind:unloaded event observed during rollback toggle'); + if (!sawLoaded) flow4.failures.push('No mind:loaded event observed after rollback toggle'); + + await closeProfileModal(page); + + // Follow-up turn: appends unstructured to log.md (no sentinel re-introduced). + const chatResult = await sendOneShotTurn(page, onMindContext.mindId, 'Reply with the single word: STOP'); + flow4.notes.push(`follow-up chat assistantText length=${chatResult.assistantText.length}, error=${chatResult.errorMessage || ''}`); + if (chatResult.errorMessage) flow4.failures.push(`Follow-up chat turn failed: ${chatResult.errorMessage}`); + await delay(1500); + + const onLogAfterFollowUp = readFileOrNull(onLogPath) ?? ''; + flow4.notes.push(`log.md after follow-up (size=${onLogAfterFollowUp.length}): ${preview(onLogAfterFollowUp)}`); + if (onLogAfterFollowUp.includes('')) { + flow4.failures.push('Follow-up turn re-introduced sentinel — DailyLogWriter not torn down'); + } + // Note: post-rollback the mind is in opted-out mode. A turn through the + // chat IPC does NOT write to log.md (no observer is attached). We simply + // verify the file did not regress. + + flow4.notes.push(`relevant main logs (sample): ${sampleRelevant(flow4Logs).join(' | ')}`); + } catch (err) { + flow4.failures.push(`Exception: ${err instanceof Error ? err.stack ?? err.message : String(err)}`); + } finally { + flow4.passed = flow4.failures.length === 0; + reportFlow(flow4); + } + + // --------------------------------------------------------------------- + // Final consolidated assertion + // --------------------------------------------------------------------- + const failedFlows = evidence.filter((e) => !e.passed); + if (failedFlows.length > 0) { + const summary = failedFlows + .map((e) => `${e.flow}\n failures:\n - ${e.failures.join('\n - ')}\n notes:\n - ${e.notes.join('\n - ')}`) + .join('\n\n'); + throw new Error(`Dream daemon validation failed:\n${summary}`); + } + }); +}); + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function snapshotLogs(app: LaunchedElectronApp | undefined): number { + return app?.logs.length ?? 0; +} + +function logsSince(app: LaunchedElectronApp | undefined, start: number): string[] { + return (app?.logs ?? []).slice(start); +} + +function readFileOrNull(p: string): string | null { + try { + return fs.readFileSync(p, 'utf-8'); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw err; + } +} + +function preview(s: string, max = 240): string { + const flat = s.replace(/\s+/g, ' ').trim(); + return flat.length <= max ? flat : flat.slice(0, max) + '…'; +} + +function sampleRelevant(lines: string[]): string[] { + return lines + .filter((l) => + /\[MindMemoryService\]|\[MindManager\]|\[rollbackToUnstructured\]|\[chamberMindConfig\]|\[DailyLogWriter\]|mind:loaded|mind:unloaded|mindMemory/i.test(l), + ) + .slice(0, 8) + .map((l) => l.replace(/\r?\n/g, ' ').slice(0, 200)); +} + +function reportFlow(f: FlowEvidence): void { + const status = f.passed ? 'PASS' : 'FAIL'; + + console.log(`[dream-daemon-bidir] ${status} — ${f.flow}`); + for (const n of f.notes) console.log(` · ${n}`); + for (const x of f.failures) console.log(` ✗ ${x}`); +} + +interface DriveGenesisOptions { + name: string; + voiceDescription: string; + purpose: string; + toggleDaemon: boolean; +} + +async function driveGenesisCustom(page: Awaited>, opts: DriveGenesisOptions): Promise { + // The genesis wizard is reachable from three different entry states: + // 1. We're already in the wizard (VoidScreen) — "Begin" button is visible. + // 2. We're on the LandingScreen (first launch, no minds) — "New Agent" button is visible. + // 3. We're in the main app (at least one mind exists) — the sidebar shows "Add Agent", + // which dispatches SHOW_LANDING and brings us to state (2). + const beginButton = page.getByRole('button', { name: 'Begin', exact: true }); + const newAgentButton = page.getByRole('button', { name: /New Agent/i }); + + const beginVisible = await beginButton.isVisible().catch(() => false); + if (!beginVisible) { + const newAgentVisible = await newAgentButton.isVisible().catch(() => false); + if (!newAgentVisible) { + // State (3) — sidebar route. + const addAgentButton = page.getByRole('button', { name: /Add Agent/i }); + await addAgentButton.waitFor({ state: 'visible', timeout: 30_000 }); + await addAgentButton.click(); + await newAgentButton.waitFor({ state: 'visible', timeout: 10_000 }); + } + await newAgentButton.click(); + } + await beginButton.waitFor({ state: 'visible', timeout: 30_000 }); + await beginButton.click(); + + // VoiceScreen — pick "Someone else..." then enter name + backstory. + await page.getByRole('button', { name: /Someone else/i }).click(); + await page.getByPlaceholder('e.g. Tony Stark, Moneypenny, Gandalf...').fill(opts.name); + await page.getByPlaceholder(/Era, source material/).fill(opts.voiceDescription); + await page.getByRole('button', { name: /Research this voice/i }).click(); + await expect(page.getByLabel('Research brief')).toHaveValue(/.+/, { timeout: 60_000 }); + await page.getByRole('button', { name: /Continue to purpose/i }).click(); + + // RoleScreen — pick "Something else..." then type purpose. Optionally toggle the dream-daemon Switch BEFORE submit. + await page.getByRole('button', { name: /Something else/i }).click(); + await page.getByPlaceholder(/Creative Director, Debate Coach/).fill(opts.purpose); + + if (opts.toggleDaemon) { + const daemonSwitch = page.getByRole('switch', { name: 'Enable dream daemon' }); + await expect(daemonSwitch).toHaveAttribute('aria-checked', 'false'); + await daemonSwitch.click(); + await expect(daemonSwitch).toHaveAttribute('aria-checked', 'true'); + } + + await page.getByRole('button', { name: /That's my purpose/i }).click(); + + // BootScreen → done. The chat input becomes visible only once Genesis completes + // and the new mind is selected. Wait long enough for SOUL generation + capability bootstrap. + await expect(page.getByPlaceholder('Message your agent… (paste an image to attach)')).toBeEnabled({ timeout: 10 * 60_000 }); +} + +async function waitForMindByName(page: Awaited>, name: string): Promise { + await expect.poll( + async () => + await page.evaluate(async (target) => { + const minds = await window.electronAPI.mind.list(); + return minds.some((m) => m.identity.name === target); + }, name), + { timeout: 30_000 }, + ).toBe(true); +} + +interface MindCtx { + mindId: string; + mindPath: string; +} + +async function getMindContext(page: Awaited>, name: string): Promise { + return await page.evaluate(async (target) => { + const minds = await window.electronAPI.mind.list(); + const mind = minds.find((m) => m.identity.name === target); + if (!mind) throw new Error(`Mind ${target} not found`); + return { mindId: mind.mindId, mindPath: mind.mindPath }; + }, name); +} + +async function openProfileModal(page: Awaited>, name: string): Promise { + const trigger = page.getByRole('button', { name: `Edit ${name} profile`, exact: true }); + // The trigger is hover-revealed (opacity-0). force:true bypasses the visibility check. + await trigger.click({ force: true }); + await expect(page.getByRole('dialog').getByText('Agent profile')).toBeVisible({ timeout: 10_000 }); +} + +async function readSwitchAria(page: Awaited>): Promise { + return await page.getByRole('switch', { name: 'Enable dream daemon' }).getAttribute('aria-checked'); +} + +async function closeProfileModal(page: Awaited>): Promise { + // The dialog has two "Close" elements (footer text button + Radix icon button with aria-label="Close"). + // Press Escape — Radix Dialog dismisses on Escape and avoids selector ambiguity. + await page.keyboard.press('Escape'); + await expect(page.getByRole('dialog')).toBeHidden({ timeout: 5_000 }).catch(() => undefined); +} + +interface ChatResult { + assistantText: string; + errorMessage: string; + doneCount: number; +} + +async function sendOneShotTurn( + page: Awaited>, + mindId: string, + prompt: string, +): Promise { + return await page.evaluate(async ({ mindId: id, prompt: text }) => { + const messageId = `dream-daemon-${Date.now()}-${Math.random().toString(36).slice(2)}`; + let assistantText = ''; + let errorMessage = ''; + let doneCount = 0; + let resolveTerminal: () => void = () => undefined; + const terminal = new Promise((resolve) => { resolveTerminal = resolve; }); + const unsubscribe = window.electronAPI.chat.onEvent((eventMindId, eventMessageId, event) => { + if (eventMindId !== id || eventMessageId !== messageId) return; + if (event.type === 'chunk' || event.type === 'message_final') { + assistantText += (event as { content?: string }).content ?? ''; + } + if (event.type === 'error') { + errorMessage = (event as { message?: string }).message ?? 'unknown error'; + resolveTerminal(); + } + if (event.type === 'done') { + doneCount += 1; + resolveTerminal(); + } + }); + try { + const send = window.electronAPI.chat.send(id, text, messageId); + const timeout = new Promise((_, reject) => { + setTimeout(() => reject(new Error('Chat turn timed out after 180s')), 180_000); + }); + await Promise.race([Promise.all([send, terminal]), timeout]); + } catch (err) { + errorMessage = err instanceof Error ? err.message : String(err); + } finally { + unsubscribe(); + } + return { assistantText, errorMessage, doneCount }; + }, { mindId, prompt }); +} + +async function removeTempRoot(root: string): Promise { + for (let attempt = 0; attempt < 10; attempt += 1) { + try { + fs.rmSync(root, { recursive: true, force: true }); + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EPERM' || attempt === 9) { + + console.warn(`[dream-daemon-bidir] Failed to remove temp root ${root}:`, error); + return; + } + await delay(250); + } + } +} diff --git a/tests/integration/mindScaffold.integration.test.ts b/tests/integration/mindScaffold.integration.test.ts index 4ca34b6f..3ca760b8 100644 --- a/tests/integration/mindScaffold.integration.test.ts +++ b/tests/integration/mindScaffold.integration.test.ts @@ -127,7 +127,7 @@ function defaultSeedFiles(paths: SoulPaths): void { } describe('MindScaffold.create — bootstrap integration', () => { - it('produces a sentinel-prefixed log.md on disk', async () => { + it('opt-in (enableDreamDaemon=true): produces a sentinel-prefixed log.md AND writes .chamber.json with consolidation.enabled=true', async () => { const callLog: RegistryCallLog = { fetchTree: [], fetchBlob: [], fetchJsonContent: [] }; const scaffold = new MindScaffold( makeFakeRegistryClient(callLog), @@ -140,6 +140,7 @@ describe('MindScaffold.create — bootstrap integration', () => { voice: 'plain', voiceDescription: 'plain', basePath: tmpRoot, + enableDreamDaemon: true, }); const logPath = path.join(mindPath, '.working-memory', 'log.md'); @@ -147,6 +148,41 @@ describe('MindScaffold.create — bootstrap integration', () => { const content = fs.readFileSync(logPath, 'utf-8'); const firstNonBlank = content.split('\n').find((l) => l.trim() !== ''); expect(firstNonBlank).toBe(STRUCTURED_LOG_SENTINEL); + + // Persist the opt-in choice so MindMemoryService.activateMind reads it + // back on the next mind load — without this, the user toggled the + // Switch but the daemon would never start. + const chamberJsonPath = path.join(mindPath, '.chamber.json'); + expect(fs.existsSync(chamberJsonPath)).toBe(true); + const chamberConfig = JSON.parse(fs.readFileSync(chamberJsonPath, 'utf-8')) as { + workingMemory?: { consolidation?: { enabled?: unknown } }; + }; + expect(chamberConfig.workingMemory?.consolidation?.enabled).toBe(true); + }); + + it('opt-out (enableDreamDaemon=false): log.md is empty AND .chamber.json is absent', async () => { + // Default flow for users who don't opt in. The mind still works (chat, + // tools, memory, rules) — it just doesn't run the dream daemon and + // doesn't materialize structured-log frames on each turn. + const callLog: RegistryCallLog = { fetchTree: [], fetchBlob: [], fetchJsonContent: [] }; + const scaffold = new MindScaffold( + makeFakeRegistryClient(callLog), + makeFakeClientFactory(defaultSeedFiles), + ); + + const mindPath = await scaffold.create({ + name: 'Quiet Mind', + role: 'integration tester', + voice: 'plain', + voiceDescription: 'plain', + basePath: tmpRoot, + enableDreamDaemon: false, + }); + + const logPath = path.join(mindPath, '.working-memory', 'log.md'); + expect(fs.existsSync(logPath)).toBe(true); + expect(fs.readFileSync(logPath, 'utf-8')).toBe(''); + expect(fs.existsSync(path.join(mindPath, '.chamber.json'))).toBe(false); }); it('records source: ianphil/genesis-frontier in registry.json', async () => { @@ -277,7 +313,7 @@ describe('MindScaffold.create — bootstrap integration', () => { const warn = vi.fn(); const info = vi.fn(); const composer = createWorkingMemoryComposer({ logger: { warn, info } }); - composer.compose(mindPath, { lastKTurns: 10, perTurnMaxBytes: 2048, memoryMaxBytes: 8192 }); + composer.compose(mindPath, { enabled: true, lastKTurns: 10, perTurnMaxBytes: 2048, memoryMaxBytes: 8192 }); expect(warn).not.toHaveBeenCalled(); // info may be called for benign reasons (e.g. memory.md truncation) but @@ -325,7 +361,7 @@ describe('MindScaffold.create — bootstrap integration', () => { const composer = createWorkingMemoryComposer({ logger: { warn, info } }); // Step 1: opening the mind triggers a system-prompt rebuild → info, never warn. - composer.compose(mindPath, { lastKTurns: 10, perTurnMaxBytes: 2048, memoryMaxBytes: 8192 }); + composer.compose(mindPath, { enabled: true, lastKTurns: 10, perTurnMaxBytes: 2048, memoryMaxBytes: 8192 }); expect(warn).not.toHaveBeenCalled(); const unstructuredInfoCalls = info.mock.calls.filter((c) => /unstructured/i.test(c[0])); expect(unstructuredInfoCalls.length).toBe(1); @@ -353,7 +389,7 @@ describe('MindScaffold.create — bootstrap integration', () => { // Step 3: subsequent prompt rebuild is silent — migration is complete. info.mockClear(); warn.mockClear(); - composer.compose(mindPath, { lastKTurns: 10, perTurnMaxBytes: 2048, memoryMaxBytes: 8192 }); + composer.compose(mindPath, { enabled: true, lastKTurns: 10, perTurnMaxBytes: 2048, memoryMaxBytes: 8192 }); expect(warn).not.toHaveBeenCalled(); for (const call of info.mock.calls) { expect(call[0]).not.toMatch(/unstructured/i); From c8bcfa30e8aafb2093fc035b7f1f0bce114f380b Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Tue, 26 May 2026 16:09:52 -0400 Subject: [PATCH 20/23] feat(feature-flags): add dreamDaemon flag wiring (dev_only rollout) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds dreamDaemon to AppFeatureFlags, DEFAULT_APP_FEATURE_FLAGS, getAppFeatureFlags, parseFeatureFlags, parseCompleteFeatureFlags. DEV defaults to true; remote policy (stable + insiders) defaults to false. Extends shared parser tests with dreamDaemon assertions and a missing-field rejection. Updates FeatureFlagService.test.ts complete-policy fixtures and renderer/test-helper fixtures so AppFeatureFlags-shaped literals satisfy the new required field. No runtime gating yet — Phase 3 wraps the dream-daemon composition in main.ts behind this flag. --- apps/desktop/src/main/devFeatureFlags.ts | 1 + .../featureFlags/FeatureFlagService.test.ts | 3 +++ .../components/layout/ActivityBar.test.tsx | 4 ++-- .../components/settings/SettingsView.test.tsx | 2 +- .../src/renderer/lib/store/reducer.test.ts | 2 +- apps/web/src/test/helpers.ts | 2 +- docs/flags/v1/flags.json | 6 +++-- packages/shared/src/feature-flags.test.ts | 24 +++++++++++++++---- packages/shared/src/feature-flags.ts | 8 ++++++- 9 files changed, 40 insertions(+), 12 deletions(-) diff --git a/apps/desktop/src/main/devFeatureFlags.ts b/apps/desktop/src/main/devFeatureFlags.ts index 4f99e942..13c27fef 100644 --- a/apps/desktop/src/main/devFeatureFlags.ts +++ b/apps/desktop/src/main/devFeatureFlags.ts @@ -11,4 +11,5 @@ export const DEV_FEATURE_FLAGS: AppFeatureFlags = { switchboardRelay: true, byoLlm: true, chamberCopilot: true, + dreamDaemon: true, }; diff --git a/apps/desktop/src/main/services/featureFlags/FeatureFlagService.test.ts b/apps/desktop/src/main/services/featureFlags/FeatureFlagService.test.ts index 968b4af7..70a77c54 100644 --- a/apps/desktop/src/main/services/featureFlags/FeatureFlagService.test.ts +++ b/apps/desktop/src/main/services/featureFlags/FeatureFlagService.test.ts @@ -9,12 +9,14 @@ const DEV_FLAGS: AppFeatureFlags = { switchboardRelay: true, byoLlm: false, chamberCopilot: true, + dreamDaemon: true, }; const REMOTE_FLAGS: AppFeatureFlags = { switchboardRelay: true, byoLlm: true, chamberCopilot: false, + dreamDaemon: false, }; describe('FeatureFlagService', () => { @@ -114,6 +116,7 @@ describe('FeatureFlagService', () => { switchboardRelay: true, byoLlm: true, chamberCopilot: true, + dreamDaemon: true, }); expect(fetched).toBe(false); }); diff --git a/apps/web/src/renderer/components/layout/ActivityBar.test.tsx b/apps/web/src/renderer/components/layout/ActivityBar.test.tsx index 1dc58064..48ad1bad 100644 --- a/apps/web/src/renderer/components/layout/ActivityBar.test.tsx +++ b/apps/web/src/renderer/components/layout/ActivityBar.test.tsx @@ -37,12 +37,12 @@ describe('ActivityBar', () => { }); it('hides the A2A Relay button when the Switchboard Relay flag is disabled', () => { - renderActivityBar({ featureFlags: { switchboardRelay: false, byoLlm: false, chamberCopilot: false } }); + renderActivityBar({ featureFlags: { switchboardRelay: false, byoLlm: false, chamberCopilot: false, dreamDaemon: false } }); expect(screen.queryByLabelText('A2A Relay')).toBeNull(); }); it('renders the A2A Relay button above settings when the Switchboard Relay flag is enabled', () => { - renderActivityBar({ featureFlags: { switchboardRelay: true, byoLlm: false, chamberCopilot: false } }); + renderActivityBar({ featureFlags: { switchboardRelay: true, byoLlm: false, chamberCopilot: false, dreamDaemon: false } }); const relayButton = screen.getByLabelText('A2A Relay'); const settingsButton = screen.getByLabelText('Settings'); const footer = relayButton.closest('[data-testid="activity-bar-footer"]'); diff --git a/apps/web/src/renderer/components/settings/SettingsView.test.tsx b/apps/web/src/renderer/components/settings/SettingsView.test.tsx index 70c6db45..b95c02a4 100644 --- a/apps/web/src/renderer/components/settings/SettingsView.test.tsx +++ b/apps/web/src/renderer/components/settings/SettingsView.test.tsx @@ -79,7 +79,7 @@ describe('SettingsView', () => { it('shows Local & Custom LLM settings when BYO LLM is feature-flagged on', async () => { render( - + , ); diff --git a/apps/web/src/renderer/lib/store/reducer.test.ts b/apps/web/src/renderer/lib/store/reducer.test.ts index 7da56042..81aa8757 100644 --- a/apps/web/src/renderer/lib/store/reducer.test.ts +++ b/apps/web/src/renderer/lib/store/reducer.test.ts @@ -626,7 +626,7 @@ describe('appReducer', () => { it('SET_FEATURE_FLAGS updates feature flags', () => { const state = appReducer(initialState, { type: 'SET_FEATURE_FLAGS', - payload: { switchboardRelay: true, byoLlm: true, chamberCopilot: true }, + payload: { switchboardRelay: true, byoLlm: true, chamberCopilot: true, dreamDaemon: true }, }); expect(state.featureFlags.switchboardRelay).toBe(true); expect(state.featureFlags.byoLlm).toBe(true); diff --git a/apps/web/src/test/helpers.ts b/apps/web/src/test/helpers.ts index bde7f8f8..b543d13a 100644 --- a/apps/web/src/test/helpers.ts +++ b/apps/web/src/test/helpers.ts @@ -353,7 +353,7 @@ export function mockElectronAPI(): ElectronAPI { close: vi.fn(), }, app: { - getFeatureFlags: vi.fn().mockResolvedValue({ switchboardRelay: false, byoLlm: false, chamberCopilot: false }), + getFeatureFlags: vi.fn().mockResolvedValue({ switchboardRelay: false, byoLlm: false, chamberCopilot: false, dreamDaemon: false }), onStartupProgress: vi.fn().mockReturnValue(vi.fn()), }, }; diff --git a/docs/flags/v1/flags.json b/docs/flags/v1/flags.json index 6261aa25..3e3b1064 100644 --- a/docs/flags/v1/flags.json +++ b/docs/flags/v1/flags.json @@ -5,12 +5,14 @@ "stable": { "switchboardRelay": false, "byoLlm": false, - "chamberCopilot": false + "chamberCopilot": false, + "dreamDaemon": false }, "insiders": { "switchboardRelay": true, "byoLlm": true, - "chamberCopilot": true + "chamberCopilot": true, + "dreamDaemon": false } } } diff --git a/packages/shared/src/feature-flags.test.ts b/packages/shared/src/feature-flags.test.ts index 9e324191..73caea4d 100644 --- a/packages/shared/src/feature-flags.test.ts +++ b/packages/shared/src/feature-flags.test.ts @@ -14,6 +14,7 @@ describe('feature flags', () => { it('keeps preview features disabled by default', () => { expect(DEFAULT_APP_FEATURE_FLAGS.switchboardRelay).toBe(false); expect(DEFAULT_APP_FEATURE_FLAGS.byoLlm).toBe(false); + expect(DEFAULT_APP_FEATURE_FLAGS.dreamDaemon).toBe(false); }); it('enables preview features for insiders versions', () => { @@ -21,6 +22,7 @@ describe('feature flags', () => { switchboardRelay: true, byoLlm: true, chamberCopilot: true, + dreamDaemon: true, }); }); @@ -33,6 +35,7 @@ describe('feature flags', () => { switchboardRelay: true, byoLlm: true, chamberCopilot: true, + dreamDaemon: true, }); }); @@ -43,11 +46,13 @@ describe('feature flags', () => { switchboardRelay: false, byoLlm: true, chamberCopilot: false, + dreamDaemon: false, }, })).toEqual({ switchboardRelay: false, byoLlm: true, chamberCopilot: false, + dreamDaemon: false, }); }); @@ -67,6 +72,7 @@ describe('feature flags', () => { switchboardRelay: true, byoLlm: false, chamberCopilot: false, + dreamDaemon: false, }); }); @@ -76,15 +82,15 @@ describe('feature flags', () => { updatedAt: '2026-05-17T21:00:00Z', ignored: true, channels: { - stable: { switchboardRelay: false, byoLlm: false, chamberCopilot: false }, - insiders: { switchboardRelay: true, byoLlm: true, chamberCopilot: true, futureFlag: true }, + stable: { switchboardRelay: false, byoLlm: false, chamberCopilot: false, dreamDaemon: false }, + insiders: { switchboardRelay: true, byoLlm: true, chamberCopilot: true, dreamDaemon: false, futureFlag: true }, }, })).toEqual({ version: 1, updatedAt: '2026-05-17T21:00:00Z', channels: { stable: DEFAULT_APP_FEATURE_FLAGS, - insiders: { switchboardRelay: true, byoLlm: true, chamberCopilot: true }, + insiders: { switchboardRelay: true, byoLlm: true, chamberCopilot: true, dreamDaemon: false }, }, }); }); @@ -95,6 +101,16 @@ describe('feature flags', () => { expect(parseRemoteFeatureFlagPolicy(null)).toBeNull(); }); + it('rejects remote policies missing the dreamDaemon field', () => { + expect(parseRemoteFeatureFlagPolicy({ + version: 1, + channels: { + stable: { switchboardRelay: false, byoLlm: false, chamberCopilot: false }, + insiders: { switchboardRelay: true, byoLlm: true, chamberCopilot: true, dreamDaemon: false }, + }, + })).toBeNull(); + }); + it('keeps the published GitHub Pages policy valid', () => { const policyPath = path.resolve(process.cwd(), 'docs', 'flags', 'v1', 'flags.json'); const policy = parseRemoteFeatureFlagPolicy(JSON.parse(fs.readFileSync(policyPath, 'utf-8')) as unknown); @@ -104,7 +120,7 @@ describe('feature flags', () => { updatedAt: '2026-05-17T21:00:00Z', channels: { stable: DEFAULT_APP_FEATURE_FLAGS, - insiders: { switchboardRelay: true, byoLlm: true, chamberCopilot: true }, + insiders: { switchboardRelay: true, byoLlm: true, chamberCopilot: true, dreamDaemon: false }, }, }); }); diff --git a/packages/shared/src/feature-flags.ts b/packages/shared/src/feature-flags.ts index 76407cf1..9e444967 100644 --- a/packages/shared/src/feature-flags.ts +++ b/packages/shared/src/feature-flags.ts @@ -2,6 +2,7 @@ export interface AppFeatureFlags { readonly switchboardRelay: boolean; readonly byoLlm: boolean; readonly chamberCopilot: boolean; + readonly dreamDaemon: boolean; } export type FeatureFlagChannel = 'stable' | 'insiders'; @@ -16,6 +17,7 @@ export const DEFAULT_APP_FEATURE_FLAGS: AppFeatureFlags = { switchboardRelay: false, byoLlm: false, chamberCopilot: false, + dreamDaemon: false, }; export function getAppFeatureFlags(options: { @@ -29,6 +31,7 @@ export function getAppFeatureFlags(options: { switchboardRelay: insiders, byoLlm: insiders, chamberCopilot: insiders, + dreamDaemon: insiders, }; } @@ -60,6 +63,7 @@ export function parseFeatureFlags(value: unknown): AppFeatureFlags | null { switchboardRelay: value.switchboardRelay === true, byoLlm: value.byoLlm === true, chamberCopilot: value.chamberCopilot === true, + dreamDaemon: value.dreamDaemon === true, }; } @@ -68,7 +72,8 @@ export function parseCompleteFeatureFlags(value: unknown): AppFeatureFlags | nul if ( typeof value.switchboardRelay !== 'boolean' || typeof value.byoLlm !== 'boolean' || - typeof value.chamberCopilot !== 'boolean' + typeof value.chamberCopilot !== 'boolean' || + typeof value.dreamDaemon !== 'boolean' ) { return null; } @@ -76,6 +81,7 @@ export function parseCompleteFeatureFlags(value: unknown): AppFeatureFlags | nul switchboardRelay: value.switchboardRelay, byoLlm: value.byoLlm, chamberCopilot: value.chamberCopilot, + dreamDaemon: value.dreamDaemon, }; } From 52582c3f793f4a9290a03163db7f43820ca058f0 Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Tue, 26 May 2026 16:33:57 -0400 Subject: [PATCH 21/23] feat(dream-daemon): gate runtime, IPC, services, and UI behind dreamDaemon flag Layered defense-in-depth so the dream-daemon stack is fully inert when the new app-level dreamDaemon feature flag is off: - apps/desktop/src/main.ts: module-level dreamDaemonFeatureEnabled() reads appFeatureFlags at call time; buildMindMemoryService composition + the __chamberMindMemoryService E2E global are wrapped in if(flag); accessor is passed positionally into IdentityLoader (arg 3), MindManager (arg 7), MindProfileService (arg 4), setupMindIPC and setupGenesisIPC configs. - main/ipc/mind.ts: SET_DREAM_DAEMON rejects when enabled && !flag, disable is always allowed so a stable build can clean up insiders opt-in state. - main/ipc/genesis.ts: new GenesisIPCOptions; CREATE handler coerces enableDreamDaemon=false in sanitizedConfig server-side when flag is off. - MindManager.doToggleDreamDaemon: throws Error('Dream Daemon is not available in this build') before mind lookup when enabled && !flag. - IdentityLoader.resolveComposerConfig: forces enabled=false in the composer config when the accessor returns false, regardless of the per-mind .chamber.json. Caps (lastKTurns, perTurnMaxBytes, memoryMaxBytes) remain faithful so a future flip-on does not lose the user's settings. - MindProfileService.getProfile: AND'd the accessor with the .chamber.json value so the (now-hidden) UI never sees a stale ON state. - renderer RoleScreen/GenesisFlow/AgentProfileModal: read featureFlags from useAppState; hide the Switch / toggle row when flag is off and coerce enableDreamDaemon=false at every emit boundary. Tests: 6 new service-layer gate tests across MindManager, IdentityLoader, and MindProfileService (146 passing). 4 new renderer flag-off tests across RoleScreen, GenesisFlow, AgentProfileModal (25 passing). Existing tests updated to pass testInitialState={ featureFlags: { dreamDaemon: true } } where they assert on the dream-daemon surface. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- apps/desktop/src/main.ts | 68 +++++++++------ apps/desktop/src/main/ipc/genesis.ts | 29 ++++++- apps/desktop/src/main/ipc/mind.ts | 19 ++++ .../components/genesis/GenesisFlow.test.tsx | 43 +++++++++- .../components/genesis/GenesisFlow.tsx | 15 +++- .../components/genesis/RoleScreen.test.tsx | 54 ++++++++++-- .../components/genesis/RoleScreen.tsx | 67 +++++++++------ .../profile/AgentProfileModal.test.tsx | 33 ++++++- .../components/profile/AgentProfileModal.tsx | 64 ++++++++------ .../services/src/chat/IdentityLoader.test.ts | 86 +++++++++++++++++++ packages/services/src/chat/IdentityLoader.ts | 18 +++- .../services/src/mind/MindManager.test.ts | 82 ++++++++++++++++++ packages/services/src/mind/MindManager.ts | 21 +++++ .../mindProfile/MindProfileService.test.ts | 74 ++++++++++++++++ .../src/mindProfile/MindProfileService.ts | 17 +++- 15 files changed, 591 insertions(+), 99 deletions(-) diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index 90508bfe..3c332c87 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -201,6 +201,12 @@ const notifier: Notifier = { }; let appFeatureFlags: AppFeatureFlags = DEFAULT_APP_FEATURE_FLAGS; +// Module-level accessor over the live `appFeatureFlags` binding. Shared by +// `initializeRuntime` (IdentityLoader, MindManager, MindProfileService) and +// the `app.on('ready')` IPC setup (setupMindIPC, setupGenesisIPC). Reading +// the property at call-time tracks future remote-policy reloads that +// re-assign `appFeatureFlags`. +const dreamDaemonFeatureEnabled = (): boolean => appFeatureFlags.dreamDaemon; let credentialStore: CredentialStore; let sharp: typeof sharpModule; let configService: ConfigService; @@ -258,7 +264,11 @@ async function initializeRuntime(): Promise { }); configService = new ConfigService(); - const identityLoader = new IdentityLoader(() => configService.load().installedTools ?? []); + const identityLoader = new IdentityLoader( + () => configService.load().installedTools ?? [], + undefined, + dreamDaemonFeatureEnabled, + ); const getGenesisMarketplaceSources = (): GenesisMindTemplateMarketplaceSource[] => configService.load().marketplaceRegistries ?? [DEFAULT_GENESIS_MIND_TEMPLATE_SOURCE]; const saveActiveLogin = (login: string | null) => { @@ -302,11 +312,12 @@ async function initializeRuntime(): Promise { viewDiscovery, () => buildProviderConfig(cachedByoLlmConfig), () => cachedByoLlmConfig?.model, + dreamDaemonFeatureEnabled, ); mindProfileService = new MindProfileService({ getMindPath: (mindId) => mindManager.getMind(mindId)?.mindPath ?? null, restartMind: (mindId) => mindManager.reloadMind(mindId), - }, identityLoader, new SharpAvatarNormalizer(sharp)); + }, identityLoader, new SharpAvatarNormalizer(sharp), dreamDaemonFeatureEnabled); userProfileService = new UserProfileService(configService); microsoftGraphProfileImporter = new MicrosoftGraphProfileImporter( userProfileService, @@ -382,35 +393,36 @@ async function initializeRuntime(): Promise { // ------------------------------------------------------------------------- // MindMemory (Dream Daemon) — per-mind background memory consolidation. - // Wires after providers so chatService observers + scheduler are ready - // before any mind:loaded event fires. The lifecycle hooks below own the - // per-mind activate/release dance; the composition root owns close() - // during quit. The better-sqlite3 ctor is injected from the shared - // `loadBetterSqlite3()` resolver so dev and packaged builds both go - // through the unified `chamber-sqlite-runtime`. + // Gated behind the `dreamDaemon` app feature flag. When the flag is off, + // `mindMemoryComposition` stays undefined and the lifecycle hooks below + // are never registered, so `mind:loaded` / `mind:unloaded` are no-ops + // for memory purposes. The composition root owns close() during quit — + // requestQuit() and before-quit both guard on `mindMemoryComposition?`. // - // NOTE (Phase 3 TODO): this will be wrapped in - // `if (appFeatureFlags.dreamDaemon) { ... }` once the `dreamDaemon` flag - // lands. Kept always-on for now so the post-merge regression baseline - // matches pre-merge behavior. + // Wires after providers so chatService observers + scheduler are ready + // before any mind:loaded event fires. The better-sqlite3 ctor is injected + // from the shared `loadBetterSqlite3()` resolver so dev and packaged + // builds both go through the unified `chamber-sqlite-runtime`. // ------------------------------------------------------------------------- - mindMemoryComposition = buildMindMemoryService({ - mindManager, - chatService, - Database: loadBetterSqlite3(), - }); - mindMemoryService = mindMemoryComposition.service; - const memoryService = mindMemoryService; - mindManager.on('mind:loaded', (ctx: MindContext) => { - memoryService.activateMind(ctx.mindId, ctx.mindPath).catch((err) => { - log.warn('mindMemory: activateMind failed', { mindId: ctx.mindId, err: String(err) }); + if (appFeatureFlags.dreamDaemon) { + mindMemoryComposition = buildMindMemoryService({ + mindManager, + chatService, + Database: loadBetterSqlite3(), }); - }); - mindManager.on('mind:unloaded', (mindId: string) => { - memoryService.releaseMind(mindId).catch((err) => { - log.warn('mindMemory: releaseMind failed', { mindId, err: String(err) }); + mindMemoryService = mindMemoryComposition.service; + const memoryService = mindMemoryService; + mindManager.on('mind:loaded', (ctx: MindContext) => { + memoryService.activateMind(ctx.mindId, ctx.mindPath).catch((err) => { + log.warn('mindMemory: activateMind failed', { mindId: ctx.mindId, err: String(err) }); + }); }); - }); + mindManager.on('mind:unloaded', (mindId: string) => { + memoryService.releaseMind(mindId).catch((err) => { + log.warn('mindMemory: releaseMind failed', { mindId, err: String(err) }); + }); + }); + } updaterService = new UpdaterService({ currentVersion: app.getVersion(), @@ -762,6 +774,7 @@ app.on('ready', async () => { devServerUrl: MAIN_WINDOW_VITE_DEV_SERVER_URL || undefined, rendererPath: MAIN_WINDOW_VITE_DEV_SERVER_URL ? undefined : path.join(__dirname, `../renderer/${MAIN_WINDOW_VITE_NAME}/index.html`), windowIcon, + dreamDaemonEnabled: dreamDaemonFeatureEnabled, }); setupMindProfileIPC(mindProfileService, mindManager, sharp); setupUserProfileIPC(userProfileService, microsoftGraphProfileImporter); @@ -780,6 +793,7 @@ app.on('ready', async () => { return result.templates; }}, genesisTemplateInstaller, + { dreamDaemonEnabled: dreamDaemonFeatureEnabled }, ); setupMarketplaceIPC(marketplaceRegistryService, { onRegistryToolsChanged: reconcileMarketplaceTools }); setupToolsIPC(toolsService); diff --git a/apps/desktop/src/main/ipc/genesis.ts b/apps/desktop/src/main/ipc/genesis.ts index 147baee1..7f0d91d5 100644 --- a/apps/desktop/src/main/ipc/genesis.ts +++ b/apps/desktop/src/main/ipc/genesis.ts @@ -29,12 +29,25 @@ const createFromTemplateSchema = z }) .strict(); +export interface GenesisIPCOptions { + // Returns the current value of the `dreamDaemon` app feature flag. When this + // returns false, `IPC.GENESIS.CREATE` server-side coerces the incoming + // `enableDreamDaemon` payload to false *before* calling `scaffold.create`, + // regardless of what the renderer claims. This guarantees that a stale or + // bypassed renderer cannot scaffold a mind with `.chamber.json + // workingMemory.consolidation.enabled: true` in a build where the feature + // is off. Defaults to always-on so tests that omit it keep current behavior. + dreamDaemonEnabled?: () => boolean; +} + export function setupGenesisIPC( mindManager: MindManager, scaffold: MindScaffold, templateCatalog: GenesisMindTemplateCatalogPort, templateInstaller: GenesisMindTemplateInstallerPort, + options: GenesisIPCOptions = {}, ): void { + const dreamDaemonEnabled = options.dreamDaemonEnabled ?? (() => true); ipcMain.handle(IPC.GENESIS.GET_DEFAULT_PATH, async () => { return getDefaultGenesisBasePath(); @@ -61,13 +74,23 @@ export function setupGenesisIPC( ipcMain.handle(IPC.GENESIS.CREATE, async (event, config: GenesisConfig) => { const win = BrowserWindow.fromWebContents(event.sender); + // Server-side enforcement of the dreamDaemon feature flag. When the flag + // is off, coerce `enableDreamDaemon` to false regardless of the renderer + // payload — this is the authoritative gate for what gets written into + // `.chamber.json workingMemory.consolidation.enabled`. RoleScreen's UI + // gate (Phase 6) hides the toggle, but a bypassed or stale renderer + // must not be able to land an opted-in mind via this channel. + const sanitizedConfig: GenesisConfig = dreamDaemonEnabled() + ? config + : { ...config, enableDreamDaemon: false }; + // Issue #44 — detect name collision BEFORE scaffolding so we never // create a directory the user can't activate. The check is // case-insensitive against currently-loaded minds; persisted-but-not- // loaded minds are not considered. - const collision = mindManager.findByName(config.name); + const collision = mindManager.findByName(sanitizedConfig.name); if (collision) { - const message = `An agent named "${config.name}" already exists. Choose a different name.`; + const message = `An agent named "${sanitizedConfig.name}" already exists. Choose a different name.`; if (win) win.webContents.send(IPC.GENESIS.PROGRESS, { step: 'error', detail: message }); return { success: false, error: message }; } @@ -77,7 +100,7 @@ export function setupGenesisIPC( }); try { - const mindPath = await scaffold.create(config); + const mindPath = await scaffold.create(sanitizedConfig); return await activateCreatedMind(mindManager, mindPath); } catch (err) { const message = err instanceof Error ? err.message : String(err); diff --git a/apps/desktop/src/main/ipc/mind.ts b/apps/desktop/src/main/ipc/mind.ts index df9d5d5c..d91d5509 100644 --- a/apps/desktop/src/main/ipc/mind.ts +++ b/apps/desktop/src/main/ipc/mind.ts @@ -16,9 +16,17 @@ export interface MindIPCConfig { devServerUrl?: string; rendererPath?: string; windowIcon?: NativeImage; + // Returns the current value of the `dreamDaemon` app feature flag. When this + // returns false, `IPC.MIND.SET_DREAM_DAEMON` rejects with a "feature + // unavailable" Error rather than calling into MindManager. The renderer's + // typed contract is `Promise` (not a result union), so a thrown + // Error surfaces through the caller's `await … catch` path. Defaults to + // always-on so test harnesses that omit it keep the existing contract. + dreamDaemonEnabled?: () => boolean; } export function setupMindIPC(mindManager: MindManager, chatService: ChatService, config: MindIPCConfig): void { + const dreamDaemonEnabled = config.dreamDaemonEnabled ?? (() => true); const windowByMind = new Map(); const listMinds = (): MindContext[] => mindManager.listMinds().map((mind) => ({ @@ -58,6 +66,17 @@ export function setupMindIPC(mindManager: MindManager, chatService: ChatService, }); ipcMain.handle(IPC.MIND.SET_DREAM_DAEMON, async (_event, mindId: string, enabled: boolean) => { + // Authoritative gate at the IPC boundary. When the app-level `dreamDaemon` + // feature flag is off, the renderer must not be able to *enable* the + // per-mind opt-in via this channel — that would activate consolidation in + // a build that disables the feature. The *disable* direction stays + // available so a future stable build that re-introduces a "clean up legacy + // state" path can route through this channel without a special case. + // The rejection surfaces via the renderer's `await … catch` path. See + // AgentProfileModal for the consuming UI. + if (enabled && !dreamDaemonEnabled()) { + throw new Error('Dream Daemon is not available in this build'); + } return enabled ? mindManager.enableDreamDaemon(mindId) : mindManager.disableDreamDaemon(mindId); diff --git a/apps/web/src/renderer/components/genesis/GenesisFlow.test.tsx b/apps/web/src/renderer/components/genesis/GenesisFlow.test.tsx index 4e3d3d52..376ad88f 100644 --- a/apps/web/src/renderer/components/genesis/GenesisFlow.test.tsx +++ b/apps/web/src/renderer/components/genesis/GenesisFlow.test.tsx @@ -6,9 +6,20 @@ import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'; import { render, screen, fireEvent, waitFor, cleanup } from '@testing-library/react'; import { GenesisFlow } from './GenesisFlow'; import { AppStateProvider, useAppState } from '../../lib/store'; +import { DEFAULT_APP_FEATURE_FLAGS } from '@chamber/shared/feature-flags'; +import type { AppFeatureFlags } from '@chamber/shared/feature-flags'; import { installElectronAPI, mockElectronAPI } from '../../../test/helpers'; import type { GenesisMindTemplate, MindContext } from '@chamber/shared/types'; +// Helper for tests that exercise the dream-daemon path: provide an +// AppStateProvider whose feature-flag slice has `dreamDaemon: true` so +// the renderer-side coercion in GenesisFlow.handleRole does not zero out +// the opt-in. Tests without an explicit flags arg get the default-off +// shape so coercion behavior under the flag-off case stays visible. +function flagsState(flags: Partial = {}) { + return { featureFlags: { ...DEFAULT_APP_FEATURE_FLAGS, ...flags } }; +} + vi.mock('./VoidScreen', () => ({ VoidScreen: ({ onBegin, @@ -232,8 +243,10 @@ describe('GenesisFlow', () => { // RoleScreen owns the Switch; GenesisFlow.handleRole must thread the // captured opt-in into the IPC call so MindScaffold sees it. Without // this the user toggles the Switch and nothing reaches the main process. + // The dreamDaemon feature flag must be ON for the coercion in + // handleRole to allow the `true` through. render( - + , ); @@ -254,6 +267,34 @@ describe('GenesisFlow', () => { }); }); + it('coerces enableDreamDaemon to false when the dreamDaemon feature flag is off', async () => { + // Renderer-side defense-in-depth: even if a child screen (test mock, + // future deep-link, stale local state) sends `true`, the GenesisFlow + // boundary must zero it out when the flag is off. The IPC layer + // (genesis.ts handler) also enforces this server-side, but the + // renderer should not depend on that. + render( + + + , + ); + + fireEvent.click(screen.getByText('Begin')); + fireEvent.click(screen.getByText('Choose voice')); + fireEvent.click(await screen.findByText('Choose role with daemon')); + + await waitFor(() => { + expect(api.genesis.create).toHaveBeenCalledWith({ + name: 'Test Agent', + role: 'Engineering Partner', + voice: 'Test Agent', + voiceDescription: 'Test voice', + basePath: 'C:\\Users\\test\\agents', + enableDreamDaemon: false, + }); + }); + }); + it('adds a marketplace from the landing page and refreshes templates', async () => { render( diff --git a/apps/web/src/renderer/components/genesis/GenesisFlow.tsx b/apps/web/src/renderer/components/genesis/GenesisFlow.tsx index 3cae2ce2..a98de5b3 100644 --- a/apps/web/src/renderer/components/genesis/GenesisFlow.tsx +++ b/apps/web/src/renderer/components/genesis/GenesisFlow.tsx @@ -1,5 +1,5 @@ import React, { useState, useCallback, useRef } from 'react'; -import { useAppDispatch } from '../../lib/store'; +import { useAppDispatch, useAppState } from '../../lib/store'; import { Logger } from '../../lib/logger'; import { VoidScreen } from './VoidScreen'; import { RoleScreen } from './RoleScreen'; @@ -28,6 +28,8 @@ export function GenesisFlow({ onComplete }: Props) { const [creationError, setCreationError] = useState(null); const creationPromiseRef = useRef | null>(null); const dispatch = useAppDispatch(); + const { featureFlags } = useAppState(); + const dreamDaemonFlag = featureFlags.dreamDaemon; const loadTemplates = useCallback(async () => { setTemplateError(null); @@ -57,6 +59,13 @@ export function GenesisFlow({ onComplete }: Props) { setStage('boot'); setCreationError(null); + // Defense-in-depth: RoleScreen already coerces its own state when the + // flag is off, but coerce again here so any future caller that + // bypasses the screen (deep-link, test harness) can't smuggle a + // `true` payload past the renderer boundary. The IPC layer also + // enforces this server-side. + const effectiveDreamDaemon = dreamDaemonFlag && enableDreamDaemon; + const defaultPath = await window.electronAPI.genesis.getDefaultPath(); const creationPromise = window.electronAPI.genesis.create({ name: name, @@ -64,7 +73,7 @@ export function GenesisFlow({ onComplete }: Props) { voice: name, voiceDescription: voiceDesc, basePath: defaultPath, - enableDreamDaemon, + enableDreamDaemon: effectiveDreamDaemon, }).catch((error: unknown) => ({ success: false, error: error instanceof Error ? error.message : String(error), @@ -76,7 +85,7 @@ export function GenesisFlow({ onComplete }: Props) { setCreationError(result.error ?? 'Genesis failed.'); log.error('Failed:', result.error); } - }, [name, voiceDesc]); + }, [name, voiceDesc, dreamDaemonFlag]); const handleVoiceWithDesc = useCallback((voiceName: string, desc: string) => { setName(voiceName); diff --git a/apps/web/src/renderer/components/genesis/RoleScreen.test.tsx b/apps/web/src/renderer/components/genesis/RoleScreen.test.tsx index c2acf294..9c95533e 100644 --- a/apps/web/src/renderer/components/genesis/RoleScreen.test.tsx +++ b/apps/web/src/renderer/components/genesis/RoleScreen.test.tsx @@ -16,6 +16,9 @@ import React from 'react'; import { describe, it, expect, vi, afterEach } from 'vitest'; import { render, screen, fireEvent, cleanup, waitFor } from '@testing-library/react'; +import { AppStateProvider } from '../../lib/store'; +import { DEFAULT_APP_FEATURE_FLAGS } from '@chamber/shared/feature-flags'; +import type { AppFeatureFlags } from '@chamber/shared/feature-flags'; // Mock TypeWriter to fire onComplete immediately — the real implementation // uses a 35ms-per-char setInterval that takes ~1.5s to finish, which would @@ -37,16 +40,32 @@ afterEach(() => { cleanup(); }); +// Helper: wrap RoleScreen with an AppStateProvider whose feature-flag slice +// has `dreamDaemon` set explicitly. Defaults to ON so the Switch is rendered +// (which is what every test in this file was originally written against). +// Flag-off cases pass `{ dreamDaemon: false }` to confirm the Switch is +// hidden and `enableDreamDaemon` is coerced to false in onSelect payloads. +function renderRoleScreen( + props: { name: string; onSelect: (role: string, enableDreamDaemon: boolean) => void }, + flags: Partial = { dreamDaemon: true }, +) { + return render( + + + , + ); +} + describe('RoleScreen — dream-daemon opt-in switch', () => { it('renders the dream-daemon Switch in the OFF position by default', async () => { - render(); + renderRoleScreen({ name: 'Test', onSelect: vi.fn() }); const toggle = await screen.findByRole('switch', { name: /dream daemon/i }); expect(toggle).not.toBeNull(); expect(toggle.getAttribute('aria-checked')).toBe('false'); }); it('toggling the Switch updates aria-checked to true', async () => { - render(); + renderRoleScreen({ name: 'Test', onSelect: vi.fn() }); const toggle = await screen.findByRole('switch', { name: /dream daemon/i }); fireEvent.click(toggle); await waitFor(() => { @@ -56,7 +75,7 @@ describe('RoleScreen — dream-daemon opt-in switch', () => { it('opt-out (default): clicking a role card calls onSelect with enableDreamDaemon=false', async () => { const onSelect = vi.fn(); - render(); + renderRoleScreen({ name: 'Test', onSelect }); const card = await screen.findByRole('button', { name: /Chief of Staff/i }); fireEvent.click(card); await waitFor(() => { @@ -67,7 +86,7 @@ describe('RoleScreen — dream-daemon opt-in switch', () => { it('opt-in: toggling the Switch ON, then clicking a card, calls onSelect with enableDreamDaemon=true', async () => { const onSelect = vi.fn(); - render(); + renderRoleScreen({ name: 'Test', onSelect }); const toggle = await screen.findByRole('switch', { name: /dream daemon/i }); fireEvent.click(toggle); const card = await screen.findByRole('button', { name: /Engineering Partner/i }); @@ -80,7 +99,7 @@ describe('RoleScreen — dream-daemon opt-in switch', () => { it('custom-role branch: opt-in propagates through "That\'s my purpose"', async () => { const onSelect = vi.fn(); - render(); + renderRoleScreen({ name: 'Test', onSelect }); const toggle = await screen.findByRole('switch', { name: /dream daemon/i }); fireEvent.click(toggle); @@ -96,3 +115,28 @@ describe('RoleScreen — dream-daemon opt-in switch', () => { expect(onSelect).toHaveBeenCalledWith('Debate Coach', true); }); }); + +describe('RoleScreen — feature-flag gate (dreamDaemon: false)', () => { + // Mirrors the gating in IdentityLoader / MindProfileService / MindManager: + // when the app-level `dreamDaemon` flag is off, the Switch is hidden and + // the renderer coerces the value forwarded into `onSelect` to false so + // a stale local state can never smuggle an opt-in past the boundary. + it('hides the dream-daemon Switch when the feature flag is off', async () => { + renderRoleScreen({ name: 'Test', onSelect: vi.fn() }, { dreamDaemon: false }); + // The role cards still render via the typewriter completion path. + await screen.findByRole('button', { name: /Chief of Staff/i }); + // The Switch must NOT be in the DOM at all. + expect(screen.queryByRole('switch', { name: /dream daemon/i })).toBeNull(); + }); + + it('forces onSelect enableDreamDaemon=false when the feature flag is off', async () => { + const onSelect = vi.fn(); + renderRoleScreen({ name: 'Test', onSelect }, { dreamDaemon: false }); + const card = await screen.findByRole('button', { name: /Research Partner/i }); + fireEvent.click(card); + await waitFor(() => { + expect(onSelect).toHaveBeenCalledTimes(1); + }); + expect(onSelect).toHaveBeenCalledWith('Research Partner', false); + }); +}); diff --git a/apps/web/src/renderer/components/genesis/RoleScreen.tsx b/apps/web/src/renderer/components/genesis/RoleScreen.tsx index 2ad27aca..4f9e4f6f 100644 --- a/apps/web/src/renderer/components/genesis/RoleScreen.tsx +++ b/apps/web/src/renderer/components/genesis/RoleScreen.tsx @@ -1,6 +1,7 @@ import React, { useState, useRef, useEffect } from 'react'; import { TypeWriter } from './TypeWriter'; import { cn } from '../../lib/utils'; +import { useAppState } from '../../lib/store'; interface Props { name: string; @@ -23,6 +24,8 @@ const ROLES = [ ]; export function RoleScreen({ name, onSelect }: Props) { + const { featureFlags } = useAppState(); + const dreamDaemonFlag = featureFlags.dreamDaemon; const [showCards, setShowCards] = useState(false); const [selected, setSelected] = useState(null); const [customRole, setCustomRole] = useState(''); @@ -39,6 +42,11 @@ export function RoleScreen({ name, onSelect }: Props) { return () => clearTimeout(t); }, [showCustomInput]); + // Defense-in-depth: even if a stale component state held `true` from + // before the flag flipped off, never forward an opt-in when the + // feature flag is disabled. The IPC layer also coerces this server-side. + const effectiveDreamDaemon = dreamDaemonFlag && enableDreamDaemon; + const handleSelect = (roleId: string) => { if (roleId === 'custom') { setSelected('custom'); @@ -48,14 +56,14 @@ export function RoleScreen({ name, onSelect }: Props) { setSelected(roleId); setTimeout(() => { const role = ROLES.find(r => r.id === roleId); - onSelect(role?.label ?? roleId, enableDreamDaemon); + onSelect(role?.label ?? roleId, effectiveDreamDaemon); }, 300); }; const handleCustomSubmit = () => { const role = customRole.trim(); if (!role) return; - onSelect(role, enableDreamDaemon); + onSelect(role, effectiveDreamDaemon); }; return ( @@ -121,35 +129,40 @@ export function RoleScreen({ name, onSelect }: Props) { decision. ARIA `switch` role + `aria-checked` is the WCAG- recommended shape for an on/off toggle (better than a raw checkbox here because the binary state is the whole UI). + Gated behind the app-level `dreamDaemon` feature flag: when + off, the Switch is hidden entirely so genesis creates a + quiet mind regardless of `.chamber.json` state. */} -
-
-
Enable dream daemon
-
- Background memory consolidation. Off by default — you can change this later. + {dreamDaemonFlag && ( +
+
+
Enable dream daemon
+
+ Background memory consolidation. Off by default — you can change this later. +
-
- -
+ > +
+ )}
)} diff --git a/apps/web/src/renderer/components/profile/AgentProfileModal.test.tsx b/apps/web/src/renderer/components/profile/AgentProfileModal.test.tsx index 774fab71..cd76933a 100644 --- a/apps/web/src/renderer/components/profile/AgentProfileModal.test.tsx +++ b/apps/web/src/renderer/components/profile/AgentProfileModal.test.tsx @@ -6,6 +6,8 @@ import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/re import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { AgentProfileModal } from './AgentProfileModal'; import { AppStateProvider } from '../../lib/store'; +import { DEFAULT_APP_FEATURE_FLAGS } from '@chamber/shared/feature-flags'; +import type { AppFeatureFlags } from '@chamber/shared/feature-flags'; import { installElectronAPI, mockElectronAPI } from '../../../test/helpers'; import type { AgentProfile, MindContext } from '@chamber/shared/types'; @@ -133,12 +135,39 @@ describe('AgentProfileModal', () => { await waitFor(() => expect(api.mind.setDreamDaemon).toHaveBeenCalledWith('mind-1', false)); }); + + describe('feature-flag gate (dreamDaemon: false)', () => { + // When the app-level flag is off the toggle row is hidden entirely. + // MindProfileService also forces `dreamDaemonEnabled: false` server-side + // in the same case, so the renderer would never even see ON — but the + // hide-on-flag-off check protects against any stale value. + it('hides the dream-daemon switch row when the feature flag is off', async () => { + (api.mindProfile.get as ReturnType).mockResolvedValue(makeProfile({ dreamDaemonEnabled: false })); + renderProfileModal({ dreamDaemon: false }); + + // The profile load + first content render must complete so the + // `Display name` label is in the DOM before we assert absence. + await screen.findByText('Display name'); + expect(screen.queryByRole('switch', { name: /dream daemon/i })).toBeNull(); + }); + + it('hides the row even if the server payload still reports dreamDaemonEnabled=true', async () => { + // Defense-in-depth: renderer must not trust a stale ON state. The + // server should force false when the flag is off, but the renderer + // gates independently. + (api.mindProfile.get as ReturnType).mockResolvedValue(makeProfile({ dreamDaemonEnabled: true })); + renderProfileModal({ dreamDaemon: false }); + + await screen.findByText('Display name'); + expect(screen.queryByRole('switch', { name: /dream daemon/i })).toBeNull(); + }); + }); }); }); -function renderProfileModal() { +function renderProfileModal(flags: Partial = { dreamDaemon: true }) { render( - + , ); diff --git a/apps/web/src/renderer/components/profile/AgentProfileModal.tsx b/apps/web/src/renderer/components/profile/AgentProfileModal.tsx index 0be95158..812fe1df 100644 --- a/apps/web/src/renderer/components/profile/AgentProfileModal.tsx +++ b/apps/web/src/renderer/components/profile/AgentProfileModal.tsx @@ -9,7 +9,7 @@ import { DialogTitle, } from '../ui/dialog'; import { cn } from '../../lib/utils'; -import { useAppDispatch } from '../../lib/store'; +import { useAppDispatch, useAppState } from '../../lib/store'; import type { AgentProfile, AgentProfileAvatarCrop, @@ -31,6 +31,8 @@ const iconButtonClass = 'inline-flex items-center justify-center rounded-md bord export function AgentProfileModal({ mind, open, onOpenChange, onProfileChanged }: AgentProfileModalProps) { const dispatch = useAppDispatch(); + const { featureFlags } = useAppState(); + const dreamDaemonFlag = featureFlags.dreamDaemon; const [profile, setProfile] = useState(null); const [loading, setLoading] = useState(false); const [error, setError] = useState(null); @@ -205,37 +207,43 @@ export function AgentProfileModal({ mind, open, onOpenChange, onProfileChanged } so the new opt-in state takes effect immediately — composer reads the gate, DailyLogWriter migrates legacy `log.md` if needed. + Gated behind the app-level `dreamDaemon` feature flag: + when off, the entire row is hidden. MindProfileService + also forces `dreamDaemonEnabled: false` server-side in + the same case, so the data and UI agree. */} -
-
-
Enable dream daemon
-
- Background memory consolidation. When enabled, this agent's chat history is structured and summarized over time. + {dreamDaemonFlag && ( +
+
+
Enable dream daemon
+
+ Background memory consolidation. When enabled, this agent's chat history is structured and summarized over time. +
-
- -
+ > +
+ )}
) : null} diff --git a/packages/services/src/chat/IdentityLoader.test.ts b/packages/services/src/chat/IdentityLoader.test.ts index 9e485b98..86eb3be4 100644 --- a/packages/services/src/chat/IdentityLoader.test.ts +++ b/packages/services/src/chat/IdentityLoader.test.ts @@ -267,5 +267,91 @@ describe('IdentityLoader', () => { const result = loader2.load('/tmp/test'); expect(result?.systemMessage).toContain('# Soul'); }); + + describe('feature-flag gate (dreamDaemonFeatureEnabled)', () => { + // The app-level `dreamDaemon` flag must be authoritative over the + // per-mind `.chamber.json workingMemory.consolidation.enabled` field. + // A stable build that picks up a mind opted-in under insiders must + // still pass `enabled: false` to the composer so the system prompt + // never references consolidated structured-log content. + const mockChamberJsonWithDaemonEnabled = () => { + vi.mocked(fs.existsSync).mockImplementation((candidate) => { + const normalized = String(candidate).replace(/\\/g, '/'); + return normalized.endsWith('SOUL.md') || normalized.endsWith('.chamber.json'); + }); + vi.mocked(fs.readFileSync).mockImplementation((candidate) => { + const normalized = String(candidate).replace(/\\/g, '/'); + if (normalized.endsWith('.chamber.json')) { + return JSON.stringify({ + workingMemory: { + consolidation: { + enabled: true, + lastKTurns: 7, + perTurnMaxBytes: 4096, + memoryMaxBytes: 16384, + }, + }, + }); + } + return '# Soul'; + }); + vi.mocked(fs.readdirSync).mockReturnValue([]); + }; + + it('forces enabled:false when the feature accessor returns false', () => { + mockChamberJsonWithDaemonEnabled(); + const composer = { compose: vi.fn(() => '') }; + const loader2 = new IdentityLoader(() => [], composer, () => false); + + loader2.load('/tmp/agents/widget'); + + expect(composer.compose).toHaveBeenCalledWith( + '/tmp/agents/widget', + // Caps come from .chamber.json (faithful to user config) so a + // future re-enable resumes with the persisted limits. Only the + // `enabled` bit is overridden by the app-level flag. + { + enabled: false, + lastKTurns: 7, + perTurnMaxBytes: 4096, + memoryMaxBytes: 16384, + }, + ); + }); + + it('honors .chamber.json enabled:true when the feature accessor returns true', () => { + mockChamberJsonWithDaemonEnabled(); + const composer = { compose: vi.fn(() => '') }; + const loader2 = new IdentityLoader(() => [], composer, () => true); + + loader2.load('/tmp/agents/widget'); + + expect(composer.compose).toHaveBeenCalledWith( + '/tmp/agents/widget', + { + enabled: true, + lastKTurns: 7, + perTurnMaxBytes: 4096, + memoryMaxBytes: 16384, + }, + ); + }); + + it('default-on accessor preserves existing behavior when omitted', () => { + // Backwards-compatibility guarantee: every existing IdentityLoader + // call site (server bin, tests, e2e harness) constructs without + // the third arg and must continue to honor .chamber.json verbatim. + mockChamberJsonWithDaemonEnabled(); + const composer = { compose: vi.fn(() => '') }; + const loader2 = new IdentityLoader(() => [], composer); + + loader2.load('/tmp/agents/widget'); + + expect(composer.compose).toHaveBeenCalledWith( + '/tmp/agents/widget', + expect.objectContaining({ enabled: true }), + ); + }); + }); }); }); diff --git a/packages/services/src/chat/IdentityLoader.ts b/packages/services/src/chat/IdentityLoader.ts index a3be7f9e..c167a4a8 100644 --- a/packages/services/src/chat/IdentityLoader.ts +++ b/packages/services/src/chat/IdentityLoader.ts @@ -24,6 +24,16 @@ export class IdentityLoader { constructor( private readonly getInstalledTools: InstalledToolsProvider = () => [], composer: WorkingMemoryComposer = createWorkingMemoryComposer(), + /** + * Returns the current value of the app-level `dreamDaemon` feature flag. + * When false, `resolveComposerConfig` forces `enabled: false` regardless + * of `.chamber.json workingMemory.consolidation.enabled` — so the system + * prompt never includes consolidated `log.md` content in a build where + * the feature is off, even if a mind was opted-in under an insiders run. + * Defaults to always-on so the services package stays decoupled from + * app-shell types and existing test fixtures keep working. + */ + private readonly dreamDaemonFeatureEnabled: () => boolean = () => true, ) { this.composer = composer; } @@ -87,10 +97,16 @@ export class IdentityLoader { // already returns DEFAULT_WORKING_MEMORY_CONSOLIDATION when the file is missing, // unparseable, or schema-invalid, so this never throws. Defaults are also // exported here so a composer-only failure path still has a fallback. + // + // App-level feature flag is authoritative over per-mind opt-in: when the + // `dreamDaemon` flag is off, force `enabled: false` regardless of what + // `.chamber.json` says. The caps (lastKTurns / perTurnMaxBytes / + // memoryMaxBytes) are kept faithfully so a future re-enable can resume + // with the user's previously-configured limits. try { const c = loadChamberMindConfig(mindPath).workingMemory.consolidation; return { - enabled: c.enabled, + enabled: this.dreamDaemonFeatureEnabled() ? c.enabled : false, lastKTurns: c.lastKTurns, perTurnMaxBytes: c.perTurnMaxBytes, memoryMaxBytes: c.memoryMaxBytes, diff --git a/packages/services/src/mind/MindManager.test.ts b/packages/services/src/mind/MindManager.test.ts index 75b27a4e..3c9fd6ad 100644 --- a/packages/services/src/mind/MindManager.test.ts +++ b/packages/services/src/mind/MindManager.test.ts @@ -1800,6 +1800,88 @@ describe('MindManager', () => { workingMemory: { consolidation: { enabled: false } }, }); }); + + describe('feature-flag gate (dreamDaemonFeatureEnabled)', () => { + // Defense-in-depth: the IPC layer is the first line of defense, but + // any internal caller (test harness, data migration, future helper) + // must also be gated. Constructing a manager with `() => false` + // exercises the gate inside `doToggleDreamDaemon` directly. + const buildGatedManager = () => { + const mgr = new MindManager( + mockClientFactory as unknown as CopilotClientFactory, + mockIdentityLoader as unknown as IdentityLoader, + mockConfigService as unknown as ConfigService, + mockViewDiscovery as unknown as ViewDiscovery, + undefined, + undefined, + () => false, + ); + mgr.setProviders([mockProvider as unknown as ChamberToolProvider]); + return mgr; + }; + + it('enableDreamDaemon throws when the feature accessor returns false', async () => { + const mgr = buildGatedManager(); + const mind = await mgr.loadMind('/tmp/agents/q'); + + await expect(mgr.enableDreamDaemon(mind.mindId)).rejects.toThrow( + /Dream Daemon is not available in this build/, + ); + expect(patchChamberMindConfig).not.toHaveBeenCalled(); + }); + + it('sequential enableDreamDaemon calls both reject when the feature is off', async () => { + // Regression guard for the daemonToggling in-flight map: if a stale + // allow ever leaked through, the SECOND call could silently resolve + // without going through doToggleDreamDaemon. We need both to throw. + const mgr = buildGatedManager(); + const mind = await mgr.loadMind('/tmp/agents/q'); + + await expect(mgr.enableDreamDaemon(mind.mindId)).rejects.toThrow( + /Dream Daemon is not available in this build/, + ); + await expect(mgr.enableDreamDaemon(mind.mindId)).rejects.toThrow( + /Dream Daemon is not available in this build/, + ); + expect(patchChamberMindConfig).not.toHaveBeenCalled(); + }); + + it('disableDreamDaemon is allowed even when the feature accessor returns false', async () => { + // A stable build that picks up a mind enabled under insiders must + // still be able to clean up the persisted opt-in. Only `enable` is + // gated; `disable` is always permitted. + const mgr = buildGatedManager(); + const mind = await mgr.loadMind('/tmp/agents/q'); + + await mgr.disableDreamDaemon(mind.mindId); + + expect(patchChamberMindConfig).toHaveBeenCalledWith('/tmp/agents/q', { + workingMemory: { consolidation: { enabled: false } }, + }); + }); + + it('enableDreamDaemon proceeds when the feature accessor returns true', async () => { + // Sanity check: explicit `() => true` matches the default-on + // behavior the rest of the test file relies on. + const mgr = new MindManager( + mockClientFactory as unknown as CopilotClientFactory, + mockIdentityLoader as unknown as IdentityLoader, + mockConfigService as unknown as ConfigService, + mockViewDiscovery as unknown as ViewDiscovery, + undefined, + undefined, + () => true, + ); + mgr.setProviders([mockProvider as unknown as ChamberToolProvider]); + const mind = await mgr.loadMind('/tmp/agents/q'); + + await mgr.enableDreamDaemon(mind.mindId); + + expect(patchChamberMindConfig).toHaveBeenCalledWith('/tmp/agents/q', { + workingMemory: { consolidation: { enabled: true } }, + }); + }); + }); }); describe('BYO LLM provider config integration (SDK-native)', () => { diff --git a/packages/services/src/mind/MindManager.ts b/packages/services/src/mind/MindManager.ts index b3b1510e..06d5d6eb 100644 --- a/packages/services/src/mind/MindManager.ts +++ b/packages/services/src/mind/MindManager.ts @@ -79,6 +79,18 @@ export class MindManager extends EventEmitter { * the SDK rejects createSession({provider}) without a model argument. */ private readonly byoDefaultModelProvider: () => string | undefined = () => undefined, + /** + * Returns the current value of the app-level `dreamDaemon` feature flag. + * Defense-in-depth gate for `enableDreamDaemon`: when this returns false, + * a call to `enableDreamDaemon(mindId)` throws "Dream Daemon is not + * available in this build" rather than patching `.chamber.json` and + * reloading the mind. `disableDreamDaemon` is intentionally NOT gated + * (a stable build must still be able to clean up the persisted opt-in + * for a mind that was opted-in under an insiders build). Defaults to + * always-on so the services package stays decoupled from app-shell types + * and existing test fixtures continue to work without modification. + */ + private readonly dreamDaemonFeatureEnabled: () => boolean = () => true, ) { super(); } @@ -371,6 +383,15 @@ export class MindManager extends EventEmitter { } private async doToggleDreamDaemon(mindId: string, enabled: boolean): Promise { + // Defense-in-depth: the IPC layer already rejects `enable` calls when the + // app-level feature flag is off, but a future internal caller (data + // migration, test harness reaching past IPC) must not be able to flip the + // opt-in either. `disable` is intentionally permitted regardless so the + // service still has a path to clean up persisted opt-in state for minds + // that were enabled under an insiders build. + if (enabled && !this.dreamDaemonFeatureEnabled()) { + throw new Error('Dream Daemon is not available in this build'); + } const context = this.minds.get(mindId); if (!context) throw new Error(`Mind ${mindId} not found`); const mindPath = context.mindPath; diff --git a/packages/services/src/mindProfile/MindProfileService.test.ts b/packages/services/src/mindProfile/MindProfileService.test.ts index 5542ef68..cac6b5e4 100644 --- a/packages/services/src/mindProfile/MindProfileService.test.ts +++ b/packages/services/src/mindProfile/MindProfileService.test.ts @@ -129,6 +129,80 @@ describe('MindProfileService', () => { fs.rmSync(root, { recursive: true, force: true }); } }); + + describe('feature-flag gate (dreamDaemonFeatureEnabled)', () => { + // Mirrors IdentityLoader's gate: app-level flag is authoritative over + // per-mind .chamber.json opt-in. When the flag is off, the profile + // payload must report `dreamDaemonEnabled: false` so the (now-hidden) + // toggle UI never sees a stale ON state. + it('forces dreamDaemonEnabled=false even when .chamber.json says true and accessor returns false', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-profile-')); + try { + fs.mkdirSync(path.join(root, '.github', 'agents'), { recursive: true }); + fs.writeFileSync(path.join(root, 'SOUL.md'), '# Moneypenny\n'); + fs.writeFileSync( + path.join(root, '.chamber.json'), + JSON.stringify({ workingMemory: { consolidation: { enabled: true } } }), + ); + + const provider: MindProfileMindProvider = { + getMindPath: () => root, + restartMind: async () => ({}), + }; + const normalizer: AvatarNormalizer = { + normalize: async ({ outputPath }) => { + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + fs.writeFileSync(outputPath, Buffer.from('avatar')); + }, + }; + const service = new MindProfileService( + provider, + new IdentityLoader(), + normalizer, + () => false, + ); + + const profile = service.getProfile('mind-1'); + expect(profile.dreamDaemonEnabled).toBe(false); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it('honors .chamber.json enabled:true when the accessor returns true', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'chamber-profile-')); + try { + fs.mkdirSync(path.join(root, '.github', 'agents'), { recursive: true }); + fs.writeFileSync(path.join(root, 'SOUL.md'), '# Moneypenny\n'); + fs.writeFileSync( + path.join(root, '.chamber.json'), + JSON.stringify({ workingMemory: { consolidation: { enabled: true } } }), + ); + + const provider: MindProfileMindProvider = { + getMindPath: () => root, + restartMind: async () => ({}), + }; + const normalizer: AvatarNormalizer = { + normalize: async ({ outputPath }) => { + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + fs.writeFileSync(outputPath, Buffer.from('avatar')); + }, + }; + const service = new MindProfileService( + provider, + new IdentityLoader(), + normalizer, + () => true, + ); + + const profile = service.getProfile('mind-1'); + expect(profile.dreamDaemonEnabled).toBe(true); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + }); }); function createProfileFixture() { diff --git a/packages/services/src/mindProfile/MindProfileService.ts b/packages/services/src/mindProfile/MindProfileService.ts index c21fd7b8..a020cd70 100644 --- a/packages/services/src/mindProfile/MindProfileService.ts +++ b/packages/services/src/mindProfile/MindProfileService.ts @@ -17,11 +17,23 @@ const AVATAR_RELATIVE_PATH = path.join('.chamber', 'avatar.png'); const MAX_PROFILE_FILE_BYTES = 512_000; export class MindProfileService { + private readonly dreamDaemonFeatureEnabled: () => boolean; + constructor( private readonly minds: MindProfileMindProvider, private readonly identityLoader: IdentityLoader, private readonly avatarNormalizer: AvatarNormalizer, - ) {} + /** + * Returns the current value of the app-level `dreamDaemon` feature flag. + * When false, `getProfile` reports `dreamDaemonEnabled: false` regardless + * of `.chamber.json workingMemory.consolidation.enabled`, so the renderer + * (and any conditional rendering keyed off this field) never observes a + * stale ON state from a mind opted-in under an insiders build. + */ + dreamDaemonFeatureEnabled: () => boolean = () => true, + ) { + this.dreamDaemonFeatureEnabled = dreamDaemonFeatureEnabled; + } getProfile(mindId: string, needsRestart = false): AgentProfile { const mindPath = this.requireMindPath(mindId); @@ -39,7 +51,8 @@ export class MindProfileService { soul: this.readProfileFile(mindPath, 'soul', 'SOUL.md'), agentFiles: this.listAgentFiles(mindPath), needsRestart, - dreamDaemonEnabled: chamberConfig.workingMemory.consolidation.enabled, + dreamDaemonEnabled: + this.dreamDaemonFeatureEnabled() && chamberConfig.workingMemory.consolidation.enabled, }; } From 645f8b3b4b2cd02dc4c62065dbf833b1492c5da4 Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Tue, 26 May 2026 16:41:41 -0400 Subject: [PATCH 22/23] docs(feature-flags): document dreamDaemon flag + fix AGENTS.md md-lint - Add dreamDaemon row to the Current flags table (dev-only rollout: stable remote=false, insiders remote=false, dev=true). - New 'Dream Daemon' subsection enumerates every off-state behavior (buildMindMemoryService not called, IPC reject, genesis.create coercion, MindManager throw, IdentityLoader enabled=false override, MindProfileService payload override, RoleScreen/GenesisFlow/AgentProfileModal UI hide). - Update DEV_FEATURE_FLAGS snippet and the docs/flags/v1/flags.json example to include dreamDaemon. - Fix MD022/MD032 in AGENTS.md (blank lines around the Dream Daemon heading added on the branch) so 'npm run lint' is clean. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- AGENTS.md | 1 + ai-docs/feature-flags.md | 47 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8eaa1354..51f8e611 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,6 +24,7 @@ Chamber is a desktop application where AI agents ("minds") operate as a Chief of - **Magentic**: Manager-driven task ledger ### Memory Consolidation (Dream Daemon — experimental, opt-in) + - **Per-mind opt-in**: `workingMemory.consolidation.enabled` in each mind's `.chamber.json`. Default OFF. - **Toggle surfaces**: Genesis wizard role screen (pre-genesis) and agent profile modal (post-genesis); never silent. - **Bidirectional**: Disabling rolls the structured `log.md` (and any `log.legacy.md`) back to unstructured turn-by-turn markdown via `rollbackToUnstructured`. Single source of truth after rollback. diff --git a/ai-docs/feature-flags.md b/ai-docs/feature-flags.md index f396862c..1b5dd76e 100644 --- a/ai-docs/feature-flags.md +++ b/ai-docs/feature-flags.md @@ -50,12 +50,14 @@ Expected shape: "stable": { "switchboardRelay": false, "byoLlm": false, - "chamberCopilot": false + "chamberCopilot": false, + "dreamDaemon": false }, "insiders": { "switchboardRelay": true, "byoLlm": true, - "chamberCopilot": true + "chamberCopilot": true, + "dreamDaemon": false } } } @@ -108,6 +110,7 @@ normal app runs or release builds. | `switchboardRelay` | remote | remote | Hides the activity-bar relay entry point and route. | | `byoLlm` | remote | remote | Hides BYO model settings and disables desktop BYO runtime/IPC usage. | | `chamberCopilot` | remote | remote | Wires the chamber-copilot ACP provider and `cli_*` tools. | +| `dreamDaemon` | remote `false` | remote `false` | Dev-only. Gates the working-memory consolidation daemon, the per-mind opt-in toggle, and the prompt-path use of consolidated memory. | ## Local development flags @@ -126,6 +129,7 @@ export const DEV_FEATURE_FLAGS = { switchboardRelay: true, byoLlm: true, chamberCopilot: true, + dreamDaemon: true, }; ``` @@ -156,6 +160,45 @@ to mind tool providers. Stable builds also ignore the legacy `chamberCopilotEnabled` key in `~/.chamber/config.json`; users cannot turn this surface on locally. +## Dream Daemon + +`dreamDaemon` gates Chamber's working-memory consolidation surface (the +"dream daemon") and the prompt-time use of the consolidated memory it produces. +The flag rolls out as **dev-only**: both `channels.stable` and +`channels.insiders` are `false` in `docs/flags/v1/flags.json` so external +testers do not see the surface yet; `DEV_FEATURE_FLAGS.dreamDaemon = true` +keeps local development behavior unchanged. + +When disabled: + +- `apps/desktop/src/main.ts` does not call `buildMindMemoryService`, so the + daemon, scheduler, and SQLite-backed memory store are never constructed. + The `__chamberMindMemoryService` E2E global is also not exposed. +- `IPC.MIND.SET_DREAM_DAEMON` rejects with `"Dream Daemon is not available in + this build"` when the renderer asks to enable. Disable requests still pass + through so a stable build can clean up persisted opt-in state from an + insiders run. +- `genesis.create` coerces `enableDreamDaemon: false` server-side before + calling `MindScaffold.create`, regardless of the renderer payload. The + newly written `.chamber.json` always has + `workingMemory.consolidation.enabled: false`. +- `MindManager.enableDreamDaemon` throws before any mind lookup as a + defense-in-depth gate behind IPC. +- `IdentityLoader.resolveComposerConfig` forces `enabled: false` in the + composer config it returns, regardless of the per-mind `.chamber.json`. + Persisted caps (`lastKTurns`, `perTurnMaxBytes`, `memoryMaxBytes`) are kept + faithful so a future flip-on does not lose the user's settings. +- `MindProfileService.getProfile` reports `dreamDaemonEnabled: false` even + when `.chamber.json` says `true`, so the (now-hidden) UI never sees a stale + ON state. +- `RoleScreen`, `GenesisFlow`, and `AgentProfileModal` hide the + dream-daemon Switch / toggle row and coerce `enableDreamDaemon: false` at + every emit boundary in the renderer. + +The asymmetry on the IPC and `MindManager` gates is deliberate: enable is +gated, disable is always allowed. A stable build must be able to clean up +opt-in state for minds that were enabled under an insiders build. + ## Adding a new feature flag Use this checklist when introducing a flag for a feature still under From d49ca006a1a1f62e89f15a2bdc49abc49473ce61 Mon Sep 17 00:00:00 2001 From: johnhain-msft Date: Fri, 29 May 2026 13:46:33 -0400 Subject: [PATCH 23/23] Fix MindScaffold test mocks: rename destroy -> disconnect for SDK 0.3.0 API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SDK 0.3.0 CopilotSession exposes disconnect() not destroy(). The merge auto-took the dream-daemon-branch mock shape (destroy:) while production code (MindScaffold.generateSoul finally block) calls session.disconnect() — same as upstream master. Aligning fakes with the real SDK surface. --- packages/services/src/genesis/MindScaffold.test.ts | 2 +- tests/integration/mindScaffold.integration.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/services/src/genesis/MindScaffold.test.ts b/packages/services/src/genesis/MindScaffold.test.ts index a9137ba1..dac43d4d 100644 --- a/packages/services/src/genesis/MindScaffold.test.ts +++ b/packages/services/src/genesis/MindScaffold.test.ts @@ -597,7 +597,7 @@ describe('MindScaffold.generateSoul — genesis prompt no longer references log. it('does not pass log.md as a write target to buildGenesisPrompt', async () => { const session = { send: vi.fn<(_: { prompt: string }) => Promise>(async () => undefined), - destroy: vi.fn(async () => undefined), + disconnect: vi.fn(async () => undefined), on: vi.fn((event: string, callback: () => void) => { if (event === 'session.idle') setTimeout(callback, 0); return vi.fn(); diff --git a/tests/integration/mindScaffold.integration.test.ts b/tests/integration/mindScaffold.integration.test.ts index 3ca760b8..dd989ad3 100644 --- a/tests/integration/mindScaffold.integration.test.ts +++ b/tests/integration/mindScaffold.integration.test.ts @@ -105,7 +105,7 @@ function makeFakeClientFactory(seedFiles: (paths: SoulPaths) => void): CopilotCl send: vi.fn(async () => { seedFiles(paths); }), - destroy: vi.fn(async () => undefined), + disconnect: vi.fn(async () => undefined), on: vi.fn((event: string, callback: () => void) => { if (event === 'session.idle') setTimeout(callback, 0); return vi.fn();