From 2c0ac49f46a7c4d234290ac6b2e7f340766008dc Mon Sep 17 00:00:00 2001 From: devlux76 Date: Sat, 14 Mar 2026 02:56:42 -0600 Subject: [PATCH 1/4] refactor: move core engine code into lib/ and runtime harness into ui/ --- BackendKind.ts | 41 +- CreateVectorBackend.ts | 29 +- PLAN.md | 2 +- Policy.ts | 148 +--- TopK.ts | 29 +- VectorBackend.ts | 50 +- WasmVectorBackend.ts | 155 +--- WebGLVectorBackend.ts | 283 +------- WebGPUVectorBackend.ts | 271 +------ WebNNTypes.d.ts | 37 +- WebNNVectorBackend.ts | 98 +-- core/BuiltInModelProfiles.ts | 58 +- core/HotpathPolicy.ts | 306 +------- core/ModelDefaults.ts | 101 +-- core/ModelProfile.ts | 57 +- core/ModelProfileResolver.ts | 113 +-- core/NumericConstants.ts | 18 +- core/SalienceEngine.ts | 421 +---------- core/crypto/hash.ts | 27 +- core/crypto/sign.ts | 70 +- core/crypto/uuid.ts | 49 +- core/crypto/verify.ts | 51 +- core/types.ts | 219 +----- cortex/KnowledgeGapDetector.ts | 67 +- cortex/MetroidBuilder.ts | 218 +----- cortex/OpenTSPSolver.ts | 63 +- cortex/Query.ts | 171 +---- cortex/QueryResult.ts | 13 +- cortex/Ranking.ts | 157 +--- daydreamer/ClusterStability.ts | 681 +----------------- daydreamer/ExperienceReplay.ts | 234 +----- daydreamer/FullNeighborRecalc.ts | 202 +----- daydreamer/HebbianUpdater.ts | 194 +---- daydreamer/IdleScheduler.ts | 173 +---- daydreamer/PrototypeRecomputer.ts | 264 +------ docs/development.md | 2 +- .../DeterministicDummyEmbeddingBackend.ts | 109 +-- embeddings/EmbeddingBackend.ts | 7 +- embeddings/EmbeddingRunner.ts | 55 +- embeddings/OrtWebglEmbeddingBackend.ts | 130 +--- embeddings/ProviderResolver.ts | 349 +-------- embeddings/TransformersJsEmbeddingBackend.ts | 162 +---- hippocampus/Chunker.ts | 80 +- hippocampus/FastNeighborInsert.ts | 223 +----- hippocampus/HierarchyBuilder.ts | 405 +---------- hippocampus/Ingest.ts | 157 +--- hippocampus/PageBuilder.ts | 89 +-- lib/BackendKind.ts | 40 + lib/CreateVectorBackend.ts | 28 + lib/Policy.ts | 147 ++++ lib/TopK.ts | 28 + lib/VectorBackend.ts | 49 ++ Vectors.glsl => lib/Vectors.glsl | 0 Vectors.wat => lib/Vectors.wat | 0 Vectors.wgsl => lib/Vectors.wgsl | 0 lib/WasmVectorBackend.ts | 154 ++++ lib/WebGLVectorBackend.ts | 282 ++++++++ lib/WebGPUVectorBackend.ts | 270 +++++++ lib/WebNNTypes.d.ts | 36 + lib/WebNNVectorBackend.ts | 97 +++ lib/core/BuiltInModelProfiles.ts | 57 ++ lib/core/HotpathPolicy.ts | 305 ++++++++ lib/core/ModelDefaults.ts | 100 +++ lib/core/ModelProfile.ts | 56 ++ lib/core/ModelProfileResolver.ts | 112 +++ lib/core/NumericConstants.ts | 17 + lib/core/SalienceEngine.ts | 420 +++++++++++ lib/core/crypto/hash.ts | 26 + lib/core/crypto/sign.ts | 69 ++ lib/core/crypto/uuid.ts | 48 ++ lib/core/crypto/verify.ts | 50 ++ lib/core/types.ts | 218 ++++++ lib/cortex/KnowledgeGapDetector.ts | 66 ++ lib/cortex/MetroidBuilder.ts | 217 ++++++ lib/cortex/OpenTSPSolver.ts | 62 ++ lib/cortex/Query.ts | 170 +++++ lib/cortex/QueryResult.ts | 12 + lib/cortex/Ranking.ts | 156 ++++ lib/daydreamer/ClusterStability.ts | 680 +++++++++++++++++ lib/daydreamer/ExperienceReplay.ts | 233 ++++++ lib/daydreamer/FullNeighborRecalc.ts | 201 ++++++ lib/daydreamer/HebbianUpdater.ts | 193 +++++ lib/daydreamer/IdleScheduler.ts | 172 +++++ lib/daydreamer/PrototypeRecomputer.ts | 263 +++++++ .../DeterministicDummyEmbeddingBackend.ts | 108 +++ lib/embeddings/EmbeddingBackend.ts | 6 + lib/embeddings/EmbeddingRunner.ts | 54 ++ lib/embeddings/OrtWebglEmbeddingBackend.ts | 129 ++++ lib/embeddings/ProviderResolver.ts | 348 +++++++++ .../TransformersJsEmbeddingBackend.ts | 161 +++++ lib/hippocampus/Chunker.ts | 79 ++ lib/hippocampus/FastNeighborInsert.ts | 222 ++++++ lib/hippocampus/HierarchyBuilder.ts | 404 +++++++++++ lib/hippocampus/Ingest.ts | 156 ++++ lib/hippocampus/PageBuilder.ts | 88 +++ lib/sharing/CuriosityBroadcaster.ts | 151 ++++ lib/sharing/EligibilityClassifier.ts | 111 +++ lib/sharing/PeerExchange.ts | 131 ++++ lib/sharing/SubgraphExporter.ts | 203 ++++++ lib/sharing/SubgraphImporter.ts | 206 ++++++ lib/sharing/types.ts | 141 ++++ lib/storage/IndexedDbMetadataStore.ts | 614 ++++++++++++++++ lib/storage/MemoryVectorStore.ts | 39 + lib/storage/OPFSVectorStore.ts | 89 +++ scripts/runtime-harness-server.mjs | 2 +- sharing/CuriosityBroadcaster.ts | 152 +--- sharing/EligibilityClassifier.ts | 112 +-- sharing/PeerExchange.ts | 132 +--- sharing/SubgraphExporter.ts | 204 +----- sharing/SubgraphImporter.ts | 207 +----- sharing/types.ts | 142 +--- storage/IndexedDbMetadataStore.ts | 615 +--------------- storage/MemoryVectorStore.ts | 40 +- storage/OPFSVectorStore.ts | 90 +-- tsconfig.json | 6 +- {runtime => ui}/harness/index.html | 0 116 files changed, 8533 insertions(+), 8481 deletions(-) create mode 100644 lib/BackendKind.ts create mode 100644 lib/CreateVectorBackend.ts create mode 100644 lib/Policy.ts create mode 100644 lib/TopK.ts create mode 100644 lib/VectorBackend.ts rename Vectors.glsl => lib/Vectors.glsl (100%) rename Vectors.wat => lib/Vectors.wat (100%) rename Vectors.wgsl => lib/Vectors.wgsl (100%) create mode 100644 lib/WasmVectorBackend.ts create mode 100644 lib/WebGLVectorBackend.ts create mode 100644 lib/WebGPUVectorBackend.ts create mode 100644 lib/WebNNTypes.d.ts create mode 100644 lib/WebNNVectorBackend.ts create mode 100644 lib/core/BuiltInModelProfiles.ts create mode 100644 lib/core/HotpathPolicy.ts create mode 100644 lib/core/ModelDefaults.ts create mode 100644 lib/core/ModelProfile.ts create mode 100644 lib/core/ModelProfileResolver.ts create mode 100644 lib/core/NumericConstants.ts create mode 100644 lib/core/SalienceEngine.ts create mode 100644 lib/core/crypto/hash.ts create mode 100644 lib/core/crypto/sign.ts create mode 100644 lib/core/crypto/uuid.ts create mode 100644 lib/core/crypto/verify.ts create mode 100644 lib/core/types.ts create mode 100644 lib/cortex/KnowledgeGapDetector.ts create mode 100644 lib/cortex/MetroidBuilder.ts create mode 100644 lib/cortex/OpenTSPSolver.ts create mode 100644 lib/cortex/Query.ts create mode 100644 lib/cortex/QueryResult.ts create mode 100644 lib/cortex/Ranking.ts create mode 100644 lib/daydreamer/ClusterStability.ts create mode 100644 lib/daydreamer/ExperienceReplay.ts create mode 100644 lib/daydreamer/FullNeighborRecalc.ts create mode 100644 lib/daydreamer/HebbianUpdater.ts create mode 100644 lib/daydreamer/IdleScheduler.ts create mode 100644 lib/daydreamer/PrototypeRecomputer.ts create mode 100644 lib/embeddings/DeterministicDummyEmbeddingBackend.ts create mode 100644 lib/embeddings/EmbeddingBackend.ts create mode 100644 lib/embeddings/EmbeddingRunner.ts create mode 100644 lib/embeddings/OrtWebglEmbeddingBackend.ts create mode 100644 lib/embeddings/ProviderResolver.ts create mode 100644 lib/embeddings/TransformersJsEmbeddingBackend.ts create mode 100644 lib/hippocampus/Chunker.ts create mode 100644 lib/hippocampus/FastNeighborInsert.ts create mode 100644 lib/hippocampus/HierarchyBuilder.ts create mode 100644 lib/hippocampus/Ingest.ts create mode 100644 lib/hippocampus/PageBuilder.ts create mode 100644 lib/sharing/CuriosityBroadcaster.ts create mode 100644 lib/sharing/EligibilityClassifier.ts create mode 100644 lib/sharing/PeerExchange.ts create mode 100644 lib/sharing/SubgraphExporter.ts create mode 100644 lib/sharing/SubgraphImporter.ts create mode 100644 lib/sharing/types.ts create mode 100644 lib/storage/IndexedDbMetadataStore.ts create mode 100644 lib/storage/MemoryVectorStore.ts create mode 100644 lib/storage/OPFSVectorStore.ts rename {runtime => ui}/harness/index.html (100%) diff --git a/BackendKind.ts b/BackendKind.ts index b23187a..cfdcda0 100644 --- a/BackendKind.ts +++ b/BackendKind.ts @@ -1,40 +1 @@ -export type BackendKind = "webnn" | "webgpu" | "webgl" | "wasm"; - -function hasWebGpuSupport(): boolean { - return ( - typeof navigator !== "undefined" && - typeof (navigator as Navigator & { gpu?: unknown }).gpu !== "undefined" - ); -} - -function hasWebGl2Support(): boolean { - if (typeof document === "undefined") { - return false; - } - - const canvas = document.createElement("canvas"); - return canvas.getContext("webgl2") !== null; -} - -function hasWebNnSupport(): boolean { - return ( - typeof navigator !== "undefined" && - typeof (navigator as Navigator & { ml?: unknown }).ml !== "undefined" - ); -} - -export function detectBackend(): BackendKind { - if (hasWebGpuSupport()) { - return "webgpu"; - } - - if (hasWebGl2Support()) { - return "webgl"; - } - - if (hasWebNnSupport()) { - return "webnn"; - } - - return "wasm"; -} +export * from "./lib/BackendKind"; diff --git a/CreateVectorBackend.ts b/CreateVectorBackend.ts index 532f9b0..7b04c1f 100644 --- a/CreateVectorBackend.ts +++ b/CreateVectorBackend.ts @@ -1,28 +1 @@ -import { detectBackend } from "./BackendKind"; -import type { VectorBackend } from "./VectorBackend"; -import { WasmVectorBackend } from "./WasmVectorBackend"; -import { WebGlVectorBackend } from "./WebGLVectorBackend"; -import { WebGpuVectorBackend } from "./WebGPUVectorBackend"; -import { WebNnVectorBackend } from "./WebNNVectorBackend"; - -export async function createVectorBackend( - wasmBytes: ArrayBuffer -): Promise { - const kind = detectBackend(); - if (kind === "webgpu") { - return WebGpuVectorBackend.create().catch(() => - WasmVectorBackend.create(wasmBytes) - ); - } - if (kind === "webgl") { - return Promise.resolve(WebGlVectorBackend.create()).catch(() => - WasmVectorBackend.create(wasmBytes) - ); - } - if (kind === "webnn") { - return WebNnVectorBackend.create(wasmBytes).catch(() => - WasmVectorBackend.create(wasmBytes) - ); - } - return WasmVectorBackend.create(wasmBytes); -} +export * from "./lib/CreateVectorBackend"; diff --git a/PLAN.md b/PLAN.md index 96c1894..91e7bb3 100644 --- a/PLAN.md +++ b/PLAN.md @@ -139,7 +139,7 @@ This document tracks the implementation status of each major module in CORTEX. I | Module | Status | Files | Notes | |--------|--------|-------|-------| -| Browser Harness | ✅ Complete | `runtime/harness/index.html`, `scripts/runtime-harness-server.mjs` | Localhost-served HTML harness for browser testing | +| Browser Harness | ✅ Complete | `ui/harness/index.html`, `scripts/runtime-harness-server.mjs` | Localhost-served HTML harness for browser testing | | Electron Wrapper | ✅ Complete | `scripts/electron-harness-main.mjs` | Thin Electron launcher for GPU-realism testing | | Playwright Tests | ✅ Complete | `tests/runtime/browser-harness.spec.mjs`, `tests/runtime/electron-harness.spec.mjs` | Browser lane passes; Electron context-sensitive | | Docker Debug Lane | ✅ Complete | `docker/electron-debug/*`, `docker-compose.electron-debug.yml` | Sandbox-isolated Electron debugging via VS Code attach | diff --git a/Policy.ts b/Policy.ts index 38e02a2..ac5f044 100644 --- a/Policy.ts +++ b/Policy.ts @@ -1,147 +1 @@ -import type { ModelProfile } from "./core/ModelProfile"; -import { - ModelProfileResolver, - type ModelProfileResolverOptions, - type ResolveModelProfileInput, -} from "./core/ModelProfileResolver"; - -export type QueryScope = "broad" | "normal" | "narrow" | "default"; - -export interface ProjectionHead { - dimIn: number; - dimOut: number; - bits?: number; - // Byte offset for the projection head in a flattened projection buffer. - offset: number; -} - -export interface RoutingPolicy { - broad: ProjectionHead; - normal: ProjectionHead; - narrow: ProjectionHead; -} - -export interface ResolvedRoutingPolicy { - modelProfile: ModelProfile; - routingPolicy: RoutingPolicy; -} - -export interface ResolveRoutingPolicyOptions { - resolver?: ModelProfileResolver; - resolverOptions?: ModelProfileResolverOptions; - routingPolicyOverrides?: Partial; -} - -export interface RoutingPolicyDerivation { - broadDimRatio: number; - normalDimRatio: number; - narrowDimRatio: number; - broadHashBits: number; - dimAlignment: number; - minProjectionDim: number; -} - -export const DEFAULT_ROUTING_POLICY_DERIVATION: RoutingPolicyDerivation = - Object.freeze({ - broadDimRatio: 1 / 8, - normalDimRatio: 1 / 4, - narrowDimRatio: 1 / 2, - broadHashBits: 128, - dimAlignment: 8, - minProjectionDim: 8, - }); - -function assertPositiveInteger(name: string, value: number): void { - if (!Number.isInteger(value) || value <= 0) { - throw new Error(`${name} must be a positive integer`); - } -} - -function assertPositiveFinite(name: string, value: number): void { - if (!Number.isFinite(value) || value <= 0) { - throw new Error(`${name} must be positive and finite`); - } -} - -function alignDown(value: number, alignment: number): number { - return Math.floor(value / alignment) * alignment; -} - -function deriveProjectionDim( - dimIn: number, - ratio: number, - derivation: RoutingPolicyDerivation, -): number { - const raw = Math.floor(dimIn * ratio); - const aligned = alignDown(raw, derivation.dimAlignment); - const bounded = Math.max(derivation.minProjectionDim, aligned); - return Math.min(dimIn, bounded); -} - -function validateDerivation(derivation: RoutingPolicyDerivation): void { - assertPositiveFinite("broadDimRatio", derivation.broadDimRatio); - assertPositiveFinite("normalDimRatio", derivation.normalDimRatio); - assertPositiveFinite("narrowDimRatio", derivation.narrowDimRatio); - assertPositiveInteger("broadHashBits", derivation.broadHashBits); - assertPositiveInteger("dimAlignment", derivation.dimAlignment); - assertPositiveInteger("minProjectionDim", derivation.minProjectionDim); -} - -export function createRoutingPolicy( - modelProfile: Pick, - overrides: Partial = {}, -): RoutingPolicy { - assertPositiveInteger("embeddingDimension", modelProfile.embeddingDimension); - - const derivation: RoutingPolicyDerivation = { - ...DEFAULT_ROUTING_POLICY_DERIVATION, - ...overrides, - }; - - validateDerivation(derivation); - - const dimIn = modelProfile.embeddingDimension; - const broadDim = deriveProjectionDim(dimIn, derivation.broadDimRatio, derivation); - const normalDim = deriveProjectionDim(dimIn, derivation.normalDimRatio, derivation); - const narrowDim = deriveProjectionDim(dimIn, derivation.narrowDimRatio, derivation); - - const broadOffset = 0; - const normalOffset = broadOffset + broadDim * dimIn; - const narrowOffset = normalOffset + normalDim * dimIn; - - return { - broad: { - dimIn, - dimOut: broadDim, - bits: derivation.broadHashBits, - offset: broadOffset, - }, - normal: { - dimIn, - dimOut: normalDim, - offset: normalOffset, - }, - narrow: { - dimIn, - dimOut: narrowDim, - offset: narrowOffset, - }, - }; -} - -export function resolveRoutingPolicyForModel( - input: ResolveModelProfileInput, - options: ResolveRoutingPolicyOptions = {}, -): ResolvedRoutingPolicy { - const resolver = - options.resolver ?? new ModelProfileResolver(options.resolverOptions); - const modelProfile = resolver.resolve(input); - - return { - modelProfile, - routingPolicy: createRoutingPolicy( - modelProfile, - options.routingPolicyOverrides, - ), - }; -} +export * from "./lib/Policy"; diff --git a/TopK.ts b/TopK.ts index 3f0467e..b72995c 100644 --- a/TopK.ts +++ b/TopK.ts @@ -1,28 +1 @@ -import type { DistanceResult, ScoreResult } from "./VectorBackend"; - -export function topKByScore(scores: Float32Array, k: number): ScoreResult[] { - const limit = Math.max(0, Math.min(k, scores.length)); - const indices = Array.from({ length: scores.length }, (_, i) => i); - - indices.sort((a, b) => scores[b] - scores[a]); - - return indices.slice(0, limit).map((index) => ({ - index, - score: scores[index] - })); -} - -export function topKByDistance( - distances: Uint32Array | Int32Array | Float32Array, - k: number -): DistanceResult[] { - const limit = Math.max(0, Math.min(k, distances.length)); - const indices = Array.from({ length: distances.length }, (_, i) => i); - - indices.sort((a, b) => Number(distances[a]) - Number(distances[b])); - - return indices.slice(0, limit).map((index) => ({ - index, - distance: Number(distances[index]) - })); -} +export * from "./lib/TopK"; diff --git a/VectorBackend.ts b/VectorBackend.ts index 6fe6737..0ff6fe6 100644 --- a/VectorBackend.ts +++ b/VectorBackend.ts @@ -1,49 +1 @@ -import type { BackendKind } from "./BackendKind"; - -export interface ScoreResult { - index: number; - score: number; -} - -export interface DistanceResult { - index: number; - distance: number; -} - -export interface VectorBackend { - kind: BackendKind; - - // Exact or high-precision dot-product scoring over row-major matrices. - dotMany( - query: Float32Array, - matrix: Float32Array, - dim: number, - count: number - ): Promise; - - // Projection helper used to reduce dimensionality for routing tiers. - project( - vector: Float32Array, - projectionMatrix: Float32Array, - dimIn: number, - dimOut: number - ): Promise; - - topKFromScores(scores: Float32Array, k: number): Promise; - - // Random-hyperplane hash from projected vectors into packed binary codes. - hashToBinary( - vector: Float32Array, - projectionMatrix: Float32Array, - dimIn: number, - bits: number - ): Promise; - - hammingTopK( - queryCode: Uint32Array, - codes: Uint32Array, - wordsPerCode: number, - count: number, - k: number - ): Promise; -} +export * from "./lib/VectorBackend"; diff --git a/WasmVectorBackend.ts b/WasmVectorBackend.ts index 8817bc7..9750be5 100644 --- a/WasmVectorBackend.ts +++ b/WasmVectorBackend.ts @@ -1,154 +1 @@ -import type { - DistanceResult, - ScoreResult, - VectorBackend -} from "./VectorBackend"; -import { - FLOAT32_BYTES, - UINT32_BITS, - UINT32_BYTES, - WASM_ALLOC_ALIGNMENT_BYTES, - WASM_ALLOC_GUARD_BYTES, - WASM_PAGE_BYTES, -} from "./core/NumericConstants"; - -interface WasmVectorExports { - mem: WebAssembly.Memory; - dot_many(qPtr: number, mPtr: number, outPtr: number, dim: number, count: number): void; - project(vecPtr: number, pPtr: number, outPtr: number, dimIn: number, dimOut: number): void; - hash_binary(vecPtr: number, pPtr: number, codePtr: number, dimIn: number, bits: number): void; - hamming_scores( - queryCodePtr: number, - codesPtr: number, - outPtr: number, - wordsPerCode: number, - count: number - ): void; - topk_i32(scoresPtr: number, outPtr: number, count: number, k: number): void; - topk_f32(scoresPtr: number, outPtr: number, count: number, k: number): void; -} - -export class WasmVectorBackend implements VectorBackend { - readonly kind = "wasm" as const; - private exports!: WasmVectorExports; - private mem!: WebAssembly.Memory; - private bump = WASM_ALLOC_GUARD_BYTES; - - static async create(wasmBytes: ArrayBuffer): Promise { - const b = new WasmVectorBackend(); - const { instance } = await WebAssembly.instantiate(wasmBytes); - b.exports = instance.exports as unknown as WasmVectorExports; - b.mem = instance.exports.mem as WebAssembly.Memory; - return b; - } - - // 16-byte-aligned bump allocator; call reset() between requests - private alloc(bytes: number): number { - const ptr = - (this.bump + (WASM_ALLOC_ALIGNMENT_BYTES - 1)) & - ~(WASM_ALLOC_ALIGNMENT_BYTES - 1); - this.bump = ptr + bytes; - if (this.bump > this.mem.buffer.byteLength) { - this.mem.grow( - Math.ceil((this.bump - this.mem.buffer.byteLength) / WASM_PAGE_BYTES) - ); - } - return ptr; - } - - reset(): void { - this.bump = WASM_ALLOC_GUARD_BYTES; - } - - private writeF32(data: Float32Array): number { - const ptr = this.alloc(data.byteLength); - new Float32Array(this.mem.buffer, ptr, data.length).set(data); - return ptr; - } - - private writeU32(data: Uint32Array): number { - const ptr = this.alloc(data.byteLength); - new Uint32Array(this.mem.buffer, ptr, data.length).set(data); - return ptr; - } - - async dotMany( - query: Float32Array, matrix: Float32Array, - dim: number, count: number - ): Promise { - this.reset(); - const q_ptr = this.writeF32(query); - const m_ptr = this.writeF32(matrix); - const out_ptr = this.alloc(count * FLOAT32_BYTES); - this.exports.dot_many(q_ptr, m_ptr, out_ptr, dim, count); - return new Float32Array( - this.mem.buffer.slice(out_ptr, out_ptr + count * FLOAT32_BYTES) - ); - } - - async project( - vec: Float32Array, - projectionMatrix: Float32Array, - dimIn: number, dimOut: number - ): Promise { - this.reset(); - const v_ptr = this.writeF32(vec); - const P_ptr = this.writeF32(projectionMatrix); - const out_ptr = this.alloc(dimOut * FLOAT32_BYTES); - this.exports.project(v_ptr, P_ptr, out_ptr, dimIn, dimOut); - return new Float32Array( - this.mem.buffer.slice(out_ptr, out_ptr + dimOut * FLOAT32_BYTES) - ); - } - - async hashToBinary( - vec: Float32Array, - projectionMatrix: Float32Array, - dimIn: number, bits: number - ): Promise { - this.reset(); - const wordsPerCode = Math.ceil(bits / UINT32_BITS); - const v_ptr = this.writeF32(vec); - const P_ptr = this.writeF32(projectionMatrix); - const code_ptr = this.alloc(wordsPerCode * UINT32_BYTES); - this.exports.hash_binary(v_ptr, P_ptr, code_ptr, dimIn, bits); - return new Uint32Array( - this.mem.buffer.slice(code_ptr, code_ptr + wordsPerCode * UINT32_BYTES) - ); - } - - async hammingTopK( - queryCode: Uint32Array, codes: Uint32Array, - wordsPerCode: number, count: number, k: number - ): Promise { - this.reset(); - const q_ptr = this.writeU32(queryCode); - const codes_ptr = this.writeU32(codes); - const scores_ptr = this.alloc(count * FLOAT32_BYTES); - const out_ptr = this.alloc(k * UINT32_BYTES); - - this.exports.hamming_scores(q_ptr, codes_ptr, scores_ptr, wordsPerCode, count); - - // Snapshot distances before topk_i32 mutates scores in-place - const distances = new Int32Array( - this.mem.buffer.slice(scores_ptr, scores_ptr + count * FLOAT32_BYTES) - ); - - this.exports.topk_i32(scores_ptr, out_ptr, count, k); - - const indices = new Int32Array(this.mem.buffer, out_ptr, k); - return Array.from(indices).map(idx => ({ index: idx, distance: distances[idx] })); - } - - async topKFromScores( - scores: Float32Array, k: number - ): Promise { - this.reset(); - // copy: topk_f32 mutates in-place - const copy_ptr = this.writeF32(new Float32Array(scores)); - const out_ptr = this.alloc(k * FLOAT32_BYTES); - this.exports.topk_f32(copy_ptr, out_ptr, scores.length, k); - const indices = new Int32Array(this.mem.buffer, out_ptr, k); - return Array.from(indices).map(idx => ({ index: idx, score: scores[idx] })); - } -} +export * from "./lib/WasmVectorBackend"; diff --git a/WebGLVectorBackend.ts b/WebGLVectorBackend.ts index af9157b..967fc35 100644 --- a/WebGLVectorBackend.ts +++ b/WebGLVectorBackend.ts @@ -1,282 +1 @@ -import { topKByDistance, topKByScore } from "./TopK"; -import type { - DistanceResult, - ScoreResult, - VectorBackend -} from "./VectorBackend"; -import { - FULLSCREEN_TRIANGLE_VERTEX_COUNT, - RGBA_CHANNELS, - UINT32_BITS, -} from "./core/NumericConstants"; - -const VERT_SRC = /* glsl */`#version 300 es -out vec2 v_uv; -void main() { - // Full-screen triangle; no geometry buffer needed - vec2 pos[3]; - pos[0] = vec2(-1.0, -1.0); - pos[1] = vec2( 3.0, -1.0); - pos[2] = vec2(-1.0, 3.0); - gl_Position = vec4(pos[gl_VertexID], 0.0, 1.0); - v_uv = pos[gl_VertexID] * 0.5 + 0.5; -}`; - -// One fragment per candidate vector; textures are RGBA32F so 4 floats per texel -const DOT_FRAG_SRC = /* glsl */`#version 300 es -precision highp float; -// matrix rows packed as RGBA32F: width = ceil(dim/4), height = count -uniform highp sampler2D u_matrix; -// query packed the same way; height = 1 -uniform highp sampler2D u_query; -uniform int u_dim_packed; // ceil(dim/4) -uniform int u_actual_dim; // true dim in floats -uniform int u_count; -out vec4 out_color; // r = score -void main() { - int row = int(gl_FragCoord.x); - if (row >= u_count) { discard; } - float sum = 0.0; - for (int j = 0; j < u_dim_packed; j++) { - vec4 q = texelFetch(u_query, ivec2(j, 0), 0); - vec4 m = texelFetch(u_matrix, ivec2(j, row), 0); - // For the last texel, zero out unused lanes beyond actual_dim - int base = j * 4; - if (base + 3 >= u_actual_dim) { - int rem = u_actual_dim - base; // 1, 2, or 3 - if (rem == 1) { q.g = 0.0; q.b = 0.0; q.a = 0.0; - m.g = 0.0; m.b = 0.0; m.a = 0.0; } - if (rem == 2) { q.b = 0.0; q.a = 0.0; - m.b = 0.0; m.a = 0.0; } - if (rem == 3) { q.a = 0.0; m.a = 0.0; } - } - sum += dot(q, m); - } - out_color = vec4(sum, 0.0, 0.0, 1.0); -}`; - -// Binary hash: one fragment per bit; writes 0.0 or 1.0; caller packs into Uint32Array on CPU -const HASH_FRAG_SRC = /* glsl */`#version 300 es -precision highp float; -uniform highp sampler2D u_vec; // [1 x 1], packed RGBA32F, width=ceil(dim/4) -uniform highp sampler2D u_hyperplanes; // width=ceil(dim/4), height=bits -uniform int u_dim_packed; -uniform int u_actual_dim; -uniform int u_bits; -out vec4 out_color; // r = 1.0 if bit set -void main() { - int b = int(gl_FragCoord.x); - if (b >= u_bits) { discard; } - float dot_val = 0.0; - for (int j = 0; j < u_dim_packed; j++) { - vec4 v = texelFetch(u_vec, ivec2(j, 0), 0); - vec4 h = texelFetch(u_hyperplanes, ivec2(j, b), 0); - int base = j * 4; - if (base + 3 >= u_actual_dim) { - int rem = u_actual_dim - base; - if (rem <= 3) { v.a = 0.0; h.a = 0.0; } - if (rem <= 2) { v.b = 0.0; h.b = 0.0; } - if (rem <= 1) { v.g = 0.0; h.g = 0.0; } - } - dot_val += dot(v, h); - } - out_color = vec4(dot_val >= 0.0 ? 1.0 : 0.0, 0.0, 0.0, 1.0); -}`; - -// ───────────────────────────────────────────────────────────────── -export class WebGlVectorBackend implements VectorBackend { - readonly kind = "webgl" as const; - private gl!: WebGL2RenderingContext; - private dotProg!: WebGLProgram; - private hashProg!: WebGLProgram; - private vao!: WebGLVertexArrayObject; // empty VAO for attribute-less draw - - static create(canvas?: HTMLCanvasElement): WebGlVectorBackend { - const b = new WebGlVectorBackend(); - const c = canvas ?? document.createElement("canvas"); - const gl = c.getContext("webgl2"); - if (!gl) throw new Error("WebGL2 not supported"); - const ext = gl.getExtension("EXT_color_buffer_float"); - if (!ext) throw new Error("EXT_color_buffer_float required"); - b.gl = gl; - b.dotProg = b.compileProgram(VERT_SRC, DOT_FRAG_SRC); - b.hashProg = b.compileProgram(VERT_SRC, HASH_FRAG_SRC); - b.vao = gl.createVertexArray()!; - return b; - } - - private compileProgram(vert: string, frag: string): WebGLProgram { - const gl = this.gl; - const compile = (type: number, src: string) => { - const s = gl.createShader(type)!; - gl.shaderSource(s, src); - gl.compileShader(s); - if (!gl.getShaderParameter(s, gl.COMPILE_STATUS)) - throw new Error(gl.getShaderInfoLog(s) ?? "Shader compile error"); - return s; - }; - const prog = gl.createProgram()!; - gl.attachShader(prog, compile(gl.VERTEX_SHADER, vert)); - gl.attachShader(prog, compile(gl.FRAGMENT_SHADER, frag)); - gl.linkProgram(prog); - if (!gl.getProgramParameter(prog, gl.LINK_STATUS)) - throw new Error(gl.getProgramInfoLog(prog) ?? "Program link error"); - return prog; - } - - // Pack a Float32Array into an RGBA32F texture of size (ceil(len/4), height) - private packF32Texture(data: Float32Array, height: number): WebGLTexture { - const gl = this.gl; - const texWidth = Math.ceil(data.length / height / RGBA_CHANNELS); - // Pad to texWidth * height * 4 - const padded = new Float32Array(texWidth * height * RGBA_CHANNELS); - padded.set(data); - const tex = gl.createTexture()!; - gl.bindTexture(gl.TEXTURE_2D, tex); - gl.texImage2D( - gl.TEXTURE_2D, 0, gl.RGBA32F, - texWidth, height, 0, - gl.RGBA, gl.FLOAT, padded - ); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); - return tex; - } - - // Render to a 1D float framebuffer, return raw pixels - private drawToFramebuffer( - prog: WebGLProgram, - width: number, // = count (one pixel per candidate) - setup: (prog: WebGLProgram) => void - ): Float32Array { - const gl = this.gl; - gl.canvas.width = width; - gl.canvas.height = 1; - - const fbo = gl.createFramebuffer()!; - const rbuf = gl.createRenderbuffer()!; - gl.bindRenderbuffer(gl.RENDERBUFFER, rbuf); - gl.renderbufferStorage(gl.RENDERBUFFER, gl.RGBA32F, width, 1); - gl.bindFramebuffer(gl.FRAMEBUFFER, fbo); - gl.framebufferRenderbuffer(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.RENDERBUFFER, rbuf); - - gl.viewport(0, 0, width, 1); - gl.useProgram(prog); - setup(prog); - gl.bindVertexArray(this.vao); - gl.drawArrays(gl.TRIANGLES, 0, FULLSCREEN_TRIANGLE_VERTEX_COUNT); - - const pixels = new Float32Array(width * RGBA_CHANNELS); - gl.readPixels(0, 0, width, 1, gl.RGBA, gl.FLOAT, pixels); - gl.bindFramebuffer(gl.FRAMEBUFFER, null); - gl.deleteFramebuffer(fbo); - gl.deleteRenderbuffer(rbuf); - return pixels; - } - - private uniform1i(prog: WebGLProgram, name: string, v: number) { - this.gl.uniform1i(this.gl.getUniformLocation(prog, name), v); - } - - private bindTex(prog: WebGLProgram, name: string, tex: WebGLTexture, unit: number) { - const gl = this.gl; - gl.activeTexture(gl.TEXTURE0 + unit); - gl.bindTexture(gl.TEXTURE_2D, tex); - gl.uniform1i(gl.getUniformLocation(prog, name), unit); - } - - async dotMany( - query: Float32Array, matrix: Float32Array, - dim: number, count: number - ): Promise { - const dimPacked = Math.ceil(dim / 4); - const qTex = this.packF32Texture(query, 1); - const mTex = this.packF32Texture(matrix, count); - - const pixels = this.drawToFramebuffer(this.dotProg, count, (prog) => { - this.bindTex(prog, "u_matrix", mTex, 0); - this.bindTex(prog, "u_query", qTex, 1); - this.uniform1i(prog, "u_dim_packed", dimPacked); - this.uniform1i(prog, "u_actual_dim", dim); - this.uniform1i(prog, "u_count", count); - }); - - this.gl.deleteTexture(qTex); - this.gl.deleteTexture(mTex); - - // Extract r channel from RGBA pixels → scores - return Float32Array.from( - { length: count }, - (_, i) => pixels[i * RGBA_CHANNELS] - ); - } - - async project( - vec: Float32Array, P: Float32Array, - dimIn: number, dimOut: number - ): Promise { - return this.dotMany(vec, P, dimIn, dimOut); - } - - async hashToBinary( - vec: Float32Array, - projectionMatrix: Float32Array, - dimIn: number, bits: number - ): Promise { - const dimPacked = Math.ceil(dimIn / 4); - const vTex = this.packF32Texture(vec, 1); - const hTex = this.packF32Texture(projectionMatrix, bits); - - const pixels = this.drawToFramebuffer(this.hashProg, bits, (prog) => { - this.bindTex(prog, "u_vec", vTex, 0); - this.bindTex(prog, "u_hyperplanes", hTex, 1); - this.uniform1i(prog, "u_dim_packed", dimPacked); - this.uniform1i(prog, "u_actual_dim", dimIn); - this.uniform1i(prog, "u_bits", bits); - }); - - this.gl.deleteTexture(vTex); - this.gl.deleteTexture(hTex); - - // Pack the per-bit float results (0.0 or 1.0) into Uint32 words - const wordsPerCode = Math.ceil(bits / UINT32_BITS); - const code = new Uint32Array(wordsPerCode); - for (let b = 0; b < bits; b++) { - if (pixels[b * RGBA_CHANNELS] >= 0.5) { - const wordIndex = Math.floor(b / UINT32_BITS); - const bitIndex = b % UINT32_BITS; - code[wordIndex] |= (1 << bitIndex); - } - } - return code; - } - - // Hamming done on CPU — WebGL2 has no integer atomic ops, - // and the XOR+popcnt kernel is not naturally expressible in GLSL for large buffers. - // For 10k items at 4–8 words each this is ~40k i32 ops and comfortably fast. - async hammingTopK( - queryCode: Uint32Array, codes: Uint32Array, - wordsPerCode: number, count: number, k: number - ): Promise { - const distances = new Uint32Array(count); - for (let i = 0; i < count; i++) { - let dist = 0; - const base = i * wordsPerCode; - for (let w = 0; w < wordsPerCode; w++) { - let xor = queryCode[w] ^ codes[base + w]; - // Popcount via Hamming weight bit trick - xor = xor - ((xor >> 1) & 0x55555555); - xor = (xor & 0x33333333) + ((xor >> 2) & 0x33333333); - dist += (((xor + (xor >> 4)) & 0x0f0f0f0f) * 0x01010101) >>> 24; - } - distances[i] = dist; - } - return topKByDistance(distances, k); - } - - async topKFromScores( - scores: Float32Array, k: number - ): Promise { - return topKByScore(scores, k); - } -} +export * from "./lib/WebGLVectorBackend"; diff --git a/WebGPUVectorBackend.ts b/WebGPUVectorBackend.ts index d455eaa..4955206 100644 --- a/WebGPUVectorBackend.ts +++ b/WebGPUVectorBackend.ts @@ -1,270 +1 @@ -import { topKByDistance, topKByScore } from "./TopK"; -import type { - DistanceResult, - ScoreResult, - VectorBackend -} from "./VectorBackend"; -import { - FLOAT32_BYTES, - UINT32_BITS, - UINT32_BYTES, - WEBGPU_DOT_WORKGROUP_SIZE, - WEBGPU_HAMMING_WORKGROUP_SIZE, - WEBGPU_HASH_WORKGROUP_SIZE, - WEBGPU_MIN_UNIFORM_BYTES, -} from "./core/NumericConstants"; - -const DOT_MANY_WGSL = /* wgsl */` -struct Params { dim: u32, count: u32, words_per_code: u32, k: u32 } -@group(0) @binding(0) var query : array; -@group(0) @binding(1) var matrix : array; -@group(0) @binding(2) var scores : array; -@group(0) @binding(3) var params : Params; -@compute @workgroup_size(${WEBGPU_DOT_WORKGROUP_SIZE}) -fn main(@builtin(global_invocation_id) gid: vec3) { - let i = gid.x; - if (i >= params.count) { return; } - var sum = 0.0; - let base = i * params.dim; - for (var j = 0u; j < params.dim; j++) { sum += query[j] * matrix[base + j]; } - scores[i] = sum; -}`; - -const HASH_BINARY_WGSL = /* wgsl */` -struct Params { dim: u32, bits: u32, words_per_code: u32, _pad: u32 } -@group(0) @binding(0) var vec_in : array; -@group(0) @binding(1) var hyperplanes: array; -@group(0) @binding(2) var code_out : array>; -@group(0) @binding(3) var params : Params; -@compute @workgroup_size(${WEBGPU_HASH_WORKGROUP_SIZE}) -fn main(@builtin(global_invocation_id) gid: vec3) { - let b = gid.x; - if (b >= params.bits) { return; } - var dot = 0.0; - let base = b * params.dim; - for (var j = 0u; j < params.dim; j++) { dot += vec_in[j] * hyperplanes[base + j]; } - if (dot >= 0.0) { atomicOr(&code_out[b >> 5u], 1u << (b & 31u)); } -}`; - -const HAMMING_WGSL = /* wgsl */` -struct Params { dim: u32, count: u32, words_per_code: u32, k: u32 } -@group(0) @binding(0) var q_code : array; -@group(0) @binding(1) var codes : array; -@group(0) @binding(2) var out_dist: array; -@group(0) @binding(3) var params : Params; -@compute @workgroup_size(${WEBGPU_HAMMING_WORKGROUP_SIZE}) -fn main(@builtin(global_invocation_id) gid: vec3) { - let i = gid.x; - if (i >= params.count) { return; } - var dist = 0u; - let base = i * params.words_per_code; - for (var w = 0u; w < params.words_per_code; w++) { - dist += countOneBits(q_code[w] ^ codes[base + w]); - } - out_dist[i] = dist; -}`; - -// ───────────────────────────────────────────────────────────────── -export class WebGpuVectorBackend implements VectorBackend { - readonly kind = "webgpu" as const; - private device!: GPUDevice; - private dotPipeline!: GPUComputePipeline; - private hashPipeline!: GPUComputePipeline; - private hammingPipeline!: GPUComputePipeline; - - static async create(): Promise { - const b = new WebGpuVectorBackend(); - const adapter = await navigator.gpu.requestAdapter(); - if (!adapter) throw new Error("No WebGPU adapter"); - b.device = await adapter.requestDevice(); - b.dotPipeline = b.makePipeline(DOT_MANY_WGSL); - b.hashPipeline = b.makePipeline(HASH_BINARY_WGSL); - b.hammingPipeline = b.makePipeline(HAMMING_WGSL); - return b; - } - - private makePipeline(wgsl: string): GPUComputePipeline { - return this.device.createComputePipeline({ - layout: "auto", - compute: { - module: this.device.createShaderModule({ code: wgsl }), - entryPoint: "main", - }, - }); - } - - private toArrayBuffer(data: ArrayBufferView): ArrayBuffer { - const bytes = new Uint8Array(data.buffer, data.byteOffset, data.byteLength); - const copy = new Uint8Array(data.byteLength); - copy.set(bytes); - return copy.buffer; - } - - // Upload a Float32Array into a GPU storage buffer (read-only) - private f32Buffer(data: Float32Array, usage = GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST): GPUBuffer { - const buf = this.device.createBuffer({ size: data.byteLength, usage }); - this.device.queue.writeBuffer(buf, 0, this.toArrayBuffer(data)); - return buf; - } - - private u32Buffer(data: Uint32Array, usage = GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST): GPUBuffer { - const buf = this.device.createBuffer({ size: data.byteLength, usage }); - this.device.queue.writeBuffer(buf, 0, this.toArrayBuffer(data)); - return buf; - } - - // Create an output storage buffer + a mapped readback buffer - private outBuffer(bytes: number): { gpu: GPUBuffer; read: GPUBuffer } { - return { - gpu: this.device.createBuffer({ - size: bytes, - usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC, - }), - read: this.device.createBuffer({ - size: bytes, - usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST, - }), - }; - } - - private uniformBuffer(data: Uint32Array): GPUBuffer { - const buf = this.device.createBuffer({ - size: Math.max(WEBGPU_MIN_UNIFORM_BYTES, data.byteLength), - usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, - }); - this.device.queue.writeBuffer(buf, 0, this.toArrayBuffer(data)); - return buf; - } - - private async readbackF32(gpu: GPUBuffer, read: GPUBuffer, count: number): Promise { - const cmd = this.device.createCommandEncoder(); - cmd.copyBufferToBuffer(gpu, 0, read, 0, count * FLOAT32_BYTES); - this.device.queue.submit([cmd.finish()]); - await read.mapAsync(GPUMapMode.READ); - const result = new Float32Array(read.getMappedRange().slice(0)); - read.unmap(); - return result; - } - - private async readbackU32(gpu: GPUBuffer, read: GPUBuffer, count: number): Promise { - const cmd = this.device.createCommandEncoder(); - cmd.copyBufferToBuffer(gpu, 0, read, 0, count * UINT32_BYTES); - this.device.queue.submit([cmd.finish()]); - await read.mapAsync(GPUMapMode.READ); - const result = new Uint32Array(read.getMappedRange().slice(0)); - read.unmap(); - return result; - } - - // ── dot_many (also used for project — caller sets dim/count accordingly) - async dotMany( - query: Float32Array, matrix: Float32Array, - dim: number, count: number - ): Promise { - const qBuf = this.f32Buffer(query); - const mBuf = this.f32Buffer(matrix); - const { gpu: oBuf, read: rBuf } = this.outBuffer(count * FLOAT32_BYTES); - const uBuf = this.uniformBuffer(new Uint32Array([dim, count, 0, 0])); - - const bg = this.device.createBindGroup({ - layout: this.dotPipeline.getBindGroupLayout(0), - entries: [ - { binding: 0, resource: { buffer: qBuf } }, - { binding: 1, resource: { buffer: mBuf } }, - { binding: 2, resource: { buffer: oBuf } }, - { binding: 3, resource: { buffer: uBuf } }, - ], - }); - - const cmd = this.device.createCommandEncoder(); - const pass = cmd.beginComputePass(); - pass.setPipeline(this.dotPipeline); - pass.setBindGroup(0, bg); - pass.dispatchWorkgroups(Math.ceil(count / WEBGPU_DOT_WORKGROUP_SIZE)); - pass.end(); - this.device.queue.submit([cmd.finish()]); - - return this.readbackF32(oBuf, rBuf, count); - } - - // project reuses dotMany — P is (dimOut × dimIn), treat each row as a "vector" - async project( - vec: Float32Array, P: Float32Array, - dimIn: number, dimOut: number - ): Promise { - return this.dotMany(vec, P, dimIn, dimOut); - } - - // ── hash_binary - async hashToBinary( - vec: Float32Array, - projectionMatrix: Float32Array, - dimIn: number, bits: number - ): Promise { - const wordsPerCode = Math.ceil(bits / UINT32_BITS); - const vBuf = this.f32Buffer(vec); - const pBuf = this.f32Buffer(projectionMatrix); - const { gpu: oBuf, read: rBuf } = this.outBuffer(wordsPerCode * UINT32_BYTES); - // zero-init the output buffer before atomicOr writes - this.device.queue.writeBuffer(oBuf, 0, new Uint32Array(wordsPerCode)); - const uBuf = this.uniformBuffer(new Uint32Array([dimIn, bits, wordsPerCode, 0])); - - const bg = this.device.createBindGroup({ - layout: this.hashPipeline.getBindGroupLayout(0), - entries: [ - { binding: 0, resource: { buffer: vBuf } }, - { binding: 1, resource: { buffer: pBuf } }, - { binding: 2, resource: { buffer: oBuf } }, - { binding: 3, resource: { buffer: uBuf } }, - ], - }); - - const cmd = this.device.createCommandEncoder(); - const pass = cmd.beginComputePass(); - pass.setPipeline(this.hashPipeline); - pass.setBindGroup(0, bg); - pass.dispatchWorkgroups(Math.ceil(bits / WEBGPU_HASH_WORKGROUP_SIZE)); - pass.end(); - this.device.queue.submit([cmd.finish()]); - - return this.readbackU32(oBuf, rBuf, wordsPerCode); - } - - // ── hamming_scores + top-k (top-k done on CPU; 10k ints is trivial to sort) - async hammingTopK( - queryCode: Uint32Array, codes: Uint32Array, - wordsPerCode: number, count: number, k: number - ): Promise { - const qBuf = this.u32Buffer(queryCode); - const cBuf = this.u32Buffer(codes); - const { gpu: oBuf, read: rBuf } = this.outBuffer(count * UINT32_BYTES); - const uBuf = this.uniformBuffer(new Uint32Array([0, count, wordsPerCode, k])); - - const bg = this.device.createBindGroup({ - layout: this.hammingPipeline.getBindGroupLayout(0), - entries: [ - { binding: 0, resource: { buffer: qBuf } }, - { binding: 1, resource: { buffer: cBuf } }, - { binding: 2, resource: { buffer: oBuf } }, - { binding: 3, resource: { buffer: uBuf } }, - ], - }); - - const cmd = this.device.createCommandEncoder(); - const pass = cmd.beginComputePass(); - pass.setPipeline(this.hammingPipeline); - pass.setBindGroup(0, bg); - pass.dispatchWorkgroups(Math.ceil(count / WEBGPU_HAMMING_WORKGROUP_SIZE)); - pass.end(); - this.device.queue.submit([cmd.finish()]); - - const distances = await this.readbackU32(oBuf, rBuf, count); - return topKByDistance(distances, k); - } - - async topKFromScores( - scores: Float32Array, k: number - ): Promise { - return topKByScore(scores, k); - } -} +export * from "./lib/WebGPUVectorBackend"; diff --git a/WebNNTypes.d.ts b/WebNNTypes.d.ts index 19e6968..ec8abb0 100644 --- a/WebNNTypes.d.ts +++ b/WebNNTypes.d.ts @@ -1,36 +1 @@ -declare global { - type MLOperand = unknown; - - type MLGraph = unknown; - - interface MLContext { - compute( - graph: MLGraph, - inputs: Record, - outputs: Record - ): Promise; - } - - interface MLOperandDescriptor { - dataType: "float32"; - dimensions: number[]; - } - - class MLGraphBuilder { - constructor(context: MLContext); - input(name: string, descriptor: MLOperandDescriptor): MLOperand; - reshape(input: MLOperand, dimensions: number[]): MLOperand; - matmul(a: MLOperand, b: MLOperand): MLOperand; - build(outputs: Record): Promise; - } - - interface Navigator { - ml?: { - createContext(options?: { - deviceType?: "cpu" | "gpu" | "npu"; - }): Promise; - }; - } -} - -export {}; +export * from "./lib/WebNNTypes.d"; diff --git a/WebNNVectorBackend.ts b/WebNNVectorBackend.ts index 6b80937..7757981 100644 --- a/WebNNVectorBackend.ts +++ b/WebNNVectorBackend.ts @@ -1,97 +1 @@ -import { topKByScore } from "./TopK"; -import type { - DistanceResult, - ScoreResult, - VectorBackend -} from "./VectorBackend"; -import { WasmVectorBackend } from "./WasmVectorBackend"; - -export class WebNnVectorBackend implements VectorBackend { - readonly kind = "webnn" as const; - private ctx!: MLContext; - private builder!: MLGraphBuilder; - // Cache compiled graphs keyed by "dimIn,dimOut" to avoid recompilation - private graphCache = new Map(); - // Fallback for binary ops (WebNN has no bitwise) - private wasmFallback!: WasmVectorBackend; - - static async create(wasmBytes: ArrayBuffer): Promise { - if (!navigator.ml) { - throw new Error("WebNN is not available in this runtime"); - } - - const b = new WebNnVectorBackend(); - b.ctx = await navigator.ml.createContext({ deviceType: "gpu" }); - b.builder = new MLGraphBuilder(b.ctx); - b.wasmFallback = await WasmVectorBackend.create(wasmBytes); - return b; - } - - // Build and cache an MLGraph for a given matmul shape. - // graph computes: output[count] = matrix[count, dim] · query[dim] - // (treating dot_many as a single matmul: output = M @ q) - private async getOrBuildGraph(dim: number, count: number): Promise { - const key = `${dim},${count}`; - - if (!this.graphCache.has(key)) { - const qDesc: MLOperandDescriptor = { dataType: "float32", dimensions: [dim] }; - const mDesc: MLOperandDescriptor = { dataType: "float32", dimensions: [count, dim] }; - - const q = this.builder.input("query", qDesc); - const M = this.builder.input("matrix", mDesc); - - // Reshape query to [dim, 1] for matmul - const qCol = this.builder.reshape(q, [dim, 1]); - // matmul: [count, dim] × [dim, 1] → [count, 1] - const out = this.builder.matmul(M, qCol); - // Flatten to [count] - const flat = this.builder.reshape(out, [count]); - - const graph = await this.builder.build({ scores: flat }); - this.graphCache.set(key, graph); - } - - return this.graphCache.get(key)!; - } - - async dotMany( - query: Float32Array, matrix: Float32Array, - dim: number, count: number - ): Promise { - const graph = await this.getOrBuildGraph(dim, count); - const outputs = { scores: new Float32Array(count) }; - await this.ctx.compute(graph, { query, matrix }, outputs); - return outputs.scores; - } - - // project: same matmul, just dimIn→dimOut; WebNN handles any shape - async project( - vec: Float32Array, - projectionMatrix: Float32Array, - dimIn: number, dimOut: number - ): Promise { - return this.dotMany(vec, projectionMatrix, dimIn, dimOut); - } - - // WebNN has no bitwise instructions — delegate to WASM - async hashToBinary( - vec: Float32Array, - projectionMatrix: Float32Array, - dimIn: number, bits: number - ): Promise { - return this.wasmFallback.hashToBinary(vec, projectionMatrix, dimIn, bits); - } - - async hammingTopK( - queryCode: Uint32Array, codes: Uint32Array, - wordsPerCode: number, count: number, k: number - ): Promise { - return this.wasmFallback.hammingTopK(queryCode, codes, wordsPerCode, count, k); - } - - async topKFromScores( - scores: Float32Array, k: number - ): Promise { - return topKByScore(scores, k); - } -} +export * from "./lib/WebNNVectorBackend"; diff --git a/core/BuiltInModelProfiles.ts b/core/BuiltInModelProfiles.ts index caca66f..3b5ab3f 100644 --- a/core/BuiltInModelProfiles.ts +++ b/core/BuiltInModelProfiles.ts @@ -1,57 +1 @@ -import type { ModelProfileRegistryEntry } from "./ModelProfileResolver"; - -/** - * Built-in model profile registry entries for known matryoshka embedding models. - * - * Numeric values here are model-derived and sourced from the original model cards. - * This file is a declared source of truth for model profile numerics and is explicitly - * allowed by the guard:model-derived script. - * - * Add new entries when wiring additional real embedding providers. - */ - -/** - * Profile for `onnx-community/embeddinggemma-300m-ONNX` (Q4 quantized). - * - * Base model: google/embeddinggemma-300m - * Architecture: Gemma 2-based matryoshka embedding model. - * - * Supported matryoshka sub-dimensions (nested, smallest-to-largest): - * 64, 128, 256, 512, 768 - * - * The default dimension registered here (768) is the full-fidelity output. - * Callers may slice to a smaller sub-dimension for compressed retrieval tiers. - * - * matryoshkaProtectedDim = 128: the most coarse-grained (smallest) sub-dimension - * officially supported by the model. MetroidBuilder uses this as the protected - * floor — dimensions below 128 are not a supported embedding granularity. - * - * Task prompts (required for best retrieval quality): - * Query prefix: "query: " - * Document prefix: "passage: " - * - * @see https://huggingface.co/google/embeddinggemma-300m - * @see https://huggingface.co/onnx-community/embeddinggemma-300m-ONNX - */ -export const EMBEDDING_GEMMA_300M_MODEL_ID = - "onnx-community/embeddinggemma-300m-ONNX"; - -export const EMBEDDING_GEMMA_300M_PROFILE: ModelProfileRegistryEntry = { - embeddingDimension: 768, - contextWindowTokens: 512, - matryoshkaProtectedDim: 128, -}; - -/** - * Canonical registry of all built-in model profiles, keyed by model ID. - * This record is used as the default registry in `ModelProfileResolver`. - * - * When adding a new Matryoshka embedding model, set `matryoshkaProtectedDim` - * to the smallest sub-dimension the model officially supports. Known values: - * - embeddinggemma-300m: 128 - * - nomic-embed-text-v1.5: 64 (to be added when nomic provider is wired) - */ -export const BUILT_IN_MODEL_REGISTRY: Record = - Object.freeze({ - [EMBEDDING_GEMMA_300M_MODEL_ID]: EMBEDDING_GEMMA_300M_PROFILE, - }); +export * from "../lib/core/BuiltInModelProfiles"; diff --git a/core/HotpathPolicy.ts b/core/HotpathPolicy.ts index d8368f7..d9d3411 100644 --- a/core/HotpathPolicy.ts +++ b/core/HotpathPolicy.ts @@ -1,305 +1 @@ -// --------------------------------------------------------------------------- -// HotpathPolicy — Williams Bound policy foundation -// --------------------------------------------------------------------------- -// -// Central source of truth for the Williams Bound architecture. -// All hotpath constants live here as a frozen default policy object. -// Policy-derived != model-derived — kept strictly separate from ModelDefaults. -// --------------------------------------------------------------------------- - -import type { SalienceWeights, TierQuotaRatios, TierQuotas } from "./types"; - -// --------------------------------------------------------------------------- -// HotpathPolicy interface -// --------------------------------------------------------------------------- - -export interface HotpathPolicy { - /** Scaling factor in H(t) = ceil(c * sqrt(t * log2(1+t))) */ - readonly c: number; - - /** Salience weights: sigma = alpha*H_in + beta*R + gamma*Q */ - readonly salienceWeights: SalienceWeights; - - /** Fractional tier quota ratios (must sum to 1.0) */ - readonly tierQuotaRatios: TierQuotaRatios; -} - -// --------------------------------------------------------------------------- -// Frozen default policy object -// --------------------------------------------------------------------------- - -export const DEFAULT_HOTPATH_POLICY: HotpathPolicy = Object.freeze({ - c: 0.5, - salienceWeights: Object.freeze({ - alpha: 0.5, // Hebbian connectivity - beta: 0.3, // recency - gamma: 0.2, // query-hit frequency - }), - tierQuotaRatios: Object.freeze({ - shelf: 0.10, - volume: 0.20, - book: 0.20, - page: 0.50, - }), -}); - -// --------------------------------------------------------------------------- -// H(t) — Resident hotpath capacity -// --------------------------------------------------------------------------- - -/** - * Compute the resident hotpath capacity H(t) = ceil(c * sqrt(t * log2(1+t))). - * - * Properties guaranteed by tests: - * - Monotonically non-decreasing - * - Sublinear growth (H(t)/t shrinks as t grows) - * - Returns a finite integer >= 1 for any non-negative finite t - */ -export function computeCapacity( - graphMass: number, - c: number = DEFAULT_HOTPATH_POLICY.c, -): number { - if (!Number.isFinite(graphMass) || graphMass < 0) { - return 1; - } - if (graphMass === 0) return 1; - - const log2 = Math.log2(1 + graphMass); - const raw = c * Math.sqrt(graphMass * log2); - - if (!Number.isFinite(raw) || raw < 1) return 1; - return Math.ceil(raw); -} - -// --------------------------------------------------------------------------- -// Node salience — sigma = alpha*H_in + beta*R + gamma*Q -// --------------------------------------------------------------------------- - -/** - * Compute node salience: sigma = alpha*H_in + beta*R + gamma*Q. - * - * @param hebbianIn Sum of incident Hebbian edge weights - * @param recency Recency score (0-1, exponential decay) - * @param queryHits Query-hit count for the node - * @param weights Tunable weights (default from policy) - */ -export function computeSalience( - hebbianIn: number, - recency: number, - queryHits: number, - weights: SalienceWeights = DEFAULT_HOTPATH_POLICY.salienceWeights, -): number { - const raw = weights.alpha * hebbianIn - + weights.beta * recency - + weights.gamma * queryHits; - - if (!Number.isFinite(raw)) return 0; - return raw; -} - -// --------------------------------------------------------------------------- -// Tier quota derivation -// --------------------------------------------------------------------------- - -/** - * Allocate H(t) across shelf/volume/book/page tiers. - * - * Uses largest-remainder method so quotas sum exactly to `capacity`. - */ -export function deriveTierQuotas( - capacity: number, - ratios: TierQuotaRatios = DEFAULT_HOTPATH_POLICY.tierQuotaRatios, -): TierQuotas { - const tiers: (keyof TierQuotas)[] = ["shelf", "volume", "book", "page"]; - - // Normalize ratios so they sum to 1 - const rawTotal = tiers.reduce((sum, t) => sum + ratios[t], 0); - const normalized = tiers.map((t) => (rawTotal > 0 ? ratios[t] / rawTotal : 0.25)); - - const cap = Math.max(0, Math.floor(capacity)); - const idealShares = normalized.map((r) => r * cap); - const floors = idealShares.map((s) => Math.floor(s)); - let remaining = cap - floors.reduce((a, b) => a + b, 0); - - // Distribute remainders by largest fractional part - const remainders = idealShares.map((s, i) => ({ - index: i, - remainder: s - floors[i], - })); - remainders.sort((a, b) => b.remainder - a.remainder); - - for (const r of remainders) { - if (remaining <= 0) break; - floors[r.index]++; - remaining--; - } - - return { - shelf: floors[0], - volume: floors[1], - book: floors[2], - page: floors[3], - }; -} - -// --------------------------------------------------------------------------- -// Community quota derivation -// --------------------------------------------------------------------------- - -/** - * Distribute a tier budget proportionally across communities. - * - * Uses largest-remainder method so quotas sum exactly to `tierBudget`. - * Each community receives a minimum of 1 slot when budget allows. - * - * Returns an empty array when `communitySizes` is empty. - * If `tierBudget` is 0, every community receives 0. - */ -export function deriveCommunityQuotas( - tierBudget: number, - communitySizes: number[], -): number[] { - const n = communitySizes.length; - if (n === 0) return []; - - const budget = Math.max(0, Math.floor(tierBudget)); - if (budget === 0) return new Array(n).fill(0) as number[]; - - const totalSize = communitySizes.reduce((a, b) => a + Math.max(0, b), 0); - - // Phase 1: assign minimum 1 to each community if budget allows - const minPerCommunity = budget >= n ? 1 : 0; - const quotas = new Array(n).fill(minPerCommunity); - const remainingBudget = budget - minPerCommunity * n; - - if (remainingBudget === 0 || totalSize === 0) return quotas; - - // Phase 2: distribute remaining proportionally (largest-remainder) - const proportional = communitySizes.map( - (s) => (Math.max(0, s) / totalSize) * remainingBudget, - ); - const floors = proportional.map(Math.floor); - let floorSum = floors.reduce((a, b) => a + b, 0); - - const remainders = proportional.map((p, i) => ({ idx: i, rem: p - floors[i] })); - remainders.sort((a, b) => b.rem - a.rem); - - let j = 0; - while (floorSum < remainingBudget) { - floors[remainders[j].idx] += 1; - floorSum += 1; - j += 1; - } - - for (let i = 0; i < n; i++) quotas[i] += floors[i]; - return quotas; -} - -// --------------------------------------------------------------------------- -// Semantic neighbor degree limit — Williams-bound derived -// --------------------------------------------------------------------------- - -// Bootstrap floor for Williams-bound log formulas: ensures t_eff ≥ 2 so that -// log₂(t_eff) > 0 and log₂(log₂(1+t_eff)) is defined and positive. -const MIN_GRAPH_MASS_FOR_LOGS = 2; - -/** - * Compute the Williams-bound-derived maximum degree for the semantic neighbor - * graph given a corpus of `graphMass` total pages. - * - * The degree limit uses the same H(t) formula as the hotpath capacity but is - * bounded by a hard cap to keep the graph sparse. At small corpora the - * Williams formula naturally returns small values (e.g. 1–5 for t < 10); - * at large corpora the `hardCap` clamps growth to prevent the graph becoming - * too dense. - * - * @param graphMass Total number of pages in the corpus. - * @param c Williams Bound scaling constant (default from policy). - * @param hardCap Maximum degree regardless of formula result. Default: 32. - */ -export function computeNeighborMaxDegree( - graphMass: number, - c: number = DEFAULT_HOTPATH_POLICY.c, - hardCap = 32, -): number { - const derived = computeCapacity(graphMass, c); - return Math.min(hardCap, Math.max(1, derived)); -} - -// --------------------------------------------------------------------------- -// Dynamic subgraph expansion bounds — Williams-bound derived -// --------------------------------------------------------------------------- - -export interface SubgraphBounds { - /** Maximum number of nodes to include in the induced subgraph. */ - maxSubgraphSize: number; - /** Maximum BFS hops from seed nodes. */ - maxHops: number; - /** Maximum fanout per hop (branching factor). */ - perHopBranching: number; -} - -/** - * Compute dynamic Williams-derived bounds for subgraph expansion (step 9 of - * the Cortex query path). - * - * Formulas from DESIGN.md "Dynamic Subgraph Expansion Bounds": - * - * t_eff = max(t, 2) - * maxSubgraphSize = min(30, ⌊√(t_eff · log₂(1+t_eff)) / log₂(t_eff)⌋) - * maxHops = max(1, ⌈log₂(log₂(1 + t_eff))⌉) - * perHopBranching = max(1, ⌊maxSubgraphSize ^ (1/maxHops)⌋) - * - * The bootstrap floor `t_eff = max(t, 2)` eliminates division-by-zero for - * t ≤ 1 and ensures a safe minimum of `maxSubgraphSize=1, maxHops=1`. - * - * @param graphMass Total number of pages in the corpus. - */ -export function computeSubgraphBounds(graphMass: number): SubgraphBounds { - const tEff = Math.max(graphMass, MIN_GRAPH_MASS_FOR_LOGS); - const log2tEff = Math.log2(tEff); - - const maxSubgraphSize = Math.min( - 30, - Math.floor(Math.sqrt(tEff * Math.log2(1 + tEff)) / log2tEff), - ); - - const maxHops = Math.max(1, Math.ceil(Math.log2(Math.log2(1 + tEff)))); - - const perHopBranching = Math.max( - 1, - Math.floor(Math.pow(maxSubgraphSize, 1 / maxHops)), - ); - - return { - maxSubgraphSize: Math.max(1, maxSubgraphSize), - maxHops, - perHopBranching, - }; -} - -// --------------------------------------------------------------------------- -// Williams-derived hierarchy fanout limit -// --------------------------------------------------------------------------- - -/** - * Compute the Williams-derived fanout limit for a hierarchy node that - * currently has `childCount` children. - * - * Per DESIGN.md "Sublinear Fanout Bounds": - * Max children = O(√(childCount · log childCount)) - * - * The formula is evaluated with a bootstrap floor of t_eff = max(t, 2) to - * avoid log(0) and returns at least 1 child. - * - * @param childCount Current number of children for the parent node. - * @param c Williams Bound scaling constant. - */ -export function computeFanoutLimit( - childCount: number, - c: number = DEFAULT_HOTPATH_POLICY.c, -): number { - const tEff = Math.max(childCount, MIN_GRAPH_MASS_FOR_LOGS); - const raw = c * Math.sqrt(tEff * Math.log2(1 + tEff)); - return Math.max(1, Math.ceil(raw)); -} +export * from "../lib/core/HotpathPolicy"; diff --git a/core/ModelDefaults.ts b/core/ModelDefaults.ts index 0f629db..d94133d 100644 --- a/core/ModelDefaults.ts +++ b/core/ModelDefaults.ts @@ -1,100 +1 @@ -import type { ModelProfile, ModelProfileSeed } from "./ModelProfile"; - -export interface ModelDerivationPolicy { - truncationRatio: number; - chunkRatio: number; - minTruncationTokens: number; - minChunkTokens: number; - maxChunkTokens: number; -} - -export const DEFAULT_MODEL_DERIVATION_POLICY: ModelDerivationPolicy = Object.freeze({ - truncationRatio: 0.75, - chunkRatio: 0.25, - minTruncationTokens: 256, - minChunkTokens: 128, - maxChunkTokens: 2048, -}); - -function assertPositiveInteger(name: string, value: number): void { - if (!Number.isInteger(value) || value <= 0) { - throw new Error(`${name} must be a positive integer`); - } -} - -function assertPositiveRatio(name: string, value: number): void { - if (!Number.isFinite(value) || value <= 0) { - throw new Error(`${name} must be a positive finite number`); - } -} - -function clamp(value: number, min: number, max: number): number { - return Math.min(max, Math.max(min, value)); -} - -function validatePolicy(policy: ModelDerivationPolicy): void { - assertPositiveRatio("truncationRatio", policy.truncationRatio); - assertPositiveRatio("chunkRatio", policy.chunkRatio); - assertPositiveInteger("minTruncationTokens", policy.minTruncationTokens); - assertPositiveInteger("minChunkTokens", policy.minChunkTokens); - assertPositiveInteger("maxChunkTokens", policy.maxChunkTokens); - - if (policy.minChunkTokens > policy.maxChunkTokens) { - throw new Error("minChunkTokens cannot exceed maxChunkTokens"); - } -} - -export function deriveTruncationTokens( - contextWindowTokens: number, - policy: ModelDerivationPolicy = DEFAULT_MODEL_DERIVATION_POLICY, -): number { - assertPositiveInteger("contextWindowTokens", contextWindowTokens); - validatePolicy(policy); - - const derived = Math.floor(contextWindowTokens * policy.truncationRatio); - return Math.max(policy.minTruncationTokens, derived); -} - -export function deriveChunkTokenLimit( - contextWindowTokens: number, - policy: ModelDerivationPolicy = DEFAULT_MODEL_DERIVATION_POLICY, -): number { - const truncationTokens = deriveTruncationTokens(contextWindowTokens, policy); - const derived = Math.floor(truncationTokens * policy.chunkRatio); - - return clamp(derived, policy.minChunkTokens, policy.maxChunkTokens); -} - -export function buildModelProfileFromSeed( - seed: ModelProfileSeed, - policy: ModelDerivationPolicy = DEFAULT_MODEL_DERIVATION_POLICY, -): ModelProfile { - const modelId = seed.modelId.trim(); - if (modelId.length === 0) { - throw new Error("modelId must be a non-empty string"); - } - - assertPositiveInteger("embeddingDimension", seed.embeddingDimension); - assertPositiveInteger("contextWindowTokens", seed.contextWindowTokens); - - if (seed.matryoshkaProtectedDim !== undefined) { - assertPositiveInteger("matryoshkaProtectedDim", seed.matryoshkaProtectedDim); - if (seed.matryoshkaProtectedDim > seed.embeddingDimension) { - throw new Error( - "matryoshkaProtectedDim cannot exceed embeddingDimension", - ); - } - } - - return { - modelId, - embeddingDimension: seed.embeddingDimension, - contextWindowTokens: seed.contextWindowTokens, - truncationTokens: deriveTruncationTokens(seed.contextWindowTokens, policy), - maxChunkTokens: deriveChunkTokenLimit(seed.contextWindowTokens, policy), - source: seed.source, - ...(seed.matryoshkaProtectedDim !== undefined - ? { matryoshkaProtectedDim: seed.matryoshkaProtectedDim } - : {}), - }; -} +export * from "../lib/core/ModelDefaults"; diff --git a/core/ModelProfile.ts b/core/ModelProfile.ts index baa4a05..fc31c9f 100644 --- a/core/ModelProfile.ts +++ b/core/ModelProfile.ts @@ -1,56 +1 @@ -export type ModelProfileSource = "metadata" | "registry" | "mixed"; - -export interface ModelProfileSeed { - modelId: string; - embeddingDimension: number; - contextWindowTokens: number; - source: ModelProfileSource; - /** - * The most coarse-grained Matryoshka sub-dimension for this model. - * - * This is the smallest nested embedding size the model officially supports. - * It defines the "protected floor" used by MetroidBuilder: lower dimensions - * encode invariant domain context and are never searched for antithesis. - * - * Known values: - * - embeddinggemma-300m: 128 - * - nomic-embed-text-v1.5: 64 - * - * `undefined` for models that do not use Matryoshka Representation Learning. - * When undefined, MetroidBuilder cannot perform dimensional unwinding and will - * always declare a knowledge gap (antithesis search is not possible). - */ - matryoshkaProtectedDim?: number; -} - -export interface PartialModelMetadata { - embeddingDimension?: number; - contextWindowTokens?: number; -} - -export interface ModelProfile { - modelId: string; - embeddingDimension: number; - contextWindowTokens: number; - truncationTokens: number; - maxChunkTokens: number; - source: ModelProfileSource; - /** - * The most coarse-grained Matryoshka sub-dimension for this model. - * - * This is the smallest nested embedding size the model officially supports. - * It defines the "protected floor" used by MetroidBuilder: dimensions below - * this boundary encode invariant domain context and are never searched for - * antithesis during Matryoshka dimensional unwinding. - * - * Known values: - * - embeddinggemma-300m: 128 - * - nomic-embed-text-v1.5: 64 - * - * `undefined` for models that do not use Matryoshka Representation Learning. - * When undefined, MetroidBuilder cannot perform dimensional unwinding and will - * always declare a knowledge gap (antithesis search is not possible without - * a protected-dimension floor). - */ - matryoshkaProtectedDim?: number; -} +export * from "../lib/core/ModelProfile"; diff --git a/core/ModelProfileResolver.ts b/core/ModelProfileResolver.ts index 3cb9384..f18cd53 100644 --- a/core/ModelProfileResolver.ts +++ b/core/ModelProfileResolver.ts @@ -1,112 +1 @@ -import { - DEFAULT_MODEL_DERIVATION_POLICY, - type ModelDerivationPolicy, - buildModelProfileFromSeed, -} from "./ModelDefaults"; -import type { - ModelProfile, - ModelProfileSource, - PartialModelMetadata, -} from "./ModelProfile"; - -export interface ModelProfileRegistryEntry { - embeddingDimension: number; - contextWindowTokens: number; - /** - * The most coarse-grained Matryoshka sub-dimension for this model. - * Required for MetroidBuilder dimensional unwinding. See `ModelProfile.matryoshkaProtectedDim`. - */ - matryoshkaProtectedDim?: number; -} - -export interface ModelProfileResolverOptions { - registry?: Record; - derivationPolicy?: ModelDerivationPolicy; -} - -export interface ResolveModelProfileInput { - modelId: string; - metadata?: PartialModelMetadata; -} - -function normalizeModelId(modelId: string): string { - return modelId.trim().toLowerCase(); -} - -export class ModelProfileResolver { - private readonly registry = new Map(); - private readonly derivationPolicy: ModelDerivationPolicy; - - constructor(options: ModelProfileResolverOptions = {}) { - this.derivationPolicy = options.derivationPolicy ?? DEFAULT_MODEL_DERIVATION_POLICY; - - if (options.registry) { - for (const [modelId, entry] of Object.entries(options.registry)) { - this.register(modelId, entry); - } - } - } - - register(modelId: string, entry: ModelProfileRegistryEntry): void { - this.registry.set(normalizeModelId(modelId), { - embeddingDimension: entry.embeddingDimension, - contextWindowTokens: entry.contextWindowTokens, - ...(entry.matryoshkaProtectedDim !== undefined - ? { matryoshkaProtectedDim: entry.matryoshkaProtectedDim } - : {}), - }); - } - - resolve(input: ResolveModelProfileInput): ModelProfile { - const modelId = input.modelId.trim(); - if (modelId.length === 0) { - throw new Error("modelId must be a non-empty string"); - } - - const normalized = normalizeModelId(modelId); - const registryEntry = this.registry.get(normalized); - - const embeddingDimension = - input.metadata?.embeddingDimension ?? registryEntry?.embeddingDimension; - const contextWindowTokens = - input.metadata?.contextWindowTokens ?? registryEntry?.contextWindowTokens; - - if (embeddingDimension === undefined || contextWindowTokens === undefined) { - throw new Error( - `Cannot resolve model profile for ${modelId}. ` + - "Provide metadata or register a profile entry.", - ); - } - - const source = this.resolveSource(input.metadata, registryEntry); - - return buildModelProfileFromSeed( - { - modelId, - embeddingDimension, - contextWindowTokens, - source, - matryoshkaProtectedDim: registryEntry?.matryoshkaProtectedDim, - }, - this.derivationPolicy, - ); - } - - private resolveSource( - metadata: PartialModelMetadata | undefined, - registryEntry: ModelProfileRegistryEntry | undefined, - ): ModelProfileSource { - const hasMetadataEmbedding = metadata?.embeddingDimension !== undefined; - const hasMetadataContext = metadata?.contextWindowTokens !== undefined; - - if (hasMetadataEmbedding && hasMetadataContext) { - return "metadata"; - } - - if (!hasMetadataEmbedding && !hasMetadataContext && registryEntry) { - return "registry"; - } - - return "mixed"; - } -} +export * from "../lib/core/ModelProfileResolver"; diff --git a/core/NumericConstants.ts b/core/NumericConstants.ts index 8056062..1c193b2 100644 --- a/core/NumericConstants.ts +++ b/core/NumericConstants.ts @@ -1,17 +1 @@ -// Runtime and memory-layout numeric constants shared across backends. - -export const FLOAT32_BYTES = 4; -export const UINT32_BYTES = 4; -export const UINT32_BITS = 32; -export const RGBA_CHANNELS = 4; - -export const WASM_ALLOC_GUARD_BYTES = 1024; -export const WASM_ALLOC_ALIGNMENT_BYTES = 16; -export const WASM_PAGE_BYTES = 64 * 1024; - -export const WEBGPU_MIN_UNIFORM_BYTES = 16; -export const WEBGPU_DOT_WORKGROUP_SIZE = 256; -export const WEBGPU_HASH_WORKGROUP_SIZE = 128; -export const WEBGPU_HAMMING_WORKGROUP_SIZE = 256; - -export const FULLSCREEN_TRIANGLE_VERTEX_COUNT = 3; +export * from "../lib/core/NumericConstants"; diff --git a/core/SalienceEngine.ts b/core/SalienceEngine.ts index 65af71a..42f485b 100644 --- a/core/SalienceEngine.ts +++ b/core/SalienceEngine.ts @@ -1,420 +1 @@ -// --------------------------------------------------------------------------- -// SalienceEngine — Decision-making layer for hotpath admission -// --------------------------------------------------------------------------- -// -// Provides per-node salience computation, promotion/eviction lifecycle -// helpers, and community-aware admission logic. -// --------------------------------------------------------------------------- - -import type { Hash, HotpathEntry, MetadataStore } from "./types"; -import { - computeCapacity, - computeSalience, - DEFAULT_HOTPATH_POLICY, - deriveCommunityQuotas, - deriveTierQuotas, - type HotpathPolicy, -} from "./HotpathPolicy"; - -// --------------------------------------------------------------------------- -// Recency helper -// --------------------------------------------------------------------------- - -/** - * Compute recency score R(v) as exponential decay from the most recent - * activity timestamp. Returns a value in [0, 1]. - * - * Uses a half-life of 7 days — after 7 days of inactivity the recency - * score drops to ~0.5; after 30 days it drops to ~0.05. - */ -function recencyScore(isoTimestamp: string | undefined, now: number): number { - if (!isoTimestamp) return 0; - const ts = Date.parse(isoTimestamp); - if (!Number.isFinite(ts)) return 0; - const ageMs = Math.max(0, now - ts); - const HALF_LIFE_MS = 7 * 24 * 60 * 60 * 1000; // 7 days - return Math.exp((-Math.LN2 * ageMs) / HALF_LIFE_MS); -} - -// --------------------------------------------------------------------------- -// P0-G1: Core salience computation -// --------------------------------------------------------------------------- - -/** - * Fetch PageActivity and incident Hebbian edges for a single page, - * then compute salience via HotpathPolicy. - */ -export async function computeNodeSalience( - pageId: Hash, - metadataStore: MetadataStore, - policy: HotpathPolicy = DEFAULT_HOTPATH_POLICY, - now: number = Date.now(), -): Promise { - const [activity, neighbors] = await Promise.all([ - metadataStore.getPageActivity(pageId), - metadataStore.getNeighbors(pageId), - ]); - - const hebbianIn = neighbors.reduce((sum, e) => sum + e.weight, 0); - - const recency = recencyScore( - activity?.lastQueryAt, - now, - ); - - const queryHits = activity?.queryHitCount ?? 0; - - return computeSalience(hebbianIn, recency, queryHits, policy.salienceWeights); -} - -/** - * Efficient batch version of `computeNodeSalience`. - */ -export async function batchComputeSalience( - pageIds: Hash[], - metadataStore: MetadataStore, - policy: HotpathPolicy = DEFAULT_HOTPATH_POLICY, - now: number = Date.now(), -): Promise> { - const results = new Map(); - - // Parallelize I/O across all pages - const entries = await Promise.all( - pageIds.map(async (id) => { - const salience = await computeNodeSalience(id, metadataStore, policy, now); - return [id, salience] as const; - }), - ); - - for (const [id, salience] of entries) { - results.set(id, salience); - } - - return results; -} - -/** - * Admission gating: should a candidate be promoted into the hotpath? - * - * - During bootstrap (capacity remaining > 0): always admit. - * - During steady-state: admit only if candidate salience exceeds - * the weakest resident salience. - */ -export function shouldPromote( - candidateSalience: number, - weakestResidentSalience: number, - capacityRemaining: number, -): boolean { - if (capacityRemaining > 0) return true; - return candidateSalience > weakestResidentSalience; -} - -/** - * Find the weakest resident in a given tier/community bucket. - * - * Returns the entityId of the weakest entry, or undefined if the - * tier/community bucket is empty. - */ -export async function selectEvictionTarget( - tier: HotpathEntry["tier"], - communityId: string | undefined, - metadataStore: MetadataStore, -): Promise { - const entries = await metadataStore.getHotpathEntries(tier); - - const filtered = communityId !== undefined - ? entries.filter((e) => e.communityId === communityId) - : entries; - - if (filtered.length === 0) return undefined; - - // Find the entry with the lowest salience (deterministic: stable sort by entityId on tie) - let weakest = filtered[0]; - for (let i = 1; i < filtered.length; i++) { - const e = filtered[i]; - if ( - e.salience < weakest.salience || - (e.salience === weakest.salience && e.entityId < weakest.entityId) - ) { - weakest = e; - } - } - - return weakest.entityId; -} - -// --------------------------------------------------------------------------- -// P0-G2: Promotion / eviction lifecycle helpers -// --------------------------------------------------------------------------- - -/** - * Bootstrap phase: fill hotpath greedily by salience while - * resident count < H(t). - * - * Computes salience for all candidate pages, then admits in - * descending salience order until the capacity is reached, - * respecting tier quotas. - */ -export async function bootstrapHotpath( - metadataStore: MetadataStore, - policy: HotpathPolicy = DEFAULT_HOTPATH_POLICY, - candidatePageIds: Hash[] = [], - now: number = Date.now(), -): Promise { - if (candidatePageIds.length === 0) return; - - // Compute salience for all candidates - const salienceMap = await batchComputeSalience( - candidatePageIds, - metadataStore, - policy, - now, - ); - - // Fetch page activities for community info - const activities = await Promise.all( - candidatePageIds.map((id) => metadataStore.getPageActivity(id)), - ); - const communityMap = new Map(); - for (let i = 0; i < candidatePageIds.length; i++) { - communityMap.set(candidatePageIds[i], activities[i]?.communityId); - } - - // Sort candidates by salience descending; break ties by entityId for determinism - const sorted = [...candidatePageIds].sort((a, b) => { - const diff = (salienceMap.get(b) ?? 0) - (salienceMap.get(a) ?? 0); - return diff !== 0 ? diff : a.localeCompare(b); - }); - - // Determine current graph mass for capacity calculation - const currentEntries = await metadataStore.getHotpathEntries(); - const currentCount = currentEntries.length; - - // Estimate graph mass: existing residents + candidates gives a lower bound - // For bootstrap, use total candidate count as graph mass estimate - const graphMass = currentCount + candidatePageIds.length; - const capacity = computeCapacity(graphMass, policy.c); - const tierQuotas = deriveTierQuotas(capacity, policy.tierQuotaRatios); - - // Track how many are already in each tier - const tierCounts: Record = { shelf: 0, volume: 0, book: 0, page: 0 }; - for (const entry of currentEntries) { - tierCounts[entry.tier] = (tierCounts[entry.tier] ?? 0) + 1; - } - - let totalResident = currentCount; - - for (const candidateId of sorted) { - if (totalResident >= capacity) break; - - const tier: HotpathEntry["tier"] = "page"; // bootstrap admits at page tier - if (tierCounts[tier] >= tierQuotas[tier]) continue; - - const salience = salienceMap.get(candidateId) ?? 0; - const entry: HotpathEntry = { - entityId: candidateId, - tier, - salience, - communityId: communityMap.get(candidateId), - }; - - await metadataStore.putHotpathEntry(entry); - tierCounts[tier]++; - totalResident++; - } -} - -/** - * Steady-state promotion sweep: for each candidate, promote if its - * salience exceeds the weakest resident in the same tier/community - * bucket. On promotion, evict the weakest. - * - * Tier quotas and community quotas are enforced regardless of whether - * overall capacity is full, preventing any single tier or community - * from monopolizing the hotpath during ramp-up. - */ -export async function runPromotionSweep( - candidateIds: Hash[], - metadataStore: MetadataStore, - policy: HotpathPolicy = DEFAULT_HOTPATH_POLICY, - now: number = Date.now(), -): Promise { - if (candidateIds.length === 0) return; - - // Compute salience for all candidates - const salienceMap = await batchComputeSalience( - candidateIds, - metadataStore, - policy, - now, - ); - - // Fetch page activities for community info - const activities = await Promise.all( - candidateIds.map((id) => metadataStore.getPageActivity(id)), - ); - const communityMap = new Map(); - for (let i = 0; i < candidateIds.length; i++) { - communityMap.set(candidateIds[i], activities[i]?.communityId); - } - - // Sort candidates by salience descending for deterministic processing - const sorted = [...candidateIds].sort((a, b) => { - const diff = (salienceMap.get(b) ?? 0) - (salienceMap.get(a) ?? 0); - return diff !== 0 ? diff : a.localeCompare(b); - }); - - // Load initial state into an in-memory cache to avoid repeated store reads - let cachedEntries = await metadataStore.getHotpathEntries(); - - for (const candidateId of sorted) { - const candidateSalience = salienceMap.get(candidateId) ?? 0; - const communityId = communityMap.get(candidateId); - const tier: HotpathEntry["tier"] = "page"; - - // Derive capacity and quotas from current state - const currentCount = cachedEntries.length; - const graphMass = currentCount + candidateIds.length; - const capacity = computeCapacity(graphMass, policy.c); - const capacityRemaining = capacity - currentCount; - const tierQuotas = deriveTierQuotas(capacity, policy.tierQuotaRatios); - const tierEntries = cachedEntries.filter((e) => e.tier === tier); - const tierFull = tierEntries.length >= tierQuotas[tier]; - - // --- Community quota check (enforced regardless of capacityRemaining) --- - if (communityId !== undefined && tierFull) { - const communitySizes = getCommunityDistribution(tierEntries, communityId); - const communityQuotas = deriveCommunityQuotas( - tierQuotas[tier], - communitySizes.sizes, - ); - const communityIdx = communitySizes.communityIndex; - const communityBudget = communityIdx < communityQuotas.length - ? communityQuotas[communityIdx] - : 0; - const communityCount = communitySizes.candidateCommunityCount; - - if (communityCount >= communityBudget && communityCount > 0) { - // Community is at quota — only promote if candidate beats weakest in community - const weakestId = findWeakestIn(tierEntries, communityId); - if (weakestId === undefined) continue; - - const weakestSalience = tierEntries.find((e) => e.entityId === weakestId)?.salience ?? 0; - - if (candidateSalience > weakestSalience) { - await metadataStore.removeHotpathEntry(weakestId); - const newEntry: HotpathEntry = { - entityId: candidateId, - tier, - salience: candidateSalience, - communityId, - }; - await metadataStore.putHotpathEntry(newEntry); - cachedEntries = cachedEntries.filter((e) => e.entityId !== weakestId); - cachedEntries.push(newEntry); - } - continue; - } - } - - // --- Tier quota check (enforced regardless of capacityRemaining) --- - if (tierFull) { - // Tier is at quota — evict the weakest in the entire tier (not scoped - // to candidate's community, so new communities can displace weak entries) - const weakestId = findWeakestIn(tierEntries, undefined); - if (weakestId === undefined) continue; - - const weakestSalience = tierEntries.find((e) => e.entityId === weakestId)?.salience ?? 0; - - if (candidateSalience <= weakestSalience) continue; - - await metadataStore.removeHotpathEntry(weakestId); - const newEntry: HotpathEntry = { - entityId: candidateId, - tier, - salience: candidateSalience, - communityId, - }; - await metadataStore.putHotpathEntry(newEntry); - cachedEntries = cachedEntries.filter((e) => e.entityId !== weakestId); - cachedEntries.push(newEntry); - } else if (capacityRemaining > 0) { - // Both tier and overall capacity available — admit directly - const newEntry: HotpathEntry = { - entityId: candidateId, - tier, - salience: candidateSalience, - communityId, - }; - await metadataStore.putHotpathEntry(newEntry); - cachedEntries.push(newEntry); - } - // else: tier has room but overall capacity is full — skip - } -} - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -/** - * Compute community distribution within a set of tier entries, - * including the candidate's community. - * - * The candidate's community is counted as having at least size 1 for - * quota derivation, so that a brand-new community can receive its - * first slot via the largest-remainder method. - */ -function getCommunityDistribution( - tierEntries: HotpathEntry[], - candidateCommunityId: string, -): { - sizes: number[]; - communityIndex: number; - candidateCommunityCount: number; -} { - const communityCountMap = new Map(); - - for (const e of tierEntries) { - const cid = e.communityId ?? "__none__"; - communityCountMap.set(cid, (communityCountMap.get(cid) ?? 0) + 1); - } - - // Ensure candidate's community is represented with at least size 1 - // so that deriveCommunityQuotas can allocate it a slot - const actualCount = communityCountMap.get(candidateCommunityId) ?? 0; - communityCountMap.set(candidateCommunityId, Math.max(1, actualCount)); - - const communities = [...communityCountMap.keys()].sort(); - const sizes = communities.map((c) => communityCountMap.get(c) ?? 0); - const communityIndex = communities.indexOf(candidateCommunityId); - - return { sizes, communityIndex, candidateCommunityCount: actualCount }; -} - -/** - * Find the weakest entry in a list, optionally filtered by communityId. - * Deterministic: breaks ties by entityId (smallest wins). - */ -function findWeakestIn( - entries: HotpathEntry[], - communityId: string | undefined, -): Hash | undefined { - const filtered = communityId !== undefined - ? entries.filter((e) => e.communityId === communityId) - : entries; - - if (filtered.length === 0) return undefined; - - let weakest = filtered[0]; - for (let i = 1; i < filtered.length; i++) { - const e = filtered[i]; - if ( - e.salience < weakest.salience || - (e.salience === weakest.salience && e.entityId < weakest.entityId) - ) { - weakest = e; - } - } - return weakest.entityId; -} +export * from "../lib/core/SalienceEngine"; diff --git a/core/crypto/hash.ts b/core/crypto/hash.ts index 2616e4e..69d23bb 100644 --- a/core/crypto/hash.ts +++ b/core/crypto/hash.ts @@ -1,26 +1 @@ -import type { Hash } from "../types.js"; - -function bufferToHex(buffer: ArrayBuffer): string { - return Array.from(new Uint8Array(buffer)) - .map(b => b.toString(16).padStart(2, "0")) - .join(""); -} - -/** - * Returns the SHA-256 hex digest of a UTF-8 encoded text string. - * Used to produce `contentHash` on Page entities. - */ -export async function hashText(content: string): Promise { - const encoded = new TextEncoder().encode(content); - const buffer = await crypto.subtle.digest("SHA-256", encoded); - return bufferToHex(buffer); -} - -/** - * Returns the SHA-256 hex digest of raw binary data. - * Used to produce `vectorHash` on Page entities. - */ -export async function hashBinary(data: BufferSource): Promise { - const buffer = await crypto.subtle.digest("SHA-256", data); - return bufferToHex(buffer); -} +export * from "../../lib/core/crypto/hash"; diff --git a/core/crypto/sign.ts b/core/crypto/sign.ts index 41b7c1b..a077d01 100644 --- a/core/crypto/sign.ts +++ b/core/crypto/sign.ts @@ -1,69 +1 @@ -import type { PublicKey, Signature } from "../types.js"; - -export interface KeyPair { - /** JWK JSON string — safe to store and share. */ - publicKey: PublicKey; - /** JWK JSON string — store securely; used to reconstruct `signingKey`. */ - privateKeyJwk: string; - /** Runtime CryptoKey ready for immediate signing operations. */ - signingKey: CryptoKey; -} - -function bufferToBase64(buffer: ArrayBuffer): Signature { - return btoa(String.fromCharCode(...new Uint8Array(buffer))); -} - -/** - * Generates a new Ed25519 key pair. - * Returns the public key as a JWK string, the private key as both a JWK - * string (for secure storage) and a runtime `CryptoKey` (for signing). - */ -export async function generateKeyPair(): Promise { - const keyPair = await crypto.subtle.generateKey( - { name: "Ed25519" } as Algorithm, - true, - ["sign", "verify"], - ) as CryptoKeyPair; - - const [publicKeyJwk, privateKeyJwk] = await Promise.all([ - crypto.subtle.exportKey("jwk", keyPair.publicKey), - crypto.subtle.exportKey("jwk", keyPair.privateKey), - ]); - - return { - publicKey: JSON.stringify(publicKeyJwk), - privateKeyJwk: JSON.stringify(privateKeyJwk), - signingKey: keyPair.privateKey, - }; -} - -/** - * Imports a private key from its JWK JSON string for use in signing. - * Call this when restoring a key pair from persistent storage. - */ -export async function importSigningKey(privateKeyJwk: string): Promise { - const jwk = JSON.parse(privateKeyJwk) as JsonWebKey; - return crypto.subtle.importKey( - "jwk", - jwk, - { name: "Ed25519" } as Algorithm, - false, - ["sign"], - ); -} - -/** - * Signs arbitrary data with an Ed25519 private key. - * Returns a base64-encoded signature string. - * - * @param data - UTF-8 string or raw bytes to sign. - * @param signingKey - CryptoKey from `generateKeyPair()` or `importSigningKey()`. - */ -export async function signData( - data: string | ArrayBuffer, - signingKey: CryptoKey, -): Promise { - const bytes = typeof data === "string" ? new TextEncoder().encode(data) : data; - const signatureBuffer = await crypto.subtle.sign("Ed25519", signingKey, bytes); - return bufferToBase64(signatureBuffer); -} +export * from "../../lib/core/crypto/sign"; diff --git a/core/crypto/uuid.ts b/core/crypto/uuid.ts index d6ad498..76aa30e 100644 --- a/core/crypto/uuid.ts +++ b/core/crypto/uuid.ts @@ -1,48 +1 @@ -// --------------------------------------------------------------------------- -// uuid.ts — Minimal UUID v4 generation utility -// --------------------------------------------------------------------------- -// -// Generates a RFC 4122 version 4 (random) UUID. -// Uses crypto.randomUUID() when available (browsers and modern Node/Bun), -// with a pure-JS fallback for environments that lack it. -// --------------------------------------------------------------------------- - -/** - * Generate a RFC 4122 version 4 UUID string. - * - * Prefers the platform's built-in crypto.randomUUID() when available, - * falling back to a pure-JS crypto.getRandomValues() implementation. - */ -export function randomUUID(): string { - if ( - typeof crypto !== "undefined" && - typeof (crypto as { randomUUID?: () => string }).randomUUID === "function" - ) { - return (crypto as { randomUUID: () => string }).randomUUID(); - } - - // Fallback: manually construct UUID v4 from random bytes - const bytes = new Uint8Array(16); - if (typeof crypto !== "undefined" && typeof crypto.getRandomValues === "function") { - crypto.getRandomValues(bytes); - } else { - // No secure RNG available: refuse to generate a UUID with weak randomness - throw new Error( - "randomUUID() requires a secure crypto.getRandomValues implementation; " + - "no suitable crypto API was found in this environment." - ); - } - - // Set version bits (v4) and variant bits (RFC 4122) - bytes[6] = (bytes[6] & 0x0f) | 0x40; - bytes[8] = (bytes[8] & 0x3f) | 0x80; - - const hex = [...bytes].map((b) => b.toString(16).padStart(2, "0")); - return ( - hex.slice(0, 4).join("") + - "-" + hex.slice(4, 6).join("") + - "-" + hex.slice(6, 8).join("") + - "-" + hex.slice(8, 10).join("") + - "-" + hex.slice(10).join("") - ); -} +export * from "../../lib/core/crypto/uuid"; diff --git a/core/crypto/verify.ts b/core/crypto/verify.ts index e31b3d7..386710e 100644 --- a/core/crypto/verify.ts +++ b/core/crypto/verify.ts @@ -1,50 +1 @@ -import type { PublicKey, Signature } from "../types.js"; - -function base64ToBuffer(base64: Signature): ArrayBuffer | null { - try { - const binary = atob(base64); - const bytes = new Uint8Array(binary.length); - for (let i = 0; i < binary.length; i++) { - bytes[i] = binary.charCodeAt(i); - } - return bytes.buffer; - } catch { - return null; - } -} - -/** - * Verifies an Ed25519 signature produced by `signData`. - * - * Returns `true` if `signature` is a valid Ed25519 signature over `data` - * by the key encoded in `publicKey` (JWK JSON string). - * Returns `false` for any signature mismatch or malformed signature. - * Throws for structurally invalid public key (malformed JWK JSON). - * - * @param data - The original data that was signed (string or bytes). - * @param signature - Base64-encoded signature from `signData()`. - * @param publicKey - JWK JSON string from `KeyPair.publicKey`. - */ -export async function verifySignature( - data: string | ArrayBuffer, - signature: Signature, - publicKey: PublicKey, -): Promise { - const signatureBuffer = base64ToBuffer(signature); - if (signatureBuffer === null) { - return false; - } - - const bytes = typeof data === "string" ? new TextEncoder().encode(data) : data; - const publicKeyJwk = JSON.parse(publicKey) as JsonWebKey; - - const cryptoKey = await crypto.subtle.importKey( - "jwk", - publicKeyJwk, - { name: "Ed25519" } as Algorithm, - false, - ["verify"], - ); - - return crypto.subtle.verify("Ed25519", cryptoKey, signatureBuffer, bytes); -} +export * from "../../lib/core/crypto/verify"; diff --git a/core/types.ts b/core/types.ts index 679de81..5744231 100644 --- a/core/types.ts +++ b/core/types.ts @@ -1,218 +1 @@ -// --------------------------------------------------------------------------- -// Primitive aliases -// --------------------------------------------------------------------------- - -export type Hash = string; // SHA-256 hex -export type Signature = string; // base64 or hex -export type PublicKey = string; // JWK or raw - -// --------------------------------------------------------------------------- -// Knowledge-hierarchy entities -// --------------------------------------------------------------------------- - -export interface Page { - pageId: Hash; // SHA-256(content) - content: string; // bounded by chunk policy derived from ModelProfile - embeddingOffset: number; // byte offset into the vector file - embeddingDim: number; // resolved embedding dimension from ModelProfile - - contentHash: Hash; // SHA-256(content) - vectorHash: Hash; // SHA-256(vector bytes) - - prevPageId?: Hash | null; - nextPageId?: Hash | null; - - creatorPubKey: PublicKey; - signature: Signature; - createdAt: string; // ISO timestamp -} - -export interface BookMetadata { - title?: string; - sourceUri?: string; - tags?: string[]; - extra?: Record; -} - -export interface Book { - bookId: Hash; // SHA-256(pageIds joined or Merkle root) - pageIds: Hash[]; - medoidPageId: Hash; // representative page selected by medoid statistic - meta: BookMetadata; -} - -export interface Volume { - volumeId: Hash; - bookIds: Hash[]; - prototypeOffsets: number[]; // byte offsets into the vector file - prototypeDim: number; // runtime policy dimension for this prototype tier - variance: number; -} - -export interface Shelf { - shelfId: Hash; - volumeIds: Hash[]; - routingPrototypeOffsets: number[]; // coarse prototype byte offsets - routingDim: number; -} - -export interface Edge { - fromPageId: Hash; - toPageId: Hash; - weight: number; // Hebbian weight - lastUpdatedAt: string; // ISO timestamp -} - -// --------------------------------------------------------------------------- -// Semantic nearest-neighbor graph -// --------------------------------------------------------------------------- - -/** A single directed proximity edge in the sparse semantic neighbor graph. */ -export interface SemanticNeighbor { - neighborPageId: Hash; - cosineSimilarity: number; // threshold is defined by runtime policy - distance: number; // 1 - cosineSimilarity (ready for TSP) -} - -/** Induced subgraph returned by BFS expansion of the semantic neighbor graph. */ -export interface SemanticNeighborSubgraph { - nodes: Hash[]; - edges: { from: Hash; to: Hash; distance: number }[]; -} - -// --------------------------------------------------------------------------- -// Hotpath / Williams Bound types -// --------------------------------------------------------------------------- - -/** Lightweight per-page activity metadata for salience computation. */ -export interface PageActivity { - pageId: Hash; - queryHitCount: number; // incremented on each query hit - lastQueryAt: string; // ISO timestamp of most recent query hit - communityId?: string; // set by Daydreamer label propagation -} - -/** Record for HOT membership — used in both RAM index and IndexedDB snapshot. */ -export interface HotpathEntry { - entityId: Hash; // pageId, bookId, volumeId, or shelfId - tier: "shelf" | "volume" | "book" | "page"; - salience: number; // salience value at last computation - communityId?: string; // community this entry counts against -} - -/** Per-tier slot budgets derived from H(t). */ -export interface TierQuotas { - shelf: number; - volume: number; - book: number; - page: number; -} - -/** Fractional quota ratios for each tier (must sum to 1.0). */ -export interface TierQuotaRatios { - shelf: number; - volume: number; - book: number; - page: number; -} - -/** Tunable weights for the salience formula: sigma = alpha*H_in + beta*R + gamma*Q. */ -export interface SalienceWeights { - alpha: number; // weight for Hebbian connectivity - beta: number; // weight for recency - gamma: number; // weight for query-hit frequency -} - -// --------------------------------------------------------------------------- -// Storage abstractions -// --------------------------------------------------------------------------- - -/** - * Append-only binary vector file. - * - * `offset` values are **byte offsets** inside the file so that mixed-dimension - * vectors (full embeddings vs. compressed prototypes) coexist without - * additional metadata. - */ -export interface VectorStore { - /** Appends `vector` to the file and returns its starting byte offset. */ - appendVector(vector: Float32Array): Promise; - - /** Reads `dim` floats starting at byte offset `offset`. */ - readVector(offset: number, dim: number): Promise; - - /** Reads multiple vectors by their individual byte offsets. */ - readVectors(offsets: number[], dim: number): Promise; -} - -/** - * Structured metadata store backed by IndexedDB (or any equivalent engine). - * - * Reverse-index helpers (`getBooksByPage`, `getVolumesByBook`, - * `getShelvesByVolume`) are maintained automatically on every `put*` call so - * callers never need to manage the mappings directly. - */ -export interface MetadataStore { - // --- Core CRUD --- - putPage(page: Page): Promise; - getPage(pageId: Hash): Promise; - /** Returns all pages in the store. Used for warm/cold fallbacks in query. */ - getAllPages(): Promise; - - putBook(book: Book): Promise; - getBook(bookId: Hash): Promise; - - putVolume(volume: Volume): Promise; - getVolume(volumeId: Hash): Promise; - /** Returns all volumes in the store. */ - getAllVolumes(): Promise; - /** - * Delete a volume record and clean up all reverse-index entries - * (`bookToVolume` for each book in the volume, and the `volumeToShelf` entry). - * Callers are responsible for removing the volume from any shelf's `volumeIds` - * list before calling this method. - */ - deleteVolume(volumeId: Hash): Promise; - - putShelf(shelf: Shelf): Promise; - getShelf(shelfId: Hash): Promise; - /** Returns all shelves in the store. */ - getAllShelves(): Promise; - - // --- Hebbian edges --- - putEdges(edges: Edge[]): Promise; - /** Remove a single directed edge. */ - deleteEdge(fromPageId: Hash, toPageId: Hash): Promise; - getNeighbors(pageId: Hash, limit?: number): Promise; - - // --- Reverse-index helpers --- - getBooksByPage(pageId: Hash): Promise; - getVolumesByBook(bookId: Hash): Promise; - getShelvesByVolume(volumeId: Hash): Promise; - - // --- Semantic neighbor radius index --- - putSemanticNeighbors(pageId: Hash, neighbors: SemanticNeighbor[]): Promise; - getSemanticNeighbors(pageId: Hash, maxDegree?: number): Promise; - - /** BFS expansion of the semantic neighbor subgraph up to `maxHops` levels deep. */ - getInducedNeighborSubgraph( - seedPageIds: Hash[], - maxHops: number, - ): Promise; - - // --- Dirty-volume recalc flags --- - needsNeighborRecalc(volumeId: Hash): Promise; - flagVolumeForNeighborRecalc(volumeId: Hash): Promise; - clearNeighborRecalcFlag(volumeId: Hash): Promise; - - // --- Hotpath index --- - putHotpathEntry(entry: HotpathEntry): Promise; - getHotpathEntries(tier?: HotpathEntry["tier"]): Promise; - removeHotpathEntry(entityId: Hash): Promise; - evictWeakest(tier: HotpathEntry["tier"], communityId?: string): Promise; - getResidentCount(): Promise; - - // --- Page activity --- - putPageActivity(activity: PageActivity): Promise; - getPageActivity(pageId: Hash): Promise; -} +export * from "../lib/core/types"; diff --git a/cortex/KnowledgeGapDetector.ts b/cortex/KnowledgeGapDetector.ts index 1ce983c..3f2b344 100644 --- a/cortex/KnowledgeGapDetector.ts +++ b/cortex/KnowledgeGapDetector.ts @@ -1,66 +1 @@ -import type { Hash } from "../core/types"; -import type { ModelProfile } from "../core/ModelProfile"; -import { hashText } from "../core/crypto/hash"; -import type { Metroid } from "./MetroidBuilder"; - -export interface KnowledgeGap { - queryText: string; - queryEmbedding: Float32Array; - knowledgeBoundary: Hash | null; - detectedAt: string; -} - -export interface CuriosityProbe { - probeId: Hash; - queryText: string; - queryEmbedding: Float32Array; - knowledgeBoundary: Hash | null; - mimeType: string; - modelUrn: string; - createdAt: string; -} - -/** - * Returns a KnowledgeGap when the metroid signals that m2 could not be found - * (i.e. the engine has no antithesis for this query). Returns null when the - * metroid is complete and no gap was detected. - */ -export async function detectKnowledgeGap( - queryText: string, - queryEmbedding: Float32Array, - metroid: Metroid, - // eslint-disable-next-line @typescript-eslint/no-unused-vars -- reserved for future model-aware gap categorisation - _modelProfile: ModelProfile, -): Promise { - if (!metroid.knowledgeGap) return null; - - return { - queryText, - queryEmbedding, - knowledgeBoundary: metroid.m1 !== "" ? metroid.m1 : null, - detectedAt: new Date().toISOString(), - }; -} - -/** - * Builds a serialisable CuriosityProbe from a detected KnowledgeGap. - * The probeId is the SHA-256 of (queryText + detectedAt) so it is - * deterministic for the same gap inputs. - */ -export async function buildCuriosityProbe( - gap: KnowledgeGap, - modelProfile: ModelProfile, - mimeType = "text/plain", -): Promise { - const probeId = await hashText(gap.queryText + gap.detectedAt); - - return { - probeId, - queryText: gap.queryText, - queryEmbedding: gap.queryEmbedding, - knowledgeBoundary: gap.knowledgeBoundary, - mimeType, - modelUrn: `urn:model:${modelProfile.modelId}`, - createdAt: new Date().toISOString(), - }; -} +export * from "../lib/cortex/KnowledgeGapDetector"; diff --git a/cortex/MetroidBuilder.ts b/cortex/MetroidBuilder.ts index 30640a7..3b4a9bb 100644 --- a/cortex/MetroidBuilder.ts +++ b/cortex/MetroidBuilder.ts @@ -1,217 +1 @@ -import type { Hash, VectorStore } from "../core/types"; -import type { ModelProfile } from "../core/ModelProfile"; - -export interface Metroid { - m1: Hash; - m2: Hash | null; - c: Float32Array | null; - knowledgeGap: boolean; -} - -export interface MetroidBuilderOptions { - modelProfile: ModelProfile; - vectorStore: VectorStore; -} - -/** Standard Matryoshka tier sizes in ascending order. */ -const MATRYOSHKA_TIERS = [32, 64, 128, 256, 512, 768, 1024, 2048] as const; - -function cosineSimilarity(a: Float32Array, b: Float32Array): number { - let dotProduct = 0; - let normA = 0; - let normB = 0; - const len = Math.min(a.length, b.length); - for (let i = 0; i < len; i++) { - dotProduct += a[i] * b[i]; - normA += a[i] * a[i]; - normB += b[i] * b[i]; - } - if (normA === 0 || normB === 0) return 0; - return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)); -} - -function cosineDistance(a: Float32Array, b: Float32Array): number { - return 1 - cosineSimilarity(a, b); -} - -/** - * Returns the index of the medoid: the element that minimises total cosine - * distance to every other element in the set. - */ -function findMedoidIndex(embeddings: Float32Array[]): number { - if (embeddings.length === 1) return 0; - - let bestIdx = 0; - let bestTotal = Infinity; - - for (let i = 0; i < embeddings.length; i++) { - let total = 0; - for (let j = 0; j < embeddings.length; j++) { - if (i !== j) { - total += cosineDistance(embeddings[i], embeddings[j]); - } - } - if (total < bestTotal) { - bestTotal = total; - bestIdx = i; - } - } - - return bestIdx; -} - -interface CandidateEntry { - pageId: Hash; - embeddingOffset: number; - embeddingDim: number; -} - -interface CandidateWithEmbedding extends CandidateEntry { - embedding: Float32Array; -} - -/** - * Searches for m2 among `others` (candidates excluding m1) using the free - * dimensions starting at `protectedDim`. - * - * Returns the selected medoid candidate or `null` if no valid opposite set - * can be assembled. - */ -function searchM2( - others: CandidateWithEmbedding[], - m1Embedding: Float32Array, - protectedDim: number, -): CandidateWithEmbedding | null { - if (others.length === 0) return null; - - const m1Free = m1Embedding.slice(protectedDim); - - const scored = others.map((c) => { - const free = c.embedding.slice(protectedDim); - return { candidate: c, score: -cosineSimilarity(free, m1Free) }; - }); - - // Prefer candidates that are genuinely opposite (score >= 0). - let oppositeSet = scored.filter((s) => s.score >= 0); - - // Fall back to the top 50% when the genuine-opposite set is too small. - if (oppositeSet.length < 2) { - const byScore = [...scored].sort((a, b) => b.score - a.score); - const topHalf = Math.max(1, Math.ceil(byScore.length / 2)); - oppositeSet = byScore.slice(0, topHalf); - } - - if (oppositeSet.length === 0) return null; - - const medoidIdx = findMedoidIndex(oppositeSet.map((s) => s.candidate.embedding.slice(protectedDim))); - return oppositeSet[medoidIdx].candidate; -} - -/** - * Builds the dialectical probe (Metroid) for a given query embedding and a - * ranked list of candidate memory nodes. - * - * Step overview - * 1. Select m1 (thesis): the candidate with highest cosine similarity to the query. - * 2. Select m2 (antithesis): the medoid of the cosine-opposite set in free dims. - * Uses Matryoshka dimensional unwinding when the initial tier yields no m2. - * 3. Compute centroid c (synthesis): protected dims copied from m1, free dims - * averaged between m1 and m2. - */ -export async function buildMetroid( - queryEmbedding: Float32Array, - candidateMedoids: Array<{ pageId: Hash; embeddingOffset: number; embeddingDim: number }>, - options: MetroidBuilderOptions, -): Promise { - const { modelProfile, vectorStore } = options; - - if (candidateMedoids.length === 0) { - return { m1: "", m2: null, c: null, knowledgeGap: true }; - } - - // Load all candidate embeddings in one pass. - const candidates: CandidateWithEmbedding[] = await Promise.all( - candidateMedoids.map(async (cand) => ({ - ...cand, - embedding: await vectorStore.readVector(cand.embeddingOffset, cand.embeddingDim), - })), - ); - - // Select m1: highest cosine similarity to the query. - let m1Candidate = candidates[0]; - let m1Score = cosineSimilarity(queryEmbedding, candidates[0].embedding); - - for (let i = 1; i < candidates.length; i++) { - const score = cosineSimilarity(queryEmbedding, candidates[i].embedding); - if (score > m1Score) { - m1Score = score; - m1Candidate = candidates[i]; - } - } - - const protectedDim = modelProfile.matryoshkaProtectedDim; - - if (protectedDim === undefined) { - // Non-Matryoshka model: antithesis search is impossible. - return { m1: m1Candidate.pageId, m2: null, c: null, knowledgeGap: true }; - } - - const others = candidates.filter((c) => c.pageId !== m1Candidate.pageId); - - // --- Matryoshka dimensional unwinding --- - // Start at modelProfile.matryoshkaProtectedDim. If m2 not found, progressively - // shrink the protected boundary (expand the free-dimension search region). - - const startingTierIndex = MATRYOSHKA_TIERS.indexOf( - protectedDim as (typeof MATRYOSHKA_TIERS)[number], - ); - - // Build the list of tier boundaries to attempt, from the configured value - // down to the smallest tier (expanding the free region at each step). - const tierBoundaries: number[] = []; - if (startingTierIndex !== -1) { - for (let i = startingTierIndex; i >= 0; i--) { - tierBoundaries.push(MATRYOSHKA_TIERS[i]); - } - } else { - // protectedDim is not a standard tier; try it as-is plus any smaller standard tiers. - tierBoundaries.push(protectedDim); - for (const t of [...MATRYOSHKA_TIERS].reverse()) { - if (t < protectedDim) tierBoundaries.push(t); - } - } - - let m2Candidate: CandidateWithEmbedding | null = null; - let usedProtectedDim = protectedDim; - - for (const tierBoundary of tierBoundaries) { - const found = searchM2(others, m1Candidate.embedding, tierBoundary); - if (found !== null) { - m2Candidate = found; - usedProtectedDim = tierBoundary; - break; - } - } - - if (m2Candidate === null) { - return { m1: m1Candidate.pageId, m2: null, c: null, knowledgeGap: true }; - } - - // Compute frozen synthesis centroid c. - const fullDim = m1Candidate.embedding.length; - const c = new Float32Array(fullDim); - - for (let i = 0; i < usedProtectedDim; i++) { - c[i] = m1Candidate.embedding[i]; - } - for (let i = usedProtectedDim; i < fullDim; i++) { - c[i] = (m1Candidate.embedding[i] + m2Candidate.embedding[i]) / 2; - } - - return { - m1: m1Candidate.pageId, - m2: m2Candidate.pageId, - c, - knowledgeGap: false, - }; -} +export * from "../lib/cortex/MetroidBuilder"; diff --git a/cortex/OpenTSPSolver.ts b/cortex/OpenTSPSolver.ts index 257ad80..3042595 100644 --- a/cortex/OpenTSPSolver.ts +++ b/cortex/OpenTSPSolver.ts @@ -1,62 +1 @@ -import type { Hash, SemanticNeighborSubgraph } from "../core/types"; - -/** - * Greedy nearest-neighbor open-path TSP heuristic. - * - * Visits every node in the subgraph exactly once, starting from the - * lexicographically smallest node ID for determinism. At each step the - * algorithm advances to the unvisited node nearest to the current one - * (using edge distance). Ties are broken lexicographically. Missing edges - * are treated as having distance Infinity. - */ -export function solveOpenTSP(subgraph: SemanticNeighborSubgraph): Hash[] { - const { nodes, edges } = subgraph; - if (nodes.length === 0) return []; - - // Build undirected adjacency map: node → (neighbor → distance). - const adj = new Map>(); - for (const node of nodes) { - adj.set(node, new Map()); - } - for (const edge of edges) { - const fromMap = adj.get(edge.from); - const toMap = adj.get(edge.to); - if (fromMap !== undefined) fromMap.set(edge.to, edge.distance); - if (toMap !== undefined) toMap.set(edge.from, edge.distance); - } - - // Pre-sort once so lexicographic tiebreaking is O(1) per step. - const sorted = [...nodes].sort(); - - const visited = new Set(); - const path: Hash[] = []; - let current = sorted[0]; - - while (path.length < nodes.length) { - visited.add(current); - path.push(current); - - if (path.length === nodes.length) break; - - const neighbors = adj.get(current)!; - let bestNode: Hash | undefined; - let bestDist = Infinity; - - for (const node of sorted) { - if (visited.has(node)) continue; - const dist = neighbors.get(node) ?? Infinity; - if ( - dist < bestDist || - (dist === bestDist && (bestNode === undefined || node < bestNode)) - ) { - bestDist = dist; - bestNode = node; - } - } - - // bestNode is always defined here because at least one unvisited node remains. - current = bestNode!; - } - - return path; -} +export * from "../lib/cortex/OpenTSPSolver"; diff --git a/cortex/Query.ts b/cortex/Query.ts index 488a1ba..75fcf86 100644 --- a/cortex/Query.ts +++ b/cortex/Query.ts @@ -1,170 +1 @@ -import type { ModelProfile } from "../core/ModelProfile"; -import type { Hash, MetadataStore, Page, VectorStore } from "../core/types"; -import type { EmbeddingRunner } from "../embeddings/EmbeddingRunner"; -import { runPromotionSweep } from "../core/SalienceEngine"; -import { computeSubgraphBounds } from "../core/HotpathPolicy"; -import type { QueryResult } from "./QueryResult"; -import { rankPages, spillToWarm } from "./Ranking"; -import { buildMetroid } from "./MetroidBuilder"; -import { detectKnowledgeGap } from "./KnowledgeGapDetector"; -import { solveOpenTSP } from "./OpenTSPSolver"; - -export interface QueryOptions { - modelProfile: ModelProfile; - embeddingRunner: EmbeddingRunner; - vectorStore: VectorStore; - metadataStore: MetadataStore; - topK?: number; - /** - * Maximum BFS depth for semantic neighbor subgraph expansion. - * - * When omitted, a dynamic Williams-derived value is computed from the - * corpus size via `computeSubgraphBounds(t)`. Providing an explicit value - * overrides the dynamic bound (useful for tests and controlled experiments). - */ - maxHops?: number; -} - -export async function query( - queryText: string, - options: QueryOptions, -): Promise { - const { - modelProfile, - embeddingRunner, - vectorStore, - metadataStore, - topK = 10, - } = options; - const nowIso = new Date().toISOString(); - - const embeddings = await embeddingRunner.embed([queryText]); - if (embeddings.length !== 1) { - throw new Error("Embedding provider returned unexpected number of embeddings"); - } - const queryEmbedding = embeddings[0]; - - const rankingOptions = { vectorStore, metadataStore }; - - // --- HOT path: score resident pages --- - const hotpathEntries = await metadataStore.getHotpathEntries("page"); - const hotpathIds = hotpathEntries.map((e) => e.entityId); - - const hotResults = await rankPages(queryEmbedding, hotpathIds, topK, rankingOptions); - const seenIds = new Set(hotResults.map((r) => r.id)); - - // --- Warm spill: fill up to topK if hot path is insufficient --- - let warmResults: Array<{ id: Hash; score: number }> = []; - if (hotResults.length < topK) { - const allWarm = await spillToWarm("page", queryEmbedding, topK, rankingOptions); - warmResults = allWarm.filter((r) => !seenIds.has(r.id)); - } - - // Merge, deduplicate, sort, and slice to topK - const merged = [...hotResults, ...warmResults]; - merged.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)); - const topResults = merged.slice(0, topK); - - // Load Page objects for the top results - const topPages = ( - await Promise.all(topResults.map((r) => metadataStore.getPage(r.id))) - ).filter((p): p is Page => p !== undefined); - - const topScores = topResults - .filter((r) => topPages.some((p) => p.pageId === r.id)) - .map((r) => r.score); - - // --- MetroidBuilder: build dialectical probe --- - // Candidates: hotpath book medoid pages + hotpath pages themselves - const hotpathBookEntries = await metadataStore.getHotpathEntries("book"); - const bookCandidates = ( - await Promise.all( - hotpathBookEntries.map(async (e) => { - const book = await metadataStore.getBook(e.entityId); - if (!book) return null; - const medoidPage = await metadataStore.getPage(book.medoidPageId); - if (!medoidPage) return null; - return { - pageId: medoidPage.pageId, - embeddingOffset: medoidPage.embeddingOffset, - embeddingDim: medoidPage.embeddingDim, - }; - }), - ) - ).filter((c): c is NonNullable => c !== null); - - const pageCandidates = topPages.map((p) => ({ - pageId: p.pageId, - embeddingOffset: p.embeddingOffset, - embeddingDim: p.embeddingDim, - })); - - // Deduplicate candidates by pageId - const candidateMap = new Map(); - for (const c of [...bookCandidates, ...pageCandidates]) { - candidateMap.set(c.pageId, c); - } - const metroidCandidates = [...candidateMap.values()]; - - const metroid = await buildMetroid(queryEmbedding, metroidCandidates, { - modelProfile, - vectorStore, - }); - - // --- KnowledgeGapDetector --- - const knowledgeGap = await detectKnowledgeGap( - queryText, - queryEmbedding, - metroid, - modelProfile, - ); - - // --- Subgraph expansion --- - // Use dynamic Williams-derived bounds unless the caller has pinned an - // explicit maxHops value. Only load all pages when we actually need to - // compute bounds — skip the full-page scan on the hot path when maxHops is - // already known. - const topPageIds = topPages.map((p) => p.pageId); - let effectiveMaxHops: number; - if (options.maxHops !== undefined) { - effectiveMaxHops = options.maxHops; - } else { - const allPages = await metadataStore.getAllPages(); - effectiveMaxHops = computeSubgraphBounds(allPages.length).maxHops; - } - const subgraph = await metadataStore.getInducedNeighborSubgraph(topPageIds, effectiveMaxHops); - - // --- TSP coherence path --- - const coherencePath = solveOpenTSP(subgraph); - - // --- Update activity for returned pages --- - await Promise.all( - topPages.map(async (page) => { - const activity = await metadataStore.getPageActivity(page.pageId); - await metadataStore.putPageActivity({ - pageId: page.pageId, - queryHitCount: (activity?.queryHitCount ?? 0) + 1, - lastQueryAt: nowIso, - communityId: activity?.communityId, - }); - }), - ); - - // --- Promotion sweep --- - await runPromotionSweep(topPageIds, metadataStore); - - return { - pages: topPages, - scores: topScores, - coherencePath, - metroid, - knowledgeGap, - metadata: { - queryText, - topK, - returned: topPages.length, - timestamp: nowIso, - modelId: modelProfile.modelId, - }, - }; -} +export * from "../lib/cortex/Query"; diff --git a/cortex/QueryResult.ts b/cortex/QueryResult.ts index 8d7406e..c459632 100644 --- a/cortex/QueryResult.ts +++ b/cortex/QueryResult.ts @@ -1,12 +1 @@ -import type { Hash, Page } from "../core/types"; -import type { Metroid } from "./MetroidBuilder"; -import type { KnowledgeGap } from "./KnowledgeGapDetector"; - -export interface QueryResult { - pages: Page[]; - scores: number[]; - coherencePath: Hash[]; - metroid: Metroid | null; - knowledgeGap: KnowledgeGap | null; - metadata: Record; -} +export * from "../lib/cortex/QueryResult"; diff --git a/cortex/Ranking.ts b/cortex/Ranking.ts index f0d9f9f..d0b4e3f 100644 --- a/cortex/Ranking.ts +++ b/cortex/Ranking.ts @@ -1,156 +1 @@ -import type { Hash, MetadataStore, VectorStore } from "../core/types"; -import type { VectorBackend } from "../VectorBackend"; - -export interface RankingOptions { - vectorStore: VectorStore; - metadataStore: MetadataStore; - vectorBackend?: VectorBackend; -} - -function cosineSimilarity(a: Float32Array, b: Float32Array): number { - let dotProduct = 0; - let normA = 0; - let normB = 0; - const len = Math.min(a.length, b.length); - for (let i = 0; i < len; i++) { - dotProduct += a[i] * b[i]; - normA += a[i] * a[i]; - normB += b[i] * b[i]; - } - if (normA === 0 || normB === 0) return 0; - return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)); -} - -function pickTopK( - scored: Array<{ id: Hash; score: number }>, - k: number, -): Array<{ id: Hash; score: number }> { - scored.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)); - return scored.slice(0, k); -} - -/** - * Ranks shelves by cosine similarity of their routing prototype to the query. - * Uses routingPrototypeOffsets[0] as the representative vector. - */ -export async function rankShelves( - queryEmbedding: Float32Array, - residentShelfIds: Hash[], - topK: number, - options: RankingOptions, -): Promise> { - if (residentShelfIds.length === 0) return []; - - const { vectorStore, metadataStore } = options; - const scored: Array<{ id: Hash; score: number }> = []; - - for (const shelfId of residentShelfIds) { - const shelf = await metadataStore.getShelf(shelfId); - if (!shelf || shelf.routingPrototypeOffsets.length === 0) continue; - const vec = await vectorStore.readVector(shelf.routingPrototypeOffsets[0], shelf.routingDim); - scored.push({ id: shelfId, score: cosineSimilarity(queryEmbedding, vec) }); - } - - return pickTopK(scored, topK); -} - -/** - * Ranks volumes by cosine similarity of their first prototype to the query. - * Uses prototypeOffsets[0] as the representative vector. - */ -export async function rankVolumes( - queryEmbedding: Float32Array, - residentVolumeIds: Hash[], - topK: number, - options: RankingOptions, -): Promise> { - if (residentVolumeIds.length === 0) return []; - - const { vectorStore, metadataStore } = options; - const scored: Array<{ id: Hash; score: number }> = []; - - for (const volumeId of residentVolumeIds) { - const volume = await metadataStore.getVolume(volumeId); - if (!volume || volume.prototypeOffsets.length === 0) continue; - const vec = await vectorStore.readVector(volume.prototypeOffsets[0], volume.prototypeDim); - scored.push({ id: volumeId, score: cosineSimilarity(queryEmbedding, vec) }); - } - - return pickTopK(scored, topK); -} - -/** - * Ranks books by cosine similarity of their medoid page embedding to the query. - */ -export async function rankBooks( - queryEmbedding: Float32Array, - residentBookIds: Hash[], - topK: number, - options: RankingOptions, -): Promise> { - if (residentBookIds.length === 0) return []; - - const { vectorStore, metadataStore } = options; - const scored: Array<{ id: Hash; score: number }> = []; - - for (const bookId of residentBookIds) { - const book = await metadataStore.getBook(bookId); - if (!book) continue; - const medoidPage = await metadataStore.getPage(book.medoidPageId); - if (!medoidPage) continue; - const vec = await vectorStore.readVector(medoidPage.embeddingOffset, medoidPage.embeddingDim); - scored.push({ id: bookId, score: cosineSimilarity(queryEmbedding, vec) }); - } - - return pickTopK(scored, topK); -} - -/** - * Ranks pages by cosine similarity of their embedding to the query. - */ -export async function rankPages( - queryEmbedding: Float32Array, - residentPageIds: Hash[], - topK: number, - options: RankingOptions, -): Promise> { - if (residentPageIds.length === 0) return []; - - const { vectorStore, metadataStore } = options; - const scored: Array<{ id: Hash; score: number }> = []; - - for (const pageId of residentPageIds) { - const page = await metadataStore.getPage(pageId); - if (!page) continue; - const vec = await vectorStore.readVector(page.embeddingOffset, page.embeddingDim); - scored.push({ id: pageId, score: cosineSimilarity(queryEmbedding, vec) }); - } - - return pickTopK(scored, topK); -} - -/** - * Spills to the warm tier when the resident set provides insufficient coverage. - * For "page": scores all pages in the store. - * For other tiers: returns [] (warm spill is only implemented for pages at this stage). - */ -export async function spillToWarm( - tier: "shelf" | "volume" | "book" | "page", - queryEmbedding: Float32Array, - topK: number, - options: RankingOptions, -): Promise> { - if (tier !== "page") return []; - - const { vectorStore, metadataStore } = options; - const allPages = await metadataStore.getAllPages(); - if (allPages.length === 0) return []; - - const scored: Array<{ id: Hash; score: number }> = []; - for (const page of allPages) { - const vec = await vectorStore.readVector(page.embeddingOffset, page.embeddingDim); - scored.push({ id: page.pageId, score: cosineSimilarity(queryEmbedding, vec) }); - } - - return pickTopK(scored, topK); -} +export * from "../lib/cortex/Ranking"; diff --git a/daydreamer/ClusterStability.ts b/daydreamer/ClusterStability.ts index e1d587d..a4be3b7 100644 --- a/daydreamer/ClusterStability.ts +++ b/daydreamer/ClusterStability.ts @@ -1,680 +1 @@ -// --------------------------------------------------------------------------- -// ClusterStability — Community detection via label propagation (P2-F) and -// volume split/merge for balanced cluster maintenance (P2-F3) -// --------------------------------------------------------------------------- -// -// Assigns community labels to pages by running lightweight label propagation -// on the semantic neighbor graph. Labels are stored in -// PageActivity.communityId and propagate into SalienceEngine community quotas. -// -// Label propagation terminates when assignments stabilise (no label changes) -// or a maximum iteration limit is reached. -// -// The Daydreamer background worker also calls ClusterStability periodically to -// detect and fix unstable volumes: -// - HIGH-VARIANCE volumes are split into two balanced sub-volumes. -// - LOW-COUNT volumes are merged into the nearest neighbour volume. -// - Community labels are updated after structural changes. -// --------------------------------------------------------------------------- - -import { hashText } from "../core/crypto/hash"; -import type { - Book, - Hash, - MetadataStore, - PageActivity, - Volume, -} from "../core/types"; - -// --------------------------------------------------------------------------- -// Label propagation options -// --------------------------------------------------------------------------- - -export interface LabelPropagationOptions { - metadataStore: MetadataStore; - /** Maximum number of label propagation iterations. Default: 20. */ - maxIterations?: number; -} - -export interface LabelPropagationResult { - /** Number of iterations until convergence (or maxIterations). */ - iterations: number; - /** True if the algorithm converged before hitting maxIterations. */ - converged: boolean; - /** Map from pageId to assigned communityId. */ - communityMap: Map; -} - -// --------------------------------------------------------------------------- -// Label propagation -// --------------------------------------------------------------------------- - -/** - * Run one pass of label propagation over all pages. - * - * Each node adopts the most frequent label among its Metroid neighbors. - * Ties are broken deterministically by choosing the lexicographically - * smallest label (consistent across runs and nodes). - * - * Returns true if any label changed during this pass. - */ -async function propagationPass( - pageIds: Hash[], - labels: Map, - metadataStore: MetadataStore, -): Promise { - let changed = false; - - // Shuffle-equivalent deterministic ordering: sort by pageId for reproducibility - const sorted = [...pageIds].sort(); - - for (const pageId of sorted) { - const neighbors = await metadataStore.getSemanticNeighbors(pageId); - if (neighbors.length === 0) continue; - - // Count neighbor labels - const counts = new Map(); - for (const n of neighbors) { - const label = labels.get(n.neighborPageId) ?? n.neighborPageId; - counts.set(label, (counts.get(label) ?? 0) + 1); - } - - // Find the most frequent label (tie-break: lexicographically smallest) - let bestLabel: string | undefined; - let bestCount = 0; - for (const [label, count] of counts) { - if ( - count > bestCount || - (count === bestCount && bestLabel !== undefined && label < bestLabel) - ) { - bestLabel = label; - bestCount = count; - } - } - - if (bestLabel !== undefined && labels.get(pageId) !== bestLabel) { - labels.set(pageId, bestLabel); - changed = true; - } - } - - return changed; -} - -// --------------------------------------------------------------------------- -// Public API -// --------------------------------------------------------------------------- - -/** - * Assign community labels to all pages via label propagation on the - * Metroid (semantic) neighbor graph. - * - * Initial labels: each page is its own community (pageId as initial label). - * Each iteration: every node adopts the most frequent label among neighbors. - * Convergence: no label changed in the most recent pass. - * - * After convergence, persists all community labels via - * `MetadataStore.putPageActivity`. - */ -export async function runLabelPropagation( - options: LabelPropagationOptions, -): Promise { - const { - metadataStore, - maxIterations = 20, - } = options; - - const allPages = await metadataStore.getAllPages(); - if (allPages.length === 0) { - return { iterations: 0, converged: true, communityMap: new Map() }; - } - - const pageIds = allPages.map((p) => p.pageId); - - // Initialise: each page is its own community - const labels = new Map(); - for (const id of pageIds) { - labels.set(id, id); - } - - let iterations = 0; - let converged = false; - - for (let iter = 0; iter < maxIterations; iter++) { - iterations++; - const changed = await propagationPass(pageIds, labels, metadataStore); - if (!changed) { - converged = true; - break; - } - } - - // Persist community labels to PageActivity - for (const pageId of pageIds) { - const communityId = labels.get(pageId) ?? pageId; - const existing = await metadataStore.getPageActivity(pageId); - const activity: PageActivity = { - pageId, - queryHitCount: existing?.queryHitCount ?? 0, - lastQueryAt: existing?.lastQueryAt ?? new Date(0).toISOString(), - communityId, - }; - await metadataStore.putPageActivity(activity); - } - - return { iterations, converged, communityMap: new Map(labels) }; -} - -/** - * Detect whether a community should be split (too large relative to graph). - * - * A community is considered oversized when it holds more than - * `maxCommunityFraction` of all pages. - * - * Returns the set of community IDs that exceed the threshold. - */ -export function detectOversizedCommunities( - communityMap: Map, - maxCommunityFraction = 0.5, -): Set { - const total = communityMap.size; - if (total === 0) return new Set(); - - const counts = new Map(); - for (const label of communityMap.values()) { - counts.set(label, (counts.get(label) ?? 0) + 1); - } - - const oversized = new Set(); - for (const [label, count] of counts) { - if (count / total > maxCommunityFraction) { - oversized.add(label); - } - } - return oversized; -} - -/** - * Detect communities that no longer have any members (empty communities). - * - * These communities should release their hotpath quota slots back to the - * page-tier budget. - * - * @param knownCommunities Full set of community IDs that have quota allocations. - * @param activeCommunities Community IDs currently assigned to at least one page. - */ -export function detectEmptyCommunities( - knownCommunities: Set, - activeCommunities: Set, -): Set { - const empty = new Set(); - for (const id of knownCommunities) { - if (!activeCommunities.has(id)) { - empty.add(id); - } - } - return empty; -} - -// --------------------------------------------------------------------------- -// ClusterStability class — Volume split/merge configuration -// --------------------------------------------------------------------------- - -// --------------------------------------------------------------------------- -// Configuration -// --------------------------------------------------------------------------- - -export interface ClusterStabilityOptions { - /** - * Volume variance threshold above which a volume is considered unstable and - * will be split. - * Defaults to 0.5. - */ - varianceThreshold?: number; - - /** - * Minimum number of books a volume must contain. Volumes with fewer books - * than this will be merged with a neighbour. - * Defaults to 2. - */ - minBooksPerVolume?: number; - - /** - * Maximum split iterations for the K-means step. - * Defaults to 10. - */ - maxKmeansIterations?: number; -} - -const DEFAULT_VARIANCE_THRESHOLD = 0.5; -const DEFAULT_MIN_BOOKS_PER_VOLUME = 2; -const DEFAULT_MAX_KMEANS_ITERATIONS = 10; - -// --------------------------------------------------------------------------- -// Result types -// --------------------------------------------------------------------------- - -export interface ClusterStabilityResult { - /** Number of volumes split into two sub-volumes. */ - splitCount: number; - - /** Number of volumes merged into a neighbour. */ - mergeCount: number; - - /** Number of PageActivity community-label updates written. */ - communityUpdates: number; - - /** ISO timestamp when the stability run completed. */ - completedAt: string; -} - -// --------------------------------------------------------------------------- -// Internal types -// --------------------------------------------------------------------------- - -// --------------------------------------------------------------------------- -// ClusterStability -// --------------------------------------------------------------------------- - -export class ClusterStability { - private readonly varianceThreshold: number; - private readonly minBooksPerVolume: number; - private readonly maxKmeansIterations: number; - - constructor(options: ClusterStabilityOptions = {}) { - this.varianceThreshold = - options.varianceThreshold ?? DEFAULT_VARIANCE_THRESHOLD; - this.minBooksPerVolume = - options.minBooksPerVolume ?? DEFAULT_MIN_BOOKS_PER_VOLUME; - this.maxKmeansIterations = - options.maxKmeansIterations ?? DEFAULT_MAX_KMEANS_ITERATIONS; - } - - /** - * Run one stability pass over all volumes in the metadata store. - * - * Scans for unstable (high-variance) volumes and undersized volumes, then - * applies the appropriate structural fix and updates community labels. - */ - async run(metadataStore: MetadataStore): Promise { - // Collect all volumes (we scan through shelves) - const shelves = await this.collectAllShelves(metadataStore); - const allVolumeIds = shelves.flatMap((s) => s.volumeIds); - - const volumes = ( - await Promise.all(allVolumeIds.map((id) => metadataStore.getVolume(id))) - ).filter((v): v is Volume => v !== undefined); - - let splitCount = 0; - let mergeCount = 0; - let communityUpdates = 0; - - // --- Pass 1: split high-variance volumes --- - for (const volume of volumes) { - if ( - volume.variance > this.varianceThreshold && - volume.bookIds.length >= 2 - ) { - const splits = await this.splitVolume(volume, metadataStore); - if (splits !== null) { - splitCount++; - communityUpdates += await this.updateCommunityLabels( - splits, - metadataStore, - ); - // Replace the old volume in shelves with the two new sub-volumes, - // then delete the orphan volume record and its reverse-index entries. - await this.replaceVolumeInShelves( - volume.volumeId, - splits, - metadataStore, - ); - await metadataStore.deleteVolume(volume.volumeId); - } - } - } - - // --- Pass 2: merge undersized volumes --- - // Re-read volumes after splits to pick up any IDs that may have changed. - // Also include newly created split volumes from Pass 1 via a fresh shelf scan. - const allShelves2 = await this.collectAllShelves(metadataStore); - const allVolumeIds2 = allShelves2.flatMap((s) => s.volumeIds); - const allVolumesNow = ( - await Promise.all(allVolumeIds2.map((id) => metadataStore.getVolume(id))) - ).filter((v): v is Volume => v !== undefined); - - // Filter to undersized volumes (skip volumes we just created by splitting) - const undersized = allVolumesNow.filter( - (v) => v.bookIds.length < this.minBooksPerVolume, - ); - - const merged = new Set(); - - for (const small of undersized) { - if (merged.has(small.volumeId)) continue; - - const neighbour = this.findNearestNeighbour( - small, - allVolumesNow.filter( - (v) => - v.volumeId !== small.volumeId && !merged.has(v.volumeId), - ), - ); - - if (neighbour === null) continue; - - const mergedVolume = await this.mergeVolumes( - small, - neighbour, - metadataStore, - ); - - merged.add(small.volumeId); - merged.add(neighbour.volumeId); - mergeCount++; - communityUpdates += await this.updateCommunityLabels( - [mergedVolume], - metadataStore, - ); - // Replace the consumed volumes in shelves with the merged volume, - // then delete their orphan records and reverse-index entries. - await this.replaceVolumeInShelves( - small.volumeId, - [mergedVolume], - metadataStore, - ); - await this.replaceVolumeInShelves( - neighbour.volumeId, - [], - metadataStore, - ); - await metadataStore.deleteVolume(small.volumeId); - await metadataStore.deleteVolume(neighbour.volumeId); - } - - return { - splitCount, - mergeCount, - communityUpdates, - completedAt: new Date().toISOString(), - }; - } - - // --------------------------------------------------------------------------- - // Split logic - // --------------------------------------------------------------------------- - - /** - * Split a high-variance volume into two sub-volumes using K-means (K=2). - * - * Returns the two new volumes, or `null` if the split cannot be performed - * (e.g. insufficient books with resolvable vectors). - */ - private async splitVolume( - volume: Volume, - metadataStore: MetadataStore, - ): Promise<[Volume, Volume] | null> { - const books = ( - await Promise.all(volume.bookIds.map((id) => metadataStore.getBook(id))) - ).filter((b): b is Book => b !== undefined); - - if (books.length < 2) return null; - - // Use only the medoid page vector as representative for each book. - // For simplicity we use the first prototype offset of the parent volume - // and the book's position (index) as a deterministic pseudo-distance. - // A full implementation would read actual medoid embeddings via VectorStore. - const assignments = this.kmeansAssign(books); - if (assignments === null) return null; - - const [groupA, groupB] = assignments; - - const volumeA = await this.buildSubVolume(groupA, volume); - const volumeB = await this.buildSubVolume(groupB, volume); - - await metadataStore.putVolume(volumeA); - await metadataStore.putVolume(volumeB); - - return [volumeA, volumeB]; - } - - /** - * Assign books to two clusters using a simple K-means initialisation: - * centroid A = first half by index, centroid B = second half. - * - * Returns `null` when it is not possible to form two non-empty clusters. - * - * The "distance" used here is the index difference (as a stable proxy when - * real vectors are not loaded), which produces a balanced split without - * requiring a live VectorStore. A production pass would replace this with - * actual cosine distances between medoid embeddings. - * - * Precomputes a `bookId → index` map so each iteration is O(n) rather than - * O(n²) (avoids repeated Array.indexOf calls inside the inner loop). - */ - private kmeansAssign(books: Book[]): [Book[], Book[]] | null { - if (books.length < 2) return null; - - const n = books.length; - // Precompute index map to avoid O(n²) indexOf calls - const indexMap = new Map( - books.map((b, i) => [b.bookId, i]), - ); - - // Centroid A = first half, centroid B = second half (index-based split) - const splitPoint = Math.ceil(n / 2); - - let groupA = books.slice(0, splitPoint); - let groupB = books.slice(splitPoint); - - if (groupA.length === 0 || groupB.length === 0) return null; - - // Run up to maxKmeansIterations assignment cycles using index centroids - for (let iter = 0; iter < this.maxKmeansIterations; iter++) { - const centroidA = this.indexCentroid(groupA, indexMap); - const centroidB = this.indexCentroid(groupB, indexMap); - - const newA: Book[] = []; - const newB: Book[] = []; - - for (const book of books) { - const idx = indexMap.get(book.bookId) ?? 0; - const distA = Math.abs(idx - centroidA); - const distB = Math.abs(idx - centroidB); - if (distA <= distB) { - newA.push(book); - } else { - newB.push(book); - } - } - - // Ensure neither cluster becomes empty - if (newA.length === 0) { - newA.push(newB.splice(0, 1)[0]); - } - if (newB.length === 0) { - newB.push(newA.splice(newA.length - 1, 1)[0]); - } - - const converged = - newA.length === groupA.length && - newA.every((b, i) => b.bookId === groupA[i]?.bookId); - - groupA = newA; - groupB = newB; - - if (converged) break; - } - - return [groupA, groupB]; - } - - /** Compute the mean index of a group using the precomputed index map. */ - private indexCentroid( - group: Book[], - indexMap: Map, - ): number { - const sum = group.reduce( - (acc, b) => acc + (indexMap.get(b.bookId) ?? 0), - 0, - ); - return sum / group.length; - } - - private async buildSubVolume( - books: Book[], - parent: Volume, - ): Promise { - const bookIds = books.map((b) => b.bookId); - const seed = `split:${parent.volumeId}:${bookIds.join(",")}`; - const volumeId = await hashText(seed); - - // Variance is approximated as half the parent's variance for each child. - // A production pass would recompute from actual embeddings. - const variance = parent.variance / 2; - - return { - volumeId, - bookIds, - prototypeOffsets: [...parent.prototypeOffsets], - prototypeDim: parent.prototypeDim, - variance, - }; - } - - // --------------------------------------------------------------------------- - // Merge logic - // --------------------------------------------------------------------------- - - private findNearestNeighbour( - target: Volume, - candidates: Volume[], - ): Volume | null { - if (candidates.length === 0) return null; - - // Use the count of shared books as a similarity proxy. - // A production pass would compare medoid embeddings. - let best = candidates[0]; - let bestShared = this.sharedBookCount(target, best); - - for (let i = 1; i < candidates.length; i++) { - const shared = this.sharedBookCount(target, candidates[i]); - if (shared > bestShared) { - best = candidates[i]; - bestShared = shared; - } - } - - return best; - } - - private sharedBookCount(a: Volume, b: Volume): number { - const setA = new Set(a.bookIds); - return b.bookIds.filter((id) => setA.has(id)).length; - } - - private async mergeVolumes( - a: Volume, - b: Volume, - metadataStore: MetadataStore, - ): Promise { - const bookIds = [...new Set([...a.bookIds, ...b.bookIds])]; - const seed = `merge:${a.volumeId}:${b.volumeId}`; - const volumeId = await hashText(seed); - - // Average the variance of the two merged volumes - const variance = (a.variance + b.variance) / 2; - - const merged: Volume = { - volumeId, - bookIds, - prototypeOffsets: [...a.prototypeOffsets, ...b.prototypeOffsets], - prototypeDim: a.prototypeDim, - variance, - }; - - await metadataStore.putVolume(merged); - return merged; - } - - // --------------------------------------------------------------------------- - // Community label updates - // --------------------------------------------------------------------------- - - /** - * After a structural change (split or merge), update the `communityId` field - * on each affected page's `PageActivity` record. - * - * The community ID is set to the new volume's `volumeId` so that the - * SalienceEngine can bucket promotions correctly. - * - * @returns The number of PageActivity records updated. - */ - private async updateCommunityLabels( - volumes: Volume[], - metadataStore: MetadataStore, - ): Promise { - let updates = 0; - - for (const volume of volumes) { - const books = ( - await Promise.all( - volume.bookIds.map((id) => metadataStore.getBook(id)), - ) - ).filter((b): b is Book => b !== undefined); - - for (const book of books) { - for (const pageId of book.pageIds) { - const activity = await metadataStore.getPageActivity(pageId); - const updated: PageActivity = { - pageId, - queryHitCount: activity?.queryHitCount ?? 0, - lastQueryAt: - activity?.lastQueryAt ?? new Date().toISOString(), - communityId: volume.volumeId, - }; - await metadataStore.putPageActivity(updated); - updates++; - } - } - } - - return updates; - } - - // --------------------------------------------------------------------------- - // Shelf update helpers - // --------------------------------------------------------------------------- - - /** - * Replace `oldVolumeId` in every shelf that references it with the IDs of - * `replacements`. Passing an empty `replacements` array removes the old - * volume from the shelf without adding a substitute. - */ - private async replaceVolumeInShelves( - oldVolumeId: Hash, - replacements: Volume[], - metadataStore: MetadataStore, - ): Promise { - const shelves = await this.collectAllShelves(metadataStore); - - for (const shelf of shelves) { - if (!shelf.volumeIds.includes(oldVolumeId)) continue; - - const newVolumeIds = shelf.volumeIds - .filter((id) => id !== oldVolumeId) - .concat(replacements.map((v) => v.volumeId)); - - await metadataStore.putShelf({ - ...shelf, - volumeIds: newVolumeIds, - }); - } - } - - private async collectAllShelves( - metadataStore: MetadataStore, - ) { - return metadataStore.getAllShelves(); - } -} +export * from "../lib/daydreamer/ClusterStability"; diff --git a/daydreamer/ExperienceReplay.ts b/daydreamer/ExperienceReplay.ts index 7ecfa1d..98410ae 100644 --- a/daydreamer/ExperienceReplay.ts +++ b/daydreamer/ExperienceReplay.ts @@ -1,233 +1 @@ -// --------------------------------------------------------------------------- -// ExperienceReplay — Idle-time query simulation for Hebbian reinforcement -// --------------------------------------------------------------------------- -// -// During idle periods the Daydreamer background worker samples recent or -// random pages, re-executes synthetic queries from their content, and -// marks traversed edges for Long-Term Potentiation (LTP). -// -// This reinforces connection patterns that were useful in the past and -// prevents them from decaying through disuse. -// --------------------------------------------------------------------------- - -import type { EmbeddingRunner } from "../embeddings/EmbeddingRunner"; -import type { ModelProfile } from "../core/ModelProfile"; -import type { MetadataStore, Page, VectorStore, Edge } from "../core/types"; -import { query as cortexQuery } from "../cortex/Query"; -import type { QueryOptions } from "../cortex/Query"; - -// --------------------------------------------------------------------------- -// Configuration -// --------------------------------------------------------------------------- - -export interface ExperienceReplayOptions { - /** - * Number of synthetic queries to execute per replay cycle. - * Defaults to 5. - */ - queriesPerCycle?: number; - - /** - * Maximum number of pages to consider as query sources. - * When set, only the most recently created pages are sampled. - * Defaults to 200 (recent-biased sampling pool). - */ - samplePoolSize?: number; - - /** - * LTP weight increment applied to edges traversed during replay. - * Defaults to 0.1. - */ - ltpIncrement?: number; - - /** - * Maximum Hebbian edge weight. Weights are clamped to this value after LTP. - * Defaults to 1.0. - */ - maxEdgeWeight?: number; - - /** - * Top-K pages to retrieve per synthetic query. - * Defaults to 5. - */ - topK?: number; -} - -const DEFAULT_QUERIES_PER_CYCLE = 5; -const DEFAULT_SAMPLE_POOL_SIZE = 200; -const DEFAULT_LTP_INCREMENT = 0.1; -const DEFAULT_MAX_EDGE_WEIGHT = 1.0; -const DEFAULT_TOP_K = 5; - -// --------------------------------------------------------------------------- -// Result types -// --------------------------------------------------------------------------- - -export interface ExperienceReplayResult { - /** Number of synthetic queries executed. */ - queriesExecuted: number; - - /** Total number of edge weight updates applied. */ - edgesStrengthened: number; - - /** ISO timestamp when the replay cycle completed. */ - completedAt: string; -} - -// --------------------------------------------------------------------------- -// ExperienceReplay -// --------------------------------------------------------------------------- - -export class ExperienceReplay { - private readonly queriesPerCycle: number; - private readonly samplePoolSize: number; - private readonly ltpIncrement: number; - private readonly maxEdgeWeight: number; - private readonly topK: number; - - constructor(options: ExperienceReplayOptions = {}) { - this.queriesPerCycle = options.queriesPerCycle ?? DEFAULT_QUERIES_PER_CYCLE; - this.samplePoolSize = options.samplePoolSize ?? DEFAULT_SAMPLE_POOL_SIZE; - this.ltpIncrement = options.ltpIncrement ?? DEFAULT_LTP_INCREMENT; - this.maxEdgeWeight = options.maxEdgeWeight ?? DEFAULT_MAX_EDGE_WEIGHT; - this.topK = options.topK ?? DEFAULT_TOP_K; - } - - /** - * Run one replay cycle. - * - * 1. Sample `queriesPerCycle` pages from the store (recent-biased). - * 2. Execute a synthetic query for each sampled page using its content. - * 3. Strengthen (LTP) Hebbian edges connecting query results to the source page. - * - * @returns Summary statistics for the cycle. - */ - async run( - modelProfile: ModelProfile, - embeddingRunner: EmbeddingRunner, - vectorStore: VectorStore, - metadataStore: MetadataStore, - ): Promise { - const allPages = await metadataStore.getAllPages(); - if (allPages.length === 0) { - return { - queriesExecuted: 0, - edgesStrengthened: 0, - completedAt: new Date().toISOString(), - }; - } - - const pool = this.buildSamplePool(allPages); - const sources = this.sampleWithoutReplacement(pool, this.queriesPerCycle); - - const queryOptions: QueryOptions = { - modelProfile, - embeddingRunner, - vectorStore, - metadataStore, - topK: this.topK, - }; - - let edgesStrengthened = 0; - - for (const sourcePage of sources) { - const result = await cortexQuery(sourcePage.content, queryOptions); - const resultPageIds = result.pages.map((p) => p.pageId); - - edgesStrengthened += await this.applyLtp( - sourcePage.pageId, - resultPageIds, - metadataStore, - ); - } - - return { - queriesExecuted: sources.length, - edgesStrengthened, - completedAt: new Date().toISOString(), - }; - } - - // --------------------------------------------------------------------------- - // Internal helpers - // --------------------------------------------------------------------------- - - /** - * Build a sample pool from `allPages`. - * - * Sorts pages by `createdAt` descending (most recent first) and caps the - * pool at `samplePoolSize` to give recent pages a higher selection probability. - */ - private buildSamplePool(allPages: Page[]): Page[] { - const sorted = [...allPages].sort((a, b) => - b.createdAt.localeCompare(a.createdAt), - ); - return sorted.slice(0, this.samplePoolSize); - } - - /** - * Sample up to `count` pages from `pool` without replacement using a - * Fisher-Yates partial shuffle. - */ - private sampleWithoutReplacement(pool: Page[], count: number): Page[] { - const arr = [...pool]; - const take = Math.min(count, arr.length); - - for (let i = 0; i < take; i++) { - const j = i + Math.floor(Math.random() * (arr.length - i)); - [arr[i], arr[j]] = [arr[j], arr[i]]; - } - - return arr.slice(0, take); - } - - /** - * Apply LTP to edges between `sourcePageId` and each page in `resultPageIds`. - * - * Fetches existing Hebbian edges, increments their weight by `ltpIncrement` - * (clamped to `maxEdgeWeight`), and writes them back. - * - * New edges are created when none exist between the source and a result page. - * - * @returns The number of edge weight updates written. - */ - private async applyLtp( - sourcePageId: string, - resultPageIds: string[], - metadataStore: MetadataStore, - ): Promise { - if (resultPageIds.length === 0) return 0; - - const existingEdges = await metadataStore.getNeighbors(sourcePageId); - const edgeMap = new Map( - existingEdges.map((e) => [e.toPageId, e]), - ); - - const now = new Date().toISOString(); - const updatedEdges: Edge[] = []; - - for (const targetId of resultPageIds) { - if (targetId === sourcePageId) continue; - - const existing = edgeMap.get(targetId); - const currentWeight = existing?.weight ?? 0; - const newWeight = Math.min( - currentWeight + this.ltpIncrement, - this.maxEdgeWeight, - ); - - updatedEdges.push({ - fromPageId: sourcePageId, - toPageId: targetId, - weight: newWeight, - lastUpdatedAt: now, - }); - } - - if (updatedEdges.length > 0) { - await metadataStore.putEdges(updatedEdges); - } - - return updatedEdges.length; - } -} +export * from "../lib/daydreamer/ExperienceReplay"; diff --git a/daydreamer/FullNeighborRecalc.ts b/daydreamer/FullNeighborRecalc.ts index acd0ecc..731ac9d 100644 --- a/daydreamer/FullNeighborRecalc.ts +++ b/daydreamer/FullNeighborRecalc.ts @@ -1,201 +1 @@ -// --------------------------------------------------------------------------- -// FullNeighborRecalc — Periodic full semantic neighbor graph recalculation (P2-C) -// --------------------------------------------------------------------------- -// -// The fast incremental neighbor insert used during ingest is approximate. -// This module performs a full pairwise recalculation for dirty volumes, -// bounded by the Williams-Bound-derived maintenance budget so the idle loop -// is not starved. -// -// Per idle cycle, the scheduler processes at most max(MIN_RECALC_PAIR_BUDGET, -// computeCapacity(graphMass)) pairwise comparisons. The minimum floor ensures -// forward progress for small corpora where the Williams formula may return a -// value smaller than a single typical volume's pair count. -// --------------------------------------------------------------------------- - -import type { Hash, MetadataStore, SemanticNeighbor, Page, VectorStore } from "../core/types"; -import { computeCapacity, DEFAULT_HOTPATH_POLICY, type HotpathPolicy } from "../core/HotpathPolicy"; -import { batchComputeSalience, runPromotionSweep } from "../core/SalienceEngine"; - -// Minimum pair budget per idle recalc cycle. -// Sized to cover the theoretical maximum for a single well-formed volume -// (BOOKS_PER_VOLUME=4 books × PAGES_PER_BOOK=8 pages = 32 pages, -// 32 × 31 = 992 pairs). Using 2048 gives a comfortable margin. -export const MIN_RECALC_PAIR_BUDGET = 2048; - -// --------------------------------------------------------------------------- -// Options -// --------------------------------------------------------------------------- - -export interface FullNeighborRecalcOptions { - metadataStore: MetadataStore; - vectorStore: VectorStore; - policy?: HotpathPolicy; - /** Maximum Metroid neighbors stored per page. Default: 16. */ - maxNeighbors?: number; - /** Current timestamp (ms since epoch). Defaults to Date.now(). */ - now?: number; -} - -export interface RecalcResult { - volumesProcessed: number; - pagesProcessed: number; - pairsComputed: number; -} - -// --------------------------------------------------------------------------- -// Cosine similarity -// --------------------------------------------------------------------------- - -function cosineSimilarity(a: Float32Array, b: Float32Array): number { - const len = Math.min(a.length, b.length); - let dot = 0; - let normA = 0; - let normB = 0; - for (let i = 0; i < len; i++) { - dot += a[i] * b[i]; - normA += a[i] * a[i]; - normB += b[i] * b[i]; - } - const denom = Math.sqrt(normA) * Math.sqrt(normB); - if (denom === 0) return 0; - return dot / denom; -} - -// --------------------------------------------------------------------------- -// Main recalc function -// --------------------------------------------------------------------------- - -/** - * Run one cycle of full neighbor graph recalculation. - * - * Finds all volumes flagged as dirty (via `needsNeighborRecalc`), loads - * their pages, computes pairwise cosine similarities, and updates the - * Metroid neighbor index. Processing is bounded by the Williams-Bound-derived - * maintenance budget to avoid blocking the idle loop. - * - * After recalculation, salience is recomputed for affected pages and a - * promotion sweep is run to keep the hotpath current. - */ -export async function runFullNeighborRecalc( - options: FullNeighborRecalcOptions, -): Promise { - const { - metadataStore, - vectorStore, - policy = DEFAULT_HOTPATH_POLICY, - maxNeighbors = 16, - now = Date.now(), - } = options; - - // Find all dirty volumes - const allVolumes = await metadataStore.getAllVolumes(); - const dirtyVolumes = ( - await Promise.all( - allVolumes.map(async (v) => ({ - volume: v, - dirty: await metadataStore.needsNeighborRecalc(v.volumeId), - })), - ) - ) - .filter((x) => x.dirty) - .map((x) => x.volume); - - if (dirtyVolumes.length === 0) { - return { volumesProcessed: 0, pagesProcessed: 0, pairsComputed: 0 }; - } - - // Compute per-cycle pair budget: max of Williams-derived capacity and - // the minimum floor so even small corpora make forward progress. - const totalGraphMass = (await metadataStore.getAllPages()).length; - const pairBudget = Math.max(MIN_RECALC_PAIR_BUDGET, computeCapacity(totalGraphMass, policy.c)); - - let totalVolumesProcessed = 0; - let totalPagesProcessed = 0; - let totalPairsComputed = 0; - - const affectedPageIds = new Set(); - - for (const volume of dirtyVolumes) { - if (totalPairsComputed >= pairBudget) break; - - // Collect all pages in this volume (via books) - const volumePages: Page[] = []; - for (const bookId of volume.bookIds) { - const book = await metadataStore.getBook(bookId); - if (!book) continue; - for (const pageId of book.pageIds) { - const page = await metadataStore.getPage(pageId); - if (page) volumePages.push(page); - } - } - - if (volumePages.length === 0) { - await metadataStore.clearNeighborRecalcFlag(volume.volumeId); - totalVolumesProcessed++; - continue; - } - - // Load all embedding vectors for this volume's pages - const vectors = await Promise.all( - volumePages.map((p) => - vectorStore.readVector(p.embeddingOffset, p.embeddingDim), - ), - ); - - // Compute pairwise similarities and build neighbor lists - const pairsInVolume = volumePages.length * (volumePages.length - 1); - const remainingBudget = pairBudget - totalPairsComputed; - const budgetExhausted = pairsInVolume > remainingBudget; - if (budgetExhausted) { - break; - } - - for (let i = 0; i < volumePages.length; i++) { - const page = volumePages[i]; - const vecI = vectors[i]; - - const neighbors: SemanticNeighbor[] = []; - - for (let j = 0; j < volumePages.length; j++) { - if (i === j) continue; - const sim = cosineSimilarity(vecI, vectors[j]); - neighbors.push({ - neighborPageId: volumePages[j].pageId, - cosineSimilarity: sim, - distance: 1 - sim, - }); - totalPairsComputed++; - } - - // Sort by similarity descending; keep top maxNeighbors - neighbors.sort( - (a, b) => - b.cosineSimilarity - a.cosineSimilarity || - a.neighborPageId.localeCompare(b.neighborPageId), - ); - const topNeighbors = neighbors.slice(0, maxNeighbors); - - await metadataStore.putSemanticNeighbors(page.pageId, topNeighbors); - affectedPageIds.add(page.pageId); - } - - // Clear the dirty flag - await metadataStore.clearNeighborRecalcFlag(volume.volumeId); - totalVolumesProcessed++; - totalPagesProcessed += volumePages.length; - } - - // Recompute salience and run promotion sweep for all affected pages - if (affectedPageIds.size > 0) { - const ids = [...affectedPageIds]; - await batchComputeSalience(ids, metadataStore, policy, now); - await runPromotionSweep(ids, metadataStore, policy, now); - } - - return { - volumesProcessed: totalVolumesProcessed, - pagesProcessed: totalPagesProcessed, - pairsComputed: totalPairsComputed, - }; -} +export * from "../lib/daydreamer/FullNeighborRecalc"; diff --git a/daydreamer/HebbianUpdater.ts b/daydreamer/HebbianUpdater.ts index 9dd710a..e75936e 100644 --- a/daydreamer/HebbianUpdater.ts +++ b/daydreamer/HebbianUpdater.ts @@ -1,193 +1 @@ -// --------------------------------------------------------------------------- -// HebbianUpdater — Edge plasticity via LTP / LTD / pruning (P2-B) -// --------------------------------------------------------------------------- -// -// Strengthens edges traversed during successful queries (Long-Term -// Potentiation), decays all edges each pass (Long-Term Depression), and -// prunes edges that fall below a threshold to keep the graph sparse. -// -// After LTP/LTD, salience is recomputed for every node whose incident edges -// changed, and a promotion/eviction sweep is run so the hotpath stays current. -// --------------------------------------------------------------------------- - -import type { Edge, Hash, MetadataStore } from "../core/types"; -import { DEFAULT_HOTPATH_POLICY, type HotpathPolicy } from "../core/HotpathPolicy"; -import { batchComputeSalience, runPromotionSweep } from "../core/SalienceEngine"; - -// --------------------------------------------------------------------------- -// Constants (policy-derived defaults; never hardcoded in callers) -// --------------------------------------------------------------------------- - -/** Default LTP step: edge weight increases by this amount on traversal. */ -export const DEFAULT_LTP_AMOUNT = 0.1; - -/** Default LTD multiplicative decay factor applied every pass (0 < decay < 1). */ -export const DEFAULT_LTD_DECAY = 0.99; - -/** Edges with weight below this threshold are removed by pruning. */ -export const DEFAULT_PRUNE_THRESHOLD = 0.01; - -/** Maximum outgoing Hebbian edges per node (degree cap). */ -export const DEFAULT_MAX_DEGREE = 16; - -// --------------------------------------------------------------------------- -// Public API -// --------------------------------------------------------------------------- - -export interface HebbianUpdaterOptions { - metadataStore: MetadataStore; - policy?: HotpathPolicy; - /** LTP step amount. Default: DEFAULT_LTP_AMOUNT. */ - ltpAmount?: number; - /** LTD multiplicative decay applied to every edge. Default: DEFAULT_LTD_DECAY. */ - ltdDecay?: number; - /** Prune edges whose weight drops below this value. Default: DEFAULT_PRUNE_THRESHOLD. */ - pruneThreshold?: number; - /** Maximum outgoing degree per node. Default: DEFAULT_MAX_DEGREE. */ - maxDegree?: number; - /** Current timestamp (ms since epoch). Defaults to Date.now(). */ - now?: number; -} - -/** - * LTP — strengthen edges that were traversed during a successful query. - * - * Clamps weights to [0, Infinity) and re-saves affected edges. - * Recomputes salience for changed nodes and triggers a promotion sweep. - */ -export async function strengthenEdges( - traversedPairs: Array<{ from: Hash; to: Hash }>, - options: HebbianUpdaterOptions, -): Promise { - if (traversedPairs.length === 0) return; - - const { - metadataStore, - policy = DEFAULT_HOTPATH_POLICY, - ltpAmount = DEFAULT_LTP_AMOUNT, - now = Date.now(), - } = options; - - // Group by source node for efficient per-node updates - const bySource = new Map>(); - for (const { from, to } of traversedPairs) { - let targets = bySource.get(from); - if (!targets) { - targets = new Set(); - bySource.set(from, targets); - } - targets.add(to); - } - - const changedNodeIds = new Set(); - - for (const [fromId, toIds] of bySource) { - const existing = await metadataStore.getNeighbors(fromId); - const edgeMap = new Map(existing.map((e) => [e.toPageId, e])); - - const timestamp = new Date(now).toISOString(); - const updatedEdges: Edge[] = []; - - for (const toId of toIds) { - const edge = edgeMap.get(toId); - if (edge) { - updatedEdges.push({ - ...edge, - weight: edge.weight + ltpAmount, - lastUpdatedAt: timestamp, - }); - } else { - // Create new edge if not yet present - updatedEdges.push({ - fromPageId: fromId, - toPageId: toId, - weight: ltpAmount, - lastUpdatedAt: timestamp, - }); - } - changedNodeIds.add(fromId); - changedNodeIds.add(toId); - } - - if (updatedEdges.length > 0) { - await metadataStore.putEdges(updatedEdges); - } - } - - if (changedNodeIds.size > 0) { - await batchComputeSalience([...changedNodeIds], metadataStore, policy, now); - await runPromotionSweep([...changedNodeIds], metadataStore, policy, now); - } -} - -/** - * LTD + pruning — decay all edges by a multiplicative factor, then remove - * edges whose weight falls below the prune threshold or that exceed the max - * degree per source node. - * - * Recomputes salience for every node whose incident edges changed. - */ -export async function decayAndPrune( - options: HebbianUpdaterOptions, -): Promise<{ decayed: number; pruned: number }> { - const { - metadataStore, - policy = DEFAULT_HOTPATH_POLICY, - ltdDecay = DEFAULT_LTD_DECAY, - pruneThreshold = DEFAULT_PRUNE_THRESHOLD, - maxDegree = DEFAULT_MAX_DEGREE, - now = Date.now(), - } = options; - - const allPages = await metadataStore.getAllPages(); - if (allPages.length === 0) return { decayed: 0, pruned: 0 }; - - const changedNodeIds = new Set(); - let totalDecayed = 0; - let totalPruned = 0; - - const timestamp = new Date(now).toISOString(); - - for (const page of allPages) { - const edges = await metadataStore.getNeighbors(page.pageId); - if (edges.length === 0) continue; - - // Apply LTD decay - const decayed: Edge[] = edges.map((e) => ({ - ...e, - weight: e.weight * ltdDecay, - lastUpdatedAt: timestamp, - })); - totalDecayed += decayed.length; - - // Separate edges to keep vs. prune - const surviving = decayed.filter((e) => e.weight >= pruneThreshold); - const pruned = decayed.filter((e) => e.weight < pruneThreshold); - - // Enforce max degree: keep the strongest surviving edges - surviving.sort((a, b) => b.weight - a.weight); - const kept = surviving.slice(0, maxDegree); - const degreeEvicted = surviving.slice(maxDegree); - - // Delete pruned edges - for (const e of [...pruned, ...degreeEvicted]) { - await metadataStore.deleteEdge(e.fromPageId, e.toPageId); - totalPruned++; - changedNodeIds.add(e.fromPageId); - changedNodeIds.add(e.toPageId); - } - - // Save decayed-but-surviving edges - if (kept.length > 0) { - await metadataStore.putEdges(kept); - changedNodeIds.add(page.pageId); - } - } - - if (changedNodeIds.size > 0) { - await batchComputeSalience([...changedNodeIds], metadataStore, policy, now); - await runPromotionSweep([...changedNodeIds], metadataStore, policy, now); - } - - return { decayed: totalDecayed, pruned: totalPruned }; -} +export * from "../lib/daydreamer/HebbianUpdater"; diff --git a/daydreamer/IdleScheduler.ts b/daydreamer/IdleScheduler.ts index 59a2ace..97e252b 100644 --- a/daydreamer/IdleScheduler.ts +++ b/daydreamer/IdleScheduler.ts @@ -1,172 +1 @@ -// --------------------------------------------------------------------------- -// IdleScheduler — Cooperative background task scheduler (P2-A) -// --------------------------------------------------------------------------- -// -// Drives background Daydreamer operations without blocking the main thread. -// Uses requestIdleCallback in browsers and setImmediate in Node/test envs. -// Tasks are prioritised by a numeric priority field (lower = higher priority). -// --------------------------------------------------------------------------- - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -/** A single schedulable background task. */ -export interface ScheduledTask { - /** Lower number = higher priority. Tasks with equal priority run FIFO. */ - priority: number; - /** The work to perform. May be called multiple times if it re-enqueues itself. */ - run(): Promise; -} - -/** Internal queue entry. */ -interface QueueEntry { - insertionOrder: number; - task: ScheduledTask; -} - -// --------------------------------------------------------------------------- -// Idle callback shim -// --------------------------------------------------------------------------- - -/** Minimum time (ms) the scheduler will attempt to do work per idle slice. */ -const DEFAULT_BUDGET_MS = 5; - -/** - * Schedule a callback for when the host is idle. - * Falls back to setImmediate (Node) or setTimeout(0) when - * requestIdleCallback is not available. - */ -function scheduleIdle(callback: (deadline: { timeRemaining(): number }) => void): void { - if (typeof requestIdleCallback === "function") { - requestIdleCallback((deadline) => callback(deadline)); - } else if (typeof setImmediate === "function") { - setImmediate(() => callback({ timeRemaining: () => DEFAULT_BUDGET_MS })); - } else { - setTimeout(() => callback({ timeRemaining: () => DEFAULT_BUDGET_MS }), 0); - } -} - -// --------------------------------------------------------------------------- -// IdleScheduler -// --------------------------------------------------------------------------- - -/** - * Cooperative background task scheduler. - * - * Tasks are run one at a time during idle slices. Each task is given a single - * idle deadline per scheduling turn; if the deadline expires the scheduler - * yields and resumes on the next idle callback. - * - * State corruption is prevented by never interrupting a task mid-execution — - * each `task.run()` call is awaited to completion before the next task starts. - */ -export class IdleScheduler { - private queue: QueueEntry[] = []; - private counter = 0; - private active = false; - private stopped = false; - private readonly budgetMs: number; - private readonly errorHandler: (error: unknown, task: ScheduledTask) => void; - - /** - * @param budgetMs Approximate milliseconds of work per idle slice. - * Defaults to 5 ms. The scheduler yields after this - * budget is consumed even if the queue is non-empty. - * @param onError Optional error handler invoked when a task throws. - * Defaults to logging to console.error (if available). - */ - constructor( - budgetMs = DEFAULT_BUDGET_MS, - onError?: (error: unknown, task: ScheduledTask) => void, - ) { - this.budgetMs = budgetMs; - this.errorHandler = - onError ?? - ((error: unknown, task: ScheduledTask): void => { - if (typeof console !== "undefined" && typeof console.error === "function") { - console.error("[IdleScheduler] Task failed", { error, task }); - } - }); - } - - /** - * Enqueue a task. The task will be run in priority order (ascending - * priority value) during the next idle callback. Enqueueing while - * the scheduler is running is safe — the task will be picked up on - * the next scheduling turn. - */ - enqueue(task: ScheduledTask): void { - this.queue.push({ insertionOrder: this.counter++, task }); - this._sortQueue(); - } - - /** - * Start the idle loop. Safe to call multiple times — extra calls are no-ops - * if the loop is already running. - */ - start(): void { - if (this.active || this.stopped) return; - this.active = true; - this._scheduleNextTurn(); - } - - /** - * Permanently stop the scheduler. After calling `stop()` no further tasks - * will be executed and `start()` becomes a no-op. Tasks already in-flight - * will complete normally. - */ - stop(): void { - this.stopped = true; - this.active = false; - } - - /** True when the task queue is empty. */ - get idle(): boolean { - return this.queue.length === 0; - } - - // --------------------------------------------------------------------------- - // Private - // --------------------------------------------------------------------------- - - private _sortQueue(): void { - this.queue.sort( - (a, b) => - a.task.priority - b.task.priority || - a.insertionOrder - b.insertionOrder, - ); - } - - private _scheduleNextTurn(): void { - if (this.stopped) return; - scheduleIdle((deadline) => { - void this._runTurn(deadline); - }); - } - - private async _runTurn(deadline: { timeRemaining(): number }): Promise { - if (this.stopped) return; - - const turnEnd = Date.now() + Math.max(deadline.timeRemaining(), this.budgetMs); - - while (this.queue.length > 0 && Date.now() < turnEnd && !this.stopped) { - const entry = this.queue.shift(); - if (!entry) break; - try { - await entry.task.run(); - } catch (error) { - // Report errors so failing tasks can be diagnosed, but do not - // allow a single bad task to crash the idle loop. - this.errorHandler(error, entry.task); - } - } - - if (!this.stopped && this.queue.length > 0) { - // More work remains — schedule another turn. - this._scheduleNextTurn(); - } else { - this.active = this.queue.length > 0; - } - } -} +export * from "../lib/daydreamer/IdleScheduler"; diff --git a/daydreamer/PrototypeRecomputer.ts b/daydreamer/PrototypeRecomputer.ts index 867fbf3..351ebc6 100644 --- a/daydreamer/PrototypeRecomputer.ts +++ b/daydreamer/PrototypeRecomputer.ts @@ -1,263 +1 @@ -// --------------------------------------------------------------------------- -// PrototypeRecomputer — Keep volume and shelf prototypes accurate (P2-D) -// --------------------------------------------------------------------------- -// -// As pages and books change, volume medoids and centroids drift. This module -// recomputes them periodically during Daydreamer idle passes. -// -// After recomputing prototypes at each level, salience is refreshed for the -// updated representative entries and a tier-scoped promotion/eviction sweep -// is run to keep the hotpath consistent. -// --------------------------------------------------------------------------- - -import type { Hash, HotpathEntry, MetadataStore, Shelf, Volume, VectorStore } from "../core/types"; -import { DEFAULT_HOTPATH_POLICY, type HotpathPolicy } from "../core/HotpathPolicy"; -import { runPromotionSweep } from "../core/SalienceEngine"; - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -/** Compute cosine similarity between two equal-length vectors. */ -function cosineSimilarity(a: Float32Array, b: Float32Array): number { - const len = Math.min(a.length, b.length); - let dot = 0; - let normA = 0; - let normB = 0; - for (let i = 0; i < len; i++) { - dot += a[i] * b[i]; - normA += a[i] * a[i]; - normB += b[i] * b[i]; - } - const denom = Math.sqrt(normA) * Math.sqrt(normB); - if (denom === 0) return 0; - return dot / denom; -} - -/** - * Select the medoid from a set of vectors: the vector that minimises the - * average distance to all others (the most "central" real member). - * - * Returns the index of the medoid in the input array, or -1 if empty. - */ -export function selectMedoidIndex(vectors: Float32Array[]): number { - if (vectors.length === 0) return -1; - if (vectors.length === 1) return 0; - - let bestIndex = 0; - let bestAvgDist = Infinity; - - for (let i = 0; i < vectors.length; i++) { - let totalDist = 0; - for (let j = 0; j < vectors.length; j++) { - if (i === j) continue; - totalDist += 1 - cosineSimilarity(vectors[i], vectors[j]); - } - const avgDist = totalDist / (vectors.length - 1); - if (avgDist < bestAvgDist) { - bestAvgDist = avgDist; - bestIndex = i; - } - } - return bestIndex; -} - -/** - * Compute the element-wise mean (centroid) of a set of equal-length vectors. - * Returns a new Float32Array of the same dimensionality. - */ -export function computeCentroid(vectors: Float32Array[]): Float32Array { - if (vectors.length === 0) return new Float32Array(0); - const dim = vectors[0].length; - const centroid = new Float32Array(dim); - for (const v of vectors) { - for (let i = 0; i < dim; i++) { - centroid[i] += v[i]; - } - } - const n = vectors.length; - for (let i = 0; i < dim; i++) { - centroid[i] /= n; - } - return centroid; -} - -// --------------------------------------------------------------------------- -// Options -// --------------------------------------------------------------------------- - -export interface PrototypeRecomputerOptions { - metadataStore: MetadataStore; - vectorStore: VectorStore; - policy?: HotpathPolicy; - /** Current timestamp (ms since epoch). Defaults to Date.now(). */ - now?: number; -} - -export interface RecomputeResult { - volumesUpdated: number; - shelvesUpdated: number; -} - -// --------------------------------------------------------------------------- -// Recompute volume prototypes -// --------------------------------------------------------------------------- - -/** - * Recompute centroid prototypes for all volumes. - * - * For each volume: - * 1. Load all page embeddings for every book in the volume. - * 2. Compute the centroid embedding across all pages. - * 3. Append updated centroid vector to VectorStore; update volume metadata. - * - * Note: Medoid selection and salience/promotion sweeps are intentionally - * omitted here. SalienceEngine methods currently assume page-tier entities; - * running them with volume IDs would produce incorrect tier assignments. - * Volume-tier salience should be wired up once SalienceEngine supports - * non-page tiers. - */ -async function recomputeVolumePrototypes( - options: PrototypeRecomputerOptions, -): Promise<{ volumeIds: Hash[]; volumesUpdated: number }> { - const { - metadataStore, - vectorStore, - } = options; - - const allVolumes = await metadataStore.getAllVolumes(); - const updatedVolumeIds: Hash[] = []; - - for (const volume of allVolumes) { - // Load all pages in this volume - const pageEntries: Array<{ pageId: Hash; vector: Float32Array }> = []; - - for (const bookId of volume.bookIds) { - const book = await metadataStore.getBook(bookId); - if (!book) continue; - for (const pageId of book.pageIds) { - const page = await metadataStore.getPage(pageId); - if (!page) continue; - const vec = await vectorStore.readVector(page.embeddingOffset, page.embeddingDim); - pageEntries.push({ pageId, vector: vec }); - } - } - - if (pageEntries.length === 0) continue; - - const vectors = pageEntries.map((e) => e.vector); - const centroidVec = computeCentroid(vectors); - - // Append centroid to vector store - const centroidOffset = await vectorStore.appendVector(centroidVec); - - // Update the volume with new medoid and prototype offsets - const updatedVolume: Volume = { - ...volume, - prototypeOffsets: [...volume.prototypeOffsets, centroidOffset], - prototypeDim: centroidVec.length, - }; - await metadataStore.putVolume(updatedVolume); - - updatedVolumeIds.push(volume.volumeId); - } - - // Note: We intentionally do not call the page-centric SalienceEngine here. - // batchComputeSalience/runPromotionSweep currently assume page-tier entities - // and hardcode `tier: "page"`. Passing volume IDs into those functions would - // compute meaningless salience values and could overwrite volume-tier - // HotpathEntry records with page-tier entries using the same entityId. - // - // Volume-tier salience/promotion should be wired up once SalienceEngine - // supports non-page tiers. For now, we only update the volume metadata and - // return the list of volumes that were recomputed. - - return { volumeIds: updatedVolumeIds, volumesUpdated: updatedVolumeIds.length }; -} - -// --------------------------------------------------------------------------- -// Recompute shelf routing prototypes -// --------------------------------------------------------------------------- - -/** - * Recompute routing prototypes for all shelves. - * - * For each shelf: - * 1. Load volume prototype embeddings. - * 2. Compute centroid across all volume prototypes. - * 3. Append new routing prototype to VectorStore; update shelf metadata. - * 4. Refresh salience and run promotion sweep for the shelf tier. - */ -async function recomputeShelfPrototypes( - options: PrototypeRecomputerOptions, -): Promise<{ shelvesUpdated: number }> { - const { - metadataStore, - vectorStore, - policy = DEFAULT_HOTPATH_POLICY, - now = Date.now(), - } = options; - - const allShelves = await metadataStore.getAllShelves(); - const updatedShelfIds: Hash[] = []; - - for (const shelf of allShelves) { - const volumeVectors: Float32Array[] = []; - - for (const volumeId of shelf.volumeIds) { - const volume = await metadataStore.getVolume(volumeId); - if (!volume || volume.prototypeOffsets.length === 0) continue; - // Use the last (most recent) prototype offset - const offset = volume.prototypeOffsets[volume.prototypeOffsets.length - 1]; - const vec = await vectorStore.readVector(offset, volume.prototypeDim); - volumeVectors.push(vec); - } - - if (volumeVectors.length === 0) continue; - - const routingPrototype = computeCentroid(volumeVectors); - const routingOffset = await vectorStore.appendVector(routingPrototype); - - const updatedShelf: Shelf = { - ...shelf, - routingPrototypeOffsets: [...shelf.routingPrototypeOffsets, routingOffset], - routingDim: routingPrototype.length, - }; - await metadataStore.putShelf(updatedShelf); - updatedShelfIds.push(shelf.shelfId); - } - - if (updatedShelfIds.length > 0) { - // Shelf-tier hotpath uses shelf IDs as entity IDs - const shelfEntries: HotpathEntry[] = updatedShelfIds.map((id) => ({ - entityId: id, - tier: "shelf" as const, - salience: 0, - communityId: undefined, - })); - for (const entry of shelfEntries) { - await metadataStore.putHotpathEntry(entry); - } - await runPromotionSweep(updatedShelfIds, metadataStore, policy, now); - } - - return { shelvesUpdated: updatedShelfIds.length }; -} - -// --------------------------------------------------------------------------- -// Public entry point -// --------------------------------------------------------------------------- - -/** - * Recompute prototypes at all hierarchy levels (volume then shelf). - * - * Volumes are processed first so shelves can reference updated volume prototypes. - */ -export async function recomputePrototypes( - options: PrototypeRecomputerOptions, -): Promise { - const { volumesUpdated } = await recomputeVolumePrototypes(options); - const { shelvesUpdated } = await recomputeShelfPrototypes(options); - - return { volumesUpdated, shelvesUpdated }; -} +export * from "../lib/daydreamer/PrototypeRecomputer"; diff --git a/docs/development.md b/docs/development.md index a08372e..fc74e5b 100644 --- a/docs/development.md +++ b/docs/development.md @@ -99,7 +99,7 @@ The harness is a thin browser page that detects runtime capabilities (WebGPU, We ```sh bun run dev:harness -# → serving runtime/harness on http://127.0.0.1:4173 +# → serving ui/harness on http://127.0.0.1:4173 ``` Open the URL in Chrome/Edge to see the capability report in the browser console. diff --git a/embeddings/DeterministicDummyEmbeddingBackend.ts b/embeddings/DeterministicDummyEmbeddingBackend.ts index a98bb43..4acf1ca 100644 --- a/embeddings/DeterministicDummyEmbeddingBackend.ts +++ b/embeddings/DeterministicDummyEmbeddingBackend.ts @@ -1,108 +1 @@ -import type { EmbeddingBackend } from "./EmbeddingBackend"; - -export const DEFAULT_DUMMY_EMBEDDING_DIMENSION = 1024; -export const SHA256_BLOCK_BYTES = 64; - -const SHA256_DIGEST_BYTES = 32; -const COUNTER_BYTES = 4; -const BYTE_TO_UNIT_SCALE = 127.5; - -export interface DeterministicDummyEmbeddingBackendOptions { - dimension?: number; - blockBytes?: number; -} - -function assertPositiveInteger(name: string, value: number): void { - if (!Number.isInteger(value) || value <= 0) { - throw new Error(`${name} must be a positive integer`); - } -} - -function getSubtleCrypto(): SubtleCrypto { - const subtle = globalThis.crypto?.subtle; - if (!subtle) { - throw new Error("SubtleCrypto is required for DeterministicDummyEmbeddingBackend"); - } - return subtle; -} - -function padBytesToBoundary(input: Uint8Array, blockBytes: number): Uint8Array { - const remainder = input.byteLength % blockBytes; - if (remainder === 0) { - return input; - } - - const padLength = blockBytes - remainder; - const padded = new Uint8Array(input.byteLength + padLength); - padded.set(input); - return padded; -} - -function byteToUnitFloat(byteValue: number): number { - return byteValue / BYTE_TO_UNIT_SCALE - 1; -} - -export class DeterministicDummyEmbeddingBackend implements EmbeddingBackend { - readonly kind = "dummy-sha256" as const; - readonly dimension: number; - - private readonly blockBytes: number; - private readonly subtle = getSubtleCrypto(); - private readonly encoder = new TextEncoder(); - - constructor(options: DeterministicDummyEmbeddingBackendOptions = {}) { - this.dimension = options.dimension ?? DEFAULT_DUMMY_EMBEDDING_DIMENSION; - this.blockBytes = options.blockBytes ?? SHA256_BLOCK_BYTES; - - assertPositiveInteger("dimension", this.dimension); - assertPositiveInteger("blockBytes", this.blockBytes); - } - - async embed(texts: string[]): Promise { - return Promise.all(texts.map((text) => this.embedOne(text))); - } - - private async embedOne(text: string): Promise { - const sourceBytes = padBytesToBoundary( - this.encoder.encode(text), - this.blockBytes, - ); - - const embedding = new Float32Array(this.dimension); - let counter = 0; - let writeIndex = 0; - - while (writeIndex < this.dimension) { - const digest = await this.digestWithCounter(sourceBytes, counter); - for ( - let digestIndex = 0; - digestIndex < SHA256_DIGEST_BYTES && writeIndex < this.dimension; - digestIndex++ - ) { - embedding[writeIndex] = byteToUnitFloat(digest[digestIndex]); - writeIndex++; - } - counter++; - } - - return embedding; - } - - private async digestWithCounter( - sourceBytes: Uint8Array, - counter: number, - ): Promise { - const payload = new Uint8Array(sourceBytes.byteLength + COUNTER_BYTES); - payload.set(sourceBytes, 0); - - const counterView = new DataView( - payload.buffer, - payload.byteOffset + sourceBytes.byteLength, - COUNTER_BYTES, - ); - counterView.setUint32(0, counter, false); - - const digest = await this.subtle.digest("SHA-256", payload); - return new Uint8Array(digest); - } -} +export * from "../lib/embeddings/DeterministicDummyEmbeddingBackend"; diff --git a/embeddings/EmbeddingBackend.ts b/embeddings/EmbeddingBackend.ts index 72bff31..836600d 100644 --- a/embeddings/EmbeddingBackend.ts +++ b/embeddings/EmbeddingBackend.ts @@ -1,6 +1 @@ -export interface EmbeddingBackend { - readonly kind: string; - readonly dimension: number; - - embed(texts: string[]): Promise; -} +export * from "../lib/embeddings/EmbeddingBackend"; diff --git a/embeddings/EmbeddingRunner.ts b/embeddings/EmbeddingRunner.ts index ea29295..4f095bd 100644 --- a/embeddings/EmbeddingRunner.ts +++ b/embeddings/EmbeddingRunner.ts @@ -1,54 +1 @@ -import type { EmbeddingBackend } from "./EmbeddingBackend"; -import { - type ResolveEmbeddingBackendOptions, - type ResolvedEmbeddingBackend, - resolveEmbeddingBackend, -} from "./ProviderResolver"; - -export type ResolveEmbeddingSelection = () => Promise; - -export class EmbeddingRunner { - private selectionPromise: Promise | undefined; - private resolvedSelection: ResolvedEmbeddingBackend | undefined; - - constructor(private readonly resolveSelection: ResolveEmbeddingSelection) {} - - static fromResolverOptions( - options: ResolveEmbeddingBackendOptions, - ): EmbeddingRunner { - return new EmbeddingRunner(() => resolveEmbeddingBackend(options)); - } - - get selectedKind(): string | undefined { - return this.resolvedSelection?.selectedKind; - } - - async getSelection(): Promise { - return this.ensureSelection(); - } - - async getBackend(): Promise { - const selection = await this.ensureSelection(); - return selection.backend; - } - - async embed(texts: string[]): Promise { - const backend = await this.getBackend(); - return backend.embed(texts); - } - - private async ensureSelection(): Promise { - if (this.resolvedSelection) { - return this.resolvedSelection; - } - - if (!this.selectionPromise) { - this.selectionPromise = this.resolveSelection().then((selection) => { - this.resolvedSelection = selection; - return selection; - }); - } - - return this.selectionPromise; - } -} +export * from "../lib/embeddings/EmbeddingRunner"; diff --git a/embeddings/OrtWebglEmbeddingBackend.ts b/embeddings/OrtWebglEmbeddingBackend.ts index 7b853ca..cdc5de0 100644 --- a/embeddings/OrtWebglEmbeddingBackend.ts +++ b/embeddings/OrtWebglEmbeddingBackend.ts @@ -1,129 +1 @@ -import type { FeatureExtractionPipeline } from "@huggingface/transformers"; - -import type { EmbeddingBackend } from "./EmbeddingBackend"; -import { - EMBEDDING_GEMMA_300M_DOCUMENT_PREFIX, - EMBEDDING_GEMMA_300M_EMBEDDING_DIMENSION, - EMBEDDING_GEMMA_300M_MODEL_ID, - EMBEDDING_GEMMA_300M_QUERY_PREFIX, -} from "./TransformersJsEmbeddingBackend"; - -export interface OrtWebglEmbeddingBackendOptions { - /** - * Hugging Face model ID to load. Must be a matryoshka-compatible embedding model. - * Defaults to `EMBEDDING_GEMMA_300M_MODEL_ID`. - */ - modelId?: string; - - /** - * Number of embedding dimensions to return. - * Defaults to `EMBEDDING_GEMMA_300M_EMBEDDING_DIMENSION`. - */ - dimension?: number; - - /** - * Prefix prepended to each text when embedding documents/passages. - * Defaults to `EMBEDDING_GEMMA_300M_DOCUMENT_PREFIX`. - */ - documentPrefix?: string; - - /** - * Prefix prepended to each text when embedding search queries. - * Defaults to `EMBEDDING_GEMMA_300M_QUERY_PREFIX`. - */ - queryPrefix?: string; -} - -/** - * Embedding backend that uses ONNX Runtime Web's explicit WebGL execution - * provider via Hugging Face Transformers.js. - * - * This backend targets systems that have WebGL but lack WebGPU or WebNN, - * providing a hardware-accelerated fallback below the WebGPU/WebNN tier. - * - * The pipeline is loaded lazily on the first `embed()` or `embedQueries()` - * call so that import cost is zero until the backend is actually needed. - */ -export class OrtWebglEmbeddingBackend implements EmbeddingBackend { - readonly kind = "webgl" as const; - readonly dimension: number; - readonly modelId: string; - readonly documentPrefix: string; - readonly queryPrefix: string; - - private pipelinePromise: Promise | undefined; - - constructor(options: OrtWebglEmbeddingBackendOptions = {}) { - this.modelId = options.modelId ?? EMBEDDING_GEMMA_300M_MODEL_ID; - this.dimension = - options.dimension ?? EMBEDDING_GEMMA_300M_EMBEDDING_DIMENSION; - this.documentPrefix = - options.documentPrefix ?? EMBEDDING_GEMMA_300M_DOCUMENT_PREFIX; - this.queryPrefix = - options.queryPrefix ?? EMBEDDING_GEMMA_300M_QUERY_PREFIX; - } - - /** - * Embeds the given texts as document/passage representations. - * Prepends `documentPrefix` before each text as required by the model. - */ - async embed(texts: string[]): Promise { - return this.embedWithPrefix(texts, this.documentPrefix); - } - - /** - * Embeds the given texts as search query representations. - * Prepends `queryPrefix` before each text as required by the model. - * - * Use this method when encoding queries for retrieval; use `embed()` for - * documents/passages being indexed. - */ - async embedQueries(texts: string[]): Promise { - return this.embedWithPrefix(texts, this.queryPrefix); - } - - private async embedWithPrefix( - texts: string[], - prefix: string, - ): Promise { - const extractor = await this.ensurePipeline(); - const prefixed = - prefix.length > 0 ? texts.map((t) => `${prefix}${t}`) : texts; - - const output = await extractor(prefixed, { - pooling: "mean", - normalize: true, - }); - - const rawData = output.data as Float32Array; - const fullDim = rawData.length / texts.length; - const sliceDim = Math.min(this.dimension, fullDim); - - const results: Float32Array[] = []; - for (let i = 0; i < texts.length; i++) { - const start = i * fullDim; - results.push(rawData.slice(start, start + sliceDim)); - } - return results; - } - - private ensurePipeline(): Promise { - if (!this.pipelinePromise) { - this.pipelinePromise = this.loadPipeline(); - } - return this.pipelinePromise; - } - - private async loadPipeline(): Promise { - const { pipeline } = await import("@huggingface/transformers"); - // Cast through unknown to work around the overloaded pipeline union type complexity. - const pipelineFn = pipeline as unknown as ( - task: string, - model: string, - options?: Record, - ) => Promise; - return pipelineFn("feature-extraction", this.modelId, { - device: "webgl", - }); - } -} +export * from "../lib/embeddings/OrtWebglEmbeddingBackend"; diff --git a/embeddings/ProviderResolver.ts b/embeddings/ProviderResolver.ts index 3e7b878..8440e17 100644 --- a/embeddings/ProviderResolver.ts +++ b/embeddings/ProviderResolver.ts @@ -1,348 +1 @@ -import { - DeterministicDummyEmbeddingBackend, - type DeterministicDummyEmbeddingBackendOptions, -} from "./DeterministicDummyEmbeddingBackend"; -import type { EmbeddingBackend } from "./EmbeddingBackend"; -import { - OrtWebglEmbeddingBackend, - type OrtWebglEmbeddingBackendOptions, -} from "./OrtWebglEmbeddingBackend"; -import { - TransformersJsEmbeddingBackend, - type TransformersJsDevice, - type TransformersJsEmbeddingBackendOptions, -} from "./TransformersJsEmbeddingBackend"; - -export type EmbeddingProviderKind = - | "webnn" - | "webgpu" - | "webgl" - | "wasm" - | "dummy" - | (string & {}); - -export interface EmbeddingProviderCandidate { - kind: EmbeddingProviderKind; - isSupported: () => boolean | Promise; - createBackend: () => EmbeddingBackend | Promise; -} - -export interface EmbeddingProviderBenchmarkPolicy { - enabled: boolean; - warmupRuns: number; - timedRuns: number; - sampleTexts: string[]; -} - -export interface EmbeddingProviderMeasurement { - kind: EmbeddingProviderKind; - meanMs: number; -} - -export type EmbeddingProviderResolveReason = - | "forced" - | "benchmark" - | "capability-order"; - -export interface ResolvedEmbeddingBackend { - backend: EmbeddingBackend; - selectedKind: EmbeddingProviderKind; - reason: EmbeddingProviderResolveReason; - supportedKinds: EmbeddingProviderKind[]; - measurements: EmbeddingProviderMeasurement[]; -} - -export const DEFAULT_PROVIDER_ORDER: ReadonlyArray = - Object.freeze([ - "webnn", - "webgpu", - "webgl", - "wasm", - "dummy", - ]); - -export const DEFAULT_PROVIDER_BENCHMARK_POLICY: EmbeddingProviderBenchmarkPolicy = - Object.freeze({ - enabled: true, - warmupRuns: 1, - timedRuns: 3, - sampleTexts: [ - "cortex benchmark probe", - "routing and coherence warmup", - "deterministic provider timing", - ], - }); - -export type BenchmarkBackendFn = ( - backend: EmbeddingBackend, - policy: EmbeddingProviderBenchmarkPolicy, -) => Promise; - -export interface ResolveEmbeddingBackendOptions { - candidates: EmbeddingProviderCandidate[]; - preferredOrder?: ReadonlyArray; - forceKind?: EmbeddingProviderKind; - benchmark?: Partial; - benchmarkBackend?: BenchmarkBackendFn; -} - -function assertPositiveInteger(name: string, value: number): void { - if (!Number.isInteger(value) || value <= 0) { - throw new Error(`${name} must be a positive integer`); - } -} - -function validateBenchmarkPolicy( - policy: EmbeddingProviderBenchmarkPolicy, -): EmbeddingProviderBenchmarkPolicy { - assertPositiveInteger("warmupRuns", policy.warmupRuns); - assertPositiveInteger("timedRuns", policy.timedRuns); - - if (policy.sampleTexts.length === 0) { - throw new Error("sampleTexts must not be empty"); - } - - return policy; -} - -function nowMs(): number { - const perfNow = globalThis.performance?.now?.bind(globalThis.performance); - if (perfNow) { - return perfNow(); - } - return Date.now(); -} - -async function defaultBenchmarkBackend( - backend: EmbeddingBackend, - policy: EmbeddingProviderBenchmarkPolicy, -): Promise { - for (let i = 0; i < policy.warmupRuns; i++) { - await backend.embed(policy.sampleTexts); - } - - let totalMs = 0; - for (let i = 0; i < policy.timedRuns; i++) { - const start = nowMs(); - await backend.embed(policy.sampleTexts); - totalMs += nowMs() - start; - } - - return totalMs / policy.timedRuns; -} - -function orderCandidates( - candidates: EmbeddingProviderCandidate[], - preferredOrder: ReadonlyArray, -): EmbeddingProviderCandidate[] { - const orderIndex = new Map(); - for (let i = 0; i < preferredOrder.length; i++) { - orderIndex.set(preferredOrder[i], i); - } - - return [...candidates].sort((a, b) => { - const aIndex = orderIndex.get(a.kind) ?? Number.MAX_SAFE_INTEGER; - const bIndex = orderIndex.get(b.kind) ?? Number.MAX_SAFE_INTEGER; - return aIndex - bIndex; - }); -} - -export async function resolveEmbeddingBackend( - options: ResolveEmbeddingBackendOptions, -): Promise { - const preferredOrder = options.preferredOrder ?? DEFAULT_PROVIDER_ORDER; - const benchmarkPolicy = validateBenchmarkPolicy({ - ...DEFAULT_PROVIDER_BENCHMARK_POLICY, - ...options.benchmark, - }); - - const capabilityChecks = await Promise.all( - options.candidates.map(async (candidate) => ({ - candidate, - supported: await candidate.isSupported(), - })), - ); - - if (options.forceKind !== undefined) { - const forcedEntry = capabilityChecks.find( - (entry) => entry.candidate.kind === options.forceKind, - ); - - if (!forcedEntry || !forcedEntry.supported) { - throw new Error(`Forced provider ${options.forceKind} is not supported`); - } - - return { - backend: await forcedEntry.candidate.createBackend(), - selectedKind: forcedEntry.candidate.kind, - reason: "forced", - supportedKinds: orderCandidates( - capabilityChecks - .filter((entry) => entry.supported) - .map((entry) => entry.candidate), - preferredOrder, - ).map((candidate) => candidate.kind), - measurements: [], - }; - } - - const supportedCandidates = orderCandidates( - capabilityChecks - .filter((entry) => entry.supported) - .map((entry) => entry.candidate), - preferredOrder, - ); - - if (supportedCandidates.length === 0) { - throw new Error("No supported embedding providers are available"); - } - - const supportedKinds = supportedCandidates.map((candidate) => candidate.kind); - - const benchmarkBackend = options.benchmarkBackend ?? defaultBenchmarkBackend; - if (benchmarkPolicy.enabled) { - const measurements: { - candidate: EmbeddingProviderCandidate; - backend: EmbeddingBackend; - meanMs: number; - }[] = []; - - for (const candidate of supportedCandidates) { - const backend = await candidate.createBackend(); - const meanMs = await benchmarkBackend(backend, benchmarkPolicy); - measurements.push({ candidate, backend, meanMs }); - } - - let winner = measurements[0]; - for (let i = 1; i < measurements.length; i++) { - if (measurements[i].meanMs < winner.meanMs) { - winner = measurements[i]; - } - } - - return { - backend: winner.backend, - selectedKind: winner.candidate.kind, - reason: "benchmark", - supportedKinds, - measurements: measurements.map((m) => ({ - kind: m.candidate.kind, - meanMs: m.meanMs, - })), - }; - } - - const selectedCandidate = supportedCandidates[0]; - return { - backend: await selectedCandidate.createBackend(), - selectedKind: selectedCandidate.kind, - reason: "capability-order", - supportedKinds, - measurements: [], - }; -} - -export function createDummyProviderCandidate( - options: DeterministicDummyEmbeddingBackendOptions = {}, -): EmbeddingProviderCandidate { - return { - kind: "dummy", - isSupported: () => globalThis.crypto?.subtle !== undefined, - createBackend: () => new DeterministicDummyEmbeddingBackend(options), - }; -} - -/** - * Checks whether a given Transformers.js ONNX device is available in the - * current runtime environment. - * - * - `"wasm"` is always considered supported (lowest common denominator). - * - `"webgpu"` requires `navigator.gpu` to be present. - * - `"webnn"` requires `navigator.ml` to be present. - */ -function isTransformersJsDeviceSupported( - device: TransformersJsDevice, -): boolean { - switch (device) { - case "webnn": - return ( - typeof globalThis.navigator !== "undefined" && - "ml" in globalThis.navigator - ); - case "webgpu": - return ( - typeof globalThis.navigator !== "undefined" && - "gpu" in globalThis.navigator - ); - case "wasm": - return true; - } -} - -/** - * Returns an `EmbeddingProviderCandidate` array for each Transformers.js - * ONNX device (`"webnn"`, `"webgpu"`, `"wasm"`), ordered from fastest to most - * widely available. - * - * Each candidate: - * - Exposes a `kind` matching the underlying device (e.g. `"webgpu"`). - * - Runs its `isSupported()` check at resolution time (no eager pipeline load). - * - Creates a `TransformersJsEmbeddingBackend` with the shared `options` plus - * the candidate-specific `device`. - * - * Pass these candidates to `resolveEmbeddingBackend` or - * `EmbeddingRunner.fromResolverOptions` to select the best available device - * at runtime. - * - * @example - * ```ts - * const runner = EmbeddingRunner.fromResolverOptions({ - * candidates: [ - * ...createTransformersJsProviderCandidates(), - * createDummyProviderCandidate(), - * ], - * }); - * ``` - */ -export function createTransformersJsProviderCandidates( - options: TransformersJsEmbeddingBackendOptions = {}, -): EmbeddingProviderCandidate[] { - const devices: TransformersJsDevice[] = ["webnn", "webgpu", "wasm"]; - - return devices.map((device) => ({ - kind: device, - isSupported: () => isTransformersJsDeviceSupported(device), - createBackend: () => - new TransformersJsEmbeddingBackend({ ...options, device }), - })); -} - -/** - * Returns an `EmbeddingProviderCandidate` for the WebGL ONNX execution - * provider via `OrtWebglEmbeddingBackend`. - * - * This candidate is supported when `WebGL2RenderingContext` is available in - * the global scope, providing a hardware-accelerated fallback for systems - * that have WebGL but lack WebGPU or WebNN. - * - * @example - * ```ts - * const runner = EmbeddingRunner.fromResolverOptions({ - * candidates: [ - * ...createTransformersJsProviderCandidates(), - * createWebglProviderCandidate(), - * createDummyProviderCandidate(), - * ], - * }); - * ``` - */ -export function createWebglProviderCandidate( - options: OrtWebglEmbeddingBackendOptions = {}, -): EmbeddingProviderCandidate { - return { - kind: "webgl", - isSupported: () => - typeof globalThis.WebGL2RenderingContext !== "undefined", - createBackend: () => new OrtWebglEmbeddingBackend(options), - }; -} +export * from "../lib/embeddings/ProviderResolver"; diff --git a/embeddings/TransformersJsEmbeddingBackend.ts b/embeddings/TransformersJsEmbeddingBackend.ts index 5ef7b23..a9ee15a 100644 --- a/embeddings/TransformersJsEmbeddingBackend.ts +++ b/embeddings/TransformersJsEmbeddingBackend.ts @@ -1,161 +1 @@ -import type { FeatureExtractionPipeline } from "@huggingface/transformers"; - -import type { EmbeddingBackend } from "./EmbeddingBackend"; - -export type TransformersJsDevice = "webnn" | "webgpu" | "wasm"; - -export interface TransformersJsEmbeddingBackendOptions { - /** - * Hugging Face model ID to load. Must be a matryoshka-compatible embedding model. - * Defaults to `EMBEDDING_GEMMA_300M_MODEL_ID`. - */ - modelId?: string; - - /** - * ONNX runtime device to use for inference. - * Defaults to `"wasm"` (always available, lowest common denominator). - */ - device?: TransformersJsDevice; - - /** - * Number of embedding dimensions to return. For matryoshka models, this may be - * any supported sub-dimension (smaller values are valid nested sub-spaces). - * Defaults to `EMBEDDING_GEMMA_300M_EMBEDDING_DIMENSION`. - */ - dimension?: number; - - /** - * Prefix prepended to each text when embedding documents/passages. - * Required by some models (e.g. EmbeddingGemma) for best retrieval quality. - * Defaults to `EMBEDDING_GEMMA_300M_DOCUMENT_PREFIX`. - */ - documentPrefix?: string; - - /** - * Prefix prepended to each text when embedding search queries. - * Required by some models (e.g. EmbeddingGemma) for best retrieval quality. - * Defaults to `EMBEDDING_GEMMA_300M_QUERY_PREFIX`. - */ - queryPrefix?: string; -} - -/** - * Default model used when no `modelId` is provided. - * Q4-quantized ONNX variant of google/embeddinggemma-300m. - */ -export const EMBEDDING_GEMMA_300M_MODEL_ID = - "onnx-community/embeddinggemma-300m-ONNX"; - -/** - * Default embedding dimension used when no `dimension` is provided. - * 768 is the full-fidelity matryoshka output dimension for EmbeddingGemma-300M. - */ -export const EMBEDDING_GEMMA_300M_EMBEDDING_DIMENSION = 768; - -/** - * Query task prefix for EmbeddingGemma-300M, as specified on the model card. - * @see https://huggingface.co/google/embeddinggemma-300m - */ -export const EMBEDDING_GEMMA_300M_QUERY_PREFIX = "query: "; - -/** - * Document/passage prefix for EmbeddingGemma-300M, as specified on the model card. - * @see https://huggingface.co/google/embeddinggemma-300m - */ -export const EMBEDDING_GEMMA_300M_DOCUMENT_PREFIX = "passage: "; - -/** - * Real embedding backend backed by `@huggingface/transformers`. - * - * Supports WebNN, WebGPU, and WASM ONNX runtime devices. The default model is - * the Q4-quantized EmbeddingGemma-300M (matryoshka). Any matryoshka-compatible - * model on the Hugging Face Hub can be substituted via `options.modelId`. - * - * The pipeline is loaded lazily on the first `embed()` or `embedQueries()` call - * so that import cost is zero until the backend is actually needed. - */ -export class TransformersJsEmbeddingBackend implements EmbeddingBackend { - readonly kind: string; - readonly dimension: number; - readonly modelId: string; - readonly device: TransformersJsDevice; - readonly documentPrefix: string; - readonly queryPrefix: string; - - private pipelinePromise: Promise | undefined; - - constructor(options: TransformersJsEmbeddingBackendOptions = {}) { - this.device = options.device ?? "wasm"; - this.modelId = options.modelId ?? EMBEDDING_GEMMA_300M_MODEL_ID; - this.dimension = - options.dimension ?? EMBEDDING_GEMMA_300M_EMBEDDING_DIMENSION; - this.documentPrefix = - options.documentPrefix ?? EMBEDDING_GEMMA_300M_DOCUMENT_PREFIX; - this.queryPrefix = options.queryPrefix ?? EMBEDDING_GEMMA_300M_QUERY_PREFIX; - this.kind = `transformers-js:${this.device}`; - } - - /** - * Embeds the given texts as document/passage representations. - * Prepends `documentPrefix` before each text as required by the model. - */ - async embed(texts: string[]): Promise { - return this.embedWithPrefix(texts, this.documentPrefix); - } - - /** - * Embeds the given texts as search query representations. - * Prepends `queryPrefix` before each text as required by the model. - * - * Use this method when encoding queries for retrieval; use `embed()` for - * documents/passages being indexed. - */ - async embedQueries(texts: string[]): Promise { - return this.embedWithPrefix(texts, this.queryPrefix); - } - - private async embedWithPrefix( - texts: string[], - prefix: string, - ): Promise { - const extractor = await this.ensurePipeline(); - const prefixed = - prefix.length > 0 ? texts.map((t) => `${prefix}${t}`) : texts; - - const output = await extractor(prefixed, { - pooling: "mean", - normalize: true, - }); - - const rawData = output.data as Float32Array; - const fullDim = rawData.length / texts.length; - const sliceDim = Math.min(this.dimension, fullDim); - - const results: Float32Array[] = []; - for (let i = 0; i < texts.length; i++) { - const start = i * fullDim; - results.push(rawData.slice(start, start + sliceDim)); - } - return results; - } - - private ensurePipeline(): Promise { - if (!this.pipelinePromise) { - this.pipelinePromise = this.loadPipeline(); - } - return this.pipelinePromise; - } - - private async loadPipeline(): Promise { - const { pipeline } = await import("@huggingface/transformers"); - // Cast through unknown to work around the overloaded pipeline union type complexity. - const pipelineFn = pipeline as unknown as ( - task: string, - model: string, - options?: Record, - ) => Promise; - return pipelineFn("feature-extraction", this.modelId, { - device: this.device, - }); - } -} +export * from "../lib/embeddings/TransformersJsEmbeddingBackend"; diff --git a/hippocampus/Chunker.ts b/hippocampus/Chunker.ts index f61b33a..147bd66 100644 --- a/hippocampus/Chunker.ts +++ b/hippocampus/Chunker.ts @@ -1,79 +1 @@ -import type { ModelProfile } from "../core/ModelProfile"; - -/** - * Splits input text into page-sized chunks based on a token budget. - * - * This is a lightweight, whitespace-token-based chunker (no external tokenizer). - * It prefers to keep sentence boundaries when possible but will split overlong - * sentences at token boundaries to respect the budget. - */ -export function chunkText(text: string, profile: ModelProfile): string[] { - return chunkTextWithMaxTokens(text, profile.maxChunkTokens); -} - -export function chunkTextWithMaxTokens( - text: string, - maxChunkTokens: number, -): string[] { - if (!Number.isInteger(maxChunkTokens) || maxChunkTokens <= 0) { // model-derived-ok - throw new Error("maxChunkTokens must be a positive integer"); - } - - const normalized = text.replace(/\s+/g, " ").trim(); - if (normalized.length === 0) { - return []; - } - - // Simple sentence boundary heuristic: split after `.`, `?`, or `!` followed by whitespace. - // This is intentionally lightweight and avoids pulling in a full NLP dependency. - const sentences = normalized - .split(/(?<=[.!?])\s+/g) - .map((s) => s.trim()) - .filter(Boolean); - - const tokenize = (s: string): string[] => { - return s.trim().split(/\s+/).filter(Boolean); - }; - - const chunks: string[] = []; - let currentTokens: string[] = []; - - const pushCurrent = () => { - if (currentTokens.length === 0) return; - chunks.push(currentTokens.join(" ")); - currentTokens = []; - }; - - const appendSentence = (sentence: string) => { - const sentenceTokens = tokenize(sentence); - if (sentenceTokens.length === 0) return; - - // Sentence is larger than budget: split it across multiple chunks. - if (sentenceTokens.length > maxChunkTokens) { - pushCurrent(); - // model-derived-ok: uses maxChunkTokens as derived from ModelProfile - for (let i = 0; i < sentenceTokens.length; i += maxChunkTokens) { // model-derived-ok - const slice = sentenceTokens.slice(i, i + maxChunkTokens); - chunks.push(slice.join(" ")); - } - return; - } - - // Try to keep sentence with current chunk. - if (currentTokens.length + sentenceTokens.length <= maxChunkTokens) { - currentTokens.push(...sentenceTokens); - return; - } - - // Otherwise, start a new chunk. - pushCurrent(); - currentTokens.push(...sentenceTokens); - }; - - for (const sentence of sentences) { - appendSentence(sentence); - } - - pushCurrent(); - return chunks; -} +export * from "../lib/hippocampus/Chunker"; diff --git a/hippocampus/FastNeighborInsert.ts b/hippocampus/FastNeighborInsert.ts index 5c27460..410a2a8 100644 --- a/hippocampus/FastNeighborInsert.ts +++ b/hippocampus/FastNeighborInsert.ts @@ -1,222 +1 @@ -import type { Hash, MetadataStore, SemanticNeighbor, VectorStore } from "../core/types"; -import type { ModelProfile } from "../core/ModelProfile"; -import type { HotpathPolicy } from "../core/HotpathPolicy"; -import { computeNeighborMaxDegree, DEFAULT_HOTPATH_POLICY } from "../core/HotpathPolicy"; -import { runPromotionSweep } from "../core/SalienceEngine"; - -// Absolute upper cap for the semantic neighbor degree. The Williams formula -// can produce larger values for very large corpora; this hard cap keeps the -// neighbor graph manageable even at scale. -const NEIGHBOR_DEGREE_HARD_CAP = 32; - -// Default cosine-distance cutoff when no policy hint is available. -// Cosine distance 0.5 ≡ cosine similarity 0.5 (≥ 0.5 similarity passes). -const DEFAULT_CUTOFF_DISTANCE = 0.5; - -export interface FastNeighborInsertOptions { - modelProfile: ModelProfile; - vectorStore: VectorStore; - metadataStore: MetadataStore; - policy?: HotpathPolicy; - maxDegree?: number; - cutoffDistance?: number; -} - -function cosineSimilarity(a: Float32Array, b: Float32Array): number { - let dot = 0; - let magA = 0; - let magB = 0; - for (let i = 0; i < a.length; i++) { - dot += a[i] * b[i]; - magA += a[i] * a[i]; - magB += b[i] * b[i]; - } - const denom = Math.sqrt(magA) * Math.sqrt(magB); - if (denom === 0) return 0; - return dot / denom; -} - -/** - * Merge a new candidate into an existing neighbor list, respecting maxDegree. - * If at capacity, evict the entry with the lowest cosineSimilarity to make room. - * Returns the updated list sorted by cosineSimilarity descending. - */ -function mergeNeighbor( - existing: SemanticNeighbor[], - candidate: SemanticNeighbor, - maxDegree: number, -): SemanticNeighbor[] { - // Avoid duplicates. - const deduped = existing.filter((n) => n.neighborPageId !== candidate.neighborPageId); - - if (deduped.length < maxDegree) { - deduped.push(candidate); - } else { - // Find weakest existing neighbor. - let weakestIdx = 0; - for (let i = 1; i < deduped.length; i++) { - if (deduped[i].cosineSimilarity < deduped[weakestIdx].cosineSimilarity) { - weakestIdx = i; - } - } - if (candidate.cosineSimilarity > deduped[weakestIdx].cosineSimilarity) { - deduped[weakestIdx] = candidate; - } - // If candidate is weaker than all existing, discard it (return unchanged). - } - - deduped.sort((a, b) => b.cosineSimilarity - a.cosineSimilarity); - return deduped; -} - -/** - * Build and persist semantic neighbor edges for `newPageIds`. - * - * Forward edges (newPage → neighbor) and reverse edges (neighbor → newPage) - * are both stored. This is NOT Hebbian — no edges_hebbian records are created. - */ -export async function insertSemanticNeighbors( - newPageIds: Hash[], - allPageIds: Hash[], - options: FastNeighborInsertOptions, -): Promise { - const { - modelProfile, - vectorStore, - metadataStore, - policy, - cutoffDistance = DEFAULT_CUTOFF_DISTANCE, - } = options; - - // Derive maxDegree from the Williams bound, scaled by corpus size. - // When a policy is provided, use its scaling constant; otherwise fall back - // to the default constant so that corpus-proportional scaling applies in - // all cases (not just when a policy object is explicitly threaded through). - // An explicit options.maxDegree always wins. - let maxDegree: number; - if (options.maxDegree !== undefined) { - maxDegree = options.maxDegree; - } else { - const c = policy?.c ?? DEFAULT_HOTPATH_POLICY.c; - maxDegree = computeNeighborMaxDegree(allPageIds.length, c, NEIGHBOR_DEGREE_HARD_CAP); - } - - if (newPageIds.length === 0) return; - - const dim = modelProfile.embeddingDimension; - - // Fetch all page records in batch for their embedding offsets. - const allPageRecords = await Promise.all( - allPageIds.map((id) => metadataStore.getPage(id)), - ); - - const offsetMap = new Map(); - for (let i = 0; i < allPageIds.length; i++) { - const p = allPageRecords[i]; - if (p) offsetMap.set(allPageIds[i], p.embeddingOffset); - } - - // (a) Throw if any newPageId is missing from the store — a missing new page - // is always a programming error (it should have been persisted before calling - // insertSemanticNeighbors) and would silently corrupt the graph. - for (const newId of newPageIds) { - if (!offsetMap.has(newId)) { - throw new Error( - `Page ${newId} not found in metadata store; persist it before inserting semantic neighbors`, - ); - } - } - - // (b) Filter allPageIds to only those that are present in the store. - // Missing entries are silently dropped — they may have been deleted between - // the getAllPages() call and this point. The vector/id arrays stay aligned. - const resolvedPageIds: Hash[] = []; - const resolvedOffsets: number[] = []; - for (const id of allPageIds) { - const offset = offsetMap.get(id); - if (offset !== undefined) { - resolvedPageIds.push(id); - resolvedOffsets.push(offset); - } - } - - const allVectors = await vectorStore.readVectors(resolvedOffsets, dim); - const vectorMap = new Map(); - for (let i = 0; i < resolvedPageIds.length; i++) { - vectorMap.set(resolvedPageIds[i], allVectors[i]); - } - - // Collect all (pageId, neighborPageId) pairs that need their stored neighbor - // lists updated, keyed by pageId. - const pendingUpdates = new Map(); - - const getOrLoadNeighbors = async (pageId: Hash): Promise => { - if (pendingUpdates.has(pageId)) return pendingUpdates.get(pageId)!; - const stored = await metadataStore.getSemanticNeighbors(pageId); - pendingUpdates.set(pageId, stored); - return stored; - }; - - for (const newId of newPageIds) { - const newVec = vectorMap.get(newId); - if (!newVec) continue; - - // Compute similarity to every other page. - const candidates: SemanticNeighbor[] = []; - for (const otherId of allPageIds) { - if (otherId === newId) continue; - const otherVec = vectorMap.get(otherId); - if (!otherVec) continue; - - const sim = cosineSimilarity(newVec, otherVec); - const dist = 1 - sim; - if (dist <= cutoffDistance) { - candidates.push({ neighborPageId: otherId, cosineSimilarity: sim, distance: dist }); - } - } - - // Sort descending and cap to maxDegree for the forward list. - candidates.sort((a, b) => b.cosineSimilarity - a.cosineSimilarity); - const forwardNeighbors = candidates.slice(0, maxDegree); - - // Merge into the new page's own neighbor list. - let newPageNeighbors = await getOrLoadNeighbors(newId); - for (const candidate of forwardNeighbors) { - newPageNeighbors = mergeNeighbor(newPageNeighbors, candidate, maxDegree); - } - pendingUpdates.set(newId, newPageNeighbors); - - // Insert reverse edges: for each accepted forward neighbor, add newId to - // that neighbor's list. - for (const fwd of forwardNeighbors) { - const reverseCandidate: SemanticNeighbor = { - neighborPageId: newId, - cosineSimilarity: fwd.cosineSimilarity, - distance: fwd.distance, - }; - let neighborList = await getOrLoadNeighbors(fwd.neighborPageId); - neighborList = mergeNeighbor(neighborList, reverseCandidate, maxDegree); - pendingUpdates.set(fwd.neighborPageId, neighborList); - } - } - - // Flush all updated neighbor lists to the store. - await Promise.all( - [...pendingUpdates.entries()].map(([pageId, neighbors]) => - metadataStore.putSemanticNeighbors(pageId, neighbors), - ), - ); - - // Mark affected volumes dirty so the Daydreamer knows to recompute. - for (const newId of newPageIds) { - const books = await metadataStore.getBooksByPage(newId); - for (const book of books) { - const vols = await metadataStore.getVolumesByBook(book.bookId); - for (const vol of vols) { - await metadataStore.flagVolumeForNeighborRecalc(vol.volumeId); - } - } - } - - await runPromotionSweep(newPageIds, metadataStore, policy); -} +export * from "../lib/hippocampus/FastNeighborInsert"; diff --git a/hippocampus/HierarchyBuilder.ts b/hippocampus/HierarchyBuilder.ts index e63aba7..fe7f8fd 100644 --- a/hippocampus/HierarchyBuilder.ts +++ b/hippocampus/HierarchyBuilder.ts @@ -1,404 +1 @@ -import type { Book, Hash, MetadataStore, SemanticNeighbor, Shelf, Volume, VectorStore } from "../core/types"; -import type { ModelProfile } from "../core/ModelProfile"; -import type { HotpathPolicy } from "../core/HotpathPolicy"; -import { computeFanoutLimit, DEFAULT_HOTPATH_POLICY } from "../core/HotpathPolicy"; -import { hashText } from "../core/crypto/hash"; -import { runPromotionSweep } from "../core/SalienceEngine"; - -// Clustering fan-out targets — policy constants, not model-derived. -// These are chosen to be consistent with the Williams Bound fanout limit: -// computeFanoutLimit(N) ≈ ceil(0.5 * sqrt(N * log2(1+N))) -// At typical early-corpus sizes these constants (4-8) sit comfortably within -// the Williams-derived quota. ClusterStability handles splits at runtime if -// volumes grow beyond their quota after the initial hierarchy build. -const PAGES_PER_BOOK = 8; -const BOOKS_PER_VOLUME = 4; -const VOLUMES_PER_SHELF = 4; - -// Max neighbors per page for the adjacency edges added by the hierarchy builder. -// Adjacency edges represent document-order contiguity and bypass the cosine -// cutoff used by FastNeighborInsert, so they must still be bounded by policy. -const ADJACENCY_MAX_DEGREE = 16; - -export interface BuildHierarchyOptions { - modelProfile: ModelProfile; - vectorStore: VectorStore; - metadataStore: MetadataStore; - policy?: HotpathPolicy; -} - -function cosineSimilarity(a: Float32Array, b: Float32Array): number { - let dot = 0; - let magA = 0; - let magB = 0; - for (let i = 0; i < a.length; i++) { - dot += a[i] * b[i]; - magA += a[i] * a[i]; - magB += b[i] * b[i]; - } - const denom = Math.sqrt(magA) * Math.sqrt(magB); - if (denom === 0) return 0; - return dot / denom; -} - -function cosineDistance(a: Float32Array, b: Float32Array): number { - return 1 - cosineSimilarity(a, b); -} - -function computeCentroid(vectors: Float32Array[]): Float32Array { - const dim = vectors[0].length; - const centroid = new Float32Array(dim); - for (const v of vectors) { - for (let i = 0; i < dim; i++) { - centroid[i] += v[i]; - } - } - for (let i = 0; i < dim; i++) { - centroid[i] /= vectors.length; - } - return centroid; -} - -/** Returns the index in `vectors` whose sum of distances to all others is minimal. */ -function selectMedoidIndex(vectors: Float32Array[]): number { - if (vectors.length === 1) return 0; - - let bestIndex = 0; - let bestTotalDistance = Infinity; - - for (let i = 0; i < vectors.length; i++) { - let totalDistance = 0; - for (let j = 0; j < vectors.length; j++) { - if (i !== j) totalDistance += cosineDistance(vectors[i], vectors[j]); - } - if (totalDistance < bestTotalDistance) { - bestTotalDistance = totalDistance; - bestIndex = i; - } - } - - return bestIndex; -} - -function chunkArray(arr: T[], size: number): T[][] { - const chunks: T[][] = []; - for (let i = 0; i < arr.length; i += size) { - chunks.push(arr.slice(i, i + size)); - } - return chunks; -} - -/** - * Merge a candidate into a neighbor list, respecting maxDegree. - * If at capacity, evicts the neighbor with the lowest cosineSimilarity. - * Returns the updated list sorted by cosineSimilarity descending. - */ -function mergeAdjacentNeighbor( - existing: SemanticNeighbor[], - candidate: SemanticNeighbor, - maxDegree: number, -): SemanticNeighbor[] { - const deduped = existing.filter((n) => n.neighborPageId !== candidate.neighborPageId); - - if (deduped.length < maxDegree) { - deduped.push(candidate); - } else { - let weakestIdx = 0; - for (let i = 1; i < deduped.length; i++) { - if (deduped[i].cosineSimilarity < deduped[weakestIdx].cosineSimilarity) { - weakestIdx = i; - } - } - if (candidate.cosineSimilarity > deduped[weakestIdx].cosineSimilarity) { - deduped[weakestIdx] = candidate; - } - } - - deduped.sort((a, b) => b.cosineSimilarity - a.cosineSimilarity); - return deduped; -} - -export async function buildHierarchy( - pageIds: Hash[], - options: BuildHierarchyOptions, -): Promise<{ books: Book[]; volumes: Volume[]; shelves: Shelf[] }> { - const { modelProfile, vectorStore, metadataStore, policy } = options; - const dim = modelProfile.embeddingDimension; - - if (pageIds.length === 0) { - return { books: [], volumes: [], shelves: [] }; - } - - // Fetch all page records to get their embedding offsets. - const pageRecords = await Promise.all(pageIds.map((id) => metadataStore.getPage(id))); - const pageOffsets = pageRecords.map((p, i) => { - if (!p) throw new Error(`Page ${pageIds[i]} not found during hierarchy build`); - return p.embeddingOffset; - }); - const pageVectors = await vectorStore.readVectors(pageOffsets, dim); - - // Build a Map for O(1) lookups throughout the hierarchy build. - const pageVectorMap = new Map(); - for (let i = 0; i < pageIds.length; i++) { - pageVectorMap.set(pageIds[i], pageVectors[i]); - } - - // ------------------------------------------------------------------------- - // Level 1: Pages → Books - // ------------------------------------------------------------------------- - const pageChunks = chunkArray(pageIds, PAGES_PER_BOOK); - const books: Book[] = []; - - for (const chunk of pageChunks) { - const sortedChunk = [...chunk].sort(); - const bookId = await hashText(sortedChunk.join("|")); - - const chunkVectors = chunk.map((id) => { - const vec = pageVectorMap.get(id); - if (!vec) throw new Error(`Vector not found for page ${id}`); - return vec; - }); - - const medoidIdx = selectMedoidIndex(chunkVectors); - const medoidPageId = chunk[medoidIdx]; - - const book: Book = { bookId, pageIds: chunk, medoidPageId, meta: {} }; - await metadataStore.putBook(book); - books.push(book); - } - - // Add SemanticNeighbor edges between consecutive pages within each book slice. - // These document-order adjacency edges are always inserted regardless of cosine - // cutoff, because adjacent text chunks of the same source are always related. - for (const book of books) { - for (let i = 0; i < book.pageIds.length - 1; i++) { - const aId = book.pageIds[i]; - const bId = book.pageIds[i + 1]; - const aVec = pageVectorMap.get(aId); - const bVec = pageVectorMap.get(bId); - if (!aVec || !bVec) continue; - - const sim = cosineSimilarity(aVec, bVec); - const dist = 1 - sim; - const forwardEdge: SemanticNeighbor = { neighborPageId: bId, cosineSimilarity: sim, distance: dist }; - const reverseEdge: SemanticNeighbor = { neighborPageId: aId, cosineSimilarity: sim, distance: dist }; - - // Forward: a → b - const existingA = await metadataStore.getSemanticNeighbors(aId); - await metadataStore.putSemanticNeighbors(aId, mergeAdjacentNeighbor(existingA, forwardEdge, ADJACENCY_MAX_DEGREE)); - - // Reverse: b → a - const existingB = await metadataStore.getSemanticNeighbors(bId); - await metadataStore.putSemanticNeighbors(bId, mergeAdjacentNeighbor(existingB, reverseEdge, ADJACENCY_MAX_DEGREE)); - } - } - - await runPromotionSweep(books.map((b) => b.bookId), metadataStore, policy); - - // ------------------------------------------------------------------------- - // Level 2: Books → Volumes - // ------------------------------------------------------------------------- - const bookChunks = chunkArray(books, BOOKS_PER_VOLUME); - const volumes: Volume[] = []; - - for (const bookChunk of bookChunks) { - const sortedBookIds = bookChunk.map((b) => b.bookId).sort(); - const volumeId = await hashText(sortedBookIds.join("|")); - - const medoidVectors = bookChunk.map((b) => { - const vec = pageVectorMap.get(b.medoidPageId); - if (!vec) throw new Error(`Vector not found for medoid page ${b.medoidPageId}`); - return vec; - }); - - const centroid = computeCentroid(medoidVectors); - const prototypeOffset = await vectorStore.appendVector(centroid); - - // Average squared cosine distance from centroid. - let variance = 0; - for (const v of medoidVectors) { - const dist = cosineDistance(v, centroid); - variance += dist * dist; - } - variance /= medoidVectors.length; - - const volume: Volume = { - volumeId, - bookIds: bookChunk.map((b) => b.bookId), - prototypeOffsets: [prototypeOffset], - prototypeDim: dim, - variance, - }; - await metadataStore.putVolume(volume); - volumes.push(volume); - } - - await runPromotionSweep(volumes.map((v) => v.volumeId), metadataStore, policy); - - // ------------------------------------------------------------------------- - // Level 3: Volumes → Shelves - // ------------------------------------------------------------------------- - const volumeChunks = chunkArray(volumes, VOLUMES_PER_SHELF); - const shelves: Shelf[] = []; - - for (const volumeChunk of volumeChunks) { - const sortedVolumeIds = volumeChunk.map((v) => v.volumeId).sort(); - const shelfId = await hashText(sortedVolumeIds.join("|")); - - const protoVectors = await Promise.all( - volumeChunk.map((v) => vectorStore.readVector(v.prototypeOffsets[0], dim)), - ); - - const routingCentroid = computeCentroid(protoVectors); - const routingOffset = await vectorStore.appendVector(routingCentroid); - - const shelf: Shelf = { - shelfId, - volumeIds: volumeChunk.map((v) => v.volumeId), - routingPrototypeOffsets: [routingOffset], - routingDim: dim, - }; - await metadataStore.putShelf(shelf); - shelves.push(shelf); - } - - await runPromotionSweep(shelves.map((s) => s.shelfId), metadataStore, policy); - - // ------------------------------------------------------------------------- - // Williams fanout quota enforcement - // ------------------------------------------------------------------------- - // Per DESIGN.md "Sublinear Fanout Bounds": when a node's child count exceeds - // its Williams-derived limit, HierarchyBuilder triggers a split. - // - // The split threshold is max(STATIC_CONSTANT, computeFanoutLimit(nodeSize)): - // - For freshly built nodes (size ≤ STATIC_CONSTANT), the Williams formula - // may return a smaller value, so we use the static constant as a floor to - // prevent splitting a structure that was just constructed correctly. - // - For nodes that have grown organically past the static constant, the - // Williams limit takes over and drives the split. - const policyC = policy?.c ?? DEFAULT_HOTPATH_POLICY.c; - - // ---- Volumes ---- - for (const volume of [...volumes]) { - const nodeLimit = Math.max(BOOKS_PER_VOLUME, computeFanoutLimit(volume.bookIds.length, policyC)); - if (volume.bookIds.length <= nodeLimit) continue; - - const subChunks = chunkArray(volume.bookIds, nodeLimit); - const subVolumes: Volume[] = []; - - for (const sub of subChunks) { - const sortedSub = [...sub].sort(); - const subVolumeId = await hashText(`split-vol:${volume.volumeId}:${sortedSub.join("|")}`); - - const subBooks = ( - await Promise.all(sub.map((id) => metadataStore.getBook(id))) - ).filter((b): b is Book => b !== undefined); - - // Compute sub-volume variance from actual medoid vectors. - const medoidVecs = subBooks - .map((b) => pageVectorMap.get(b.medoidPageId)) - .filter((v): v is Float32Array => v !== undefined); - const centroid = medoidVecs.length > 0 ? computeCentroid(medoidVecs) : new Float32Array(dim); - const protoOffset = await vectorStore.appendVector(centroid); - - let subVariance = 0; - for (const v of medoidVecs) { - const d = cosineDistance(v, centroid); - subVariance += d * d; - } - if (medoidVecs.length > 0) subVariance /= medoidVecs.length; - - const subVol: Volume = { - volumeId: subVolumeId, - bookIds: sub, - prototypeOffsets: [protoOffset], - prototypeDim: dim, - variance: subVariance, - }; - await metadataStore.putVolume(subVol); - subVolumes.push(subVol); - } - - // Replace the oversized volume in every shelf that references it. - for (const shelf of shelves) { - const idx = shelf.volumeIds.indexOf(volume.volumeId); - if (idx === -1) continue; - const newVolumeIds = [ - ...shelf.volumeIds.slice(0, idx), - ...subVolumes.map((v) => v.volumeId), - ...shelf.volumeIds.slice(idx + 1), - ]; - const updated: Shelf = { ...shelf, volumeIds: newVolumeIds }; - await metadataStore.putShelf(updated); - Object.assign(shelf, updated); - } - - // Delete the original oversized volume (and its reverse-index entries). - await metadataStore.deleteVolume(volume.volumeId); - volumes.splice(volumes.indexOf(volume), 1, ...subVolumes); - } - - // ---- Shelves ---- - for (const shelf of [...shelves]) { - const nodeLimit = Math.max(VOLUMES_PER_SHELF, computeFanoutLimit(shelf.volumeIds.length, policyC)); - if (shelf.volumeIds.length <= nodeLimit) continue; - - const subChunks = chunkArray(shelf.volumeIds, nodeLimit); - - // Update the existing shelf record to hold only the FIRST sub-chunk's - // volumes. All remaining sub-chunks get fresh shelf records. - // Note: volumes that move to new shelves keep a stale volumeToShelf entry - // pointing at this shelf's ID; that entry will be cleaned up by a future - // Daydreamer ClusterStability pass (deleteShelf is not yet on the interface). - const newShelves: Shelf[] = []; - for (let ci = 0; ci < subChunks.length; ci++) { - const sub = subChunks[ci]; - const sortedSub = [...sub].sort(); - - if (ci === 0) { - // Re-use the original shelfId for the first sub-chunk. - const subShelfVols = ( - await Promise.all(sub.map((id) => metadataStore.getVolume(id))) - ).filter((v): v is Volume => v !== undefined); - const protoVecs = await Promise.all( - subShelfVols.map((v) => vectorStore.readVector(v.prototypeOffsets[0], dim)), - ); - const centroid = protoVecs.length > 0 ? computeCentroid(protoVecs) : new Float32Array(dim); - const routingOffset = await vectorStore.appendVector(centroid); - - const updated: Shelf = { - shelfId: shelf.shelfId, - volumeIds: sub, - routingPrototypeOffsets: [routingOffset], - routingDim: dim, - }; - await metadataStore.putShelf(updated); - Object.assign(shelf, updated); - newShelves.push(updated); - } else { - const subShelfId = await hashText(`split-shelf:${shelf.shelfId}:${sortedSub.join("|")}`); - const subShelfVols = ( - await Promise.all(sub.map((id) => metadataStore.getVolume(id))) - ).filter((v): v is Volume => v !== undefined); - const protoVecs = await Promise.all( - subShelfVols.map((v) => vectorStore.readVector(v.prototypeOffsets[0], dim)), - ); - const centroid = protoVecs.length > 0 ? computeCentroid(protoVecs) : new Float32Array(dim); - const routingOffset = await vectorStore.appendVector(centroid); - - const newShelf: Shelf = { - shelfId: subShelfId, - volumeIds: sub, - routingPrototypeOffsets: [routingOffset], - routingDim: dim, - }; - await metadataStore.putShelf(newShelf); - newShelves.push(newShelf); - } - } - - shelves.splice(shelves.indexOf(shelf), 1, ...newShelves); - } - - return { books, volumes, shelves }; -} +export * from "../lib/hippocampus/HierarchyBuilder"; diff --git a/hippocampus/Ingest.ts b/hippocampus/Ingest.ts index f79b4da..57a1089 100644 --- a/hippocampus/Ingest.ts +++ b/hippocampus/Ingest.ts @@ -1,156 +1 @@ -import type { Book, MetadataStore, VectorStore } from "../core/types"; -import type { ModelProfile } from "../core/ModelProfile"; -import { hashText } from "../core/crypto/hash"; -import type { KeyPair } from "../core/crypto/sign"; -import { EmbeddingRunner } from "../embeddings/EmbeddingRunner"; -import { chunkText } from "./Chunker"; -import { buildPage } from "./PageBuilder"; -import { runPromotionSweep } from "../core/SalienceEngine"; -import { insertSemanticNeighbors } from "./FastNeighborInsert"; - -export interface IngestOptions { - modelProfile: ModelProfile; - embeddingRunner: EmbeddingRunner; - vectorStore: VectorStore; - metadataStore: MetadataStore; - keyPair: KeyPair; - now?: number; -} - -export interface IngestResult { - pages: Array>>; - /** The single Book representing everything ingested by this call. - * One ingest call = one Book, always. All pages are members. - * A collection of Books becomes a Volume; a collection of Volumes - * becomes a Shelf — those tiers are assembled by the Daydreamer. */ - book?: Book; -} - -function cosineDistance(a: Float32Array, b: Float32Array): number { - let dot = 0; - let normA = 0; - let normB = 0; - for (let i = 0; i < a.length; i++) { - dot += a[i] * b[i]; - normA += a[i] * a[i]; - normB += b[i] * b[i]; - } - const denom = Math.sqrt(normA) * Math.sqrt(normB); - if (denom === 0) return 0; - return 1 - dot / denom; -} - -/** - * Selects the index of the medoid: the element that minimises total cosine - * distance to every other element in the set. - */ -function selectMedoidIndex(vectors: Float32Array[]): number { - if (vectors.length === 1) return 0; - let bestIdx = 0; - let bestTotal = Infinity; - for (let i = 0; i < vectors.length; i++) { - let total = 0; - for (let j = 0; j < vectors.length; j++) { - if (i !== j) total += cosineDistance(vectors[i], vectors[j]); - } - if (total < bestTotal) { - bestTotal = total; - bestIdx = i; - } - } - return bestIdx; -} - -export async function ingestText( - text: string, - options: IngestOptions, -): Promise { - const { - modelProfile, - embeddingRunner, - vectorStore, - metadataStore, - keyPair, - now = Date.now(), - } = options; - - const chunks = chunkText(text, modelProfile); - if (chunks.length === 0) { - return { pages: [], book: undefined }; - } - - const createdAt = new Date(now).toISOString(); - - // Precompute page IDs (content hashes) so we can link prev/next before signing. - const pageIds = await Promise.all(chunks.map((c) => hashText(c))); - - const embeddings = await embeddingRunner.embed(chunks); - if (embeddings.length !== chunks.length) { - throw new Error("Embedding provider returned unexpected number of embeddings"); - } - - const offsets: number[] = []; - for (const embedding of embeddings) { - const offset = await vectorStore.appendVector(embedding); - offsets.push(offset); - } - - const pages = await Promise.all( - chunks.map(async (content, idx) => { - const prevPageId = idx > 0 ? pageIds[idx - 1] : null; - const nextPageId = idx < pageIds.length - 1 ? pageIds[idx + 1] : null; - - return buildPage({ - content, - embedding: embeddings[idx], - embeddingOffset: offsets[idx], - embeddingDim: modelProfile.embeddingDimension, - creatorPubKey: keyPair.publicKey, - signingKey: keyPair.signingKey, - prevPageId, - nextPageId, - createdAt, - }); - }), - ); - - // Persist pages and activity records. - for (const page of pages) { - await metadataStore.putPage(page); - await metadataStore.putPageActivity({ - pageId: page.pageId, - queryHitCount: 0, - lastQueryAt: createdAt, - }); - } - - // Build ONE Book for the entire ingest. - // A Book = the document we just ingested; its identity is the sorted set of - // its pages. Its representative is the page whose embedding is the medoid - // (minimum total cosine distance to all other pages in the document). - const medoidIdx = selectMedoidIndex(embeddings); - const sortedPageIds = [...pageIds].sort(); - const bookId = await hashText(sortedPageIds.join("|")); - const book: Book = { - bookId, - pageIds, - medoidPageId: pageIds[medoidIdx], - meta: {}, - }; - await metadataStore.putBook(book); - - // Insert semantic neighbor edges for the new pages against all stored pages. - // Volumes and Shelves are assembled by the Daydreamer from accumulated Books. - const allPages = await metadataStore.getAllPages(); - const allPageIds = allPages.map((p) => p.pageId); - await insertSemanticNeighbors(pageIds, allPageIds, { - modelProfile, - vectorStore, - metadataStore, - }); - - // Run hotpath promotion for the newly ingested pages and book. - await runPromotionSweep([...pageIds, bookId], metadataStore); - - return { pages, book }; -} +export * from "../lib/hippocampus/Ingest"; diff --git a/hippocampus/PageBuilder.ts b/hippocampus/PageBuilder.ts index d03e9e3..733f1f4 100644 --- a/hippocampus/PageBuilder.ts +++ b/hippocampus/PageBuilder.ts @@ -1,88 +1 @@ -import type { Hash, Page } from "../core/types"; -import { hashBinary, hashText } from "../core/crypto/hash"; -import { signData } from "../core/crypto/sign"; - -export interface BuildPageOptions { - content: string; - embedding: Float32Array; - embeddingOffset: number; - embeddingDim: number; - creatorPubKey: string; - signingKey: CryptoKey; - prevPageId?: Hash | null; - nextPageId?: Hash | null; - createdAt?: string; -} - -/** - * Build a Page entity from content + embedding. - * - * Creates deterministic `pageId`/`contentHash` from content, a `vectorHash` from - * the raw embedding bytes, and signs the page using the provided key. - */ -export async function buildPage(options: BuildPageOptions): Promise { - const { - content, - embedding, - embeddingOffset, - embeddingDim, - creatorPubKey, - signingKey, - prevPageId = null, - nextPageId = null, - createdAt = new Date().toISOString(), - } = options; - - if (embedding.length !== embeddingDim) { - throw new Error( - `Embedding dimension mismatch: expected ${embeddingDim}, got ${embedding.length}`, - ); - } - - const contentHash = await hashText(content); - const pageId = contentHash; - - // Copy into a new ArrayBuffer-backed view so we never pass a SharedArrayBuffer - // into WebCrypto (and keep TypeScript happy). - const rawVector = new Uint8Array(embedding.byteLength); - rawVector.set(new Uint8Array(embedding.buffer, embedding.byteOffset, embedding.byteLength)); - const vectorHash = await hashBinary(rawVector); - - const unsignedPage = { - pageId, - content, - embeddingOffset, - embeddingDim, - contentHash, - vectorHash, - prevPageId: prevPageId ?? null, - nextPageId: nextPageId ?? null, - creatorPubKey, - createdAt, - } as const; - - // Deterministic canonical representation used for signing. - const canonical = canonicalizePageForSigning(unsignedPage); - const signature = await signData(canonical, signingKey); - - return { - ...unsignedPage, - signature, - }; -} - -function canonicalizePageForSigning(page: Omit): string { - // Keep key order stable for deterministic signing. - return JSON.stringify({ - pageId: page.pageId, - content: page.content, - embeddingOffset: page.embeddingOffset, - embeddingDim: page.embeddingDim, - contentHash: page.contentHash, - vectorHash: page.vectorHash, - prevPageId: page.prevPageId ?? null, - nextPageId: page.nextPageId ?? null, - creatorPubKey: page.creatorPubKey, - createdAt: page.createdAt, - }); -} +export * from "../lib/hippocampus/PageBuilder"; diff --git a/lib/BackendKind.ts b/lib/BackendKind.ts new file mode 100644 index 0000000..b23187a --- /dev/null +++ b/lib/BackendKind.ts @@ -0,0 +1,40 @@ +export type BackendKind = "webnn" | "webgpu" | "webgl" | "wasm"; + +function hasWebGpuSupport(): boolean { + return ( + typeof navigator !== "undefined" && + typeof (navigator as Navigator & { gpu?: unknown }).gpu !== "undefined" + ); +} + +function hasWebGl2Support(): boolean { + if (typeof document === "undefined") { + return false; + } + + const canvas = document.createElement("canvas"); + return canvas.getContext("webgl2") !== null; +} + +function hasWebNnSupport(): boolean { + return ( + typeof navigator !== "undefined" && + typeof (navigator as Navigator & { ml?: unknown }).ml !== "undefined" + ); +} + +export function detectBackend(): BackendKind { + if (hasWebGpuSupport()) { + return "webgpu"; + } + + if (hasWebGl2Support()) { + return "webgl"; + } + + if (hasWebNnSupport()) { + return "webnn"; + } + + return "wasm"; +} diff --git a/lib/CreateVectorBackend.ts b/lib/CreateVectorBackend.ts new file mode 100644 index 0000000..532f9b0 --- /dev/null +++ b/lib/CreateVectorBackend.ts @@ -0,0 +1,28 @@ +import { detectBackend } from "./BackendKind"; +import type { VectorBackend } from "./VectorBackend"; +import { WasmVectorBackend } from "./WasmVectorBackend"; +import { WebGlVectorBackend } from "./WebGLVectorBackend"; +import { WebGpuVectorBackend } from "./WebGPUVectorBackend"; +import { WebNnVectorBackend } from "./WebNNVectorBackend"; + +export async function createVectorBackend( + wasmBytes: ArrayBuffer +): Promise { + const kind = detectBackend(); + if (kind === "webgpu") { + return WebGpuVectorBackend.create().catch(() => + WasmVectorBackend.create(wasmBytes) + ); + } + if (kind === "webgl") { + return Promise.resolve(WebGlVectorBackend.create()).catch(() => + WasmVectorBackend.create(wasmBytes) + ); + } + if (kind === "webnn") { + return WebNnVectorBackend.create(wasmBytes).catch(() => + WasmVectorBackend.create(wasmBytes) + ); + } + return WasmVectorBackend.create(wasmBytes); +} diff --git a/lib/Policy.ts b/lib/Policy.ts new file mode 100644 index 0000000..38e02a2 --- /dev/null +++ b/lib/Policy.ts @@ -0,0 +1,147 @@ +import type { ModelProfile } from "./core/ModelProfile"; +import { + ModelProfileResolver, + type ModelProfileResolverOptions, + type ResolveModelProfileInput, +} from "./core/ModelProfileResolver"; + +export type QueryScope = "broad" | "normal" | "narrow" | "default"; + +export interface ProjectionHead { + dimIn: number; + dimOut: number; + bits?: number; + // Byte offset for the projection head in a flattened projection buffer. + offset: number; +} + +export interface RoutingPolicy { + broad: ProjectionHead; + normal: ProjectionHead; + narrow: ProjectionHead; +} + +export interface ResolvedRoutingPolicy { + modelProfile: ModelProfile; + routingPolicy: RoutingPolicy; +} + +export interface ResolveRoutingPolicyOptions { + resolver?: ModelProfileResolver; + resolverOptions?: ModelProfileResolverOptions; + routingPolicyOverrides?: Partial; +} + +export interface RoutingPolicyDerivation { + broadDimRatio: number; + normalDimRatio: number; + narrowDimRatio: number; + broadHashBits: number; + dimAlignment: number; + minProjectionDim: number; +} + +export const DEFAULT_ROUTING_POLICY_DERIVATION: RoutingPolicyDerivation = + Object.freeze({ + broadDimRatio: 1 / 8, + normalDimRatio: 1 / 4, + narrowDimRatio: 1 / 2, + broadHashBits: 128, + dimAlignment: 8, + minProjectionDim: 8, + }); + +function assertPositiveInteger(name: string, value: number): void { + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer`); + } +} + +function assertPositiveFinite(name: string, value: number): void { + if (!Number.isFinite(value) || value <= 0) { + throw new Error(`${name} must be positive and finite`); + } +} + +function alignDown(value: number, alignment: number): number { + return Math.floor(value / alignment) * alignment; +} + +function deriveProjectionDim( + dimIn: number, + ratio: number, + derivation: RoutingPolicyDerivation, +): number { + const raw = Math.floor(dimIn * ratio); + const aligned = alignDown(raw, derivation.dimAlignment); + const bounded = Math.max(derivation.minProjectionDim, aligned); + return Math.min(dimIn, bounded); +} + +function validateDerivation(derivation: RoutingPolicyDerivation): void { + assertPositiveFinite("broadDimRatio", derivation.broadDimRatio); + assertPositiveFinite("normalDimRatio", derivation.normalDimRatio); + assertPositiveFinite("narrowDimRatio", derivation.narrowDimRatio); + assertPositiveInteger("broadHashBits", derivation.broadHashBits); + assertPositiveInteger("dimAlignment", derivation.dimAlignment); + assertPositiveInteger("minProjectionDim", derivation.minProjectionDim); +} + +export function createRoutingPolicy( + modelProfile: Pick, + overrides: Partial = {}, +): RoutingPolicy { + assertPositiveInteger("embeddingDimension", modelProfile.embeddingDimension); + + const derivation: RoutingPolicyDerivation = { + ...DEFAULT_ROUTING_POLICY_DERIVATION, + ...overrides, + }; + + validateDerivation(derivation); + + const dimIn = modelProfile.embeddingDimension; + const broadDim = deriveProjectionDim(dimIn, derivation.broadDimRatio, derivation); + const normalDim = deriveProjectionDim(dimIn, derivation.normalDimRatio, derivation); + const narrowDim = deriveProjectionDim(dimIn, derivation.narrowDimRatio, derivation); + + const broadOffset = 0; + const normalOffset = broadOffset + broadDim * dimIn; + const narrowOffset = normalOffset + normalDim * dimIn; + + return { + broad: { + dimIn, + dimOut: broadDim, + bits: derivation.broadHashBits, + offset: broadOffset, + }, + normal: { + dimIn, + dimOut: normalDim, + offset: normalOffset, + }, + narrow: { + dimIn, + dimOut: narrowDim, + offset: narrowOffset, + }, + }; +} + +export function resolveRoutingPolicyForModel( + input: ResolveModelProfileInput, + options: ResolveRoutingPolicyOptions = {}, +): ResolvedRoutingPolicy { + const resolver = + options.resolver ?? new ModelProfileResolver(options.resolverOptions); + const modelProfile = resolver.resolve(input); + + return { + modelProfile, + routingPolicy: createRoutingPolicy( + modelProfile, + options.routingPolicyOverrides, + ), + }; +} diff --git a/lib/TopK.ts b/lib/TopK.ts new file mode 100644 index 0000000..3f0467e --- /dev/null +++ b/lib/TopK.ts @@ -0,0 +1,28 @@ +import type { DistanceResult, ScoreResult } from "./VectorBackend"; + +export function topKByScore(scores: Float32Array, k: number): ScoreResult[] { + const limit = Math.max(0, Math.min(k, scores.length)); + const indices = Array.from({ length: scores.length }, (_, i) => i); + + indices.sort((a, b) => scores[b] - scores[a]); + + return indices.slice(0, limit).map((index) => ({ + index, + score: scores[index] + })); +} + +export function topKByDistance( + distances: Uint32Array | Int32Array | Float32Array, + k: number +): DistanceResult[] { + const limit = Math.max(0, Math.min(k, distances.length)); + const indices = Array.from({ length: distances.length }, (_, i) => i); + + indices.sort((a, b) => Number(distances[a]) - Number(distances[b])); + + return indices.slice(0, limit).map((index) => ({ + index, + distance: Number(distances[index]) + })); +} diff --git a/lib/VectorBackend.ts b/lib/VectorBackend.ts new file mode 100644 index 0000000..6fe6737 --- /dev/null +++ b/lib/VectorBackend.ts @@ -0,0 +1,49 @@ +import type { BackendKind } from "./BackendKind"; + +export interface ScoreResult { + index: number; + score: number; +} + +export interface DistanceResult { + index: number; + distance: number; +} + +export interface VectorBackend { + kind: BackendKind; + + // Exact or high-precision dot-product scoring over row-major matrices. + dotMany( + query: Float32Array, + matrix: Float32Array, + dim: number, + count: number + ): Promise; + + // Projection helper used to reduce dimensionality for routing tiers. + project( + vector: Float32Array, + projectionMatrix: Float32Array, + dimIn: number, + dimOut: number + ): Promise; + + topKFromScores(scores: Float32Array, k: number): Promise; + + // Random-hyperplane hash from projected vectors into packed binary codes. + hashToBinary( + vector: Float32Array, + projectionMatrix: Float32Array, + dimIn: number, + bits: number + ): Promise; + + hammingTopK( + queryCode: Uint32Array, + codes: Uint32Array, + wordsPerCode: number, + count: number, + k: number + ): Promise; +} diff --git a/Vectors.glsl b/lib/Vectors.glsl similarity index 100% rename from Vectors.glsl rename to lib/Vectors.glsl diff --git a/Vectors.wat b/lib/Vectors.wat similarity index 100% rename from Vectors.wat rename to lib/Vectors.wat diff --git a/Vectors.wgsl b/lib/Vectors.wgsl similarity index 100% rename from Vectors.wgsl rename to lib/Vectors.wgsl diff --git a/lib/WasmVectorBackend.ts b/lib/WasmVectorBackend.ts new file mode 100644 index 0000000..8817bc7 --- /dev/null +++ b/lib/WasmVectorBackend.ts @@ -0,0 +1,154 @@ +import type { + DistanceResult, + ScoreResult, + VectorBackend +} from "./VectorBackend"; +import { + FLOAT32_BYTES, + UINT32_BITS, + UINT32_BYTES, + WASM_ALLOC_ALIGNMENT_BYTES, + WASM_ALLOC_GUARD_BYTES, + WASM_PAGE_BYTES, +} from "./core/NumericConstants"; + +interface WasmVectorExports { + mem: WebAssembly.Memory; + dot_many(qPtr: number, mPtr: number, outPtr: number, dim: number, count: number): void; + project(vecPtr: number, pPtr: number, outPtr: number, dimIn: number, dimOut: number): void; + hash_binary(vecPtr: number, pPtr: number, codePtr: number, dimIn: number, bits: number): void; + hamming_scores( + queryCodePtr: number, + codesPtr: number, + outPtr: number, + wordsPerCode: number, + count: number + ): void; + topk_i32(scoresPtr: number, outPtr: number, count: number, k: number): void; + topk_f32(scoresPtr: number, outPtr: number, count: number, k: number): void; +} + +export class WasmVectorBackend implements VectorBackend { + readonly kind = "wasm" as const; + private exports!: WasmVectorExports; + private mem!: WebAssembly.Memory; + private bump = WASM_ALLOC_GUARD_BYTES; + + static async create(wasmBytes: ArrayBuffer): Promise { + const b = new WasmVectorBackend(); + const { instance } = await WebAssembly.instantiate(wasmBytes); + b.exports = instance.exports as unknown as WasmVectorExports; + b.mem = instance.exports.mem as WebAssembly.Memory; + return b; + } + + // 16-byte-aligned bump allocator; call reset() between requests + private alloc(bytes: number): number { + const ptr = + (this.bump + (WASM_ALLOC_ALIGNMENT_BYTES - 1)) & + ~(WASM_ALLOC_ALIGNMENT_BYTES - 1); + this.bump = ptr + bytes; + if (this.bump > this.mem.buffer.byteLength) { + this.mem.grow( + Math.ceil((this.bump - this.mem.buffer.byteLength) / WASM_PAGE_BYTES) + ); + } + return ptr; + } + + reset(): void { + this.bump = WASM_ALLOC_GUARD_BYTES; + } + + private writeF32(data: Float32Array): number { + const ptr = this.alloc(data.byteLength); + new Float32Array(this.mem.buffer, ptr, data.length).set(data); + return ptr; + } + + private writeU32(data: Uint32Array): number { + const ptr = this.alloc(data.byteLength); + new Uint32Array(this.mem.buffer, ptr, data.length).set(data); + return ptr; + } + + async dotMany( + query: Float32Array, matrix: Float32Array, + dim: number, count: number + ): Promise { + this.reset(); + const q_ptr = this.writeF32(query); + const m_ptr = this.writeF32(matrix); + const out_ptr = this.alloc(count * FLOAT32_BYTES); + this.exports.dot_many(q_ptr, m_ptr, out_ptr, dim, count); + return new Float32Array( + this.mem.buffer.slice(out_ptr, out_ptr + count * FLOAT32_BYTES) + ); + } + + async project( + vec: Float32Array, + projectionMatrix: Float32Array, + dimIn: number, dimOut: number + ): Promise { + this.reset(); + const v_ptr = this.writeF32(vec); + const P_ptr = this.writeF32(projectionMatrix); + const out_ptr = this.alloc(dimOut * FLOAT32_BYTES); + this.exports.project(v_ptr, P_ptr, out_ptr, dimIn, dimOut); + return new Float32Array( + this.mem.buffer.slice(out_ptr, out_ptr + dimOut * FLOAT32_BYTES) + ); + } + + async hashToBinary( + vec: Float32Array, + projectionMatrix: Float32Array, + dimIn: number, bits: number + ): Promise { + this.reset(); + const wordsPerCode = Math.ceil(bits / UINT32_BITS); + const v_ptr = this.writeF32(vec); + const P_ptr = this.writeF32(projectionMatrix); + const code_ptr = this.alloc(wordsPerCode * UINT32_BYTES); + this.exports.hash_binary(v_ptr, P_ptr, code_ptr, dimIn, bits); + return new Uint32Array( + this.mem.buffer.slice(code_ptr, code_ptr + wordsPerCode * UINT32_BYTES) + ); + } + + async hammingTopK( + queryCode: Uint32Array, codes: Uint32Array, + wordsPerCode: number, count: number, k: number + ): Promise { + this.reset(); + const q_ptr = this.writeU32(queryCode); + const codes_ptr = this.writeU32(codes); + const scores_ptr = this.alloc(count * FLOAT32_BYTES); + const out_ptr = this.alloc(k * UINT32_BYTES); + + this.exports.hamming_scores(q_ptr, codes_ptr, scores_ptr, wordsPerCode, count); + + // Snapshot distances before topk_i32 mutates scores in-place + const distances = new Int32Array( + this.mem.buffer.slice(scores_ptr, scores_ptr + count * FLOAT32_BYTES) + ); + + this.exports.topk_i32(scores_ptr, out_ptr, count, k); + + const indices = new Int32Array(this.mem.buffer, out_ptr, k); + return Array.from(indices).map(idx => ({ index: idx, distance: distances[idx] })); + } + + async topKFromScores( + scores: Float32Array, k: number + ): Promise { + this.reset(); + // copy: topk_f32 mutates in-place + const copy_ptr = this.writeF32(new Float32Array(scores)); + const out_ptr = this.alloc(k * FLOAT32_BYTES); + this.exports.topk_f32(copy_ptr, out_ptr, scores.length, k); + const indices = new Int32Array(this.mem.buffer, out_ptr, k); + return Array.from(indices).map(idx => ({ index: idx, score: scores[idx] })); + } +} diff --git a/lib/WebGLVectorBackend.ts b/lib/WebGLVectorBackend.ts new file mode 100644 index 0000000..af9157b --- /dev/null +++ b/lib/WebGLVectorBackend.ts @@ -0,0 +1,282 @@ +import { topKByDistance, topKByScore } from "./TopK"; +import type { + DistanceResult, + ScoreResult, + VectorBackend +} from "./VectorBackend"; +import { + FULLSCREEN_TRIANGLE_VERTEX_COUNT, + RGBA_CHANNELS, + UINT32_BITS, +} from "./core/NumericConstants"; + +const VERT_SRC = /* glsl */`#version 300 es +out vec2 v_uv; +void main() { + // Full-screen triangle; no geometry buffer needed + vec2 pos[3]; + pos[0] = vec2(-1.0, -1.0); + pos[1] = vec2( 3.0, -1.0); + pos[2] = vec2(-1.0, 3.0); + gl_Position = vec4(pos[gl_VertexID], 0.0, 1.0); + v_uv = pos[gl_VertexID] * 0.5 + 0.5; +}`; + +// One fragment per candidate vector; textures are RGBA32F so 4 floats per texel +const DOT_FRAG_SRC = /* glsl */`#version 300 es +precision highp float; +// matrix rows packed as RGBA32F: width = ceil(dim/4), height = count +uniform highp sampler2D u_matrix; +// query packed the same way; height = 1 +uniform highp sampler2D u_query; +uniform int u_dim_packed; // ceil(dim/4) +uniform int u_actual_dim; // true dim in floats +uniform int u_count; +out vec4 out_color; // r = score +void main() { + int row = int(gl_FragCoord.x); + if (row >= u_count) { discard; } + float sum = 0.0; + for (int j = 0; j < u_dim_packed; j++) { + vec4 q = texelFetch(u_query, ivec2(j, 0), 0); + vec4 m = texelFetch(u_matrix, ivec2(j, row), 0); + // For the last texel, zero out unused lanes beyond actual_dim + int base = j * 4; + if (base + 3 >= u_actual_dim) { + int rem = u_actual_dim - base; // 1, 2, or 3 + if (rem == 1) { q.g = 0.0; q.b = 0.0; q.a = 0.0; + m.g = 0.0; m.b = 0.0; m.a = 0.0; } + if (rem == 2) { q.b = 0.0; q.a = 0.0; + m.b = 0.0; m.a = 0.0; } + if (rem == 3) { q.a = 0.0; m.a = 0.0; } + } + sum += dot(q, m); + } + out_color = vec4(sum, 0.0, 0.0, 1.0); +}`; + +// Binary hash: one fragment per bit; writes 0.0 or 1.0; caller packs into Uint32Array on CPU +const HASH_FRAG_SRC = /* glsl */`#version 300 es +precision highp float; +uniform highp sampler2D u_vec; // [1 x 1], packed RGBA32F, width=ceil(dim/4) +uniform highp sampler2D u_hyperplanes; // width=ceil(dim/4), height=bits +uniform int u_dim_packed; +uniform int u_actual_dim; +uniform int u_bits; +out vec4 out_color; // r = 1.0 if bit set +void main() { + int b = int(gl_FragCoord.x); + if (b >= u_bits) { discard; } + float dot_val = 0.0; + for (int j = 0; j < u_dim_packed; j++) { + vec4 v = texelFetch(u_vec, ivec2(j, 0), 0); + vec4 h = texelFetch(u_hyperplanes, ivec2(j, b), 0); + int base = j * 4; + if (base + 3 >= u_actual_dim) { + int rem = u_actual_dim - base; + if (rem <= 3) { v.a = 0.0; h.a = 0.0; } + if (rem <= 2) { v.b = 0.0; h.b = 0.0; } + if (rem <= 1) { v.g = 0.0; h.g = 0.0; } + } + dot_val += dot(v, h); + } + out_color = vec4(dot_val >= 0.0 ? 1.0 : 0.0, 0.0, 0.0, 1.0); +}`; + +// ───────────────────────────────────────────────────────────────── +export class WebGlVectorBackend implements VectorBackend { + readonly kind = "webgl" as const; + private gl!: WebGL2RenderingContext; + private dotProg!: WebGLProgram; + private hashProg!: WebGLProgram; + private vao!: WebGLVertexArrayObject; // empty VAO for attribute-less draw + + static create(canvas?: HTMLCanvasElement): WebGlVectorBackend { + const b = new WebGlVectorBackend(); + const c = canvas ?? document.createElement("canvas"); + const gl = c.getContext("webgl2"); + if (!gl) throw new Error("WebGL2 not supported"); + const ext = gl.getExtension("EXT_color_buffer_float"); + if (!ext) throw new Error("EXT_color_buffer_float required"); + b.gl = gl; + b.dotProg = b.compileProgram(VERT_SRC, DOT_FRAG_SRC); + b.hashProg = b.compileProgram(VERT_SRC, HASH_FRAG_SRC); + b.vao = gl.createVertexArray()!; + return b; + } + + private compileProgram(vert: string, frag: string): WebGLProgram { + const gl = this.gl; + const compile = (type: number, src: string) => { + const s = gl.createShader(type)!; + gl.shaderSource(s, src); + gl.compileShader(s); + if (!gl.getShaderParameter(s, gl.COMPILE_STATUS)) + throw new Error(gl.getShaderInfoLog(s) ?? "Shader compile error"); + return s; + }; + const prog = gl.createProgram()!; + gl.attachShader(prog, compile(gl.VERTEX_SHADER, vert)); + gl.attachShader(prog, compile(gl.FRAGMENT_SHADER, frag)); + gl.linkProgram(prog); + if (!gl.getProgramParameter(prog, gl.LINK_STATUS)) + throw new Error(gl.getProgramInfoLog(prog) ?? "Program link error"); + return prog; + } + + // Pack a Float32Array into an RGBA32F texture of size (ceil(len/4), height) + private packF32Texture(data: Float32Array, height: number): WebGLTexture { + const gl = this.gl; + const texWidth = Math.ceil(data.length / height / RGBA_CHANNELS); + // Pad to texWidth * height * 4 + const padded = new Float32Array(texWidth * height * RGBA_CHANNELS); + padded.set(data); + const tex = gl.createTexture()!; + gl.bindTexture(gl.TEXTURE_2D, tex); + gl.texImage2D( + gl.TEXTURE_2D, 0, gl.RGBA32F, + texWidth, height, 0, + gl.RGBA, gl.FLOAT, padded + ); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + return tex; + } + + // Render to a 1D float framebuffer, return raw pixels + private drawToFramebuffer( + prog: WebGLProgram, + width: number, // = count (one pixel per candidate) + setup: (prog: WebGLProgram) => void + ): Float32Array { + const gl = this.gl; + gl.canvas.width = width; + gl.canvas.height = 1; + + const fbo = gl.createFramebuffer()!; + const rbuf = gl.createRenderbuffer()!; + gl.bindRenderbuffer(gl.RENDERBUFFER, rbuf); + gl.renderbufferStorage(gl.RENDERBUFFER, gl.RGBA32F, width, 1); + gl.bindFramebuffer(gl.FRAMEBUFFER, fbo); + gl.framebufferRenderbuffer(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.RENDERBUFFER, rbuf); + + gl.viewport(0, 0, width, 1); + gl.useProgram(prog); + setup(prog); + gl.bindVertexArray(this.vao); + gl.drawArrays(gl.TRIANGLES, 0, FULLSCREEN_TRIANGLE_VERTEX_COUNT); + + const pixels = new Float32Array(width * RGBA_CHANNELS); + gl.readPixels(0, 0, width, 1, gl.RGBA, gl.FLOAT, pixels); + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.deleteFramebuffer(fbo); + gl.deleteRenderbuffer(rbuf); + return pixels; + } + + private uniform1i(prog: WebGLProgram, name: string, v: number) { + this.gl.uniform1i(this.gl.getUniformLocation(prog, name), v); + } + + private bindTex(prog: WebGLProgram, name: string, tex: WebGLTexture, unit: number) { + const gl = this.gl; + gl.activeTexture(gl.TEXTURE0 + unit); + gl.bindTexture(gl.TEXTURE_2D, tex); + gl.uniform1i(gl.getUniformLocation(prog, name), unit); + } + + async dotMany( + query: Float32Array, matrix: Float32Array, + dim: number, count: number + ): Promise { + const dimPacked = Math.ceil(dim / 4); + const qTex = this.packF32Texture(query, 1); + const mTex = this.packF32Texture(matrix, count); + + const pixels = this.drawToFramebuffer(this.dotProg, count, (prog) => { + this.bindTex(prog, "u_matrix", mTex, 0); + this.bindTex(prog, "u_query", qTex, 1); + this.uniform1i(prog, "u_dim_packed", dimPacked); + this.uniform1i(prog, "u_actual_dim", dim); + this.uniform1i(prog, "u_count", count); + }); + + this.gl.deleteTexture(qTex); + this.gl.deleteTexture(mTex); + + // Extract r channel from RGBA pixels → scores + return Float32Array.from( + { length: count }, + (_, i) => pixels[i * RGBA_CHANNELS] + ); + } + + async project( + vec: Float32Array, P: Float32Array, + dimIn: number, dimOut: number + ): Promise { + return this.dotMany(vec, P, dimIn, dimOut); + } + + async hashToBinary( + vec: Float32Array, + projectionMatrix: Float32Array, + dimIn: number, bits: number + ): Promise { + const dimPacked = Math.ceil(dimIn / 4); + const vTex = this.packF32Texture(vec, 1); + const hTex = this.packF32Texture(projectionMatrix, bits); + + const pixels = this.drawToFramebuffer(this.hashProg, bits, (prog) => { + this.bindTex(prog, "u_vec", vTex, 0); + this.bindTex(prog, "u_hyperplanes", hTex, 1); + this.uniform1i(prog, "u_dim_packed", dimPacked); + this.uniform1i(prog, "u_actual_dim", dimIn); + this.uniform1i(prog, "u_bits", bits); + }); + + this.gl.deleteTexture(vTex); + this.gl.deleteTexture(hTex); + + // Pack the per-bit float results (0.0 or 1.0) into Uint32 words + const wordsPerCode = Math.ceil(bits / UINT32_BITS); + const code = new Uint32Array(wordsPerCode); + for (let b = 0; b < bits; b++) { + if (pixels[b * RGBA_CHANNELS] >= 0.5) { + const wordIndex = Math.floor(b / UINT32_BITS); + const bitIndex = b % UINT32_BITS; + code[wordIndex] |= (1 << bitIndex); + } + } + return code; + } + + // Hamming done on CPU — WebGL2 has no integer atomic ops, + // and the XOR+popcnt kernel is not naturally expressible in GLSL for large buffers. + // For 10k items at 4–8 words each this is ~40k i32 ops and comfortably fast. + async hammingTopK( + queryCode: Uint32Array, codes: Uint32Array, + wordsPerCode: number, count: number, k: number + ): Promise { + const distances = new Uint32Array(count); + for (let i = 0; i < count; i++) { + let dist = 0; + const base = i * wordsPerCode; + for (let w = 0; w < wordsPerCode; w++) { + let xor = queryCode[w] ^ codes[base + w]; + // Popcount via Hamming weight bit trick + xor = xor - ((xor >> 1) & 0x55555555); + xor = (xor & 0x33333333) + ((xor >> 2) & 0x33333333); + dist += (((xor + (xor >> 4)) & 0x0f0f0f0f) * 0x01010101) >>> 24; + } + distances[i] = dist; + } + return topKByDistance(distances, k); + } + + async topKFromScores( + scores: Float32Array, k: number + ): Promise { + return topKByScore(scores, k); + } +} diff --git a/lib/WebGPUVectorBackend.ts b/lib/WebGPUVectorBackend.ts new file mode 100644 index 0000000..d455eaa --- /dev/null +++ b/lib/WebGPUVectorBackend.ts @@ -0,0 +1,270 @@ +import { topKByDistance, topKByScore } from "./TopK"; +import type { + DistanceResult, + ScoreResult, + VectorBackend +} from "./VectorBackend"; +import { + FLOAT32_BYTES, + UINT32_BITS, + UINT32_BYTES, + WEBGPU_DOT_WORKGROUP_SIZE, + WEBGPU_HAMMING_WORKGROUP_SIZE, + WEBGPU_HASH_WORKGROUP_SIZE, + WEBGPU_MIN_UNIFORM_BYTES, +} from "./core/NumericConstants"; + +const DOT_MANY_WGSL = /* wgsl */` +struct Params { dim: u32, count: u32, words_per_code: u32, k: u32 } +@group(0) @binding(0) var query : array; +@group(0) @binding(1) var matrix : array; +@group(0) @binding(2) var scores : array; +@group(0) @binding(3) var params : Params; +@compute @workgroup_size(${WEBGPU_DOT_WORKGROUP_SIZE}) +fn main(@builtin(global_invocation_id) gid: vec3) { + let i = gid.x; + if (i >= params.count) { return; } + var sum = 0.0; + let base = i * params.dim; + for (var j = 0u; j < params.dim; j++) { sum += query[j] * matrix[base + j]; } + scores[i] = sum; +}`; + +const HASH_BINARY_WGSL = /* wgsl */` +struct Params { dim: u32, bits: u32, words_per_code: u32, _pad: u32 } +@group(0) @binding(0) var vec_in : array; +@group(0) @binding(1) var hyperplanes: array; +@group(0) @binding(2) var code_out : array>; +@group(0) @binding(3) var params : Params; +@compute @workgroup_size(${WEBGPU_HASH_WORKGROUP_SIZE}) +fn main(@builtin(global_invocation_id) gid: vec3) { + let b = gid.x; + if (b >= params.bits) { return; } + var dot = 0.0; + let base = b * params.dim; + for (var j = 0u; j < params.dim; j++) { dot += vec_in[j] * hyperplanes[base + j]; } + if (dot >= 0.0) { atomicOr(&code_out[b >> 5u], 1u << (b & 31u)); } +}`; + +const HAMMING_WGSL = /* wgsl */` +struct Params { dim: u32, count: u32, words_per_code: u32, k: u32 } +@group(0) @binding(0) var q_code : array; +@group(0) @binding(1) var codes : array; +@group(0) @binding(2) var out_dist: array; +@group(0) @binding(3) var params : Params; +@compute @workgroup_size(${WEBGPU_HAMMING_WORKGROUP_SIZE}) +fn main(@builtin(global_invocation_id) gid: vec3) { + let i = gid.x; + if (i >= params.count) { return; } + var dist = 0u; + let base = i * params.words_per_code; + for (var w = 0u; w < params.words_per_code; w++) { + dist += countOneBits(q_code[w] ^ codes[base + w]); + } + out_dist[i] = dist; +}`; + +// ───────────────────────────────────────────────────────────────── +export class WebGpuVectorBackend implements VectorBackend { + readonly kind = "webgpu" as const; + private device!: GPUDevice; + private dotPipeline!: GPUComputePipeline; + private hashPipeline!: GPUComputePipeline; + private hammingPipeline!: GPUComputePipeline; + + static async create(): Promise { + const b = new WebGpuVectorBackend(); + const adapter = await navigator.gpu.requestAdapter(); + if (!adapter) throw new Error("No WebGPU adapter"); + b.device = await adapter.requestDevice(); + b.dotPipeline = b.makePipeline(DOT_MANY_WGSL); + b.hashPipeline = b.makePipeline(HASH_BINARY_WGSL); + b.hammingPipeline = b.makePipeline(HAMMING_WGSL); + return b; + } + + private makePipeline(wgsl: string): GPUComputePipeline { + return this.device.createComputePipeline({ + layout: "auto", + compute: { + module: this.device.createShaderModule({ code: wgsl }), + entryPoint: "main", + }, + }); + } + + private toArrayBuffer(data: ArrayBufferView): ArrayBuffer { + const bytes = new Uint8Array(data.buffer, data.byteOffset, data.byteLength); + const copy = new Uint8Array(data.byteLength); + copy.set(bytes); + return copy.buffer; + } + + // Upload a Float32Array into a GPU storage buffer (read-only) + private f32Buffer(data: Float32Array, usage = GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST): GPUBuffer { + const buf = this.device.createBuffer({ size: data.byteLength, usage }); + this.device.queue.writeBuffer(buf, 0, this.toArrayBuffer(data)); + return buf; + } + + private u32Buffer(data: Uint32Array, usage = GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST): GPUBuffer { + const buf = this.device.createBuffer({ size: data.byteLength, usage }); + this.device.queue.writeBuffer(buf, 0, this.toArrayBuffer(data)); + return buf; + } + + // Create an output storage buffer + a mapped readback buffer + private outBuffer(bytes: number): { gpu: GPUBuffer; read: GPUBuffer } { + return { + gpu: this.device.createBuffer({ + size: bytes, + usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC, + }), + read: this.device.createBuffer({ + size: bytes, + usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST, + }), + }; + } + + private uniformBuffer(data: Uint32Array): GPUBuffer { + const buf = this.device.createBuffer({ + size: Math.max(WEBGPU_MIN_UNIFORM_BYTES, data.byteLength), + usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, + }); + this.device.queue.writeBuffer(buf, 0, this.toArrayBuffer(data)); + return buf; + } + + private async readbackF32(gpu: GPUBuffer, read: GPUBuffer, count: number): Promise { + const cmd = this.device.createCommandEncoder(); + cmd.copyBufferToBuffer(gpu, 0, read, 0, count * FLOAT32_BYTES); + this.device.queue.submit([cmd.finish()]); + await read.mapAsync(GPUMapMode.READ); + const result = new Float32Array(read.getMappedRange().slice(0)); + read.unmap(); + return result; + } + + private async readbackU32(gpu: GPUBuffer, read: GPUBuffer, count: number): Promise { + const cmd = this.device.createCommandEncoder(); + cmd.copyBufferToBuffer(gpu, 0, read, 0, count * UINT32_BYTES); + this.device.queue.submit([cmd.finish()]); + await read.mapAsync(GPUMapMode.READ); + const result = new Uint32Array(read.getMappedRange().slice(0)); + read.unmap(); + return result; + } + + // ── dot_many (also used for project — caller sets dim/count accordingly) + async dotMany( + query: Float32Array, matrix: Float32Array, + dim: number, count: number + ): Promise { + const qBuf = this.f32Buffer(query); + const mBuf = this.f32Buffer(matrix); + const { gpu: oBuf, read: rBuf } = this.outBuffer(count * FLOAT32_BYTES); + const uBuf = this.uniformBuffer(new Uint32Array([dim, count, 0, 0])); + + const bg = this.device.createBindGroup({ + layout: this.dotPipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: { buffer: qBuf } }, + { binding: 1, resource: { buffer: mBuf } }, + { binding: 2, resource: { buffer: oBuf } }, + { binding: 3, resource: { buffer: uBuf } }, + ], + }); + + const cmd = this.device.createCommandEncoder(); + const pass = cmd.beginComputePass(); + pass.setPipeline(this.dotPipeline); + pass.setBindGroup(0, bg); + pass.dispatchWorkgroups(Math.ceil(count / WEBGPU_DOT_WORKGROUP_SIZE)); + pass.end(); + this.device.queue.submit([cmd.finish()]); + + return this.readbackF32(oBuf, rBuf, count); + } + + // project reuses dotMany — P is (dimOut × dimIn), treat each row as a "vector" + async project( + vec: Float32Array, P: Float32Array, + dimIn: number, dimOut: number + ): Promise { + return this.dotMany(vec, P, dimIn, dimOut); + } + + // ── hash_binary + async hashToBinary( + vec: Float32Array, + projectionMatrix: Float32Array, + dimIn: number, bits: number + ): Promise { + const wordsPerCode = Math.ceil(bits / UINT32_BITS); + const vBuf = this.f32Buffer(vec); + const pBuf = this.f32Buffer(projectionMatrix); + const { gpu: oBuf, read: rBuf } = this.outBuffer(wordsPerCode * UINT32_BYTES); + // zero-init the output buffer before atomicOr writes + this.device.queue.writeBuffer(oBuf, 0, new Uint32Array(wordsPerCode)); + const uBuf = this.uniformBuffer(new Uint32Array([dimIn, bits, wordsPerCode, 0])); + + const bg = this.device.createBindGroup({ + layout: this.hashPipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: { buffer: vBuf } }, + { binding: 1, resource: { buffer: pBuf } }, + { binding: 2, resource: { buffer: oBuf } }, + { binding: 3, resource: { buffer: uBuf } }, + ], + }); + + const cmd = this.device.createCommandEncoder(); + const pass = cmd.beginComputePass(); + pass.setPipeline(this.hashPipeline); + pass.setBindGroup(0, bg); + pass.dispatchWorkgroups(Math.ceil(bits / WEBGPU_HASH_WORKGROUP_SIZE)); + pass.end(); + this.device.queue.submit([cmd.finish()]); + + return this.readbackU32(oBuf, rBuf, wordsPerCode); + } + + // ── hamming_scores + top-k (top-k done on CPU; 10k ints is trivial to sort) + async hammingTopK( + queryCode: Uint32Array, codes: Uint32Array, + wordsPerCode: number, count: number, k: number + ): Promise { + const qBuf = this.u32Buffer(queryCode); + const cBuf = this.u32Buffer(codes); + const { gpu: oBuf, read: rBuf } = this.outBuffer(count * UINT32_BYTES); + const uBuf = this.uniformBuffer(new Uint32Array([0, count, wordsPerCode, k])); + + const bg = this.device.createBindGroup({ + layout: this.hammingPipeline.getBindGroupLayout(0), + entries: [ + { binding: 0, resource: { buffer: qBuf } }, + { binding: 1, resource: { buffer: cBuf } }, + { binding: 2, resource: { buffer: oBuf } }, + { binding: 3, resource: { buffer: uBuf } }, + ], + }); + + const cmd = this.device.createCommandEncoder(); + const pass = cmd.beginComputePass(); + pass.setPipeline(this.hammingPipeline); + pass.setBindGroup(0, bg); + pass.dispatchWorkgroups(Math.ceil(count / WEBGPU_HAMMING_WORKGROUP_SIZE)); + pass.end(); + this.device.queue.submit([cmd.finish()]); + + const distances = await this.readbackU32(oBuf, rBuf, count); + return topKByDistance(distances, k); + } + + async topKFromScores( + scores: Float32Array, k: number + ): Promise { + return topKByScore(scores, k); + } +} diff --git a/lib/WebNNTypes.d.ts b/lib/WebNNTypes.d.ts new file mode 100644 index 0000000..19e6968 --- /dev/null +++ b/lib/WebNNTypes.d.ts @@ -0,0 +1,36 @@ +declare global { + type MLOperand = unknown; + + type MLGraph = unknown; + + interface MLContext { + compute( + graph: MLGraph, + inputs: Record, + outputs: Record + ): Promise; + } + + interface MLOperandDescriptor { + dataType: "float32"; + dimensions: number[]; + } + + class MLGraphBuilder { + constructor(context: MLContext); + input(name: string, descriptor: MLOperandDescriptor): MLOperand; + reshape(input: MLOperand, dimensions: number[]): MLOperand; + matmul(a: MLOperand, b: MLOperand): MLOperand; + build(outputs: Record): Promise; + } + + interface Navigator { + ml?: { + createContext(options?: { + deviceType?: "cpu" | "gpu" | "npu"; + }): Promise; + }; + } +} + +export {}; diff --git a/lib/WebNNVectorBackend.ts b/lib/WebNNVectorBackend.ts new file mode 100644 index 0000000..6b80937 --- /dev/null +++ b/lib/WebNNVectorBackend.ts @@ -0,0 +1,97 @@ +import { topKByScore } from "./TopK"; +import type { + DistanceResult, + ScoreResult, + VectorBackend +} from "./VectorBackend"; +import { WasmVectorBackend } from "./WasmVectorBackend"; + +export class WebNnVectorBackend implements VectorBackend { + readonly kind = "webnn" as const; + private ctx!: MLContext; + private builder!: MLGraphBuilder; + // Cache compiled graphs keyed by "dimIn,dimOut" to avoid recompilation + private graphCache = new Map(); + // Fallback for binary ops (WebNN has no bitwise) + private wasmFallback!: WasmVectorBackend; + + static async create(wasmBytes: ArrayBuffer): Promise { + if (!navigator.ml) { + throw new Error("WebNN is not available in this runtime"); + } + + const b = new WebNnVectorBackend(); + b.ctx = await navigator.ml.createContext({ deviceType: "gpu" }); + b.builder = new MLGraphBuilder(b.ctx); + b.wasmFallback = await WasmVectorBackend.create(wasmBytes); + return b; + } + + // Build and cache an MLGraph for a given matmul shape. + // graph computes: output[count] = matrix[count, dim] · query[dim] + // (treating dot_many as a single matmul: output = M @ q) + private async getOrBuildGraph(dim: number, count: number): Promise { + const key = `${dim},${count}`; + + if (!this.graphCache.has(key)) { + const qDesc: MLOperandDescriptor = { dataType: "float32", dimensions: [dim] }; + const mDesc: MLOperandDescriptor = { dataType: "float32", dimensions: [count, dim] }; + + const q = this.builder.input("query", qDesc); + const M = this.builder.input("matrix", mDesc); + + // Reshape query to [dim, 1] for matmul + const qCol = this.builder.reshape(q, [dim, 1]); + // matmul: [count, dim] × [dim, 1] → [count, 1] + const out = this.builder.matmul(M, qCol); + // Flatten to [count] + const flat = this.builder.reshape(out, [count]); + + const graph = await this.builder.build({ scores: flat }); + this.graphCache.set(key, graph); + } + + return this.graphCache.get(key)!; + } + + async dotMany( + query: Float32Array, matrix: Float32Array, + dim: number, count: number + ): Promise { + const graph = await this.getOrBuildGraph(dim, count); + const outputs = { scores: new Float32Array(count) }; + await this.ctx.compute(graph, { query, matrix }, outputs); + return outputs.scores; + } + + // project: same matmul, just dimIn→dimOut; WebNN handles any shape + async project( + vec: Float32Array, + projectionMatrix: Float32Array, + dimIn: number, dimOut: number + ): Promise { + return this.dotMany(vec, projectionMatrix, dimIn, dimOut); + } + + // WebNN has no bitwise instructions — delegate to WASM + async hashToBinary( + vec: Float32Array, + projectionMatrix: Float32Array, + dimIn: number, bits: number + ): Promise { + return this.wasmFallback.hashToBinary(vec, projectionMatrix, dimIn, bits); + } + + async hammingTopK( + queryCode: Uint32Array, codes: Uint32Array, + wordsPerCode: number, count: number, k: number + ): Promise { + return this.wasmFallback.hammingTopK(queryCode, codes, wordsPerCode, count, k); + } + + async topKFromScores( + scores: Float32Array, k: number + ): Promise { + return topKByScore(scores, k); + } +} diff --git a/lib/core/BuiltInModelProfiles.ts b/lib/core/BuiltInModelProfiles.ts new file mode 100644 index 0000000..caca66f --- /dev/null +++ b/lib/core/BuiltInModelProfiles.ts @@ -0,0 +1,57 @@ +import type { ModelProfileRegistryEntry } from "./ModelProfileResolver"; + +/** + * Built-in model profile registry entries for known matryoshka embedding models. + * + * Numeric values here are model-derived and sourced from the original model cards. + * This file is a declared source of truth for model profile numerics and is explicitly + * allowed by the guard:model-derived script. + * + * Add new entries when wiring additional real embedding providers. + */ + +/** + * Profile for `onnx-community/embeddinggemma-300m-ONNX` (Q4 quantized). + * + * Base model: google/embeddinggemma-300m + * Architecture: Gemma 2-based matryoshka embedding model. + * + * Supported matryoshka sub-dimensions (nested, smallest-to-largest): + * 64, 128, 256, 512, 768 + * + * The default dimension registered here (768) is the full-fidelity output. + * Callers may slice to a smaller sub-dimension for compressed retrieval tiers. + * + * matryoshkaProtectedDim = 128: the most coarse-grained (smallest) sub-dimension + * officially supported by the model. MetroidBuilder uses this as the protected + * floor — dimensions below 128 are not a supported embedding granularity. + * + * Task prompts (required for best retrieval quality): + * Query prefix: "query: " + * Document prefix: "passage: " + * + * @see https://huggingface.co/google/embeddinggemma-300m + * @see https://huggingface.co/onnx-community/embeddinggemma-300m-ONNX + */ +export const EMBEDDING_GEMMA_300M_MODEL_ID = + "onnx-community/embeddinggemma-300m-ONNX"; + +export const EMBEDDING_GEMMA_300M_PROFILE: ModelProfileRegistryEntry = { + embeddingDimension: 768, + contextWindowTokens: 512, + matryoshkaProtectedDim: 128, +}; + +/** + * Canonical registry of all built-in model profiles, keyed by model ID. + * This record is used as the default registry in `ModelProfileResolver`. + * + * When adding a new Matryoshka embedding model, set `matryoshkaProtectedDim` + * to the smallest sub-dimension the model officially supports. Known values: + * - embeddinggemma-300m: 128 + * - nomic-embed-text-v1.5: 64 (to be added when nomic provider is wired) + */ +export const BUILT_IN_MODEL_REGISTRY: Record = + Object.freeze({ + [EMBEDDING_GEMMA_300M_MODEL_ID]: EMBEDDING_GEMMA_300M_PROFILE, + }); diff --git a/lib/core/HotpathPolicy.ts b/lib/core/HotpathPolicy.ts new file mode 100644 index 0000000..d8368f7 --- /dev/null +++ b/lib/core/HotpathPolicy.ts @@ -0,0 +1,305 @@ +// --------------------------------------------------------------------------- +// HotpathPolicy — Williams Bound policy foundation +// --------------------------------------------------------------------------- +// +// Central source of truth for the Williams Bound architecture. +// All hotpath constants live here as a frozen default policy object. +// Policy-derived != model-derived — kept strictly separate from ModelDefaults. +// --------------------------------------------------------------------------- + +import type { SalienceWeights, TierQuotaRatios, TierQuotas } from "./types"; + +// --------------------------------------------------------------------------- +// HotpathPolicy interface +// --------------------------------------------------------------------------- + +export interface HotpathPolicy { + /** Scaling factor in H(t) = ceil(c * sqrt(t * log2(1+t))) */ + readonly c: number; + + /** Salience weights: sigma = alpha*H_in + beta*R + gamma*Q */ + readonly salienceWeights: SalienceWeights; + + /** Fractional tier quota ratios (must sum to 1.0) */ + readonly tierQuotaRatios: TierQuotaRatios; +} + +// --------------------------------------------------------------------------- +// Frozen default policy object +// --------------------------------------------------------------------------- + +export const DEFAULT_HOTPATH_POLICY: HotpathPolicy = Object.freeze({ + c: 0.5, + salienceWeights: Object.freeze({ + alpha: 0.5, // Hebbian connectivity + beta: 0.3, // recency + gamma: 0.2, // query-hit frequency + }), + tierQuotaRatios: Object.freeze({ + shelf: 0.10, + volume: 0.20, + book: 0.20, + page: 0.50, + }), +}); + +// --------------------------------------------------------------------------- +// H(t) — Resident hotpath capacity +// --------------------------------------------------------------------------- + +/** + * Compute the resident hotpath capacity H(t) = ceil(c * sqrt(t * log2(1+t))). + * + * Properties guaranteed by tests: + * - Monotonically non-decreasing + * - Sublinear growth (H(t)/t shrinks as t grows) + * - Returns a finite integer >= 1 for any non-negative finite t + */ +export function computeCapacity( + graphMass: number, + c: number = DEFAULT_HOTPATH_POLICY.c, +): number { + if (!Number.isFinite(graphMass) || graphMass < 0) { + return 1; + } + if (graphMass === 0) return 1; + + const log2 = Math.log2(1 + graphMass); + const raw = c * Math.sqrt(graphMass * log2); + + if (!Number.isFinite(raw) || raw < 1) return 1; + return Math.ceil(raw); +} + +// --------------------------------------------------------------------------- +// Node salience — sigma = alpha*H_in + beta*R + gamma*Q +// --------------------------------------------------------------------------- + +/** + * Compute node salience: sigma = alpha*H_in + beta*R + gamma*Q. + * + * @param hebbianIn Sum of incident Hebbian edge weights + * @param recency Recency score (0-1, exponential decay) + * @param queryHits Query-hit count for the node + * @param weights Tunable weights (default from policy) + */ +export function computeSalience( + hebbianIn: number, + recency: number, + queryHits: number, + weights: SalienceWeights = DEFAULT_HOTPATH_POLICY.salienceWeights, +): number { + const raw = weights.alpha * hebbianIn + + weights.beta * recency + + weights.gamma * queryHits; + + if (!Number.isFinite(raw)) return 0; + return raw; +} + +// --------------------------------------------------------------------------- +// Tier quota derivation +// --------------------------------------------------------------------------- + +/** + * Allocate H(t) across shelf/volume/book/page tiers. + * + * Uses largest-remainder method so quotas sum exactly to `capacity`. + */ +export function deriveTierQuotas( + capacity: number, + ratios: TierQuotaRatios = DEFAULT_HOTPATH_POLICY.tierQuotaRatios, +): TierQuotas { + const tiers: (keyof TierQuotas)[] = ["shelf", "volume", "book", "page"]; + + // Normalize ratios so they sum to 1 + const rawTotal = tiers.reduce((sum, t) => sum + ratios[t], 0); + const normalized = tiers.map((t) => (rawTotal > 0 ? ratios[t] / rawTotal : 0.25)); + + const cap = Math.max(0, Math.floor(capacity)); + const idealShares = normalized.map((r) => r * cap); + const floors = idealShares.map((s) => Math.floor(s)); + let remaining = cap - floors.reduce((a, b) => a + b, 0); + + // Distribute remainders by largest fractional part + const remainders = idealShares.map((s, i) => ({ + index: i, + remainder: s - floors[i], + })); + remainders.sort((a, b) => b.remainder - a.remainder); + + for (const r of remainders) { + if (remaining <= 0) break; + floors[r.index]++; + remaining--; + } + + return { + shelf: floors[0], + volume: floors[1], + book: floors[2], + page: floors[3], + }; +} + +// --------------------------------------------------------------------------- +// Community quota derivation +// --------------------------------------------------------------------------- + +/** + * Distribute a tier budget proportionally across communities. + * + * Uses largest-remainder method so quotas sum exactly to `tierBudget`. + * Each community receives a minimum of 1 slot when budget allows. + * + * Returns an empty array when `communitySizes` is empty. + * If `tierBudget` is 0, every community receives 0. + */ +export function deriveCommunityQuotas( + tierBudget: number, + communitySizes: number[], +): number[] { + const n = communitySizes.length; + if (n === 0) return []; + + const budget = Math.max(0, Math.floor(tierBudget)); + if (budget === 0) return new Array(n).fill(0) as number[]; + + const totalSize = communitySizes.reduce((a, b) => a + Math.max(0, b), 0); + + // Phase 1: assign minimum 1 to each community if budget allows + const minPerCommunity = budget >= n ? 1 : 0; + const quotas = new Array(n).fill(minPerCommunity); + const remainingBudget = budget - minPerCommunity * n; + + if (remainingBudget === 0 || totalSize === 0) return quotas; + + // Phase 2: distribute remaining proportionally (largest-remainder) + const proportional = communitySizes.map( + (s) => (Math.max(0, s) / totalSize) * remainingBudget, + ); + const floors = proportional.map(Math.floor); + let floorSum = floors.reduce((a, b) => a + b, 0); + + const remainders = proportional.map((p, i) => ({ idx: i, rem: p - floors[i] })); + remainders.sort((a, b) => b.rem - a.rem); + + let j = 0; + while (floorSum < remainingBudget) { + floors[remainders[j].idx] += 1; + floorSum += 1; + j += 1; + } + + for (let i = 0; i < n; i++) quotas[i] += floors[i]; + return quotas; +} + +// --------------------------------------------------------------------------- +// Semantic neighbor degree limit — Williams-bound derived +// --------------------------------------------------------------------------- + +// Bootstrap floor for Williams-bound log formulas: ensures t_eff ≥ 2 so that +// log₂(t_eff) > 0 and log₂(log₂(1+t_eff)) is defined and positive. +const MIN_GRAPH_MASS_FOR_LOGS = 2; + +/** + * Compute the Williams-bound-derived maximum degree for the semantic neighbor + * graph given a corpus of `graphMass` total pages. + * + * The degree limit uses the same H(t) formula as the hotpath capacity but is + * bounded by a hard cap to keep the graph sparse. At small corpora the + * Williams formula naturally returns small values (e.g. 1–5 for t < 10); + * at large corpora the `hardCap` clamps growth to prevent the graph becoming + * too dense. + * + * @param graphMass Total number of pages in the corpus. + * @param c Williams Bound scaling constant (default from policy). + * @param hardCap Maximum degree regardless of formula result. Default: 32. + */ +export function computeNeighborMaxDegree( + graphMass: number, + c: number = DEFAULT_HOTPATH_POLICY.c, + hardCap = 32, +): number { + const derived = computeCapacity(graphMass, c); + return Math.min(hardCap, Math.max(1, derived)); +} + +// --------------------------------------------------------------------------- +// Dynamic subgraph expansion bounds — Williams-bound derived +// --------------------------------------------------------------------------- + +export interface SubgraphBounds { + /** Maximum number of nodes to include in the induced subgraph. */ + maxSubgraphSize: number; + /** Maximum BFS hops from seed nodes. */ + maxHops: number; + /** Maximum fanout per hop (branching factor). */ + perHopBranching: number; +} + +/** + * Compute dynamic Williams-derived bounds for subgraph expansion (step 9 of + * the Cortex query path). + * + * Formulas from DESIGN.md "Dynamic Subgraph Expansion Bounds": + * + * t_eff = max(t, 2) + * maxSubgraphSize = min(30, ⌊√(t_eff · log₂(1+t_eff)) / log₂(t_eff)⌋) + * maxHops = max(1, ⌈log₂(log₂(1 + t_eff))⌉) + * perHopBranching = max(1, ⌊maxSubgraphSize ^ (1/maxHops)⌋) + * + * The bootstrap floor `t_eff = max(t, 2)` eliminates division-by-zero for + * t ≤ 1 and ensures a safe minimum of `maxSubgraphSize=1, maxHops=1`. + * + * @param graphMass Total number of pages in the corpus. + */ +export function computeSubgraphBounds(graphMass: number): SubgraphBounds { + const tEff = Math.max(graphMass, MIN_GRAPH_MASS_FOR_LOGS); + const log2tEff = Math.log2(tEff); + + const maxSubgraphSize = Math.min( + 30, + Math.floor(Math.sqrt(tEff * Math.log2(1 + tEff)) / log2tEff), + ); + + const maxHops = Math.max(1, Math.ceil(Math.log2(Math.log2(1 + tEff)))); + + const perHopBranching = Math.max( + 1, + Math.floor(Math.pow(maxSubgraphSize, 1 / maxHops)), + ); + + return { + maxSubgraphSize: Math.max(1, maxSubgraphSize), + maxHops, + perHopBranching, + }; +} + +// --------------------------------------------------------------------------- +// Williams-derived hierarchy fanout limit +// --------------------------------------------------------------------------- + +/** + * Compute the Williams-derived fanout limit for a hierarchy node that + * currently has `childCount` children. + * + * Per DESIGN.md "Sublinear Fanout Bounds": + * Max children = O(√(childCount · log childCount)) + * + * The formula is evaluated with a bootstrap floor of t_eff = max(t, 2) to + * avoid log(0) and returns at least 1 child. + * + * @param childCount Current number of children for the parent node. + * @param c Williams Bound scaling constant. + */ +export function computeFanoutLimit( + childCount: number, + c: number = DEFAULT_HOTPATH_POLICY.c, +): number { + const tEff = Math.max(childCount, MIN_GRAPH_MASS_FOR_LOGS); + const raw = c * Math.sqrt(tEff * Math.log2(1 + tEff)); + return Math.max(1, Math.ceil(raw)); +} diff --git a/lib/core/ModelDefaults.ts b/lib/core/ModelDefaults.ts new file mode 100644 index 0000000..0f629db --- /dev/null +++ b/lib/core/ModelDefaults.ts @@ -0,0 +1,100 @@ +import type { ModelProfile, ModelProfileSeed } from "./ModelProfile"; + +export interface ModelDerivationPolicy { + truncationRatio: number; + chunkRatio: number; + minTruncationTokens: number; + minChunkTokens: number; + maxChunkTokens: number; +} + +export const DEFAULT_MODEL_DERIVATION_POLICY: ModelDerivationPolicy = Object.freeze({ + truncationRatio: 0.75, + chunkRatio: 0.25, + minTruncationTokens: 256, + minChunkTokens: 128, + maxChunkTokens: 2048, +}); + +function assertPositiveInteger(name: string, value: number): void { + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer`); + } +} + +function assertPositiveRatio(name: string, value: number): void { + if (!Number.isFinite(value) || value <= 0) { + throw new Error(`${name} must be a positive finite number`); + } +} + +function clamp(value: number, min: number, max: number): number { + return Math.min(max, Math.max(min, value)); +} + +function validatePolicy(policy: ModelDerivationPolicy): void { + assertPositiveRatio("truncationRatio", policy.truncationRatio); + assertPositiveRatio("chunkRatio", policy.chunkRatio); + assertPositiveInteger("minTruncationTokens", policy.minTruncationTokens); + assertPositiveInteger("minChunkTokens", policy.minChunkTokens); + assertPositiveInteger("maxChunkTokens", policy.maxChunkTokens); + + if (policy.minChunkTokens > policy.maxChunkTokens) { + throw new Error("minChunkTokens cannot exceed maxChunkTokens"); + } +} + +export function deriveTruncationTokens( + contextWindowTokens: number, + policy: ModelDerivationPolicy = DEFAULT_MODEL_DERIVATION_POLICY, +): number { + assertPositiveInteger("contextWindowTokens", contextWindowTokens); + validatePolicy(policy); + + const derived = Math.floor(contextWindowTokens * policy.truncationRatio); + return Math.max(policy.minTruncationTokens, derived); +} + +export function deriveChunkTokenLimit( + contextWindowTokens: number, + policy: ModelDerivationPolicy = DEFAULT_MODEL_DERIVATION_POLICY, +): number { + const truncationTokens = deriveTruncationTokens(contextWindowTokens, policy); + const derived = Math.floor(truncationTokens * policy.chunkRatio); + + return clamp(derived, policy.minChunkTokens, policy.maxChunkTokens); +} + +export function buildModelProfileFromSeed( + seed: ModelProfileSeed, + policy: ModelDerivationPolicy = DEFAULT_MODEL_DERIVATION_POLICY, +): ModelProfile { + const modelId = seed.modelId.trim(); + if (modelId.length === 0) { + throw new Error("modelId must be a non-empty string"); + } + + assertPositiveInteger("embeddingDimension", seed.embeddingDimension); + assertPositiveInteger("contextWindowTokens", seed.contextWindowTokens); + + if (seed.matryoshkaProtectedDim !== undefined) { + assertPositiveInteger("matryoshkaProtectedDim", seed.matryoshkaProtectedDim); + if (seed.matryoshkaProtectedDim > seed.embeddingDimension) { + throw new Error( + "matryoshkaProtectedDim cannot exceed embeddingDimension", + ); + } + } + + return { + modelId, + embeddingDimension: seed.embeddingDimension, + contextWindowTokens: seed.contextWindowTokens, + truncationTokens: deriveTruncationTokens(seed.contextWindowTokens, policy), + maxChunkTokens: deriveChunkTokenLimit(seed.contextWindowTokens, policy), + source: seed.source, + ...(seed.matryoshkaProtectedDim !== undefined + ? { matryoshkaProtectedDim: seed.matryoshkaProtectedDim } + : {}), + }; +} diff --git a/lib/core/ModelProfile.ts b/lib/core/ModelProfile.ts new file mode 100644 index 0000000..baa4a05 --- /dev/null +++ b/lib/core/ModelProfile.ts @@ -0,0 +1,56 @@ +export type ModelProfileSource = "metadata" | "registry" | "mixed"; + +export interface ModelProfileSeed { + modelId: string; + embeddingDimension: number; + contextWindowTokens: number; + source: ModelProfileSource; + /** + * The most coarse-grained Matryoshka sub-dimension for this model. + * + * This is the smallest nested embedding size the model officially supports. + * It defines the "protected floor" used by MetroidBuilder: lower dimensions + * encode invariant domain context and are never searched for antithesis. + * + * Known values: + * - embeddinggemma-300m: 128 + * - nomic-embed-text-v1.5: 64 + * + * `undefined` for models that do not use Matryoshka Representation Learning. + * When undefined, MetroidBuilder cannot perform dimensional unwinding and will + * always declare a knowledge gap (antithesis search is not possible). + */ + matryoshkaProtectedDim?: number; +} + +export interface PartialModelMetadata { + embeddingDimension?: number; + contextWindowTokens?: number; +} + +export interface ModelProfile { + modelId: string; + embeddingDimension: number; + contextWindowTokens: number; + truncationTokens: number; + maxChunkTokens: number; + source: ModelProfileSource; + /** + * The most coarse-grained Matryoshka sub-dimension for this model. + * + * This is the smallest nested embedding size the model officially supports. + * It defines the "protected floor" used by MetroidBuilder: dimensions below + * this boundary encode invariant domain context and are never searched for + * antithesis during Matryoshka dimensional unwinding. + * + * Known values: + * - embeddinggemma-300m: 128 + * - nomic-embed-text-v1.5: 64 + * + * `undefined` for models that do not use Matryoshka Representation Learning. + * When undefined, MetroidBuilder cannot perform dimensional unwinding and will + * always declare a knowledge gap (antithesis search is not possible without + * a protected-dimension floor). + */ + matryoshkaProtectedDim?: number; +} diff --git a/lib/core/ModelProfileResolver.ts b/lib/core/ModelProfileResolver.ts new file mode 100644 index 0000000..3cb9384 --- /dev/null +++ b/lib/core/ModelProfileResolver.ts @@ -0,0 +1,112 @@ +import { + DEFAULT_MODEL_DERIVATION_POLICY, + type ModelDerivationPolicy, + buildModelProfileFromSeed, +} from "./ModelDefaults"; +import type { + ModelProfile, + ModelProfileSource, + PartialModelMetadata, +} from "./ModelProfile"; + +export interface ModelProfileRegistryEntry { + embeddingDimension: number; + contextWindowTokens: number; + /** + * The most coarse-grained Matryoshka sub-dimension for this model. + * Required for MetroidBuilder dimensional unwinding. See `ModelProfile.matryoshkaProtectedDim`. + */ + matryoshkaProtectedDim?: number; +} + +export interface ModelProfileResolverOptions { + registry?: Record; + derivationPolicy?: ModelDerivationPolicy; +} + +export interface ResolveModelProfileInput { + modelId: string; + metadata?: PartialModelMetadata; +} + +function normalizeModelId(modelId: string): string { + return modelId.trim().toLowerCase(); +} + +export class ModelProfileResolver { + private readonly registry = new Map(); + private readonly derivationPolicy: ModelDerivationPolicy; + + constructor(options: ModelProfileResolverOptions = {}) { + this.derivationPolicy = options.derivationPolicy ?? DEFAULT_MODEL_DERIVATION_POLICY; + + if (options.registry) { + for (const [modelId, entry] of Object.entries(options.registry)) { + this.register(modelId, entry); + } + } + } + + register(modelId: string, entry: ModelProfileRegistryEntry): void { + this.registry.set(normalizeModelId(modelId), { + embeddingDimension: entry.embeddingDimension, + contextWindowTokens: entry.contextWindowTokens, + ...(entry.matryoshkaProtectedDim !== undefined + ? { matryoshkaProtectedDim: entry.matryoshkaProtectedDim } + : {}), + }); + } + + resolve(input: ResolveModelProfileInput): ModelProfile { + const modelId = input.modelId.trim(); + if (modelId.length === 0) { + throw new Error("modelId must be a non-empty string"); + } + + const normalized = normalizeModelId(modelId); + const registryEntry = this.registry.get(normalized); + + const embeddingDimension = + input.metadata?.embeddingDimension ?? registryEntry?.embeddingDimension; + const contextWindowTokens = + input.metadata?.contextWindowTokens ?? registryEntry?.contextWindowTokens; + + if (embeddingDimension === undefined || contextWindowTokens === undefined) { + throw new Error( + `Cannot resolve model profile for ${modelId}. ` + + "Provide metadata or register a profile entry.", + ); + } + + const source = this.resolveSource(input.metadata, registryEntry); + + return buildModelProfileFromSeed( + { + modelId, + embeddingDimension, + contextWindowTokens, + source, + matryoshkaProtectedDim: registryEntry?.matryoshkaProtectedDim, + }, + this.derivationPolicy, + ); + } + + private resolveSource( + metadata: PartialModelMetadata | undefined, + registryEntry: ModelProfileRegistryEntry | undefined, + ): ModelProfileSource { + const hasMetadataEmbedding = metadata?.embeddingDimension !== undefined; + const hasMetadataContext = metadata?.contextWindowTokens !== undefined; + + if (hasMetadataEmbedding && hasMetadataContext) { + return "metadata"; + } + + if (!hasMetadataEmbedding && !hasMetadataContext && registryEntry) { + return "registry"; + } + + return "mixed"; + } +} diff --git a/lib/core/NumericConstants.ts b/lib/core/NumericConstants.ts new file mode 100644 index 0000000..8056062 --- /dev/null +++ b/lib/core/NumericConstants.ts @@ -0,0 +1,17 @@ +// Runtime and memory-layout numeric constants shared across backends. + +export const FLOAT32_BYTES = 4; +export const UINT32_BYTES = 4; +export const UINT32_BITS = 32; +export const RGBA_CHANNELS = 4; + +export const WASM_ALLOC_GUARD_BYTES = 1024; +export const WASM_ALLOC_ALIGNMENT_BYTES = 16; +export const WASM_PAGE_BYTES = 64 * 1024; + +export const WEBGPU_MIN_UNIFORM_BYTES = 16; +export const WEBGPU_DOT_WORKGROUP_SIZE = 256; +export const WEBGPU_HASH_WORKGROUP_SIZE = 128; +export const WEBGPU_HAMMING_WORKGROUP_SIZE = 256; + +export const FULLSCREEN_TRIANGLE_VERTEX_COUNT = 3; diff --git a/lib/core/SalienceEngine.ts b/lib/core/SalienceEngine.ts new file mode 100644 index 0000000..65af71a --- /dev/null +++ b/lib/core/SalienceEngine.ts @@ -0,0 +1,420 @@ +// --------------------------------------------------------------------------- +// SalienceEngine — Decision-making layer for hotpath admission +// --------------------------------------------------------------------------- +// +// Provides per-node salience computation, promotion/eviction lifecycle +// helpers, and community-aware admission logic. +// --------------------------------------------------------------------------- + +import type { Hash, HotpathEntry, MetadataStore } from "./types"; +import { + computeCapacity, + computeSalience, + DEFAULT_HOTPATH_POLICY, + deriveCommunityQuotas, + deriveTierQuotas, + type HotpathPolicy, +} from "./HotpathPolicy"; + +// --------------------------------------------------------------------------- +// Recency helper +// --------------------------------------------------------------------------- + +/** + * Compute recency score R(v) as exponential decay from the most recent + * activity timestamp. Returns a value in [0, 1]. + * + * Uses a half-life of 7 days — after 7 days of inactivity the recency + * score drops to ~0.5; after 30 days it drops to ~0.05. + */ +function recencyScore(isoTimestamp: string | undefined, now: number): number { + if (!isoTimestamp) return 0; + const ts = Date.parse(isoTimestamp); + if (!Number.isFinite(ts)) return 0; + const ageMs = Math.max(0, now - ts); + const HALF_LIFE_MS = 7 * 24 * 60 * 60 * 1000; // 7 days + return Math.exp((-Math.LN2 * ageMs) / HALF_LIFE_MS); +} + +// --------------------------------------------------------------------------- +// P0-G1: Core salience computation +// --------------------------------------------------------------------------- + +/** + * Fetch PageActivity and incident Hebbian edges for a single page, + * then compute salience via HotpathPolicy. + */ +export async function computeNodeSalience( + pageId: Hash, + metadataStore: MetadataStore, + policy: HotpathPolicy = DEFAULT_HOTPATH_POLICY, + now: number = Date.now(), +): Promise { + const [activity, neighbors] = await Promise.all([ + metadataStore.getPageActivity(pageId), + metadataStore.getNeighbors(pageId), + ]); + + const hebbianIn = neighbors.reduce((sum, e) => sum + e.weight, 0); + + const recency = recencyScore( + activity?.lastQueryAt, + now, + ); + + const queryHits = activity?.queryHitCount ?? 0; + + return computeSalience(hebbianIn, recency, queryHits, policy.salienceWeights); +} + +/** + * Efficient batch version of `computeNodeSalience`. + */ +export async function batchComputeSalience( + pageIds: Hash[], + metadataStore: MetadataStore, + policy: HotpathPolicy = DEFAULT_HOTPATH_POLICY, + now: number = Date.now(), +): Promise> { + const results = new Map(); + + // Parallelize I/O across all pages + const entries = await Promise.all( + pageIds.map(async (id) => { + const salience = await computeNodeSalience(id, metadataStore, policy, now); + return [id, salience] as const; + }), + ); + + for (const [id, salience] of entries) { + results.set(id, salience); + } + + return results; +} + +/** + * Admission gating: should a candidate be promoted into the hotpath? + * + * - During bootstrap (capacity remaining > 0): always admit. + * - During steady-state: admit only if candidate salience exceeds + * the weakest resident salience. + */ +export function shouldPromote( + candidateSalience: number, + weakestResidentSalience: number, + capacityRemaining: number, +): boolean { + if (capacityRemaining > 0) return true; + return candidateSalience > weakestResidentSalience; +} + +/** + * Find the weakest resident in a given tier/community bucket. + * + * Returns the entityId of the weakest entry, or undefined if the + * tier/community bucket is empty. + */ +export async function selectEvictionTarget( + tier: HotpathEntry["tier"], + communityId: string | undefined, + metadataStore: MetadataStore, +): Promise { + const entries = await metadataStore.getHotpathEntries(tier); + + const filtered = communityId !== undefined + ? entries.filter((e) => e.communityId === communityId) + : entries; + + if (filtered.length === 0) return undefined; + + // Find the entry with the lowest salience (deterministic: stable sort by entityId on tie) + let weakest = filtered[0]; + for (let i = 1; i < filtered.length; i++) { + const e = filtered[i]; + if ( + e.salience < weakest.salience || + (e.salience === weakest.salience && e.entityId < weakest.entityId) + ) { + weakest = e; + } + } + + return weakest.entityId; +} + +// --------------------------------------------------------------------------- +// P0-G2: Promotion / eviction lifecycle helpers +// --------------------------------------------------------------------------- + +/** + * Bootstrap phase: fill hotpath greedily by salience while + * resident count < H(t). + * + * Computes salience for all candidate pages, then admits in + * descending salience order until the capacity is reached, + * respecting tier quotas. + */ +export async function bootstrapHotpath( + metadataStore: MetadataStore, + policy: HotpathPolicy = DEFAULT_HOTPATH_POLICY, + candidatePageIds: Hash[] = [], + now: number = Date.now(), +): Promise { + if (candidatePageIds.length === 0) return; + + // Compute salience for all candidates + const salienceMap = await batchComputeSalience( + candidatePageIds, + metadataStore, + policy, + now, + ); + + // Fetch page activities for community info + const activities = await Promise.all( + candidatePageIds.map((id) => metadataStore.getPageActivity(id)), + ); + const communityMap = new Map(); + for (let i = 0; i < candidatePageIds.length; i++) { + communityMap.set(candidatePageIds[i], activities[i]?.communityId); + } + + // Sort candidates by salience descending; break ties by entityId for determinism + const sorted = [...candidatePageIds].sort((a, b) => { + const diff = (salienceMap.get(b) ?? 0) - (salienceMap.get(a) ?? 0); + return diff !== 0 ? diff : a.localeCompare(b); + }); + + // Determine current graph mass for capacity calculation + const currentEntries = await metadataStore.getHotpathEntries(); + const currentCount = currentEntries.length; + + // Estimate graph mass: existing residents + candidates gives a lower bound + // For bootstrap, use total candidate count as graph mass estimate + const graphMass = currentCount + candidatePageIds.length; + const capacity = computeCapacity(graphMass, policy.c); + const tierQuotas = deriveTierQuotas(capacity, policy.tierQuotaRatios); + + // Track how many are already in each tier + const tierCounts: Record = { shelf: 0, volume: 0, book: 0, page: 0 }; + for (const entry of currentEntries) { + tierCounts[entry.tier] = (tierCounts[entry.tier] ?? 0) + 1; + } + + let totalResident = currentCount; + + for (const candidateId of sorted) { + if (totalResident >= capacity) break; + + const tier: HotpathEntry["tier"] = "page"; // bootstrap admits at page tier + if (tierCounts[tier] >= tierQuotas[tier]) continue; + + const salience = salienceMap.get(candidateId) ?? 0; + const entry: HotpathEntry = { + entityId: candidateId, + tier, + salience, + communityId: communityMap.get(candidateId), + }; + + await metadataStore.putHotpathEntry(entry); + tierCounts[tier]++; + totalResident++; + } +} + +/** + * Steady-state promotion sweep: for each candidate, promote if its + * salience exceeds the weakest resident in the same tier/community + * bucket. On promotion, evict the weakest. + * + * Tier quotas and community quotas are enforced regardless of whether + * overall capacity is full, preventing any single tier or community + * from monopolizing the hotpath during ramp-up. + */ +export async function runPromotionSweep( + candidateIds: Hash[], + metadataStore: MetadataStore, + policy: HotpathPolicy = DEFAULT_HOTPATH_POLICY, + now: number = Date.now(), +): Promise { + if (candidateIds.length === 0) return; + + // Compute salience for all candidates + const salienceMap = await batchComputeSalience( + candidateIds, + metadataStore, + policy, + now, + ); + + // Fetch page activities for community info + const activities = await Promise.all( + candidateIds.map((id) => metadataStore.getPageActivity(id)), + ); + const communityMap = new Map(); + for (let i = 0; i < candidateIds.length; i++) { + communityMap.set(candidateIds[i], activities[i]?.communityId); + } + + // Sort candidates by salience descending for deterministic processing + const sorted = [...candidateIds].sort((a, b) => { + const diff = (salienceMap.get(b) ?? 0) - (salienceMap.get(a) ?? 0); + return diff !== 0 ? diff : a.localeCompare(b); + }); + + // Load initial state into an in-memory cache to avoid repeated store reads + let cachedEntries = await metadataStore.getHotpathEntries(); + + for (const candidateId of sorted) { + const candidateSalience = salienceMap.get(candidateId) ?? 0; + const communityId = communityMap.get(candidateId); + const tier: HotpathEntry["tier"] = "page"; + + // Derive capacity and quotas from current state + const currentCount = cachedEntries.length; + const graphMass = currentCount + candidateIds.length; + const capacity = computeCapacity(graphMass, policy.c); + const capacityRemaining = capacity - currentCount; + const tierQuotas = deriveTierQuotas(capacity, policy.tierQuotaRatios); + const tierEntries = cachedEntries.filter((e) => e.tier === tier); + const tierFull = tierEntries.length >= tierQuotas[tier]; + + // --- Community quota check (enforced regardless of capacityRemaining) --- + if (communityId !== undefined && tierFull) { + const communitySizes = getCommunityDistribution(tierEntries, communityId); + const communityQuotas = deriveCommunityQuotas( + tierQuotas[tier], + communitySizes.sizes, + ); + const communityIdx = communitySizes.communityIndex; + const communityBudget = communityIdx < communityQuotas.length + ? communityQuotas[communityIdx] + : 0; + const communityCount = communitySizes.candidateCommunityCount; + + if (communityCount >= communityBudget && communityCount > 0) { + // Community is at quota — only promote if candidate beats weakest in community + const weakestId = findWeakestIn(tierEntries, communityId); + if (weakestId === undefined) continue; + + const weakestSalience = tierEntries.find((e) => e.entityId === weakestId)?.salience ?? 0; + + if (candidateSalience > weakestSalience) { + await metadataStore.removeHotpathEntry(weakestId); + const newEntry: HotpathEntry = { + entityId: candidateId, + tier, + salience: candidateSalience, + communityId, + }; + await metadataStore.putHotpathEntry(newEntry); + cachedEntries = cachedEntries.filter((e) => e.entityId !== weakestId); + cachedEntries.push(newEntry); + } + continue; + } + } + + // --- Tier quota check (enforced regardless of capacityRemaining) --- + if (tierFull) { + // Tier is at quota — evict the weakest in the entire tier (not scoped + // to candidate's community, so new communities can displace weak entries) + const weakestId = findWeakestIn(tierEntries, undefined); + if (weakestId === undefined) continue; + + const weakestSalience = tierEntries.find((e) => e.entityId === weakestId)?.salience ?? 0; + + if (candidateSalience <= weakestSalience) continue; + + await metadataStore.removeHotpathEntry(weakestId); + const newEntry: HotpathEntry = { + entityId: candidateId, + tier, + salience: candidateSalience, + communityId, + }; + await metadataStore.putHotpathEntry(newEntry); + cachedEntries = cachedEntries.filter((e) => e.entityId !== weakestId); + cachedEntries.push(newEntry); + } else if (capacityRemaining > 0) { + // Both tier and overall capacity available — admit directly + const newEntry: HotpathEntry = { + entityId: candidateId, + tier, + salience: candidateSalience, + communityId, + }; + await metadataStore.putHotpathEntry(newEntry); + cachedEntries.push(newEntry); + } + // else: tier has room but overall capacity is full — skip + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** + * Compute community distribution within a set of tier entries, + * including the candidate's community. + * + * The candidate's community is counted as having at least size 1 for + * quota derivation, so that a brand-new community can receive its + * first slot via the largest-remainder method. + */ +function getCommunityDistribution( + tierEntries: HotpathEntry[], + candidateCommunityId: string, +): { + sizes: number[]; + communityIndex: number; + candidateCommunityCount: number; +} { + const communityCountMap = new Map(); + + for (const e of tierEntries) { + const cid = e.communityId ?? "__none__"; + communityCountMap.set(cid, (communityCountMap.get(cid) ?? 0) + 1); + } + + // Ensure candidate's community is represented with at least size 1 + // so that deriveCommunityQuotas can allocate it a slot + const actualCount = communityCountMap.get(candidateCommunityId) ?? 0; + communityCountMap.set(candidateCommunityId, Math.max(1, actualCount)); + + const communities = [...communityCountMap.keys()].sort(); + const sizes = communities.map((c) => communityCountMap.get(c) ?? 0); + const communityIndex = communities.indexOf(candidateCommunityId); + + return { sizes, communityIndex, candidateCommunityCount: actualCount }; +} + +/** + * Find the weakest entry in a list, optionally filtered by communityId. + * Deterministic: breaks ties by entityId (smallest wins). + */ +function findWeakestIn( + entries: HotpathEntry[], + communityId: string | undefined, +): Hash | undefined { + const filtered = communityId !== undefined + ? entries.filter((e) => e.communityId === communityId) + : entries; + + if (filtered.length === 0) return undefined; + + let weakest = filtered[0]; + for (let i = 1; i < filtered.length; i++) { + const e = filtered[i]; + if ( + e.salience < weakest.salience || + (e.salience === weakest.salience && e.entityId < weakest.entityId) + ) { + weakest = e; + } + } + return weakest.entityId; +} diff --git a/lib/core/crypto/hash.ts b/lib/core/crypto/hash.ts new file mode 100644 index 0000000..2616e4e --- /dev/null +++ b/lib/core/crypto/hash.ts @@ -0,0 +1,26 @@ +import type { Hash } from "../types.js"; + +function bufferToHex(buffer: ArrayBuffer): string { + return Array.from(new Uint8Array(buffer)) + .map(b => b.toString(16).padStart(2, "0")) + .join(""); +} + +/** + * Returns the SHA-256 hex digest of a UTF-8 encoded text string. + * Used to produce `contentHash` on Page entities. + */ +export async function hashText(content: string): Promise { + const encoded = new TextEncoder().encode(content); + const buffer = await crypto.subtle.digest("SHA-256", encoded); + return bufferToHex(buffer); +} + +/** + * Returns the SHA-256 hex digest of raw binary data. + * Used to produce `vectorHash` on Page entities. + */ +export async function hashBinary(data: BufferSource): Promise { + const buffer = await crypto.subtle.digest("SHA-256", data); + return bufferToHex(buffer); +} diff --git a/lib/core/crypto/sign.ts b/lib/core/crypto/sign.ts new file mode 100644 index 0000000..41b7c1b --- /dev/null +++ b/lib/core/crypto/sign.ts @@ -0,0 +1,69 @@ +import type { PublicKey, Signature } from "../types.js"; + +export interface KeyPair { + /** JWK JSON string — safe to store and share. */ + publicKey: PublicKey; + /** JWK JSON string — store securely; used to reconstruct `signingKey`. */ + privateKeyJwk: string; + /** Runtime CryptoKey ready for immediate signing operations. */ + signingKey: CryptoKey; +} + +function bufferToBase64(buffer: ArrayBuffer): Signature { + return btoa(String.fromCharCode(...new Uint8Array(buffer))); +} + +/** + * Generates a new Ed25519 key pair. + * Returns the public key as a JWK string, the private key as both a JWK + * string (for secure storage) and a runtime `CryptoKey` (for signing). + */ +export async function generateKeyPair(): Promise { + const keyPair = await crypto.subtle.generateKey( + { name: "Ed25519" } as Algorithm, + true, + ["sign", "verify"], + ) as CryptoKeyPair; + + const [publicKeyJwk, privateKeyJwk] = await Promise.all([ + crypto.subtle.exportKey("jwk", keyPair.publicKey), + crypto.subtle.exportKey("jwk", keyPair.privateKey), + ]); + + return { + publicKey: JSON.stringify(publicKeyJwk), + privateKeyJwk: JSON.stringify(privateKeyJwk), + signingKey: keyPair.privateKey, + }; +} + +/** + * Imports a private key from its JWK JSON string for use in signing. + * Call this when restoring a key pair from persistent storage. + */ +export async function importSigningKey(privateKeyJwk: string): Promise { + const jwk = JSON.parse(privateKeyJwk) as JsonWebKey; + return crypto.subtle.importKey( + "jwk", + jwk, + { name: "Ed25519" } as Algorithm, + false, + ["sign"], + ); +} + +/** + * Signs arbitrary data with an Ed25519 private key. + * Returns a base64-encoded signature string. + * + * @param data - UTF-8 string or raw bytes to sign. + * @param signingKey - CryptoKey from `generateKeyPair()` or `importSigningKey()`. + */ +export async function signData( + data: string | ArrayBuffer, + signingKey: CryptoKey, +): Promise { + const bytes = typeof data === "string" ? new TextEncoder().encode(data) : data; + const signatureBuffer = await crypto.subtle.sign("Ed25519", signingKey, bytes); + return bufferToBase64(signatureBuffer); +} diff --git a/lib/core/crypto/uuid.ts b/lib/core/crypto/uuid.ts new file mode 100644 index 0000000..d6ad498 --- /dev/null +++ b/lib/core/crypto/uuid.ts @@ -0,0 +1,48 @@ +// --------------------------------------------------------------------------- +// uuid.ts — Minimal UUID v4 generation utility +// --------------------------------------------------------------------------- +// +// Generates a RFC 4122 version 4 (random) UUID. +// Uses crypto.randomUUID() when available (browsers and modern Node/Bun), +// with a pure-JS fallback for environments that lack it. +// --------------------------------------------------------------------------- + +/** + * Generate a RFC 4122 version 4 UUID string. + * + * Prefers the platform's built-in crypto.randomUUID() when available, + * falling back to a pure-JS crypto.getRandomValues() implementation. + */ +export function randomUUID(): string { + if ( + typeof crypto !== "undefined" && + typeof (crypto as { randomUUID?: () => string }).randomUUID === "function" + ) { + return (crypto as { randomUUID: () => string }).randomUUID(); + } + + // Fallback: manually construct UUID v4 from random bytes + const bytes = new Uint8Array(16); + if (typeof crypto !== "undefined" && typeof crypto.getRandomValues === "function") { + crypto.getRandomValues(bytes); + } else { + // No secure RNG available: refuse to generate a UUID with weak randomness + throw new Error( + "randomUUID() requires a secure crypto.getRandomValues implementation; " + + "no suitable crypto API was found in this environment." + ); + } + + // Set version bits (v4) and variant bits (RFC 4122) + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + + const hex = [...bytes].map((b) => b.toString(16).padStart(2, "0")); + return ( + hex.slice(0, 4).join("") + + "-" + hex.slice(4, 6).join("") + + "-" + hex.slice(6, 8).join("") + + "-" + hex.slice(8, 10).join("") + + "-" + hex.slice(10).join("") + ); +} diff --git a/lib/core/crypto/verify.ts b/lib/core/crypto/verify.ts new file mode 100644 index 0000000..e31b3d7 --- /dev/null +++ b/lib/core/crypto/verify.ts @@ -0,0 +1,50 @@ +import type { PublicKey, Signature } from "../types.js"; + +function base64ToBuffer(base64: Signature): ArrayBuffer | null { + try { + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + return bytes.buffer; + } catch { + return null; + } +} + +/** + * Verifies an Ed25519 signature produced by `signData`. + * + * Returns `true` if `signature` is a valid Ed25519 signature over `data` + * by the key encoded in `publicKey` (JWK JSON string). + * Returns `false` for any signature mismatch or malformed signature. + * Throws for structurally invalid public key (malformed JWK JSON). + * + * @param data - The original data that was signed (string or bytes). + * @param signature - Base64-encoded signature from `signData()`. + * @param publicKey - JWK JSON string from `KeyPair.publicKey`. + */ +export async function verifySignature( + data: string | ArrayBuffer, + signature: Signature, + publicKey: PublicKey, +): Promise { + const signatureBuffer = base64ToBuffer(signature); + if (signatureBuffer === null) { + return false; + } + + const bytes = typeof data === "string" ? new TextEncoder().encode(data) : data; + const publicKeyJwk = JSON.parse(publicKey) as JsonWebKey; + + const cryptoKey = await crypto.subtle.importKey( + "jwk", + publicKeyJwk, + { name: "Ed25519" } as Algorithm, + false, + ["verify"], + ); + + return crypto.subtle.verify("Ed25519", cryptoKey, signatureBuffer, bytes); +} diff --git a/lib/core/types.ts b/lib/core/types.ts new file mode 100644 index 0000000..679de81 --- /dev/null +++ b/lib/core/types.ts @@ -0,0 +1,218 @@ +// --------------------------------------------------------------------------- +// Primitive aliases +// --------------------------------------------------------------------------- + +export type Hash = string; // SHA-256 hex +export type Signature = string; // base64 or hex +export type PublicKey = string; // JWK or raw + +// --------------------------------------------------------------------------- +// Knowledge-hierarchy entities +// --------------------------------------------------------------------------- + +export interface Page { + pageId: Hash; // SHA-256(content) + content: string; // bounded by chunk policy derived from ModelProfile + embeddingOffset: number; // byte offset into the vector file + embeddingDim: number; // resolved embedding dimension from ModelProfile + + contentHash: Hash; // SHA-256(content) + vectorHash: Hash; // SHA-256(vector bytes) + + prevPageId?: Hash | null; + nextPageId?: Hash | null; + + creatorPubKey: PublicKey; + signature: Signature; + createdAt: string; // ISO timestamp +} + +export interface BookMetadata { + title?: string; + sourceUri?: string; + tags?: string[]; + extra?: Record; +} + +export interface Book { + bookId: Hash; // SHA-256(pageIds joined or Merkle root) + pageIds: Hash[]; + medoidPageId: Hash; // representative page selected by medoid statistic + meta: BookMetadata; +} + +export interface Volume { + volumeId: Hash; + bookIds: Hash[]; + prototypeOffsets: number[]; // byte offsets into the vector file + prototypeDim: number; // runtime policy dimension for this prototype tier + variance: number; +} + +export interface Shelf { + shelfId: Hash; + volumeIds: Hash[]; + routingPrototypeOffsets: number[]; // coarse prototype byte offsets + routingDim: number; +} + +export interface Edge { + fromPageId: Hash; + toPageId: Hash; + weight: number; // Hebbian weight + lastUpdatedAt: string; // ISO timestamp +} + +// --------------------------------------------------------------------------- +// Semantic nearest-neighbor graph +// --------------------------------------------------------------------------- + +/** A single directed proximity edge in the sparse semantic neighbor graph. */ +export interface SemanticNeighbor { + neighborPageId: Hash; + cosineSimilarity: number; // threshold is defined by runtime policy + distance: number; // 1 - cosineSimilarity (ready for TSP) +} + +/** Induced subgraph returned by BFS expansion of the semantic neighbor graph. */ +export interface SemanticNeighborSubgraph { + nodes: Hash[]; + edges: { from: Hash; to: Hash; distance: number }[]; +} + +// --------------------------------------------------------------------------- +// Hotpath / Williams Bound types +// --------------------------------------------------------------------------- + +/** Lightweight per-page activity metadata for salience computation. */ +export interface PageActivity { + pageId: Hash; + queryHitCount: number; // incremented on each query hit + lastQueryAt: string; // ISO timestamp of most recent query hit + communityId?: string; // set by Daydreamer label propagation +} + +/** Record for HOT membership — used in both RAM index and IndexedDB snapshot. */ +export interface HotpathEntry { + entityId: Hash; // pageId, bookId, volumeId, or shelfId + tier: "shelf" | "volume" | "book" | "page"; + salience: number; // salience value at last computation + communityId?: string; // community this entry counts against +} + +/** Per-tier slot budgets derived from H(t). */ +export interface TierQuotas { + shelf: number; + volume: number; + book: number; + page: number; +} + +/** Fractional quota ratios for each tier (must sum to 1.0). */ +export interface TierQuotaRatios { + shelf: number; + volume: number; + book: number; + page: number; +} + +/** Tunable weights for the salience formula: sigma = alpha*H_in + beta*R + gamma*Q. */ +export interface SalienceWeights { + alpha: number; // weight for Hebbian connectivity + beta: number; // weight for recency + gamma: number; // weight for query-hit frequency +} + +// --------------------------------------------------------------------------- +// Storage abstractions +// --------------------------------------------------------------------------- + +/** + * Append-only binary vector file. + * + * `offset` values are **byte offsets** inside the file so that mixed-dimension + * vectors (full embeddings vs. compressed prototypes) coexist without + * additional metadata. + */ +export interface VectorStore { + /** Appends `vector` to the file and returns its starting byte offset. */ + appendVector(vector: Float32Array): Promise; + + /** Reads `dim` floats starting at byte offset `offset`. */ + readVector(offset: number, dim: number): Promise; + + /** Reads multiple vectors by their individual byte offsets. */ + readVectors(offsets: number[], dim: number): Promise; +} + +/** + * Structured metadata store backed by IndexedDB (or any equivalent engine). + * + * Reverse-index helpers (`getBooksByPage`, `getVolumesByBook`, + * `getShelvesByVolume`) are maintained automatically on every `put*` call so + * callers never need to manage the mappings directly. + */ +export interface MetadataStore { + // --- Core CRUD --- + putPage(page: Page): Promise; + getPage(pageId: Hash): Promise; + /** Returns all pages in the store. Used for warm/cold fallbacks in query. */ + getAllPages(): Promise; + + putBook(book: Book): Promise; + getBook(bookId: Hash): Promise; + + putVolume(volume: Volume): Promise; + getVolume(volumeId: Hash): Promise; + /** Returns all volumes in the store. */ + getAllVolumes(): Promise; + /** + * Delete a volume record and clean up all reverse-index entries + * (`bookToVolume` for each book in the volume, and the `volumeToShelf` entry). + * Callers are responsible for removing the volume from any shelf's `volumeIds` + * list before calling this method. + */ + deleteVolume(volumeId: Hash): Promise; + + putShelf(shelf: Shelf): Promise; + getShelf(shelfId: Hash): Promise; + /** Returns all shelves in the store. */ + getAllShelves(): Promise; + + // --- Hebbian edges --- + putEdges(edges: Edge[]): Promise; + /** Remove a single directed edge. */ + deleteEdge(fromPageId: Hash, toPageId: Hash): Promise; + getNeighbors(pageId: Hash, limit?: number): Promise; + + // --- Reverse-index helpers --- + getBooksByPage(pageId: Hash): Promise; + getVolumesByBook(bookId: Hash): Promise; + getShelvesByVolume(volumeId: Hash): Promise; + + // --- Semantic neighbor radius index --- + putSemanticNeighbors(pageId: Hash, neighbors: SemanticNeighbor[]): Promise; + getSemanticNeighbors(pageId: Hash, maxDegree?: number): Promise; + + /** BFS expansion of the semantic neighbor subgraph up to `maxHops` levels deep. */ + getInducedNeighborSubgraph( + seedPageIds: Hash[], + maxHops: number, + ): Promise; + + // --- Dirty-volume recalc flags --- + needsNeighborRecalc(volumeId: Hash): Promise; + flagVolumeForNeighborRecalc(volumeId: Hash): Promise; + clearNeighborRecalcFlag(volumeId: Hash): Promise; + + // --- Hotpath index --- + putHotpathEntry(entry: HotpathEntry): Promise; + getHotpathEntries(tier?: HotpathEntry["tier"]): Promise; + removeHotpathEntry(entityId: Hash): Promise; + evictWeakest(tier: HotpathEntry["tier"], communityId?: string): Promise; + getResidentCount(): Promise; + + // --- Page activity --- + putPageActivity(activity: PageActivity): Promise; + getPageActivity(pageId: Hash): Promise; +} diff --git a/lib/cortex/KnowledgeGapDetector.ts b/lib/cortex/KnowledgeGapDetector.ts new file mode 100644 index 0000000..1ce983c --- /dev/null +++ b/lib/cortex/KnowledgeGapDetector.ts @@ -0,0 +1,66 @@ +import type { Hash } from "../core/types"; +import type { ModelProfile } from "../core/ModelProfile"; +import { hashText } from "../core/crypto/hash"; +import type { Metroid } from "./MetroidBuilder"; + +export interface KnowledgeGap { + queryText: string; + queryEmbedding: Float32Array; + knowledgeBoundary: Hash | null; + detectedAt: string; +} + +export interface CuriosityProbe { + probeId: Hash; + queryText: string; + queryEmbedding: Float32Array; + knowledgeBoundary: Hash | null; + mimeType: string; + modelUrn: string; + createdAt: string; +} + +/** + * Returns a KnowledgeGap when the metroid signals that m2 could not be found + * (i.e. the engine has no antithesis for this query). Returns null when the + * metroid is complete and no gap was detected. + */ +export async function detectKnowledgeGap( + queryText: string, + queryEmbedding: Float32Array, + metroid: Metroid, + // eslint-disable-next-line @typescript-eslint/no-unused-vars -- reserved for future model-aware gap categorisation + _modelProfile: ModelProfile, +): Promise { + if (!metroid.knowledgeGap) return null; + + return { + queryText, + queryEmbedding, + knowledgeBoundary: metroid.m1 !== "" ? metroid.m1 : null, + detectedAt: new Date().toISOString(), + }; +} + +/** + * Builds a serialisable CuriosityProbe from a detected KnowledgeGap. + * The probeId is the SHA-256 of (queryText + detectedAt) so it is + * deterministic for the same gap inputs. + */ +export async function buildCuriosityProbe( + gap: KnowledgeGap, + modelProfile: ModelProfile, + mimeType = "text/plain", +): Promise { + const probeId = await hashText(gap.queryText + gap.detectedAt); + + return { + probeId, + queryText: gap.queryText, + queryEmbedding: gap.queryEmbedding, + knowledgeBoundary: gap.knowledgeBoundary, + mimeType, + modelUrn: `urn:model:${modelProfile.modelId}`, + createdAt: new Date().toISOString(), + }; +} diff --git a/lib/cortex/MetroidBuilder.ts b/lib/cortex/MetroidBuilder.ts new file mode 100644 index 0000000..30640a7 --- /dev/null +++ b/lib/cortex/MetroidBuilder.ts @@ -0,0 +1,217 @@ +import type { Hash, VectorStore } from "../core/types"; +import type { ModelProfile } from "../core/ModelProfile"; + +export interface Metroid { + m1: Hash; + m2: Hash | null; + c: Float32Array | null; + knowledgeGap: boolean; +} + +export interface MetroidBuilderOptions { + modelProfile: ModelProfile; + vectorStore: VectorStore; +} + +/** Standard Matryoshka tier sizes in ascending order. */ +const MATRYOSHKA_TIERS = [32, 64, 128, 256, 512, 768, 1024, 2048] as const; + +function cosineSimilarity(a: Float32Array, b: Float32Array): number { + let dotProduct = 0; + let normA = 0; + let normB = 0; + const len = Math.min(a.length, b.length); + for (let i = 0; i < len; i++) { + dotProduct += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + if (normA === 0 || normB === 0) return 0; + return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)); +} + +function cosineDistance(a: Float32Array, b: Float32Array): number { + return 1 - cosineSimilarity(a, b); +} + +/** + * Returns the index of the medoid: the element that minimises total cosine + * distance to every other element in the set. + */ +function findMedoidIndex(embeddings: Float32Array[]): number { + if (embeddings.length === 1) return 0; + + let bestIdx = 0; + let bestTotal = Infinity; + + for (let i = 0; i < embeddings.length; i++) { + let total = 0; + for (let j = 0; j < embeddings.length; j++) { + if (i !== j) { + total += cosineDistance(embeddings[i], embeddings[j]); + } + } + if (total < bestTotal) { + bestTotal = total; + bestIdx = i; + } + } + + return bestIdx; +} + +interface CandidateEntry { + pageId: Hash; + embeddingOffset: number; + embeddingDim: number; +} + +interface CandidateWithEmbedding extends CandidateEntry { + embedding: Float32Array; +} + +/** + * Searches for m2 among `others` (candidates excluding m1) using the free + * dimensions starting at `protectedDim`. + * + * Returns the selected medoid candidate or `null` if no valid opposite set + * can be assembled. + */ +function searchM2( + others: CandidateWithEmbedding[], + m1Embedding: Float32Array, + protectedDim: number, +): CandidateWithEmbedding | null { + if (others.length === 0) return null; + + const m1Free = m1Embedding.slice(protectedDim); + + const scored = others.map((c) => { + const free = c.embedding.slice(protectedDim); + return { candidate: c, score: -cosineSimilarity(free, m1Free) }; + }); + + // Prefer candidates that are genuinely opposite (score >= 0). + let oppositeSet = scored.filter((s) => s.score >= 0); + + // Fall back to the top 50% when the genuine-opposite set is too small. + if (oppositeSet.length < 2) { + const byScore = [...scored].sort((a, b) => b.score - a.score); + const topHalf = Math.max(1, Math.ceil(byScore.length / 2)); + oppositeSet = byScore.slice(0, topHalf); + } + + if (oppositeSet.length === 0) return null; + + const medoidIdx = findMedoidIndex(oppositeSet.map((s) => s.candidate.embedding.slice(protectedDim))); + return oppositeSet[medoidIdx].candidate; +} + +/** + * Builds the dialectical probe (Metroid) for a given query embedding and a + * ranked list of candidate memory nodes. + * + * Step overview + * 1. Select m1 (thesis): the candidate with highest cosine similarity to the query. + * 2. Select m2 (antithesis): the medoid of the cosine-opposite set in free dims. + * Uses Matryoshka dimensional unwinding when the initial tier yields no m2. + * 3. Compute centroid c (synthesis): protected dims copied from m1, free dims + * averaged between m1 and m2. + */ +export async function buildMetroid( + queryEmbedding: Float32Array, + candidateMedoids: Array<{ pageId: Hash; embeddingOffset: number; embeddingDim: number }>, + options: MetroidBuilderOptions, +): Promise { + const { modelProfile, vectorStore } = options; + + if (candidateMedoids.length === 0) { + return { m1: "", m2: null, c: null, knowledgeGap: true }; + } + + // Load all candidate embeddings in one pass. + const candidates: CandidateWithEmbedding[] = await Promise.all( + candidateMedoids.map(async (cand) => ({ + ...cand, + embedding: await vectorStore.readVector(cand.embeddingOffset, cand.embeddingDim), + })), + ); + + // Select m1: highest cosine similarity to the query. + let m1Candidate = candidates[0]; + let m1Score = cosineSimilarity(queryEmbedding, candidates[0].embedding); + + for (let i = 1; i < candidates.length; i++) { + const score = cosineSimilarity(queryEmbedding, candidates[i].embedding); + if (score > m1Score) { + m1Score = score; + m1Candidate = candidates[i]; + } + } + + const protectedDim = modelProfile.matryoshkaProtectedDim; + + if (protectedDim === undefined) { + // Non-Matryoshka model: antithesis search is impossible. + return { m1: m1Candidate.pageId, m2: null, c: null, knowledgeGap: true }; + } + + const others = candidates.filter((c) => c.pageId !== m1Candidate.pageId); + + // --- Matryoshka dimensional unwinding --- + // Start at modelProfile.matryoshkaProtectedDim. If m2 not found, progressively + // shrink the protected boundary (expand the free-dimension search region). + + const startingTierIndex = MATRYOSHKA_TIERS.indexOf( + protectedDim as (typeof MATRYOSHKA_TIERS)[number], + ); + + // Build the list of tier boundaries to attempt, from the configured value + // down to the smallest tier (expanding the free region at each step). + const tierBoundaries: number[] = []; + if (startingTierIndex !== -1) { + for (let i = startingTierIndex; i >= 0; i--) { + tierBoundaries.push(MATRYOSHKA_TIERS[i]); + } + } else { + // protectedDim is not a standard tier; try it as-is plus any smaller standard tiers. + tierBoundaries.push(protectedDim); + for (const t of [...MATRYOSHKA_TIERS].reverse()) { + if (t < protectedDim) tierBoundaries.push(t); + } + } + + let m2Candidate: CandidateWithEmbedding | null = null; + let usedProtectedDim = protectedDim; + + for (const tierBoundary of tierBoundaries) { + const found = searchM2(others, m1Candidate.embedding, tierBoundary); + if (found !== null) { + m2Candidate = found; + usedProtectedDim = tierBoundary; + break; + } + } + + if (m2Candidate === null) { + return { m1: m1Candidate.pageId, m2: null, c: null, knowledgeGap: true }; + } + + // Compute frozen synthesis centroid c. + const fullDim = m1Candidate.embedding.length; + const c = new Float32Array(fullDim); + + for (let i = 0; i < usedProtectedDim; i++) { + c[i] = m1Candidate.embedding[i]; + } + for (let i = usedProtectedDim; i < fullDim; i++) { + c[i] = (m1Candidate.embedding[i] + m2Candidate.embedding[i]) / 2; + } + + return { + m1: m1Candidate.pageId, + m2: m2Candidate.pageId, + c, + knowledgeGap: false, + }; +} diff --git a/lib/cortex/OpenTSPSolver.ts b/lib/cortex/OpenTSPSolver.ts new file mode 100644 index 0000000..257ad80 --- /dev/null +++ b/lib/cortex/OpenTSPSolver.ts @@ -0,0 +1,62 @@ +import type { Hash, SemanticNeighborSubgraph } from "../core/types"; + +/** + * Greedy nearest-neighbor open-path TSP heuristic. + * + * Visits every node in the subgraph exactly once, starting from the + * lexicographically smallest node ID for determinism. At each step the + * algorithm advances to the unvisited node nearest to the current one + * (using edge distance). Ties are broken lexicographically. Missing edges + * are treated as having distance Infinity. + */ +export function solveOpenTSP(subgraph: SemanticNeighborSubgraph): Hash[] { + const { nodes, edges } = subgraph; + if (nodes.length === 0) return []; + + // Build undirected adjacency map: node → (neighbor → distance). + const adj = new Map>(); + for (const node of nodes) { + adj.set(node, new Map()); + } + for (const edge of edges) { + const fromMap = adj.get(edge.from); + const toMap = adj.get(edge.to); + if (fromMap !== undefined) fromMap.set(edge.to, edge.distance); + if (toMap !== undefined) toMap.set(edge.from, edge.distance); + } + + // Pre-sort once so lexicographic tiebreaking is O(1) per step. + const sorted = [...nodes].sort(); + + const visited = new Set(); + const path: Hash[] = []; + let current = sorted[0]; + + while (path.length < nodes.length) { + visited.add(current); + path.push(current); + + if (path.length === nodes.length) break; + + const neighbors = adj.get(current)!; + let bestNode: Hash | undefined; + let bestDist = Infinity; + + for (const node of sorted) { + if (visited.has(node)) continue; + const dist = neighbors.get(node) ?? Infinity; + if ( + dist < bestDist || + (dist === bestDist && (bestNode === undefined || node < bestNode)) + ) { + bestDist = dist; + bestNode = node; + } + } + + // bestNode is always defined here because at least one unvisited node remains. + current = bestNode!; + } + + return path; +} diff --git a/lib/cortex/Query.ts b/lib/cortex/Query.ts new file mode 100644 index 0000000..488a1ba --- /dev/null +++ b/lib/cortex/Query.ts @@ -0,0 +1,170 @@ +import type { ModelProfile } from "../core/ModelProfile"; +import type { Hash, MetadataStore, Page, VectorStore } from "../core/types"; +import type { EmbeddingRunner } from "../embeddings/EmbeddingRunner"; +import { runPromotionSweep } from "../core/SalienceEngine"; +import { computeSubgraphBounds } from "../core/HotpathPolicy"; +import type { QueryResult } from "./QueryResult"; +import { rankPages, spillToWarm } from "./Ranking"; +import { buildMetroid } from "./MetroidBuilder"; +import { detectKnowledgeGap } from "./KnowledgeGapDetector"; +import { solveOpenTSP } from "./OpenTSPSolver"; + +export interface QueryOptions { + modelProfile: ModelProfile; + embeddingRunner: EmbeddingRunner; + vectorStore: VectorStore; + metadataStore: MetadataStore; + topK?: number; + /** + * Maximum BFS depth for semantic neighbor subgraph expansion. + * + * When omitted, a dynamic Williams-derived value is computed from the + * corpus size via `computeSubgraphBounds(t)`. Providing an explicit value + * overrides the dynamic bound (useful for tests and controlled experiments). + */ + maxHops?: number; +} + +export async function query( + queryText: string, + options: QueryOptions, +): Promise { + const { + modelProfile, + embeddingRunner, + vectorStore, + metadataStore, + topK = 10, + } = options; + const nowIso = new Date().toISOString(); + + const embeddings = await embeddingRunner.embed([queryText]); + if (embeddings.length !== 1) { + throw new Error("Embedding provider returned unexpected number of embeddings"); + } + const queryEmbedding = embeddings[0]; + + const rankingOptions = { vectorStore, metadataStore }; + + // --- HOT path: score resident pages --- + const hotpathEntries = await metadataStore.getHotpathEntries("page"); + const hotpathIds = hotpathEntries.map((e) => e.entityId); + + const hotResults = await rankPages(queryEmbedding, hotpathIds, topK, rankingOptions); + const seenIds = new Set(hotResults.map((r) => r.id)); + + // --- Warm spill: fill up to topK if hot path is insufficient --- + let warmResults: Array<{ id: Hash; score: number }> = []; + if (hotResults.length < topK) { + const allWarm = await spillToWarm("page", queryEmbedding, topK, rankingOptions); + warmResults = allWarm.filter((r) => !seenIds.has(r.id)); + } + + // Merge, deduplicate, sort, and slice to topK + const merged = [...hotResults, ...warmResults]; + merged.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)); + const topResults = merged.slice(0, topK); + + // Load Page objects for the top results + const topPages = ( + await Promise.all(topResults.map((r) => metadataStore.getPage(r.id))) + ).filter((p): p is Page => p !== undefined); + + const topScores = topResults + .filter((r) => topPages.some((p) => p.pageId === r.id)) + .map((r) => r.score); + + // --- MetroidBuilder: build dialectical probe --- + // Candidates: hotpath book medoid pages + hotpath pages themselves + const hotpathBookEntries = await metadataStore.getHotpathEntries("book"); + const bookCandidates = ( + await Promise.all( + hotpathBookEntries.map(async (e) => { + const book = await metadataStore.getBook(e.entityId); + if (!book) return null; + const medoidPage = await metadataStore.getPage(book.medoidPageId); + if (!medoidPage) return null; + return { + pageId: medoidPage.pageId, + embeddingOffset: medoidPage.embeddingOffset, + embeddingDim: medoidPage.embeddingDim, + }; + }), + ) + ).filter((c): c is NonNullable => c !== null); + + const pageCandidates = topPages.map((p) => ({ + pageId: p.pageId, + embeddingOffset: p.embeddingOffset, + embeddingDim: p.embeddingDim, + })); + + // Deduplicate candidates by pageId + const candidateMap = new Map(); + for (const c of [...bookCandidates, ...pageCandidates]) { + candidateMap.set(c.pageId, c); + } + const metroidCandidates = [...candidateMap.values()]; + + const metroid = await buildMetroid(queryEmbedding, metroidCandidates, { + modelProfile, + vectorStore, + }); + + // --- KnowledgeGapDetector --- + const knowledgeGap = await detectKnowledgeGap( + queryText, + queryEmbedding, + metroid, + modelProfile, + ); + + // --- Subgraph expansion --- + // Use dynamic Williams-derived bounds unless the caller has pinned an + // explicit maxHops value. Only load all pages when we actually need to + // compute bounds — skip the full-page scan on the hot path when maxHops is + // already known. + const topPageIds = topPages.map((p) => p.pageId); + let effectiveMaxHops: number; + if (options.maxHops !== undefined) { + effectiveMaxHops = options.maxHops; + } else { + const allPages = await metadataStore.getAllPages(); + effectiveMaxHops = computeSubgraphBounds(allPages.length).maxHops; + } + const subgraph = await metadataStore.getInducedNeighborSubgraph(topPageIds, effectiveMaxHops); + + // --- TSP coherence path --- + const coherencePath = solveOpenTSP(subgraph); + + // --- Update activity for returned pages --- + await Promise.all( + topPages.map(async (page) => { + const activity = await metadataStore.getPageActivity(page.pageId); + await metadataStore.putPageActivity({ + pageId: page.pageId, + queryHitCount: (activity?.queryHitCount ?? 0) + 1, + lastQueryAt: nowIso, + communityId: activity?.communityId, + }); + }), + ); + + // --- Promotion sweep --- + await runPromotionSweep(topPageIds, metadataStore); + + return { + pages: topPages, + scores: topScores, + coherencePath, + metroid, + knowledgeGap, + metadata: { + queryText, + topK, + returned: topPages.length, + timestamp: nowIso, + modelId: modelProfile.modelId, + }, + }; +} diff --git a/lib/cortex/QueryResult.ts b/lib/cortex/QueryResult.ts new file mode 100644 index 0000000..8d7406e --- /dev/null +++ b/lib/cortex/QueryResult.ts @@ -0,0 +1,12 @@ +import type { Hash, Page } from "../core/types"; +import type { Metroid } from "./MetroidBuilder"; +import type { KnowledgeGap } from "./KnowledgeGapDetector"; + +export interface QueryResult { + pages: Page[]; + scores: number[]; + coherencePath: Hash[]; + metroid: Metroid | null; + knowledgeGap: KnowledgeGap | null; + metadata: Record; +} diff --git a/lib/cortex/Ranking.ts b/lib/cortex/Ranking.ts new file mode 100644 index 0000000..f0d9f9f --- /dev/null +++ b/lib/cortex/Ranking.ts @@ -0,0 +1,156 @@ +import type { Hash, MetadataStore, VectorStore } from "../core/types"; +import type { VectorBackend } from "../VectorBackend"; + +export interface RankingOptions { + vectorStore: VectorStore; + metadataStore: MetadataStore; + vectorBackend?: VectorBackend; +} + +function cosineSimilarity(a: Float32Array, b: Float32Array): number { + let dotProduct = 0; + let normA = 0; + let normB = 0; + const len = Math.min(a.length, b.length); + for (let i = 0; i < len; i++) { + dotProduct += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + if (normA === 0 || normB === 0) return 0; + return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)); +} + +function pickTopK( + scored: Array<{ id: Hash; score: number }>, + k: number, +): Array<{ id: Hash; score: number }> { + scored.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)); + return scored.slice(0, k); +} + +/** + * Ranks shelves by cosine similarity of their routing prototype to the query. + * Uses routingPrototypeOffsets[0] as the representative vector. + */ +export async function rankShelves( + queryEmbedding: Float32Array, + residentShelfIds: Hash[], + topK: number, + options: RankingOptions, +): Promise> { + if (residentShelfIds.length === 0) return []; + + const { vectorStore, metadataStore } = options; + const scored: Array<{ id: Hash; score: number }> = []; + + for (const shelfId of residentShelfIds) { + const shelf = await metadataStore.getShelf(shelfId); + if (!shelf || shelf.routingPrototypeOffsets.length === 0) continue; + const vec = await vectorStore.readVector(shelf.routingPrototypeOffsets[0], shelf.routingDim); + scored.push({ id: shelfId, score: cosineSimilarity(queryEmbedding, vec) }); + } + + return pickTopK(scored, topK); +} + +/** + * Ranks volumes by cosine similarity of their first prototype to the query. + * Uses prototypeOffsets[0] as the representative vector. + */ +export async function rankVolumes( + queryEmbedding: Float32Array, + residentVolumeIds: Hash[], + topK: number, + options: RankingOptions, +): Promise> { + if (residentVolumeIds.length === 0) return []; + + const { vectorStore, metadataStore } = options; + const scored: Array<{ id: Hash; score: number }> = []; + + for (const volumeId of residentVolumeIds) { + const volume = await metadataStore.getVolume(volumeId); + if (!volume || volume.prototypeOffsets.length === 0) continue; + const vec = await vectorStore.readVector(volume.prototypeOffsets[0], volume.prototypeDim); + scored.push({ id: volumeId, score: cosineSimilarity(queryEmbedding, vec) }); + } + + return pickTopK(scored, topK); +} + +/** + * Ranks books by cosine similarity of their medoid page embedding to the query. + */ +export async function rankBooks( + queryEmbedding: Float32Array, + residentBookIds: Hash[], + topK: number, + options: RankingOptions, +): Promise> { + if (residentBookIds.length === 0) return []; + + const { vectorStore, metadataStore } = options; + const scored: Array<{ id: Hash; score: number }> = []; + + for (const bookId of residentBookIds) { + const book = await metadataStore.getBook(bookId); + if (!book) continue; + const medoidPage = await metadataStore.getPage(book.medoidPageId); + if (!medoidPage) continue; + const vec = await vectorStore.readVector(medoidPage.embeddingOffset, medoidPage.embeddingDim); + scored.push({ id: bookId, score: cosineSimilarity(queryEmbedding, vec) }); + } + + return pickTopK(scored, topK); +} + +/** + * Ranks pages by cosine similarity of their embedding to the query. + */ +export async function rankPages( + queryEmbedding: Float32Array, + residentPageIds: Hash[], + topK: number, + options: RankingOptions, +): Promise> { + if (residentPageIds.length === 0) return []; + + const { vectorStore, metadataStore } = options; + const scored: Array<{ id: Hash; score: number }> = []; + + for (const pageId of residentPageIds) { + const page = await metadataStore.getPage(pageId); + if (!page) continue; + const vec = await vectorStore.readVector(page.embeddingOffset, page.embeddingDim); + scored.push({ id: pageId, score: cosineSimilarity(queryEmbedding, vec) }); + } + + return pickTopK(scored, topK); +} + +/** + * Spills to the warm tier when the resident set provides insufficient coverage. + * For "page": scores all pages in the store. + * For other tiers: returns [] (warm spill is only implemented for pages at this stage). + */ +export async function spillToWarm( + tier: "shelf" | "volume" | "book" | "page", + queryEmbedding: Float32Array, + topK: number, + options: RankingOptions, +): Promise> { + if (tier !== "page") return []; + + const { vectorStore, metadataStore } = options; + const allPages = await metadataStore.getAllPages(); + if (allPages.length === 0) return []; + + const scored: Array<{ id: Hash; score: number }> = []; + for (const page of allPages) { + const vec = await vectorStore.readVector(page.embeddingOffset, page.embeddingDim); + scored.push({ id: page.pageId, score: cosineSimilarity(queryEmbedding, vec) }); + } + + return pickTopK(scored, topK); +} diff --git a/lib/daydreamer/ClusterStability.ts b/lib/daydreamer/ClusterStability.ts new file mode 100644 index 0000000..e1d587d --- /dev/null +++ b/lib/daydreamer/ClusterStability.ts @@ -0,0 +1,680 @@ +// --------------------------------------------------------------------------- +// ClusterStability — Community detection via label propagation (P2-F) and +// volume split/merge for balanced cluster maintenance (P2-F3) +// --------------------------------------------------------------------------- +// +// Assigns community labels to pages by running lightweight label propagation +// on the semantic neighbor graph. Labels are stored in +// PageActivity.communityId and propagate into SalienceEngine community quotas. +// +// Label propagation terminates when assignments stabilise (no label changes) +// or a maximum iteration limit is reached. +// +// The Daydreamer background worker also calls ClusterStability periodically to +// detect and fix unstable volumes: +// - HIGH-VARIANCE volumes are split into two balanced sub-volumes. +// - LOW-COUNT volumes are merged into the nearest neighbour volume. +// - Community labels are updated after structural changes. +// --------------------------------------------------------------------------- + +import { hashText } from "../core/crypto/hash"; +import type { + Book, + Hash, + MetadataStore, + PageActivity, + Volume, +} from "../core/types"; + +// --------------------------------------------------------------------------- +// Label propagation options +// --------------------------------------------------------------------------- + +export interface LabelPropagationOptions { + metadataStore: MetadataStore; + /** Maximum number of label propagation iterations. Default: 20. */ + maxIterations?: number; +} + +export interface LabelPropagationResult { + /** Number of iterations until convergence (or maxIterations). */ + iterations: number; + /** True if the algorithm converged before hitting maxIterations. */ + converged: boolean; + /** Map from pageId to assigned communityId. */ + communityMap: Map; +} + +// --------------------------------------------------------------------------- +// Label propagation +// --------------------------------------------------------------------------- + +/** + * Run one pass of label propagation over all pages. + * + * Each node adopts the most frequent label among its Metroid neighbors. + * Ties are broken deterministically by choosing the lexicographically + * smallest label (consistent across runs and nodes). + * + * Returns true if any label changed during this pass. + */ +async function propagationPass( + pageIds: Hash[], + labels: Map, + metadataStore: MetadataStore, +): Promise { + let changed = false; + + // Shuffle-equivalent deterministic ordering: sort by pageId for reproducibility + const sorted = [...pageIds].sort(); + + for (const pageId of sorted) { + const neighbors = await metadataStore.getSemanticNeighbors(pageId); + if (neighbors.length === 0) continue; + + // Count neighbor labels + const counts = new Map(); + for (const n of neighbors) { + const label = labels.get(n.neighborPageId) ?? n.neighborPageId; + counts.set(label, (counts.get(label) ?? 0) + 1); + } + + // Find the most frequent label (tie-break: lexicographically smallest) + let bestLabel: string | undefined; + let bestCount = 0; + for (const [label, count] of counts) { + if ( + count > bestCount || + (count === bestCount && bestLabel !== undefined && label < bestLabel) + ) { + bestLabel = label; + bestCount = count; + } + } + + if (bestLabel !== undefined && labels.get(pageId) !== bestLabel) { + labels.set(pageId, bestLabel); + changed = true; + } + } + + return changed; +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/** + * Assign community labels to all pages via label propagation on the + * Metroid (semantic) neighbor graph. + * + * Initial labels: each page is its own community (pageId as initial label). + * Each iteration: every node adopts the most frequent label among neighbors. + * Convergence: no label changed in the most recent pass. + * + * After convergence, persists all community labels via + * `MetadataStore.putPageActivity`. + */ +export async function runLabelPropagation( + options: LabelPropagationOptions, +): Promise { + const { + metadataStore, + maxIterations = 20, + } = options; + + const allPages = await metadataStore.getAllPages(); + if (allPages.length === 0) { + return { iterations: 0, converged: true, communityMap: new Map() }; + } + + const pageIds = allPages.map((p) => p.pageId); + + // Initialise: each page is its own community + const labels = new Map(); + for (const id of pageIds) { + labels.set(id, id); + } + + let iterations = 0; + let converged = false; + + for (let iter = 0; iter < maxIterations; iter++) { + iterations++; + const changed = await propagationPass(pageIds, labels, metadataStore); + if (!changed) { + converged = true; + break; + } + } + + // Persist community labels to PageActivity + for (const pageId of pageIds) { + const communityId = labels.get(pageId) ?? pageId; + const existing = await metadataStore.getPageActivity(pageId); + const activity: PageActivity = { + pageId, + queryHitCount: existing?.queryHitCount ?? 0, + lastQueryAt: existing?.lastQueryAt ?? new Date(0).toISOString(), + communityId, + }; + await metadataStore.putPageActivity(activity); + } + + return { iterations, converged, communityMap: new Map(labels) }; +} + +/** + * Detect whether a community should be split (too large relative to graph). + * + * A community is considered oversized when it holds more than + * `maxCommunityFraction` of all pages. + * + * Returns the set of community IDs that exceed the threshold. + */ +export function detectOversizedCommunities( + communityMap: Map, + maxCommunityFraction = 0.5, +): Set { + const total = communityMap.size; + if (total === 0) return new Set(); + + const counts = new Map(); + for (const label of communityMap.values()) { + counts.set(label, (counts.get(label) ?? 0) + 1); + } + + const oversized = new Set(); + for (const [label, count] of counts) { + if (count / total > maxCommunityFraction) { + oversized.add(label); + } + } + return oversized; +} + +/** + * Detect communities that no longer have any members (empty communities). + * + * These communities should release their hotpath quota slots back to the + * page-tier budget. + * + * @param knownCommunities Full set of community IDs that have quota allocations. + * @param activeCommunities Community IDs currently assigned to at least one page. + */ +export function detectEmptyCommunities( + knownCommunities: Set, + activeCommunities: Set, +): Set { + const empty = new Set(); + for (const id of knownCommunities) { + if (!activeCommunities.has(id)) { + empty.add(id); + } + } + return empty; +} + +// --------------------------------------------------------------------------- +// ClusterStability class — Volume split/merge configuration +// --------------------------------------------------------------------------- + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +export interface ClusterStabilityOptions { + /** + * Volume variance threshold above which a volume is considered unstable and + * will be split. + * Defaults to 0.5. + */ + varianceThreshold?: number; + + /** + * Minimum number of books a volume must contain. Volumes with fewer books + * than this will be merged with a neighbour. + * Defaults to 2. + */ + minBooksPerVolume?: number; + + /** + * Maximum split iterations for the K-means step. + * Defaults to 10. + */ + maxKmeansIterations?: number; +} + +const DEFAULT_VARIANCE_THRESHOLD = 0.5; +const DEFAULT_MIN_BOOKS_PER_VOLUME = 2; +const DEFAULT_MAX_KMEANS_ITERATIONS = 10; + +// --------------------------------------------------------------------------- +// Result types +// --------------------------------------------------------------------------- + +export interface ClusterStabilityResult { + /** Number of volumes split into two sub-volumes. */ + splitCount: number; + + /** Number of volumes merged into a neighbour. */ + mergeCount: number; + + /** Number of PageActivity community-label updates written. */ + communityUpdates: number; + + /** ISO timestamp when the stability run completed. */ + completedAt: string; +} + +// --------------------------------------------------------------------------- +// Internal types +// --------------------------------------------------------------------------- + +// --------------------------------------------------------------------------- +// ClusterStability +// --------------------------------------------------------------------------- + +export class ClusterStability { + private readonly varianceThreshold: number; + private readonly minBooksPerVolume: number; + private readonly maxKmeansIterations: number; + + constructor(options: ClusterStabilityOptions = {}) { + this.varianceThreshold = + options.varianceThreshold ?? DEFAULT_VARIANCE_THRESHOLD; + this.minBooksPerVolume = + options.minBooksPerVolume ?? DEFAULT_MIN_BOOKS_PER_VOLUME; + this.maxKmeansIterations = + options.maxKmeansIterations ?? DEFAULT_MAX_KMEANS_ITERATIONS; + } + + /** + * Run one stability pass over all volumes in the metadata store. + * + * Scans for unstable (high-variance) volumes and undersized volumes, then + * applies the appropriate structural fix and updates community labels. + */ + async run(metadataStore: MetadataStore): Promise { + // Collect all volumes (we scan through shelves) + const shelves = await this.collectAllShelves(metadataStore); + const allVolumeIds = shelves.flatMap((s) => s.volumeIds); + + const volumes = ( + await Promise.all(allVolumeIds.map((id) => metadataStore.getVolume(id))) + ).filter((v): v is Volume => v !== undefined); + + let splitCount = 0; + let mergeCount = 0; + let communityUpdates = 0; + + // --- Pass 1: split high-variance volumes --- + for (const volume of volumes) { + if ( + volume.variance > this.varianceThreshold && + volume.bookIds.length >= 2 + ) { + const splits = await this.splitVolume(volume, metadataStore); + if (splits !== null) { + splitCount++; + communityUpdates += await this.updateCommunityLabels( + splits, + metadataStore, + ); + // Replace the old volume in shelves with the two new sub-volumes, + // then delete the orphan volume record and its reverse-index entries. + await this.replaceVolumeInShelves( + volume.volumeId, + splits, + metadataStore, + ); + await metadataStore.deleteVolume(volume.volumeId); + } + } + } + + // --- Pass 2: merge undersized volumes --- + // Re-read volumes after splits to pick up any IDs that may have changed. + // Also include newly created split volumes from Pass 1 via a fresh shelf scan. + const allShelves2 = await this.collectAllShelves(metadataStore); + const allVolumeIds2 = allShelves2.flatMap((s) => s.volumeIds); + const allVolumesNow = ( + await Promise.all(allVolumeIds2.map((id) => metadataStore.getVolume(id))) + ).filter((v): v is Volume => v !== undefined); + + // Filter to undersized volumes (skip volumes we just created by splitting) + const undersized = allVolumesNow.filter( + (v) => v.bookIds.length < this.minBooksPerVolume, + ); + + const merged = new Set(); + + for (const small of undersized) { + if (merged.has(small.volumeId)) continue; + + const neighbour = this.findNearestNeighbour( + small, + allVolumesNow.filter( + (v) => + v.volumeId !== small.volumeId && !merged.has(v.volumeId), + ), + ); + + if (neighbour === null) continue; + + const mergedVolume = await this.mergeVolumes( + small, + neighbour, + metadataStore, + ); + + merged.add(small.volumeId); + merged.add(neighbour.volumeId); + mergeCount++; + communityUpdates += await this.updateCommunityLabels( + [mergedVolume], + metadataStore, + ); + // Replace the consumed volumes in shelves with the merged volume, + // then delete their orphan records and reverse-index entries. + await this.replaceVolumeInShelves( + small.volumeId, + [mergedVolume], + metadataStore, + ); + await this.replaceVolumeInShelves( + neighbour.volumeId, + [], + metadataStore, + ); + await metadataStore.deleteVolume(small.volumeId); + await metadataStore.deleteVolume(neighbour.volumeId); + } + + return { + splitCount, + mergeCount, + communityUpdates, + completedAt: new Date().toISOString(), + }; + } + + // --------------------------------------------------------------------------- + // Split logic + // --------------------------------------------------------------------------- + + /** + * Split a high-variance volume into two sub-volumes using K-means (K=2). + * + * Returns the two new volumes, or `null` if the split cannot be performed + * (e.g. insufficient books with resolvable vectors). + */ + private async splitVolume( + volume: Volume, + metadataStore: MetadataStore, + ): Promise<[Volume, Volume] | null> { + const books = ( + await Promise.all(volume.bookIds.map((id) => metadataStore.getBook(id))) + ).filter((b): b is Book => b !== undefined); + + if (books.length < 2) return null; + + // Use only the medoid page vector as representative for each book. + // For simplicity we use the first prototype offset of the parent volume + // and the book's position (index) as a deterministic pseudo-distance. + // A full implementation would read actual medoid embeddings via VectorStore. + const assignments = this.kmeansAssign(books); + if (assignments === null) return null; + + const [groupA, groupB] = assignments; + + const volumeA = await this.buildSubVolume(groupA, volume); + const volumeB = await this.buildSubVolume(groupB, volume); + + await metadataStore.putVolume(volumeA); + await metadataStore.putVolume(volumeB); + + return [volumeA, volumeB]; + } + + /** + * Assign books to two clusters using a simple K-means initialisation: + * centroid A = first half by index, centroid B = second half. + * + * Returns `null` when it is not possible to form two non-empty clusters. + * + * The "distance" used here is the index difference (as a stable proxy when + * real vectors are not loaded), which produces a balanced split without + * requiring a live VectorStore. A production pass would replace this with + * actual cosine distances between medoid embeddings. + * + * Precomputes a `bookId → index` map so each iteration is O(n) rather than + * O(n²) (avoids repeated Array.indexOf calls inside the inner loop). + */ + private kmeansAssign(books: Book[]): [Book[], Book[]] | null { + if (books.length < 2) return null; + + const n = books.length; + // Precompute index map to avoid O(n²) indexOf calls + const indexMap = new Map( + books.map((b, i) => [b.bookId, i]), + ); + + // Centroid A = first half, centroid B = second half (index-based split) + const splitPoint = Math.ceil(n / 2); + + let groupA = books.slice(0, splitPoint); + let groupB = books.slice(splitPoint); + + if (groupA.length === 0 || groupB.length === 0) return null; + + // Run up to maxKmeansIterations assignment cycles using index centroids + for (let iter = 0; iter < this.maxKmeansIterations; iter++) { + const centroidA = this.indexCentroid(groupA, indexMap); + const centroidB = this.indexCentroid(groupB, indexMap); + + const newA: Book[] = []; + const newB: Book[] = []; + + for (const book of books) { + const idx = indexMap.get(book.bookId) ?? 0; + const distA = Math.abs(idx - centroidA); + const distB = Math.abs(idx - centroidB); + if (distA <= distB) { + newA.push(book); + } else { + newB.push(book); + } + } + + // Ensure neither cluster becomes empty + if (newA.length === 0) { + newA.push(newB.splice(0, 1)[0]); + } + if (newB.length === 0) { + newB.push(newA.splice(newA.length - 1, 1)[0]); + } + + const converged = + newA.length === groupA.length && + newA.every((b, i) => b.bookId === groupA[i]?.bookId); + + groupA = newA; + groupB = newB; + + if (converged) break; + } + + return [groupA, groupB]; + } + + /** Compute the mean index of a group using the precomputed index map. */ + private indexCentroid( + group: Book[], + indexMap: Map, + ): number { + const sum = group.reduce( + (acc, b) => acc + (indexMap.get(b.bookId) ?? 0), + 0, + ); + return sum / group.length; + } + + private async buildSubVolume( + books: Book[], + parent: Volume, + ): Promise { + const bookIds = books.map((b) => b.bookId); + const seed = `split:${parent.volumeId}:${bookIds.join(",")}`; + const volumeId = await hashText(seed); + + // Variance is approximated as half the parent's variance for each child. + // A production pass would recompute from actual embeddings. + const variance = parent.variance / 2; + + return { + volumeId, + bookIds, + prototypeOffsets: [...parent.prototypeOffsets], + prototypeDim: parent.prototypeDim, + variance, + }; + } + + // --------------------------------------------------------------------------- + // Merge logic + // --------------------------------------------------------------------------- + + private findNearestNeighbour( + target: Volume, + candidates: Volume[], + ): Volume | null { + if (candidates.length === 0) return null; + + // Use the count of shared books as a similarity proxy. + // A production pass would compare medoid embeddings. + let best = candidates[0]; + let bestShared = this.sharedBookCount(target, best); + + for (let i = 1; i < candidates.length; i++) { + const shared = this.sharedBookCount(target, candidates[i]); + if (shared > bestShared) { + best = candidates[i]; + bestShared = shared; + } + } + + return best; + } + + private sharedBookCount(a: Volume, b: Volume): number { + const setA = new Set(a.bookIds); + return b.bookIds.filter((id) => setA.has(id)).length; + } + + private async mergeVolumes( + a: Volume, + b: Volume, + metadataStore: MetadataStore, + ): Promise { + const bookIds = [...new Set([...a.bookIds, ...b.bookIds])]; + const seed = `merge:${a.volumeId}:${b.volumeId}`; + const volumeId = await hashText(seed); + + // Average the variance of the two merged volumes + const variance = (a.variance + b.variance) / 2; + + const merged: Volume = { + volumeId, + bookIds, + prototypeOffsets: [...a.prototypeOffsets, ...b.prototypeOffsets], + prototypeDim: a.prototypeDim, + variance, + }; + + await metadataStore.putVolume(merged); + return merged; + } + + // --------------------------------------------------------------------------- + // Community label updates + // --------------------------------------------------------------------------- + + /** + * After a structural change (split or merge), update the `communityId` field + * on each affected page's `PageActivity` record. + * + * The community ID is set to the new volume's `volumeId` so that the + * SalienceEngine can bucket promotions correctly. + * + * @returns The number of PageActivity records updated. + */ + private async updateCommunityLabels( + volumes: Volume[], + metadataStore: MetadataStore, + ): Promise { + let updates = 0; + + for (const volume of volumes) { + const books = ( + await Promise.all( + volume.bookIds.map((id) => metadataStore.getBook(id)), + ) + ).filter((b): b is Book => b !== undefined); + + for (const book of books) { + for (const pageId of book.pageIds) { + const activity = await metadataStore.getPageActivity(pageId); + const updated: PageActivity = { + pageId, + queryHitCount: activity?.queryHitCount ?? 0, + lastQueryAt: + activity?.lastQueryAt ?? new Date().toISOString(), + communityId: volume.volumeId, + }; + await metadataStore.putPageActivity(updated); + updates++; + } + } + } + + return updates; + } + + // --------------------------------------------------------------------------- + // Shelf update helpers + // --------------------------------------------------------------------------- + + /** + * Replace `oldVolumeId` in every shelf that references it with the IDs of + * `replacements`. Passing an empty `replacements` array removes the old + * volume from the shelf without adding a substitute. + */ + private async replaceVolumeInShelves( + oldVolumeId: Hash, + replacements: Volume[], + metadataStore: MetadataStore, + ): Promise { + const shelves = await this.collectAllShelves(metadataStore); + + for (const shelf of shelves) { + if (!shelf.volumeIds.includes(oldVolumeId)) continue; + + const newVolumeIds = shelf.volumeIds + .filter((id) => id !== oldVolumeId) + .concat(replacements.map((v) => v.volumeId)); + + await metadataStore.putShelf({ + ...shelf, + volumeIds: newVolumeIds, + }); + } + } + + private async collectAllShelves( + metadataStore: MetadataStore, + ) { + return metadataStore.getAllShelves(); + } +} diff --git a/lib/daydreamer/ExperienceReplay.ts b/lib/daydreamer/ExperienceReplay.ts new file mode 100644 index 0000000..7ecfa1d --- /dev/null +++ b/lib/daydreamer/ExperienceReplay.ts @@ -0,0 +1,233 @@ +// --------------------------------------------------------------------------- +// ExperienceReplay — Idle-time query simulation for Hebbian reinforcement +// --------------------------------------------------------------------------- +// +// During idle periods the Daydreamer background worker samples recent or +// random pages, re-executes synthetic queries from their content, and +// marks traversed edges for Long-Term Potentiation (LTP). +// +// This reinforces connection patterns that were useful in the past and +// prevents them from decaying through disuse. +// --------------------------------------------------------------------------- + +import type { EmbeddingRunner } from "../embeddings/EmbeddingRunner"; +import type { ModelProfile } from "../core/ModelProfile"; +import type { MetadataStore, Page, VectorStore, Edge } from "../core/types"; +import { query as cortexQuery } from "../cortex/Query"; +import type { QueryOptions } from "../cortex/Query"; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +export interface ExperienceReplayOptions { + /** + * Number of synthetic queries to execute per replay cycle. + * Defaults to 5. + */ + queriesPerCycle?: number; + + /** + * Maximum number of pages to consider as query sources. + * When set, only the most recently created pages are sampled. + * Defaults to 200 (recent-biased sampling pool). + */ + samplePoolSize?: number; + + /** + * LTP weight increment applied to edges traversed during replay. + * Defaults to 0.1. + */ + ltpIncrement?: number; + + /** + * Maximum Hebbian edge weight. Weights are clamped to this value after LTP. + * Defaults to 1.0. + */ + maxEdgeWeight?: number; + + /** + * Top-K pages to retrieve per synthetic query. + * Defaults to 5. + */ + topK?: number; +} + +const DEFAULT_QUERIES_PER_CYCLE = 5; +const DEFAULT_SAMPLE_POOL_SIZE = 200; +const DEFAULT_LTP_INCREMENT = 0.1; +const DEFAULT_MAX_EDGE_WEIGHT = 1.0; +const DEFAULT_TOP_K = 5; + +// --------------------------------------------------------------------------- +// Result types +// --------------------------------------------------------------------------- + +export interface ExperienceReplayResult { + /** Number of synthetic queries executed. */ + queriesExecuted: number; + + /** Total number of edge weight updates applied. */ + edgesStrengthened: number; + + /** ISO timestamp when the replay cycle completed. */ + completedAt: string; +} + +// --------------------------------------------------------------------------- +// ExperienceReplay +// --------------------------------------------------------------------------- + +export class ExperienceReplay { + private readonly queriesPerCycle: number; + private readonly samplePoolSize: number; + private readonly ltpIncrement: number; + private readonly maxEdgeWeight: number; + private readonly topK: number; + + constructor(options: ExperienceReplayOptions = {}) { + this.queriesPerCycle = options.queriesPerCycle ?? DEFAULT_QUERIES_PER_CYCLE; + this.samplePoolSize = options.samplePoolSize ?? DEFAULT_SAMPLE_POOL_SIZE; + this.ltpIncrement = options.ltpIncrement ?? DEFAULT_LTP_INCREMENT; + this.maxEdgeWeight = options.maxEdgeWeight ?? DEFAULT_MAX_EDGE_WEIGHT; + this.topK = options.topK ?? DEFAULT_TOP_K; + } + + /** + * Run one replay cycle. + * + * 1. Sample `queriesPerCycle` pages from the store (recent-biased). + * 2. Execute a synthetic query for each sampled page using its content. + * 3. Strengthen (LTP) Hebbian edges connecting query results to the source page. + * + * @returns Summary statistics for the cycle. + */ + async run( + modelProfile: ModelProfile, + embeddingRunner: EmbeddingRunner, + vectorStore: VectorStore, + metadataStore: MetadataStore, + ): Promise { + const allPages = await metadataStore.getAllPages(); + if (allPages.length === 0) { + return { + queriesExecuted: 0, + edgesStrengthened: 0, + completedAt: new Date().toISOString(), + }; + } + + const pool = this.buildSamplePool(allPages); + const sources = this.sampleWithoutReplacement(pool, this.queriesPerCycle); + + const queryOptions: QueryOptions = { + modelProfile, + embeddingRunner, + vectorStore, + metadataStore, + topK: this.topK, + }; + + let edgesStrengthened = 0; + + for (const sourcePage of sources) { + const result = await cortexQuery(sourcePage.content, queryOptions); + const resultPageIds = result.pages.map((p) => p.pageId); + + edgesStrengthened += await this.applyLtp( + sourcePage.pageId, + resultPageIds, + metadataStore, + ); + } + + return { + queriesExecuted: sources.length, + edgesStrengthened, + completedAt: new Date().toISOString(), + }; + } + + // --------------------------------------------------------------------------- + // Internal helpers + // --------------------------------------------------------------------------- + + /** + * Build a sample pool from `allPages`. + * + * Sorts pages by `createdAt` descending (most recent first) and caps the + * pool at `samplePoolSize` to give recent pages a higher selection probability. + */ + private buildSamplePool(allPages: Page[]): Page[] { + const sorted = [...allPages].sort((a, b) => + b.createdAt.localeCompare(a.createdAt), + ); + return sorted.slice(0, this.samplePoolSize); + } + + /** + * Sample up to `count` pages from `pool` without replacement using a + * Fisher-Yates partial shuffle. + */ + private sampleWithoutReplacement(pool: Page[], count: number): Page[] { + const arr = [...pool]; + const take = Math.min(count, arr.length); + + for (let i = 0; i < take; i++) { + const j = i + Math.floor(Math.random() * (arr.length - i)); + [arr[i], arr[j]] = [arr[j], arr[i]]; + } + + return arr.slice(0, take); + } + + /** + * Apply LTP to edges between `sourcePageId` and each page in `resultPageIds`. + * + * Fetches existing Hebbian edges, increments their weight by `ltpIncrement` + * (clamped to `maxEdgeWeight`), and writes them back. + * + * New edges are created when none exist between the source and a result page. + * + * @returns The number of edge weight updates written. + */ + private async applyLtp( + sourcePageId: string, + resultPageIds: string[], + metadataStore: MetadataStore, + ): Promise { + if (resultPageIds.length === 0) return 0; + + const existingEdges = await metadataStore.getNeighbors(sourcePageId); + const edgeMap = new Map( + existingEdges.map((e) => [e.toPageId, e]), + ); + + const now = new Date().toISOString(); + const updatedEdges: Edge[] = []; + + for (const targetId of resultPageIds) { + if (targetId === sourcePageId) continue; + + const existing = edgeMap.get(targetId); + const currentWeight = existing?.weight ?? 0; + const newWeight = Math.min( + currentWeight + this.ltpIncrement, + this.maxEdgeWeight, + ); + + updatedEdges.push({ + fromPageId: sourcePageId, + toPageId: targetId, + weight: newWeight, + lastUpdatedAt: now, + }); + } + + if (updatedEdges.length > 0) { + await metadataStore.putEdges(updatedEdges); + } + + return updatedEdges.length; + } +} diff --git a/lib/daydreamer/FullNeighborRecalc.ts b/lib/daydreamer/FullNeighborRecalc.ts new file mode 100644 index 0000000..acd0ecc --- /dev/null +++ b/lib/daydreamer/FullNeighborRecalc.ts @@ -0,0 +1,201 @@ +// --------------------------------------------------------------------------- +// FullNeighborRecalc — Periodic full semantic neighbor graph recalculation (P2-C) +// --------------------------------------------------------------------------- +// +// The fast incremental neighbor insert used during ingest is approximate. +// This module performs a full pairwise recalculation for dirty volumes, +// bounded by the Williams-Bound-derived maintenance budget so the idle loop +// is not starved. +// +// Per idle cycle, the scheduler processes at most max(MIN_RECALC_PAIR_BUDGET, +// computeCapacity(graphMass)) pairwise comparisons. The minimum floor ensures +// forward progress for small corpora where the Williams formula may return a +// value smaller than a single typical volume's pair count. +// --------------------------------------------------------------------------- + +import type { Hash, MetadataStore, SemanticNeighbor, Page, VectorStore } from "../core/types"; +import { computeCapacity, DEFAULT_HOTPATH_POLICY, type HotpathPolicy } from "../core/HotpathPolicy"; +import { batchComputeSalience, runPromotionSweep } from "../core/SalienceEngine"; + +// Minimum pair budget per idle recalc cycle. +// Sized to cover the theoretical maximum for a single well-formed volume +// (BOOKS_PER_VOLUME=4 books × PAGES_PER_BOOK=8 pages = 32 pages, +// 32 × 31 = 992 pairs). Using 2048 gives a comfortable margin. +export const MIN_RECALC_PAIR_BUDGET = 2048; + +// --------------------------------------------------------------------------- +// Options +// --------------------------------------------------------------------------- + +export interface FullNeighborRecalcOptions { + metadataStore: MetadataStore; + vectorStore: VectorStore; + policy?: HotpathPolicy; + /** Maximum Metroid neighbors stored per page. Default: 16. */ + maxNeighbors?: number; + /** Current timestamp (ms since epoch). Defaults to Date.now(). */ + now?: number; +} + +export interface RecalcResult { + volumesProcessed: number; + pagesProcessed: number; + pairsComputed: number; +} + +// --------------------------------------------------------------------------- +// Cosine similarity +// --------------------------------------------------------------------------- + +function cosineSimilarity(a: Float32Array, b: Float32Array): number { + const len = Math.min(a.length, b.length); + let dot = 0; + let normA = 0; + let normB = 0; + for (let i = 0; i < len; i++) { + dot += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + const denom = Math.sqrt(normA) * Math.sqrt(normB); + if (denom === 0) return 0; + return dot / denom; +} + +// --------------------------------------------------------------------------- +// Main recalc function +// --------------------------------------------------------------------------- + +/** + * Run one cycle of full neighbor graph recalculation. + * + * Finds all volumes flagged as dirty (via `needsNeighborRecalc`), loads + * their pages, computes pairwise cosine similarities, and updates the + * Metroid neighbor index. Processing is bounded by the Williams-Bound-derived + * maintenance budget to avoid blocking the idle loop. + * + * After recalculation, salience is recomputed for affected pages and a + * promotion sweep is run to keep the hotpath current. + */ +export async function runFullNeighborRecalc( + options: FullNeighborRecalcOptions, +): Promise { + const { + metadataStore, + vectorStore, + policy = DEFAULT_HOTPATH_POLICY, + maxNeighbors = 16, + now = Date.now(), + } = options; + + // Find all dirty volumes + const allVolumes = await metadataStore.getAllVolumes(); + const dirtyVolumes = ( + await Promise.all( + allVolumes.map(async (v) => ({ + volume: v, + dirty: await metadataStore.needsNeighborRecalc(v.volumeId), + })), + ) + ) + .filter((x) => x.dirty) + .map((x) => x.volume); + + if (dirtyVolumes.length === 0) { + return { volumesProcessed: 0, pagesProcessed: 0, pairsComputed: 0 }; + } + + // Compute per-cycle pair budget: max of Williams-derived capacity and + // the minimum floor so even small corpora make forward progress. + const totalGraphMass = (await metadataStore.getAllPages()).length; + const pairBudget = Math.max(MIN_RECALC_PAIR_BUDGET, computeCapacity(totalGraphMass, policy.c)); + + let totalVolumesProcessed = 0; + let totalPagesProcessed = 0; + let totalPairsComputed = 0; + + const affectedPageIds = new Set(); + + for (const volume of dirtyVolumes) { + if (totalPairsComputed >= pairBudget) break; + + // Collect all pages in this volume (via books) + const volumePages: Page[] = []; + for (const bookId of volume.bookIds) { + const book = await metadataStore.getBook(bookId); + if (!book) continue; + for (const pageId of book.pageIds) { + const page = await metadataStore.getPage(pageId); + if (page) volumePages.push(page); + } + } + + if (volumePages.length === 0) { + await metadataStore.clearNeighborRecalcFlag(volume.volumeId); + totalVolumesProcessed++; + continue; + } + + // Load all embedding vectors for this volume's pages + const vectors = await Promise.all( + volumePages.map((p) => + vectorStore.readVector(p.embeddingOffset, p.embeddingDim), + ), + ); + + // Compute pairwise similarities and build neighbor lists + const pairsInVolume = volumePages.length * (volumePages.length - 1); + const remainingBudget = pairBudget - totalPairsComputed; + const budgetExhausted = pairsInVolume > remainingBudget; + if (budgetExhausted) { + break; + } + + for (let i = 0; i < volumePages.length; i++) { + const page = volumePages[i]; + const vecI = vectors[i]; + + const neighbors: SemanticNeighbor[] = []; + + for (let j = 0; j < volumePages.length; j++) { + if (i === j) continue; + const sim = cosineSimilarity(vecI, vectors[j]); + neighbors.push({ + neighborPageId: volumePages[j].pageId, + cosineSimilarity: sim, + distance: 1 - sim, + }); + totalPairsComputed++; + } + + // Sort by similarity descending; keep top maxNeighbors + neighbors.sort( + (a, b) => + b.cosineSimilarity - a.cosineSimilarity || + a.neighborPageId.localeCompare(b.neighborPageId), + ); + const topNeighbors = neighbors.slice(0, maxNeighbors); + + await metadataStore.putSemanticNeighbors(page.pageId, topNeighbors); + affectedPageIds.add(page.pageId); + } + + // Clear the dirty flag + await metadataStore.clearNeighborRecalcFlag(volume.volumeId); + totalVolumesProcessed++; + totalPagesProcessed += volumePages.length; + } + + // Recompute salience and run promotion sweep for all affected pages + if (affectedPageIds.size > 0) { + const ids = [...affectedPageIds]; + await batchComputeSalience(ids, metadataStore, policy, now); + await runPromotionSweep(ids, metadataStore, policy, now); + } + + return { + volumesProcessed: totalVolumesProcessed, + pagesProcessed: totalPagesProcessed, + pairsComputed: totalPairsComputed, + }; +} diff --git a/lib/daydreamer/HebbianUpdater.ts b/lib/daydreamer/HebbianUpdater.ts new file mode 100644 index 0000000..9dd710a --- /dev/null +++ b/lib/daydreamer/HebbianUpdater.ts @@ -0,0 +1,193 @@ +// --------------------------------------------------------------------------- +// HebbianUpdater — Edge plasticity via LTP / LTD / pruning (P2-B) +// --------------------------------------------------------------------------- +// +// Strengthens edges traversed during successful queries (Long-Term +// Potentiation), decays all edges each pass (Long-Term Depression), and +// prunes edges that fall below a threshold to keep the graph sparse. +// +// After LTP/LTD, salience is recomputed for every node whose incident edges +// changed, and a promotion/eviction sweep is run so the hotpath stays current. +// --------------------------------------------------------------------------- + +import type { Edge, Hash, MetadataStore } from "../core/types"; +import { DEFAULT_HOTPATH_POLICY, type HotpathPolicy } from "../core/HotpathPolicy"; +import { batchComputeSalience, runPromotionSweep } from "../core/SalienceEngine"; + +// --------------------------------------------------------------------------- +// Constants (policy-derived defaults; never hardcoded in callers) +// --------------------------------------------------------------------------- + +/** Default LTP step: edge weight increases by this amount on traversal. */ +export const DEFAULT_LTP_AMOUNT = 0.1; + +/** Default LTD multiplicative decay factor applied every pass (0 < decay < 1). */ +export const DEFAULT_LTD_DECAY = 0.99; + +/** Edges with weight below this threshold are removed by pruning. */ +export const DEFAULT_PRUNE_THRESHOLD = 0.01; + +/** Maximum outgoing Hebbian edges per node (degree cap). */ +export const DEFAULT_MAX_DEGREE = 16; + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +export interface HebbianUpdaterOptions { + metadataStore: MetadataStore; + policy?: HotpathPolicy; + /** LTP step amount. Default: DEFAULT_LTP_AMOUNT. */ + ltpAmount?: number; + /** LTD multiplicative decay applied to every edge. Default: DEFAULT_LTD_DECAY. */ + ltdDecay?: number; + /** Prune edges whose weight drops below this value. Default: DEFAULT_PRUNE_THRESHOLD. */ + pruneThreshold?: number; + /** Maximum outgoing degree per node. Default: DEFAULT_MAX_DEGREE. */ + maxDegree?: number; + /** Current timestamp (ms since epoch). Defaults to Date.now(). */ + now?: number; +} + +/** + * LTP — strengthen edges that were traversed during a successful query. + * + * Clamps weights to [0, Infinity) and re-saves affected edges. + * Recomputes salience for changed nodes and triggers a promotion sweep. + */ +export async function strengthenEdges( + traversedPairs: Array<{ from: Hash; to: Hash }>, + options: HebbianUpdaterOptions, +): Promise { + if (traversedPairs.length === 0) return; + + const { + metadataStore, + policy = DEFAULT_HOTPATH_POLICY, + ltpAmount = DEFAULT_LTP_AMOUNT, + now = Date.now(), + } = options; + + // Group by source node for efficient per-node updates + const bySource = new Map>(); + for (const { from, to } of traversedPairs) { + let targets = bySource.get(from); + if (!targets) { + targets = new Set(); + bySource.set(from, targets); + } + targets.add(to); + } + + const changedNodeIds = new Set(); + + for (const [fromId, toIds] of bySource) { + const existing = await metadataStore.getNeighbors(fromId); + const edgeMap = new Map(existing.map((e) => [e.toPageId, e])); + + const timestamp = new Date(now).toISOString(); + const updatedEdges: Edge[] = []; + + for (const toId of toIds) { + const edge = edgeMap.get(toId); + if (edge) { + updatedEdges.push({ + ...edge, + weight: edge.weight + ltpAmount, + lastUpdatedAt: timestamp, + }); + } else { + // Create new edge if not yet present + updatedEdges.push({ + fromPageId: fromId, + toPageId: toId, + weight: ltpAmount, + lastUpdatedAt: timestamp, + }); + } + changedNodeIds.add(fromId); + changedNodeIds.add(toId); + } + + if (updatedEdges.length > 0) { + await metadataStore.putEdges(updatedEdges); + } + } + + if (changedNodeIds.size > 0) { + await batchComputeSalience([...changedNodeIds], metadataStore, policy, now); + await runPromotionSweep([...changedNodeIds], metadataStore, policy, now); + } +} + +/** + * LTD + pruning — decay all edges by a multiplicative factor, then remove + * edges whose weight falls below the prune threshold or that exceed the max + * degree per source node. + * + * Recomputes salience for every node whose incident edges changed. + */ +export async function decayAndPrune( + options: HebbianUpdaterOptions, +): Promise<{ decayed: number; pruned: number }> { + const { + metadataStore, + policy = DEFAULT_HOTPATH_POLICY, + ltdDecay = DEFAULT_LTD_DECAY, + pruneThreshold = DEFAULT_PRUNE_THRESHOLD, + maxDegree = DEFAULT_MAX_DEGREE, + now = Date.now(), + } = options; + + const allPages = await metadataStore.getAllPages(); + if (allPages.length === 0) return { decayed: 0, pruned: 0 }; + + const changedNodeIds = new Set(); + let totalDecayed = 0; + let totalPruned = 0; + + const timestamp = new Date(now).toISOString(); + + for (const page of allPages) { + const edges = await metadataStore.getNeighbors(page.pageId); + if (edges.length === 0) continue; + + // Apply LTD decay + const decayed: Edge[] = edges.map((e) => ({ + ...e, + weight: e.weight * ltdDecay, + lastUpdatedAt: timestamp, + })); + totalDecayed += decayed.length; + + // Separate edges to keep vs. prune + const surviving = decayed.filter((e) => e.weight >= pruneThreshold); + const pruned = decayed.filter((e) => e.weight < pruneThreshold); + + // Enforce max degree: keep the strongest surviving edges + surviving.sort((a, b) => b.weight - a.weight); + const kept = surviving.slice(0, maxDegree); + const degreeEvicted = surviving.slice(maxDegree); + + // Delete pruned edges + for (const e of [...pruned, ...degreeEvicted]) { + await metadataStore.deleteEdge(e.fromPageId, e.toPageId); + totalPruned++; + changedNodeIds.add(e.fromPageId); + changedNodeIds.add(e.toPageId); + } + + // Save decayed-but-surviving edges + if (kept.length > 0) { + await metadataStore.putEdges(kept); + changedNodeIds.add(page.pageId); + } + } + + if (changedNodeIds.size > 0) { + await batchComputeSalience([...changedNodeIds], metadataStore, policy, now); + await runPromotionSweep([...changedNodeIds], metadataStore, policy, now); + } + + return { decayed: totalDecayed, pruned: totalPruned }; +} diff --git a/lib/daydreamer/IdleScheduler.ts b/lib/daydreamer/IdleScheduler.ts new file mode 100644 index 0000000..59a2ace --- /dev/null +++ b/lib/daydreamer/IdleScheduler.ts @@ -0,0 +1,172 @@ +// --------------------------------------------------------------------------- +// IdleScheduler — Cooperative background task scheduler (P2-A) +// --------------------------------------------------------------------------- +// +// Drives background Daydreamer operations without blocking the main thread. +// Uses requestIdleCallback in browsers and setImmediate in Node/test envs. +// Tasks are prioritised by a numeric priority field (lower = higher priority). +// --------------------------------------------------------------------------- + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +/** A single schedulable background task. */ +export interface ScheduledTask { + /** Lower number = higher priority. Tasks with equal priority run FIFO. */ + priority: number; + /** The work to perform. May be called multiple times if it re-enqueues itself. */ + run(): Promise; +} + +/** Internal queue entry. */ +interface QueueEntry { + insertionOrder: number; + task: ScheduledTask; +} + +// --------------------------------------------------------------------------- +// Idle callback shim +// --------------------------------------------------------------------------- + +/** Minimum time (ms) the scheduler will attempt to do work per idle slice. */ +const DEFAULT_BUDGET_MS = 5; + +/** + * Schedule a callback for when the host is idle. + * Falls back to setImmediate (Node) or setTimeout(0) when + * requestIdleCallback is not available. + */ +function scheduleIdle(callback: (deadline: { timeRemaining(): number }) => void): void { + if (typeof requestIdleCallback === "function") { + requestIdleCallback((deadline) => callback(deadline)); + } else if (typeof setImmediate === "function") { + setImmediate(() => callback({ timeRemaining: () => DEFAULT_BUDGET_MS })); + } else { + setTimeout(() => callback({ timeRemaining: () => DEFAULT_BUDGET_MS }), 0); + } +} + +// --------------------------------------------------------------------------- +// IdleScheduler +// --------------------------------------------------------------------------- + +/** + * Cooperative background task scheduler. + * + * Tasks are run one at a time during idle slices. Each task is given a single + * idle deadline per scheduling turn; if the deadline expires the scheduler + * yields and resumes on the next idle callback. + * + * State corruption is prevented by never interrupting a task mid-execution — + * each `task.run()` call is awaited to completion before the next task starts. + */ +export class IdleScheduler { + private queue: QueueEntry[] = []; + private counter = 0; + private active = false; + private stopped = false; + private readonly budgetMs: number; + private readonly errorHandler: (error: unknown, task: ScheduledTask) => void; + + /** + * @param budgetMs Approximate milliseconds of work per idle slice. + * Defaults to 5 ms. The scheduler yields after this + * budget is consumed even if the queue is non-empty. + * @param onError Optional error handler invoked when a task throws. + * Defaults to logging to console.error (if available). + */ + constructor( + budgetMs = DEFAULT_BUDGET_MS, + onError?: (error: unknown, task: ScheduledTask) => void, + ) { + this.budgetMs = budgetMs; + this.errorHandler = + onError ?? + ((error: unknown, task: ScheduledTask): void => { + if (typeof console !== "undefined" && typeof console.error === "function") { + console.error("[IdleScheduler] Task failed", { error, task }); + } + }); + } + + /** + * Enqueue a task. The task will be run in priority order (ascending + * priority value) during the next idle callback. Enqueueing while + * the scheduler is running is safe — the task will be picked up on + * the next scheduling turn. + */ + enqueue(task: ScheduledTask): void { + this.queue.push({ insertionOrder: this.counter++, task }); + this._sortQueue(); + } + + /** + * Start the idle loop. Safe to call multiple times — extra calls are no-ops + * if the loop is already running. + */ + start(): void { + if (this.active || this.stopped) return; + this.active = true; + this._scheduleNextTurn(); + } + + /** + * Permanently stop the scheduler. After calling `stop()` no further tasks + * will be executed and `start()` becomes a no-op. Tasks already in-flight + * will complete normally. + */ + stop(): void { + this.stopped = true; + this.active = false; + } + + /** True when the task queue is empty. */ + get idle(): boolean { + return this.queue.length === 0; + } + + // --------------------------------------------------------------------------- + // Private + // --------------------------------------------------------------------------- + + private _sortQueue(): void { + this.queue.sort( + (a, b) => + a.task.priority - b.task.priority || + a.insertionOrder - b.insertionOrder, + ); + } + + private _scheduleNextTurn(): void { + if (this.stopped) return; + scheduleIdle((deadline) => { + void this._runTurn(deadline); + }); + } + + private async _runTurn(deadline: { timeRemaining(): number }): Promise { + if (this.stopped) return; + + const turnEnd = Date.now() + Math.max(deadline.timeRemaining(), this.budgetMs); + + while (this.queue.length > 0 && Date.now() < turnEnd && !this.stopped) { + const entry = this.queue.shift(); + if (!entry) break; + try { + await entry.task.run(); + } catch (error) { + // Report errors so failing tasks can be diagnosed, but do not + // allow a single bad task to crash the idle loop. + this.errorHandler(error, entry.task); + } + } + + if (!this.stopped && this.queue.length > 0) { + // More work remains — schedule another turn. + this._scheduleNextTurn(); + } else { + this.active = this.queue.length > 0; + } + } +} diff --git a/lib/daydreamer/PrototypeRecomputer.ts b/lib/daydreamer/PrototypeRecomputer.ts new file mode 100644 index 0000000..867fbf3 --- /dev/null +++ b/lib/daydreamer/PrototypeRecomputer.ts @@ -0,0 +1,263 @@ +// --------------------------------------------------------------------------- +// PrototypeRecomputer — Keep volume and shelf prototypes accurate (P2-D) +// --------------------------------------------------------------------------- +// +// As pages and books change, volume medoids and centroids drift. This module +// recomputes them periodically during Daydreamer idle passes. +// +// After recomputing prototypes at each level, salience is refreshed for the +// updated representative entries and a tier-scoped promotion/eviction sweep +// is run to keep the hotpath consistent. +// --------------------------------------------------------------------------- + +import type { Hash, HotpathEntry, MetadataStore, Shelf, Volume, VectorStore } from "../core/types"; +import { DEFAULT_HOTPATH_POLICY, type HotpathPolicy } from "../core/HotpathPolicy"; +import { runPromotionSweep } from "../core/SalienceEngine"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** Compute cosine similarity between two equal-length vectors. */ +function cosineSimilarity(a: Float32Array, b: Float32Array): number { + const len = Math.min(a.length, b.length); + let dot = 0; + let normA = 0; + let normB = 0; + for (let i = 0; i < len; i++) { + dot += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + const denom = Math.sqrt(normA) * Math.sqrt(normB); + if (denom === 0) return 0; + return dot / denom; +} + +/** + * Select the medoid from a set of vectors: the vector that minimises the + * average distance to all others (the most "central" real member). + * + * Returns the index of the medoid in the input array, or -1 if empty. + */ +export function selectMedoidIndex(vectors: Float32Array[]): number { + if (vectors.length === 0) return -1; + if (vectors.length === 1) return 0; + + let bestIndex = 0; + let bestAvgDist = Infinity; + + for (let i = 0; i < vectors.length; i++) { + let totalDist = 0; + for (let j = 0; j < vectors.length; j++) { + if (i === j) continue; + totalDist += 1 - cosineSimilarity(vectors[i], vectors[j]); + } + const avgDist = totalDist / (vectors.length - 1); + if (avgDist < bestAvgDist) { + bestAvgDist = avgDist; + bestIndex = i; + } + } + return bestIndex; +} + +/** + * Compute the element-wise mean (centroid) of a set of equal-length vectors. + * Returns a new Float32Array of the same dimensionality. + */ +export function computeCentroid(vectors: Float32Array[]): Float32Array { + if (vectors.length === 0) return new Float32Array(0); + const dim = vectors[0].length; + const centroid = new Float32Array(dim); + for (const v of vectors) { + for (let i = 0; i < dim; i++) { + centroid[i] += v[i]; + } + } + const n = vectors.length; + for (let i = 0; i < dim; i++) { + centroid[i] /= n; + } + return centroid; +} + +// --------------------------------------------------------------------------- +// Options +// --------------------------------------------------------------------------- + +export interface PrototypeRecomputerOptions { + metadataStore: MetadataStore; + vectorStore: VectorStore; + policy?: HotpathPolicy; + /** Current timestamp (ms since epoch). Defaults to Date.now(). */ + now?: number; +} + +export interface RecomputeResult { + volumesUpdated: number; + shelvesUpdated: number; +} + +// --------------------------------------------------------------------------- +// Recompute volume prototypes +// --------------------------------------------------------------------------- + +/** + * Recompute centroid prototypes for all volumes. + * + * For each volume: + * 1. Load all page embeddings for every book in the volume. + * 2. Compute the centroid embedding across all pages. + * 3. Append updated centroid vector to VectorStore; update volume metadata. + * + * Note: Medoid selection and salience/promotion sweeps are intentionally + * omitted here. SalienceEngine methods currently assume page-tier entities; + * running them with volume IDs would produce incorrect tier assignments. + * Volume-tier salience should be wired up once SalienceEngine supports + * non-page tiers. + */ +async function recomputeVolumePrototypes( + options: PrototypeRecomputerOptions, +): Promise<{ volumeIds: Hash[]; volumesUpdated: number }> { + const { + metadataStore, + vectorStore, + } = options; + + const allVolumes = await metadataStore.getAllVolumes(); + const updatedVolumeIds: Hash[] = []; + + for (const volume of allVolumes) { + // Load all pages in this volume + const pageEntries: Array<{ pageId: Hash; vector: Float32Array }> = []; + + for (const bookId of volume.bookIds) { + const book = await metadataStore.getBook(bookId); + if (!book) continue; + for (const pageId of book.pageIds) { + const page = await metadataStore.getPage(pageId); + if (!page) continue; + const vec = await vectorStore.readVector(page.embeddingOffset, page.embeddingDim); + pageEntries.push({ pageId, vector: vec }); + } + } + + if (pageEntries.length === 0) continue; + + const vectors = pageEntries.map((e) => e.vector); + const centroidVec = computeCentroid(vectors); + + // Append centroid to vector store + const centroidOffset = await vectorStore.appendVector(centroidVec); + + // Update the volume with new medoid and prototype offsets + const updatedVolume: Volume = { + ...volume, + prototypeOffsets: [...volume.prototypeOffsets, centroidOffset], + prototypeDim: centroidVec.length, + }; + await metadataStore.putVolume(updatedVolume); + + updatedVolumeIds.push(volume.volumeId); + } + + // Note: We intentionally do not call the page-centric SalienceEngine here. + // batchComputeSalience/runPromotionSweep currently assume page-tier entities + // and hardcode `tier: "page"`. Passing volume IDs into those functions would + // compute meaningless salience values and could overwrite volume-tier + // HotpathEntry records with page-tier entries using the same entityId. + // + // Volume-tier salience/promotion should be wired up once SalienceEngine + // supports non-page tiers. For now, we only update the volume metadata and + // return the list of volumes that were recomputed. + + return { volumeIds: updatedVolumeIds, volumesUpdated: updatedVolumeIds.length }; +} + +// --------------------------------------------------------------------------- +// Recompute shelf routing prototypes +// --------------------------------------------------------------------------- + +/** + * Recompute routing prototypes for all shelves. + * + * For each shelf: + * 1. Load volume prototype embeddings. + * 2. Compute centroid across all volume prototypes. + * 3. Append new routing prototype to VectorStore; update shelf metadata. + * 4. Refresh salience and run promotion sweep for the shelf tier. + */ +async function recomputeShelfPrototypes( + options: PrototypeRecomputerOptions, +): Promise<{ shelvesUpdated: number }> { + const { + metadataStore, + vectorStore, + policy = DEFAULT_HOTPATH_POLICY, + now = Date.now(), + } = options; + + const allShelves = await metadataStore.getAllShelves(); + const updatedShelfIds: Hash[] = []; + + for (const shelf of allShelves) { + const volumeVectors: Float32Array[] = []; + + for (const volumeId of shelf.volumeIds) { + const volume = await metadataStore.getVolume(volumeId); + if (!volume || volume.prototypeOffsets.length === 0) continue; + // Use the last (most recent) prototype offset + const offset = volume.prototypeOffsets[volume.prototypeOffsets.length - 1]; + const vec = await vectorStore.readVector(offset, volume.prototypeDim); + volumeVectors.push(vec); + } + + if (volumeVectors.length === 0) continue; + + const routingPrototype = computeCentroid(volumeVectors); + const routingOffset = await vectorStore.appendVector(routingPrototype); + + const updatedShelf: Shelf = { + ...shelf, + routingPrototypeOffsets: [...shelf.routingPrototypeOffsets, routingOffset], + routingDim: routingPrototype.length, + }; + await metadataStore.putShelf(updatedShelf); + updatedShelfIds.push(shelf.shelfId); + } + + if (updatedShelfIds.length > 0) { + // Shelf-tier hotpath uses shelf IDs as entity IDs + const shelfEntries: HotpathEntry[] = updatedShelfIds.map((id) => ({ + entityId: id, + tier: "shelf" as const, + salience: 0, + communityId: undefined, + })); + for (const entry of shelfEntries) { + await metadataStore.putHotpathEntry(entry); + } + await runPromotionSweep(updatedShelfIds, metadataStore, policy, now); + } + + return { shelvesUpdated: updatedShelfIds.length }; +} + +// --------------------------------------------------------------------------- +// Public entry point +// --------------------------------------------------------------------------- + +/** + * Recompute prototypes at all hierarchy levels (volume then shelf). + * + * Volumes are processed first so shelves can reference updated volume prototypes. + */ +export async function recomputePrototypes( + options: PrototypeRecomputerOptions, +): Promise { + const { volumesUpdated } = await recomputeVolumePrototypes(options); + const { shelvesUpdated } = await recomputeShelfPrototypes(options); + + return { volumesUpdated, shelvesUpdated }; +} diff --git a/lib/embeddings/DeterministicDummyEmbeddingBackend.ts b/lib/embeddings/DeterministicDummyEmbeddingBackend.ts new file mode 100644 index 0000000..a98bb43 --- /dev/null +++ b/lib/embeddings/DeterministicDummyEmbeddingBackend.ts @@ -0,0 +1,108 @@ +import type { EmbeddingBackend } from "./EmbeddingBackend"; + +export const DEFAULT_DUMMY_EMBEDDING_DIMENSION = 1024; +export const SHA256_BLOCK_BYTES = 64; + +const SHA256_DIGEST_BYTES = 32; +const COUNTER_BYTES = 4; +const BYTE_TO_UNIT_SCALE = 127.5; + +export interface DeterministicDummyEmbeddingBackendOptions { + dimension?: number; + blockBytes?: number; +} + +function assertPositiveInteger(name: string, value: number): void { + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer`); + } +} + +function getSubtleCrypto(): SubtleCrypto { + const subtle = globalThis.crypto?.subtle; + if (!subtle) { + throw new Error("SubtleCrypto is required for DeterministicDummyEmbeddingBackend"); + } + return subtle; +} + +function padBytesToBoundary(input: Uint8Array, blockBytes: number): Uint8Array { + const remainder = input.byteLength % blockBytes; + if (remainder === 0) { + return input; + } + + const padLength = blockBytes - remainder; + const padded = new Uint8Array(input.byteLength + padLength); + padded.set(input); + return padded; +} + +function byteToUnitFloat(byteValue: number): number { + return byteValue / BYTE_TO_UNIT_SCALE - 1; +} + +export class DeterministicDummyEmbeddingBackend implements EmbeddingBackend { + readonly kind = "dummy-sha256" as const; + readonly dimension: number; + + private readonly blockBytes: number; + private readonly subtle = getSubtleCrypto(); + private readonly encoder = new TextEncoder(); + + constructor(options: DeterministicDummyEmbeddingBackendOptions = {}) { + this.dimension = options.dimension ?? DEFAULT_DUMMY_EMBEDDING_DIMENSION; + this.blockBytes = options.blockBytes ?? SHA256_BLOCK_BYTES; + + assertPositiveInteger("dimension", this.dimension); + assertPositiveInteger("blockBytes", this.blockBytes); + } + + async embed(texts: string[]): Promise { + return Promise.all(texts.map((text) => this.embedOne(text))); + } + + private async embedOne(text: string): Promise { + const sourceBytes = padBytesToBoundary( + this.encoder.encode(text), + this.blockBytes, + ); + + const embedding = new Float32Array(this.dimension); + let counter = 0; + let writeIndex = 0; + + while (writeIndex < this.dimension) { + const digest = await this.digestWithCounter(sourceBytes, counter); + for ( + let digestIndex = 0; + digestIndex < SHA256_DIGEST_BYTES && writeIndex < this.dimension; + digestIndex++ + ) { + embedding[writeIndex] = byteToUnitFloat(digest[digestIndex]); + writeIndex++; + } + counter++; + } + + return embedding; + } + + private async digestWithCounter( + sourceBytes: Uint8Array, + counter: number, + ): Promise { + const payload = new Uint8Array(sourceBytes.byteLength + COUNTER_BYTES); + payload.set(sourceBytes, 0); + + const counterView = new DataView( + payload.buffer, + payload.byteOffset + sourceBytes.byteLength, + COUNTER_BYTES, + ); + counterView.setUint32(0, counter, false); + + const digest = await this.subtle.digest("SHA-256", payload); + return new Uint8Array(digest); + } +} diff --git a/lib/embeddings/EmbeddingBackend.ts b/lib/embeddings/EmbeddingBackend.ts new file mode 100644 index 0000000..72bff31 --- /dev/null +++ b/lib/embeddings/EmbeddingBackend.ts @@ -0,0 +1,6 @@ +export interface EmbeddingBackend { + readonly kind: string; + readonly dimension: number; + + embed(texts: string[]): Promise; +} diff --git a/lib/embeddings/EmbeddingRunner.ts b/lib/embeddings/EmbeddingRunner.ts new file mode 100644 index 0000000..ea29295 --- /dev/null +++ b/lib/embeddings/EmbeddingRunner.ts @@ -0,0 +1,54 @@ +import type { EmbeddingBackend } from "./EmbeddingBackend"; +import { + type ResolveEmbeddingBackendOptions, + type ResolvedEmbeddingBackend, + resolveEmbeddingBackend, +} from "./ProviderResolver"; + +export type ResolveEmbeddingSelection = () => Promise; + +export class EmbeddingRunner { + private selectionPromise: Promise | undefined; + private resolvedSelection: ResolvedEmbeddingBackend | undefined; + + constructor(private readonly resolveSelection: ResolveEmbeddingSelection) {} + + static fromResolverOptions( + options: ResolveEmbeddingBackendOptions, + ): EmbeddingRunner { + return new EmbeddingRunner(() => resolveEmbeddingBackend(options)); + } + + get selectedKind(): string | undefined { + return this.resolvedSelection?.selectedKind; + } + + async getSelection(): Promise { + return this.ensureSelection(); + } + + async getBackend(): Promise { + const selection = await this.ensureSelection(); + return selection.backend; + } + + async embed(texts: string[]): Promise { + const backend = await this.getBackend(); + return backend.embed(texts); + } + + private async ensureSelection(): Promise { + if (this.resolvedSelection) { + return this.resolvedSelection; + } + + if (!this.selectionPromise) { + this.selectionPromise = this.resolveSelection().then((selection) => { + this.resolvedSelection = selection; + return selection; + }); + } + + return this.selectionPromise; + } +} diff --git a/lib/embeddings/OrtWebglEmbeddingBackend.ts b/lib/embeddings/OrtWebglEmbeddingBackend.ts new file mode 100644 index 0000000..7b853ca --- /dev/null +++ b/lib/embeddings/OrtWebglEmbeddingBackend.ts @@ -0,0 +1,129 @@ +import type { FeatureExtractionPipeline } from "@huggingface/transformers"; + +import type { EmbeddingBackend } from "./EmbeddingBackend"; +import { + EMBEDDING_GEMMA_300M_DOCUMENT_PREFIX, + EMBEDDING_GEMMA_300M_EMBEDDING_DIMENSION, + EMBEDDING_GEMMA_300M_MODEL_ID, + EMBEDDING_GEMMA_300M_QUERY_PREFIX, +} from "./TransformersJsEmbeddingBackend"; + +export interface OrtWebglEmbeddingBackendOptions { + /** + * Hugging Face model ID to load. Must be a matryoshka-compatible embedding model. + * Defaults to `EMBEDDING_GEMMA_300M_MODEL_ID`. + */ + modelId?: string; + + /** + * Number of embedding dimensions to return. + * Defaults to `EMBEDDING_GEMMA_300M_EMBEDDING_DIMENSION`. + */ + dimension?: number; + + /** + * Prefix prepended to each text when embedding documents/passages. + * Defaults to `EMBEDDING_GEMMA_300M_DOCUMENT_PREFIX`. + */ + documentPrefix?: string; + + /** + * Prefix prepended to each text when embedding search queries. + * Defaults to `EMBEDDING_GEMMA_300M_QUERY_PREFIX`. + */ + queryPrefix?: string; +} + +/** + * Embedding backend that uses ONNX Runtime Web's explicit WebGL execution + * provider via Hugging Face Transformers.js. + * + * This backend targets systems that have WebGL but lack WebGPU or WebNN, + * providing a hardware-accelerated fallback below the WebGPU/WebNN tier. + * + * The pipeline is loaded lazily on the first `embed()` or `embedQueries()` + * call so that import cost is zero until the backend is actually needed. + */ +export class OrtWebglEmbeddingBackend implements EmbeddingBackend { + readonly kind = "webgl" as const; + readonly dimension: number; + readonly modelId: string; + readonly documentPrefix: string; + readonly queryPrefix: string; + + private pipelinePromise: Promise | undefined; + + constructor(options: OrtWebglEmbeddingBackendOptions = {}) { + this.modelId = options.modelId ?? EMBEDDING_GEMMA_300M_MODEL_ID; + this.dimension = + options.dimension ?? EMBEDDING_GEMMA_300M_EMBEDDING_DIMENSION; + this.documentPrefix = + options.documentPrefix ?? EMBEDDING_GEMMA_300M_DOCUMENT_PREFIX; + this.queryPrefix = + options.queryPrefix ?? EMBEDDING_GEMMA_300M_QUERY_PREFIX; + } + + /** + * Embeds the given texts as document/passage representations. + * Prepends `documentPrefix` before each text as required by the model. + */ + async embed(texts: string[]): Promise { + return this.embedWithPrefix(texts, this.documentPrefix); + } + + /** + * Embeds the given texts as search query representations. + * Prepends `queryPrefix` before each text as required by the model. + * + * Use this method when encoding queries for retrieval; use `embed()` for + * documents/passages being indexed. + */ + async embedQueries(texts: string[]): Promise { + return this.embedWithPrefix(texts, this.queryPrefix); + } + + private async embedWithPrefix( + texts: string[], + prefix: string, + ): Promise { + const extractor = await this.ensurePipeline(); + const prefixed = + prefix.length > 0 ? texts.map((t) => `${prefix}${t}`) : texts; + + const output = await extractor(prefixed, { + pooling: "mean", + normalize: true, + }); + + const rawData = output.data as Float32Array; + const fullDim = rawData.length / texts.length; + const sliceDim = Math.min(this.dimension, fullDim); + + const results: Float32Array[] = []; + for (let i = 0; i < texts.length; i++) { + const start = i * fullDim; + results.push(rawData.slice(start, start + sliceDim)); + } + return results; + } + + private ensurePipeline(): Promise { + if (!this.pipelinePromise) { + this.pipelinePromise = this.loadPipeline(); + } + return this.pipelinePromise; + } + + private async loadPipeline(): Promise { + const { pipeline } = await import("@huggingface/transformers"); + // Cast through unknown to work around the overloaded pipeline union type complexity. + const pipelineFn = pipeline as unknown as ( + task: string, + model: string, + options?: Record, + ) => Promise; + return pipelineFn("feature-extraction", this.modelId, { + device: "webgl", + }); + } +} diff --git a/lib/embeddings/ProviderResolver.ts b/lib/embeddings/ProviderResolver.ts new file mode 100644 index 0000000..3e7b878 --- /dev/null +++ b/lib/embeddings/ProviderResolver.ts @@ -0,0 +1,348 @@ +import { + DeterministicDummyEmbeddingBackend, + type DeterministicDummyEmbeddingBackendOptions, +} from "./DeterministicDummyEmbeddingBackend"; +import type { EmbeddingBackend } from "./EmbeddingBackend"; +import { + OrtWebglEmbeddingBackend, + type OrtWebglEmbeddingBackendOptions, +} from "./OrtWebglEmbeddingBackend"; +import { + TransformersJsEmbeddingBackend, + type TransformersJsDevice, + type TransformersJsEmbeddingBackendOptions, +} from "./TransformersJsEmbeddingBackend"; + +export type EmbeddingProviderKind = + | "webnn" + | "webgpu" + | "webgl" + | "wasm" + | "dummy" + | (string & {}); + +export interface EmbeddingProviderCandidate { + kind: EmbeddingProviderKind; + isSupported: () => boolean | Promise; + createBackend: () => EmbeddingBackend | Promise; +} + +export interface EmbeddingProviderBenchmarkPolicy { + enabled: boolean; + warmupRuns: number; + timedRuns: number; + sampleTexts: string[]; +} + +export interface EmbeddingProviderMeasurement { + kind: EmbeddingProviderKind; + meanMs: number; +} + +export type EmbeddingProviderResolveReason = + | "forced" + | "benchmark" + | "capability-order"; + +export interface ResolvedEmbeddingBackend { + backend: EmbeddingBackend; + selectedKind: EmbeddingProviderKind; + reason: EmbeddingProviderResolveReason; + supportedKinds: EmbeddingProviderKind[]; + measurements: EmbeddingProviderMeasurement[]; +} + +export const DEFAULT_PROVIDER_ORDER: ReadonlyArray = + Object.freeze([ + "webnn", + "webgpu", + "webgl", + "wasm", + "dummy", + ]); + +export const DEFAULT_PROVIDER_BENCHMARK_POLICY: EmbeddingProviderBenchmarkPolicy = + Object.freeze({ + enabled: true, + warmupRuns: 1, + timedRuns: 3, + sampleTexts: [ + "cortex benchmark probe", + "routing and coherence warmup", + "deterministic provider timing", + ], + }); + +export type BenchmarkBackendFn = ( + backend: EmbeddingBackend, + policy: EmbeddingProviderBenchmarkPolicy, +) => Promise; + +export interface ResolveEmbeddingBackendOptions { + candidates: EmbeddingProviderCandidate[]; + preferredOrder?: ReadonlyArray; + forceKind?: EmbeddingProviderKind; + benchmark?: Partial; + benchmarkBackend?: BenchmarkBackendFn; +} + +function assertPositiveInteger(name: string, value: number): void { + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer`); + } +} + +function validateBenchmarkPolicy( + policy: EmbeddingProviderBenchmarkPolicy, +): EmbeddingProviderBenchmarkPolicy { + assertPositiveInteger("warmupRuns", policy.warmupRuns); + assertPositiveInteger("timedRuns", policy.timedRuns); + + if (policy.sampleTexts.length === 0) { + throw new Error("sampleTexts must not be empty"); + } + + return policy; +} + +function nowMs(): number { + const perfNow = globalThis.performance?.now?.bind(globalThis.performance); + if (perfNow) { + return perfNow(); + } + return Date.now(); +} + +async function defaultBenchmarkBackend( + backend: EmbeddingBackend, + policy: EmbeddingProviderBenchmarkPolicy, +): Promise { + for (let i = 0; i < policy.warmupRuns; i++) { + await backend.embed(policy.sampleTexts); + } + + let totalMs = 0; + for (let i = 0; i < policy.timedRuns; i++) { + const start = nowMs(); + await backend.embed(policy.sampleTexts); + totalMs += nowMs() - start; + } + + return totalMs / policy.timedRuns; +} + +function orderCandidates( + candidates: EmbeddingProviderCandidate[], + preferredOrder: ReadonlyArray, +): EmbeddingProviderCandidate[] { + const orderIndex = new Map(); + for (let i = 0; i < preferredOrder.length; i++) { + orderIndex.set(preferredOrder[i], i); + } + + return [...candidates].sort((a, b) => { + const aIndex = orderIndex.get(a.kind) ?? Number.MAX_SAFE_INTEGER; + const bIndex = orderIndex.get(b.kind) ?? Number.MAX_SAFE_INTEGER; + return aIndex - bIndex; + }); +} + +export async function resolveEmbeddingBackend( + options: ResolveEmbeddingBackendOptions, +): Promise { + const preferredOrder = options.preferredOrder ?? DEFAULT_PROVIDER_ORDER; + const benchmarkPolicy = validateBenchmarkPolicy({ + ...DEFAULT_PROVIDER_BENCHMARK_POLICY, + ...options.benchmark, + }); + + const capabilityChecks = await Promise.all( + options.candidates.map(async (candidate) => ({ + candidate, + supported: await candidate.isSupported(), + })), + ); + + if (options.forceKind !== undefined) { + const forcedEntry = capabilityChecks.find( + (entry) => entry.candidate.kind === options.forceKind, + ); + + if (!forcedEntry || !forcedEntry.supported) { + throw new Error(`Forced provider ${options.forceKind} is not supported`); + } + + return { + backend: await forcedEntry.candidate.createBackend(), + selectedKind: forcedEntry.candidate.kind, + reason: "forced", + supportedKinds: orderCandidates( + capabilityChecks + .filter((entry) => entry.supported) + .map((entry) => entry.candidate), + preferredOrder, + ).map((candidate) => candidate.kind), + measurements: [], + }; + } + + const supportedCandidates = orderCandidates( + capabilityChecks + .filter((entry) => entry.supported) + .map((entry) => entry.candidate), + preferredOrder, + ); + + if (supportedCandidates.length === 0) { + throw new Error("No supported embedding providers are available"); + } + + const supportedKinds = supportedCandidates.map((candidate) => candidate.kind); + + const benchmarkBackend = options.benchmarkBackend ?? defaultBenchmarkBackend; + if (benchmarkPolicy.enabled) { + const measurements: { + candidate: EmbeddingProviderCandidate; + backend: EmbeddingBackend; + meanMs: number; + }[] = []; + + for (const candidate of supportedCandidates) { + const backend = await candidate.createBackend(); + const meanMs = await benchmarkBackend(backend, benchmarkPolicy); + measurements.push({ candidate, backend, meanMs }); + } + + let winner = measurements[0]; + for (let i = 1; i < measurements.length; i++) { + if (measurements[i].meanMs < winner.meanMs) { + winner = measurements[i]; + } + } + + return { + backend: winner.backend, + selectedKind: winner.candidate.kind, + reason: "benchmark", + supportedKinds, + measurements: measurements.map((m) => ({ + kind: m.candidate.kind, + meanMs: m.meanMs, + })), + }; + } + + const selectedCandidate = supportedCandidates[0]; + return { + backend: await selectedCandidate.createBackend(), + selectedKind: selectedCandidate.kind, + reason: "capability-order", + supportedKinds, + measurements: [], + }; +} + +export function createDummyProviderCandidate( + options: DeterministicDummyEmbeddingBackendOptions = {}, +): EmbeddingProviderCandidate { + return { + kind: "dummy", + isSupported: () => globalThis.crypto?.subtle !== undefined, + createBackend: () => new DeterministicDummyEmbeddingBackend(options), + }; +} + +/** + * Checks whether a given Transformers.js ONNX device is available in the + * current runtime environment. + * + * - `"wasm"` is always considered supported (lowest common denominator). + * - `"webgpu"` requires `navigator.gpu` to be present. + * - `"webnn"` requires `navigator.ml` to be present. + */ +function isTransformersJsDeviceSupported( + device: TransformersJsDevice, +): boolean { + switch (device) { + case "webnn": + return ( + typeof globalThis.navigator !== "undefined" && + "ml" in globalThis.navigator + ); + case "webgpu": + return ( + typeof globalThis.navigator !== "undefined" && + "gpu" in globalThis.navigator + ); + case "wasm": + return true; + } +} + +/** + * Returns an `EmbeddingProviderCandidate` array for each Transformers.js + * ONNX device (`"webnn"`, `"webgpu"`, `"wasm"`), ordered from fastest to most + * widely available. + * + * Each candidate: + * - Exposes a `kind` matching the underlying device (e.g. `"webgpu"`). + * - Runs its `isSupported()` check at resolution time (no eager pipeline load). + * - Creates a `TransformersJsEmbeddingBackend` with the shared `options` plus + * the candidate-specific `device`. + * + * Pass these candidates to `resolveEmbeddingBackend` or + * `EmbeddingRunner.fromResolverOptions` to select the best available device + * at runtime. + * + * @example + * ```ts + * const runner = EmbeddingRunner.fromResolverOptions({ + * candidates: [ + * ...createTransformersJsProviderCandidates(), + * createDummyProviderCandidate(), + * ], + * }); + * ``` + */ +export function createTransformersJsProviderCandidates( + options: TransformersJsEmbeddingBackendOptions = {}, +): EmbeddingProviderCandidate[] { + const devices: TransformersJsDevice[] = ["webnn", "webgpu", "wasm"]; + + return devices.map((device) => ({ + kind: device, + isSupported: () => isTransformersJsDeviceSupported(device), + createBackend: () => + new TransformersJsEmbeddingBackend({ ...options, device }), + })); +} + +/** + * Returns an `EmbeddingProviderCandidate` for the WebGL ONNX execution + * provider via `OrtWebglEmbeddingBackend`. + * + * This candidate is supported when `WebGL2RenderingContext` is available in + * the global scope, providing a hardware-accelerated fallback for systems + * that have WebGL but lack WebGPU or WebNN. + * + * @example + * ```ts + * const runner = EmbeddingRunner.fromResolverOptions({ + * candidates: [ + * ...createTransformersJsProviderCandidates(), + * createWebglProviderCandidate(), + * createDummyProviderCandidate(), + * ], + * }); + * ``` + */ +export function createWebglProviderCandidate( + options: OrtWebglEmbeddingBackendOptions = {}, +): EmbeddingProviderCandidate { + return { + kind: "webgl", + isSupported: () => + typeof globalThis.WebGL2RenderingContext !== "undefined", + createBackend: () => new OrtWebglEmbeddingBackend(options), + }; +} diff --git a/lib/embeddings/TransformersJsEmbeddingBackend.ts b/lib/embeddings/TransformersJsEmbeddingBackend.ts new file mode 100644 index 0000000..5ef7b23 --- /dev/null +++ b/lib/embeddings/TransformersJsEmbeddingBackend.ts @@ -0,0 +1,161 @@ +import type { FeatureExtractionPipeline } from "@huggingface/transformers"; + +import type { EmbeddingBackend } from "./EmbeddingBackend"; + +export type TransformersJsDevice = "webnn" | "webgpu" | "wasm"; + +export interface TransformersJsEmbeddingBackendOptions { + /** + * Hugging Face model ID to load. Must be a matryoshka-compatible embedding model. + * Defaults to `EMBEDDING_GEMMA_300M_MODEL_ID`. + */ + modelId?: string; + + /** + * ONNX runtime device to use for inference. + * Defaults to `"wasm"` (always available, lowest common denominator). + */ + device?: TransformersJsDevice; + + /** + * Number of embedding dimensions to return. For matryoshka models, this may be + * any supported sub-dimension (smaller values are valid nested sub-spaces). + * Defaults to `EMBEDDING_GEMMA_300M_EMBEDDING_DIMENSION`. + */ + dimension?: number; + + /** + * Prefix prepended to each text when embedding documents/passages. + * Required by some models (e.g. EmbeddingGemma) for best retrieval quality. + * Defaults to `EMBEDDING_GEMMA_300M_DOCUMENT_PREFIX`. + */ + documentPrefix?: string; + + /** + * Prefix prepended to each text when embedding search queries. + * Required by some models (e.g. EmbeddingGemma) for best retrieval quality. + * Defaults to `EMBEDDING_GEMMA_300M_QUERY_PREFIX`. + */ + queryPrefix?: string; +} + +/** + * Default model used when no `modelId` is provided. + * Q4-quantized ONNX variant of google/embeddinggemma-300m. + */ +export const EMBEDDING_GEMMA_300M_MODEL_ID = + "onnx-community/embeddinggemma-300m-ONNX"; + +/** + * Default embedding dimension used when no `dimension` is provided. + * 768 is the full-fidelity matryoshka output dimension for EmbeddingGemma-300M. + */ +export const EMBEDDING_GEMMA_300M_EMBEDDING_DIMENSION = 768; + +/** + * Query task prefix for EmbeddingGemma-300M, as specified on the model card. + * @see https://huggingface.co/google/embeddinggemma-300m + */ +export const EMBEDDING_GEMMA_300M_QUERY_PREFIX = "query: "; + +/** + * Document/passage prefix for EmbeddingGemma-300M, as specified on the model card. + * @see https://huggingface.co/google/embeddinggemma-300m + */ +export const EMBEDDING_GEMMA_300M_DOCUMENT_PREFIX = "passage: "; + +/** + * Real embedding backend backed by `@huggingface/transformers`. + * + * Supports WebNN, WebGPU, and WASM ONNX runtime devices. The default model is + * the Q4-quantized EmbeddingGemma-300M (matryoshka). Any matryoshka-compatible + * model on the Hugging Face Hub can be substituted via `options.modelId`. + * + * The pipeline is loaded lazily on the first `embed()` or `embedQueries()` call + * so that import cost is zero until the backend is actually needed. + */ +export class TransformersJsEmbeddingBackend implements EmbeddingBackend { + readonly kind: string; + readonly dimension: number; + readonly modelId: string; + readonly device: TransformersJsDevice; + readonly documentPrefix: string; + readonly queryPrefix: string; + + private pipelinePromise: Promise | undefined; + + constructor(options: TransformersJsEmbeddingBackendOptions = {}) { + this.device = options.device ?? "wasm"; + this.modelId = options.modelId ?? EMBEDDING_GEMMA_300M_MODEL_ID; + this.dimension = + options.dimension ?? EMBEDDING_GEMMA_300M_EMBEDDING_DIMENSION; + this.documentPrefix = + options.documentPrefix ?? EMBEDDING_GEMMA_300M_DOCUMENT_PREFIX; + this.queryPrefix = options.queryPrefix ?? EMBEDDING_GEMMA_300M_QUERY_PREFIX; + this.kind = `transformers-js:${this.device}`; + } + + /** + * Embeds the given texts as document/passage representations. + * Prepends `documentPrefix` before each text as required by the model. + */ + async embed(texts: string[]): Promise { + return this.embedWithPrefix(texts, this.documentPrefix); + } + + /** + * Embeds the given texts as search query representations. + * Prepends `queryPrefix` before each text as required by the model. + * + * Use this method when encoding queries for retrieval; use `embed()` for + * documents/passages being indexed. + */ + async embedQueries(texts: string[]): Promise { + return this.embedWithPrefix(texts, this.queryPrefix); + } + + private async embedWithPrefix( + texts: string[], + prefix: string, + ): Promise { + const extractor = await this.ensurePipeline(); + const prefixed = + prefix.length > 0 ? texts.map((t) => `${prefix}${t}`) : texts; + + const output = await extractor(prefixed, { + pooling: "mean", + normalize: true, + }); + + const rawData = output.data as Float32Array; + const fullDim = rawData.length / texts.length; + const sliceDim = Math.min(this.dimension, fullDim); + + const results: Float32Array[] = []; + for (let i = 0; i < texts.length; i++) { + const start = i * fullDim; + results.push(rawData.slice(start, start + sliceDim)); + } + return results; + } + + private ensurePipeline(): Promise { + if (!this.pipelinePromise) { + this.pipelinePromise = this.loadPipeline(); + } + return this.pipelinePromise; + } + + private async loadPipeline(): Promise { + const { pipeline } = await import("@huggingface/transformers"); + // Cast through unknown to work around the overloaded pipeline union type complexity. + const pipelineFn = pipeline as unknown as ( + task: string, + model: string, + options?: Record, + ) => Promise; + return pipelineFn("feature-extraction", this.modelId, { + device: this.device, + }); + } +} diff --git a/lib/hippocampus/Chunker.ts b/lib/hippocampus/Chunker.ts new file mode 100644 index 0000000..f61b33a --- /dev/null +++ b/lib/hippocampus/Chunker.ts @@ -0,0 +1,79 @@ +import type { ModelProfile } from "../core/ModelProfile"; + +/** + * Splits input text into page-sized chunks based on a token budget. + * + * This is a lightweight, whitespace-token-based chunker (no external tokenizer). + * It prefers to keep sentence boundaries when possible but will split overlong + * sentences at token boundaries to respect the budget. + */ +export function chunkText(text: string, profile: ModelProfile): string[] { + return chunkTextWithMaxTokens(text, profile.maxChunkTokens); +} + +export function chunkTextWithMaxTokens( + text: string, + maxChunkTokens: number, +): string[] { + if (!Number.isInteger(maxChunkTokens) || maxChunkTokens <= 0) { // model-derived-ok + throw new Error("maxChunkTokens must be a positive integer"); + } + + const normalized = text.replace(/\s+/g, " ").trim(); + if (normalized.length === 0) { + return []; + } + + // Simple sentence boundary heuristic: split after `.`, `?`, or `!` followed by whitespace. + // This is intentionally lightweight and avoids pulling in a full NLP dependency. + const sentences = normalized + .split(/(?<=[.!?])\s+/g) + .map((s) => s.trim()) + .filter(Boolean); + + const tokenize = (s: string): string[] => { + return s.trim().split(/\s+/).filter(Boolean); + }; + + const chunks: string[] = []; + let currentTokens: string[] = []; + + const pushCurrent = () => { + if (currentTokens.length === 0) return; + chunks.push(currentTokens.join(" ")); + currentTokens = []; + }; + + const appendSentence = (sentence: string) => { + const sentenceTokens = tokenize(sentence); + if (sentenceTokens.length === 0) return; + + // Sentence is larger than budget: split it across multiple chunks. + if (sentenceTokens.length > maxChunkTokens) { + pushCurrent(); + // model-derived-ok: uses maxChunkTokens as derived from ModelProfile + for (let i = 0; i < sentenceTokens.length; i += maxChunkTokens) { // model-derived-ok + const slice = sentenceTokens.slice(i, i + maxChunkTokens); + chunks.push(slice.join(" ")); + } + return; + } + + // Try to keep sentence with current chunk. + if (currentTokens.length + sentenceTokens.length <= maxChunkTokens) { + currentTokens.push(...sentenceTokens); + return; + } + + // Otherwise, start a new chunk. + pushCurrent(); + currentTokens.push(...sentenceTokens); + }; + + for (const sentence of sentences) { + appendSentence(sentence); + } + + pushCurrent(); + return chunks; +} diff --git a/lib/hippocampus/FastNeighborInsert.ts b/lib/hippocampus/FastNeighborInsert.ts new file mode 100644 index 0000000..5c27460 --- /dev/null +++ b/lib/hippocampus/FastNeighborInsert.ts @@ -0,0 +1,222 @@ +import type { Hash, MetadataStore, SemanticNeighbor, VectorStore } from "../core/types"; +import type { ModelProfile } from "../core/ModelProfile"; +import type { HotpathPolicy } from "../core/HotpathPolicy"; +import { computeNeighborMaxDegree, DEFAULT_HOTPATH_POLICY } from "../core/HotpathPolicy"; +import { runPromotionSweep } from "../core/SalienceEngine"; + +// Absolute upper cap for the semantic neighbor degree. The Williams formula +// can produce larger values for very large corpora; this hard cap keeps the +// neighbor graph manageable even at scale. +const NEIGHBOR_DEGREE_HARD_CAP = 32; + +// Default cosine-distance cutoff when no policy hint is available. +// Cosine distance 0.5 ≡ cosine similarity 0.5 (≥ 0.5 similarity passes). +const DEFAULT_CUTOFF_DISTANCE = 0.5; + +export interface FastNeighborInsertOptions { + modelProfile: ModelProfile; + vectorStore: VectorStore; + metadataStore: MetadataStore; + policy?: HotpathPolicy; + maxDegree?: number; + cutoffDistance?: number; +} + +function cosineSimilarity(a: Float32Array, b: Float32Array): number { + let dot = 0; + let magA = 0; + let magB = 0; + for (let i = 0; i < a.length; i++) { + dot += a[i] * b[i]; + magA += a[i] * a[i]; + magB += b[i] * b[i]; + } + const denom = Math.sqrt(magA) * Math.sqrt(magB); + if (denom === 0) return 0; + return dot / denom; +} + +/** + * Merge a new candidate into an existing neighbor list, respecting maxDegree. + * If at capacity, evict the entry with the lowest cosineSimilarity to make room. + * Returns the updated list sorted by cosineSimilarity descending. + */ +function mergeNeighbor( + existing: SemanticNeighbor[], + candidate: SemanticNeighbor, + maxDegree: number, +): SemanticNeighbor[] { + // Avoid duplicates. + const deduped = existing.filter((n) => n.neighborPageId !== candidate.neighborPageId); + + if (deduped.length < maxDegree) { + deduped.push(candidate); + } else { + // Find weakest existing neighbor. + let weakestIdx = 0; + for (let i = 1; i < deduped.length; i++) { + if (deduped[i].cosineSimilarity < deduped[weakestIdx].cosineSimilarity) { + weakestIdx = i; + } + } + if (candidate.cosineSimilarity > deduped[weakestIdx].cosineSimilarity) { + deduped[weakestIdx] = candidate; + } + // If candidate is weaker than all existing, discard it (return unchanged). + } + + deduped.sort((a, b) => b.cosineSimilarity - a.cosineSimilarity); + return deduped; +} + +/** + * Build and persist semantic neighbor edges for `newPageIds`. + * + * Forward edges (newPage → neighbor) and reverse edges (neighbor → newPage) + * are both stored. This is NOT Hebbian — no edges_hebbian records are created. + */ +export async function insertSemanticNeighbors( + newPageIds: Hash[], + allPageIds: Hash[], + options: FastNeighborInsertOptions, +): Promise { + const { + modelProfile, + vectorStore, + metadataStore, + policy, + cutoffDistance = DEFAULT_CUTOFF_DISTANCE, + } = options; + + // Derive maxDegree from the Williams bound, scaled by corpus size. + // When a policy is provided, use its scaling constant; otherwise fall back + // to the default constant so that corpus-proportional scaling applies in + // all cases (not just when a policy object is explicitly threaded through). + // An explicit options.maxDegree always wins. + let maxDegree: number; + if (options.maxDegree !== undefined) { + maxDegree = options.maxDegree; + } else { + const c = policy?.c ?? DEFAULT_HOTPATH_POLICY.c; + maxDegree = computeNeighborMaxDegree(allPageIds.length, c, NEIGHBOR_DEGREE_HARD_CAP); + } + + if (newPageIds.length === 0) return; + + const dim = modelProfile.embeddingDimension; + + // Fetch all page records in batch for their embedding offsets. + const allPageRecords = await Promise.all( + allPageIds.map((id) => metadataStore.getPage(id)), + ); + + const offsetMap = new Map(); + for (let i = 0; i < allPageIds.length; i++) { + const p = allPageRecords[i]; + if (p) offsetMap.set(allPageIds[i], p.embeddingOffset); + } + + // (a) Throw if any newPageId is missing from the store — a missing new page + // is always a programming error (it should have been persisted before calling + // insertSemanticNeighbors) and would silently corrupt the graph. + for (const newId of newPageIds) { + if (!offsetMap.has(newId)) { + throw new Error( + `Page ${newId} not found in metadata store; persist it before inserting semantic neighbors`, + ); + } + } + + // (b) Filter allPageIds to only those that are present in the store. + // Missing entries are silently dropped — they may have been deleted between + // the getAllPages() call and this point. The vector/id arrays stay aligned. + const resolvedPageIds: Hash[] = []; + const resolvedOffsets: number[] = []; + for (const id of allPageIds) { + const offset = offsetMap.get(id); + if (offset !== undefined) { + resolvedPageIds.push(id); + resolvedOffsets.push(offset); + } + } + + const allVectors = await vectorStore.readVectors(resolvedOffsets, dim); + const vectorMap = new Map(); + for (let i = 0; i < resolvedPageIds.length; i++) { + vectorMap.set(resolvedPageIds[i], allVectors[i]); + } + + // Collect all (pageId, neighborPageId) pairs that need their stored neighbor + // lists updated, keyed by pageId. + const pendingUpdates = new Map(); + + const getOrLoadNeighbors = async (pageId: Hash): Promise => { + if (pendingUpdates.has(pageId)) return pendingUpdates.get(pageId)!; + const stored = await metadataStore.getSemanticNeighbors(pageId); + pendingUpdates.set(pageId, stored); + return stored; + }; + + for (const newId of newPageIds) { + const newVec = vectorMap.get(newId); + if (!newVec) continue; + + // Compute similarity to every other page. + const candidates: SemanticNeighbor[] = []; + for (const otherId of allPageIds) { + if (otherId === newId) continue; + const otherVec = vectorMap.get(otherId); + if (!otherVec) continue; + + const sim = cosineSimilarity(newVec, otherVec); + const dist = 1 - sim; + if (dist <= cutoffDistance) { + candidates.push({ neighborPageId: otherId, cosineSimilarity: sim, distance: dist }); + } + } + + // Sort descending and cap to maxDegree for the forward list. + candidates.sort((a, b) => b.cosineSimilarity - a.cosineSimilarity); + const forwardNeighbors = candidates.slice(0, maxDegree); + + // Merge into the new page's own neighbor list. + let newPageNeighbors = await getOrLoadNeighbors(newId); + for (const candidate of forwardNeighbors) { + newPageNeighbors = mergeNeighbor(newPageNeighbors, candidate, maxDegree); + } + pendingUpdates.set(newId, newPageNeighbors); + + // Insert reverse edges: for each accepted forward neighbor, add newId to + // that neighbor's list. + for (const fwd of forwardNeighbors) { + const reverseCandidate: SemanticNeighbor = { + neighborPageId: newId, + cosineSimilarity: fwd.cosineSimilarity, + distance: fwd.distance, + }; + let neighborList = await getOrLoadNeighbors(fwd.neighborPageId); + neighborList = mergeNeighbor(neighborList, reverseCandidate, maxDegree); + pendingUpdates.set(fwd.neighborPageId, neighborList); + } + } + + // Flush all updated neighbor lists to the store. + await Promise.all( + [...pendingUpdates.entries()].map(([pageId, neighbors]) => + metadataStore.putSemanticNeighbors(pageId, neighbors), + ), + ); + + // Mark affected volumes dirty so the Daydreamer knows to recompute. + for (const newId of newPageIds) { + const books = await metadataStore.getBooksByPage(newId); + for (const book of books) { + const vols = await metadataStore.getVolumesByBook(book.bookId); + for (const vol of vols) { + await metadataStore.flagVolumeForNeighborRecalc(vol.volumeId); + } + } + } + + await runPromotionSweep(newPageIds, metadataStore, policy); +} diff --git a/lib/hippocampus/HierarchyBuilder.ts b/lib/hippocampus/HierarchyBuilder.ts new file mode 100644 index 0000000..e63aba7 --- /dev/null +++ b/lib/hippocampus/HierarchyBuilder.ts @@ -0,0 +1,404 @@ +import type { Book, Hash, MetadataStore, SemanticNeighbor, Shelf, Volume, VectorStore } from "../core/types"; +import type { ModelProfile } from "../core/ModelProfile"; +import type { HotpathPolicy } from "../core/HotpathPolicy"; +import { computeFanoutLimit, DEFAULT_HOTPATH_POLICY } from "../core/HotpathPolicy"; +import { hashText } from "../core/crypto/hash"; +import { runPromotionSweep } from "../core/SalienceEngine"; + +// Clustering fan-out targets — policy constants, not model-derived. +// These are chosen to be consistent with the Williams Bound fanout limit: +// computeFanoutLimit(N) ≈ ceil(0.5 * sqrt(N * log2(1+N))) +// At typical early-corpus sizes these constants (4-8) sit comfortably within +// the Williams-derived quota. ClusterStability handles splits at runtime if +// volumes grow beyond their quota after the initial hierarchy build. +const PAGES_PER_BOOK = 8; +const BOOKS_PER_VOLUME = 4; +const VOLUMES_PER_SHELF = 4; + +// Max neighbors per page for the adjacency edges added by the hierarchy builder. +// Adjacency edges represent document-order contiguity and bypass the cosine +// cutoff used by FastNeighborInsert, so they must still be bounded by policy. +const ADJACENCY_MAX_DEGREE = 16; + +export interface BuildHierarchyOptions { + modelProfile: ModelProfile; + vectorStore: VectorStore; + metadataStore: MetadataStore; + policy?: HotpathPolicy; +} + +function cosineSimilarity(a: Float32Array, b: Float32Array): number { + let dot = 0; + let magA = 0; + let magB = 0; + for (let i = 0; i < a.length; i++) { + dot += a[i] * b[i]; + magA += a[i] * a[i]; + magB += b[i] * b[i]; + } + const denom = Math.sqrt(magA) * Math.sqrt(magB); + if (denom === 0) return 0; + return dot / denom; +} + +function cosineDistance(a: Float32Array, b: Float32Array): number { + return 1 - cosineSimilarity(a, b); +} + +function computeCentroid(vectors: Float32Array[]): Float32Array { + const dim = vectors[0].length; + const centroid = new Float32Array(dim); + for (const v of vectors) { + for (let i = 0; i < dim; i++) { + centroid[i] += v[i]; + } + } + for (let i = 0; i < dim; i++) { + centroid[i] /= vectors.length; + } + return centroid; +} + +/** Returns the index in `vectors` whose sum of distances to all others is minimal. */ +function selectMedoidIndex(vectors: Float32Array[]): number { + if (vectors.length === 1) return 0; + + let bestIndex = 0; + let bestTotalDistance = Infinity; + + for (let i = 0; i < vectors.length; i++) { + let totalDistance = 0; + for (let j = 0; j < vectors.length; j++) { + if (i !== j) totalDistance += cosineDistance(vectors[i], vectors[j]); + } + if (totalDistance < bestTotalDistance) { + bestTotalDistance = totalDistance; + bestIndex = i; + } + } + + return bestIndex; +} + +function chunkArray(arr: T[], size: number): T[][] { + const chunks: T[][] = []; + for (let i = 0; i < arr.length; i += size) { + chunks.push(arr.slice(i, i + size)); + } + return chunks; +} + +/** + * Merge a candidate into a neighbor list, respecting maxDegree. + * If at capacity, evicts the neighbor with the lowest cosineSimilarity. + * Returns the updated list sorted by cosineSimilarity descending. + */ +function mergeAdjacentNeighbor( + existing: SemanticNeighbor[], + candidate: SemanticNeighbor, + maxDegree: number, +): SemanticNeighbor[] { + const deduped = existing.filter((n) => n.neighborPageId !== candidate.neighborPageId); + + if (deduped.length < maxDegree) { + deduped.push(candidate); + } else { + let weakestIdx = 0; + for (let i = 1; i < deduped.length; i++) { + if (deduped[i].cosineSimilarity < deduped[weakestIdx].cosineSimilarity) { + weakestIdx = i; + } + } + if (candidate.cosineSimilarity > deduped[weakestIdx].cosineSimilarity) { + deduped[weakestIdx] = candidate; + } + } + + deduped.sort((a, b) => b.cosineSimilarity - a.cosineSimilarity); + return deduped; +} + +export async function buildHierarchy( + pageIds: Hash[], + options: BuildHierarchyOptions, +): Promise<{ books: Book[]; volumes: Volume[]; shelves: Shelf[] }> { + const { modelProfile, vectorStore, metadataStore, policy } = options; + const dim = modelProfile.embeddingDimension; + + if (pageIds.length === 0) { + return { books: [], volumes: [], shelves: [] }; + } + + // Fetch all page records to get their embedding offsets. + const pageRecords = await Promise.all(pageIds.map((id) => metadataStore.getPage(id))); + const pageOffsets = pageRecords.map((p, i) => { + if (!p) throw new Error(`Page ${pageIds[i]} not found during hierarchy build`); + return p.embeddingOffset; + }); + const pageVectors = await vectorStore.readVectors(pageOffsets, dim); + + // Build a Map for O(1) lookups throughout the hierarchy build. + const pageVectorMap = new Map(); + for (let i = 0; i < pageIds.length; i++) { + pageVectorMap.set(pageIds[i], pageVectors[i]); + } + + // ------------------------------------------------------------------------- + // Level 1: Pages → Books + // ------------------------------------------------------------------------- + const pageChunks = chunkArray(pageIds, PAGES_PER_BOOK); + const books: Book[] = []; + + for (const chunk of pageChunks) { + const sortedChunk = [...chunk].sort(); + const bookId = await hashText(sortedChunk.join("|")); + + const chunkVectors = chunk.map((id) => { + const vec = pageVectorMap.get(id); + if (!vec) throw new Error(`Vector not found for page ${id}`); + return vec; + }); + + const medoidIdx = selectMedoidIndex(chunkVectors); + const medoidPageId = chunk[medoidIdx]; + + const book: Book = { bookId, pageIds: chunk, medoidPageId, meta: {} }; + await metadataStore.putBook(book); + books.push(book); + } + + // Add SemanticNeighbor edges between consecutive pages within each book slice. + // These document-order adjacency edges are always inserted regardless of cosine + // cutoff, because adjacent text chunks of the same source are always related. + for (const book of books) { + for (let i = 0; i < book.pageIds.length - 1; i++) { + const aId = book.pageIds[i]; + const bId = book.pageIds[i + 1]; + const aVec = pageVectorMap.get(aId); + const bVec = pageVectorMap.get(bId); + if (!aVec || !bVec) continue; + + const sim = cosineSimilarity(aVec, bVec); + const dist = 1 - sim; + const forwardEdge: SemanticNeighbor = { neighborPageId: bId, cosineSimilarity: sim, distance: dist }; + const reverseEdge: SemanticNeighbor = { neighborPageId: aId, cosineSimilarity: sim, distance: dist }; + + // Forward: a → b + const existingA = await metadataStore.getSemanticNeighbors(aId); + await metadataStore.putSemanticNeighbors(aId, mergeAdjacentNeighbor(existingA, forwardEdge, ADJACENCY_MAX_DEGREE)); + + // Reverse: b → a + const existingB = await metadataStore.getSemanticNeighbors(bId); + await metadataStore.putSemanticNeighbors(bId, mergeAdjacentNeighbor(existingB, reverseEdge, ADJACENCY_MAX_DEGREE)); + } + } + + await runPromotionSweep(books.map((b) => b.bookId), metadataStore, policy); + + // ------------------------------------------------------------------------- + // Level 2: Books → Volumes + // ------------------------------------------------------------------------- + const bookChunks = chunkArray(books, BOOKS_PER_VOLUME); + const volumes: Volume[] = []; + + for (const bookChunk of bookChunks) { + const sortedBookIds = bookChunk.map((b) => b.bookId).sort(); + const volumeId = await hashText(sortedBookIds.join("|")); + + const medoidVectors = bookChunk.map((b) => { + const vec = pageVectorMap.get(b.medoidPageId); + if (!vec) throw new Error(`Vector not found for medoid page ${b.medoidPageId}`); + return vec; + }); + + const centroid = computeCentroid(medoidVectors); + const prototypeOffset = await vectorStore.appendVector(centroid); + + // Average squared cosine distance from centroid. + let variance = 0; + for (const v of medoidVectors) { + const dist = cosineDistance(v, centroid); + variance += dist * dist; + } + variance /= medoidVectors.length; + + const volume: Volume = { + volumeId, + bookIds: bookChunk.map((b) => b.bookId), + prototypeOffsets: [prototypeOffset], + prototypeDim: dim, + variance, + }; + await metadataStore.putVolume(volume); + volumes.push(volume); + } + + await runPromotionSweep(volumes.map((v) => v.volumeId), metadataStore, policy); + + // ------------------------------------------------------------------------- + // Level 3: Volumes → Shelves + // ------------------------------------------------------------------------- + const volumeChunks = chunkArray(volumes, VOLUMES_PER_SHELF); + const shelves: Shelf[] = []; + + for (const volumeChunk of volumeChunks) { + const sortedVolumeIds = volumeChunk.map((v) => v.volumeId).sort(); + const shelfId = await hashText(sortedVolumeIds.join("|")); + + const protoVectors = await Promise.all( + volumeChunk.map((v) => vectorStore.readVector(v.prototypeOffsets[0], dim)), + ); + + const routingCentroid = computeCentroid(protoVectors); + const routingOffset = await vectorStore.appendVector(routingCentroid); + + const shelf: Shelf = { + shelfId, + volumeIds: volumeChunk.map((v) => v.volumeId), + routingPrototypeOffsets: [routingOffset], + routingDim: dim, + }; + await metadataStore.putShelf(shelf); + shelves.push(shelf); + } + + await runPromotionSweep(shelves.map((s) => s.shelfId), metadataStore, policy); + + // ------------------------------------------------------------------------- + // Williams fanout quota enforcement + // ------------------------------------------------------------------------- + // Per DESIGN.md "Sublinear Fanout Bounds": when a node's child count exceeds + // its Williams-derived limit, HierarchyBuilder triggers a split. + // + // The split threshold is max(STATIC_CONSTANT, computeFanoutLimit(nodeSize)): + // - For freshly built nodes (size ≤ STATIC_CONSTANT), the Williams formula + // may return a smaller value, so we use the static constant as a floor to + // prevent splitting a structure that was just constructed correctly. + // - For nodes that have grown organically past the static constant, the + // Williams limit takes over and drives the split. + const policyC = policy?.c ?? DEFAULT_HOTPATH_POLICY.c; + + // ---- Volumes ---- + for (const volume of [...volumes]) { + const nodeLimit = Math.max(BOOKS_PER_VOLUME, computeFanoutLimit(volume.bookIds.length, policyC)); + if (volume.bookIds.length <= nodeLimit) continue; + + const subChunks = chunkArray(volume.bookIds, nodeLimit); + const subVolumes: Volume[] = []; + + for (const sub of subChunks) { + const sortedSub = [...sub].sort(); + const subVolumeId = await hashText(`split-vol:${volume.volumeId}:${sortedSub.join("|")}`); + + const subBooks = ( + await Promise.all(sub.map((id) => metadataStore.getBook(id))) + ).filter((b): b is Book => b !== undefined); + + // Compute sub-volume variance from actual medoid vectors. + const medoidVecs = subBooks + .map((b) => pageVectorMap.get(b.medoidPageId)) + .filter((v): v is Float32Array => v !== undefined); + const centroid = medoidVecs.length > 0 ? computeCentroid(medoidVecs) : new Float32Array(dim); + const protoOffset = await vectorStore.appendVector(centroid); + + let subVariance = 0; + for (const v of medoidVecs) { + const d = cosineDistance(v, centroid); + subVariance += d * d; + } + if (medoidVecs.length > 0) subVariance /= medoidVecs.length; + + const subVol: Volume = { + volumeId: subVolumeId, + bookIds: sub, + prototypeOffsets: [protoOffset], + prototypeDim: dim, + variance: subVariance, + }; + await metadataStore.putVolume(subVol); + subVolumes.push(subVol); + } + + // Replace the oversized volume in every shelf that references it. + for (const shelf of shelves) { + const idx = shelf.volumeIds.indexOf(volume.volumeId); + if (idx === -1) continue; + const newVolumeIds = [ + ...shelf.volumeIds.slice(0, idx), + ...subVolumes.map((v) => v.volumeId), + ...shelf.volumeIds.slice(idx + 1), + ]; + const updated: Shelf = { ...shelf, volumeIds: newVolumeIds }; + await metadataStore.putShelf(updated); + Object.assign(shelf, updated); + } + + // Delete the original oversized volume (and its reverse-index entries). + await metadataStore.deleteVolume(volume.volumeId); + volumes.splice(volumes.indexOf(volume), 1, ...subVolumes); + } + + // ---- Shelves ---- + for (const shelf of [...shelves]) { + const nodeLimit = Math.max(VOLUMES_PER_SHELF, computeFanoutLimit(shelf.volumeIds.length, policyC)); + if (shelf.volumeIds.length <= nodeLimit) continue; + + const subChunks = chunkArray(shelf.volumeIds, nodeLimit); + + // Update the existing shelf record to hold only the FIRST sub-chunk's + // volumes. All remaining sub-chunks get fresh shelf records. + // Note: volumes that move to new shelves keep a stale volumeToShelf entry + // pointing at this shelf's ID; that entry will be cleaned up by a future + // Daydreamer ClusterStability pass (deleteShelf is not yet on the interface). + const newShelves: Shelf[] = []; + for (let ci = 0; ci < subChunks.length; ci++) { + const sub = subChunks[ci]; + const sortedSub = [...sub].sort(); + + if (ci === 0) { + // Re-use the original shelfId for the first sub-chunk. + const subShelfVols = ( + await Promise.all(sub.map((id) => metadataStore.getVolume(id))) + ).filter((v): v is Volume => v !== undefined); + const protoVecs = await Promise.all( + subShelfVols.map((v) => vectorStore.readVector(v.prototypeOffsets[0], dim)), + ); + const centroid = protoVecs.length > 0 ? computeCentroid(protoVecs) : new Float32Array(dim); + const routingOffset = await vectorStore.appendVector(centroid); + + const updated: Shelf = { + shelfId: shelf.shelfId, + volumeIds: sub, + routingPrototypeOffsets: [routingOffset], + routingDim: dim, + }; + await metadataStore.putShelf(updated); + Object.assign(shelf, updated); + newShelves.push(updated); + } else { + const subShelfId = await hashText(`split-shelf:${shelf.shelfId}:${sortedSub.join("|")}`); + const subShelfVols = ( + await Promise.all(sub.map((id) => metadataStore.getVolume(id))) + ).filter((v): v is Volume => v !== undefined); + const protoVecs = await Promise.all( + subShelfVols.map((v) => vectorStore.readVector(v.prototypeOffsets[0], dim)), + ); + const centroid = protoVecs.length > 0 ? computeCentroid(protoVecs) : new Float32Array(dim); + const routingOffset = await vectorStore.appendVector(centroid); + + const newShelf: Shelf = { + shelfId: subShelfId, + volumeIds: sub, + routingPrototypeOffsets: [routingOffset], + routingDim: dim, + }; + await metadataStore.putShelf(newShelf); + newShelves.push(newShelf); + } + } + + shelves.splice(shelves.indexOf(shelf), 1, ...newShelves); + } + + return { books, volumes, shelves }; +} diff --git a/lib/hippocampus/Ingest.ts b/lib/hippocampus/Ingest.ts new file mode 100644 index 0000000..f79b4da --- /dev/null +++ b/lib/hippocampus/Ingest.ts @@ -0,0 +1,156 @@ +import type { Book, MetadataStore, VectorStore } from "../core/types"; +import type { ModelProfile } from "../core/ModelProfile"; +import { hashText } from "../core/crypto/hash"; +import type { KeyPair } from "../core/crypto/sign"; +import { EmbeddingRunner } from "../embeddings/EmbeddingRunner"; +import { chunkText } from "./Chunker"; +import { buildPage } from "./PageBuilder"; +import { runPromotionSweep } from "../core/SalienceEngine"; +import { insertSemanticNeighbors } from "./FastNeighborInsert"; + +export interface IngestOptions { + modelProfile: ModelProfile; + embeddingRunner: EmbeddingRunner; + vectorStore: VectorStore; + metadataStore: MetadataStore; + keyPair: KeyPair; + now?: number; +} + +export interface IngestResult { + pages: Array>>; + /** The single Book representing everything ingested by this call. + * One ingest call = one Book, always. All pages are members. + * A collection of Books becomes a Volume; a collection of Volumes + * becomes a Shelf — those tiers are assembled by the Daydreamer. */ + book?: Book; +} + +function cosineDistance(a: Float32Array, b: Float32Array): number { + let dot = 0; + let normA = 0; + let normB = 0; + for (let i = 0; i < a.length; i++) { + dot += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + const denom = Math.sqrt(normA) * Math.sqrt(normB); + if (denom === 0) return 0; + return 1 - dot / denom; +} + +/** + * Selects the index of the medoid: the element that minimises total cosine + * distance to every other element in the set. + */ +function selectMedoidIndex(vectors: Float32Array[]): number { + if (vectors.length === 1) return 0; + let bestIdx = 0; + let bestTotal = Infinity; + for (let i = 0; i < vectors.length; i++) { + let total = 0; + for (let j = 0; j < vectors.length; j++) { + if (i !== j) total += cosineDistance(vectors[i], vectors[j]); + } + if (total < bestTotal) { + bestTotal = total; + bestIdx = i; + } + } + return bestIdx; +} + +export async function ingestText( + text: string, + options: IngestOptions, +): Promise { + const { + modelProfile, + embeddingRunner, + vectorStore, + metadataStore, + keyPair, + now = Date.now(), + } = options; + + const chunks = chunkText(text, modelProfile); + if (chunks.length === 0) { + return { pages: [], book: undefined }; + } + + const createdAt = new Date(now).toISOString(); + + // Precompute page IDs (content hashes) so we can link prev/next before signing. + const pageIds = await Promise.all(chunks.map((c) => hashText(c))); + + const embeddings = await embeddingRunner.embed(chunks); + if (embeddings.length !== chunks.length) { + throw new Error("Embedding provider returned unexpected number of embeddings"); + } + + const offsets: number[] = []; + for (const embedding of embeddings) { + const offset = await vectorStore.appendVector(embedding); + offsets.push(offset); + } + + const pages = await Promise.all( + chunks.map(async (content, idx) => { + const prevPageId = idx > 0 ? pageIds[idx - 1] : null; + const nextPageId = idx < pageIds.length - 1 ? pageIds[idx + 1] : null; + + return buildPage({ + content, + embedding: embeddings[idx], + embeddingOffset: offsets[idx], + embeddingDim: modelProfile.embeddingDimension, + creatorPubKey: keyPair.publicKey, + signingKey: keyPair.signingKey, + prevPageId, + nextPageId, + createdAt, + }); + }), + ); + + // Persist pages and activity records. + for (const page of pages) { + await metadataStore.putPage(page); + await metadataStore.putPageActivity({ + pageId: page.pageId, + queryHitCount: 0, + lastQueryAt: createdAt, + }); + } + + // Build ONE Book for the entire ingest. + // A Book = the document we just ingested; its identity is the sorted set of + // its pages. Its representative is the page whose embedding is the medoid + // (minimum total cosine distance to all other pages in the document). + const medoidIdx = selectMedoidIndex(embeddings); + const sortedPageIds = [...pageIds].sort(); + const bookId = await hashText(sortedPageIds.join("|")); + const book: Book = { + bookId, + pageIds, + medoidPageId: pageIds[medoidIdx], + meta: {}, + }; + await metadataStore.putBook(book); + + // Insert semantic neighbor edges for the new pages against all stored pages. + // Volumes and Shelves are assembled by the Daydreamer from accumulated Books. + const allPages = await metadataStore.getAllPages(); + const allPageIds = allPages.map((p) => p.pageId); + await insertSemanticNeighbors(pageIds, allPageIds, { + modelProfile, + vectorStore, + metadataStore, + }); + + // Run hotpath promotion for the newly ingested pages and book. + await runPromotionSweep([...pageIds, bookId], metadataStore); + + return { pages, book }; +} diff --git a/lib/hippocampus/PageBuilder.ts b/lib/hippocampus/PageBuilder.ts new file mode 100644 index 0000000..d03e9e3 --- /dev/null +++ b/lib/hippocampus/PageBuilder.ts @@ -0,0 +1,88 @@ +import type { Hash, Page } from "../core/types"; +import { hashBinary, hashText } from "../core/crypto/hash"; +import { signData } from "../core/crypto/sign"; + +export interface BuildPageOptions { + content: string; + embedding: Float32Array; + embeddingOffset: number; + embeddingDim: number; + creatorPubKey: string; + signingKey: CryptoKey; + prevPageId?: Hash | null; + nextPageId?: Hash | null; + createdAt?: string; +} + +/** + * Build a Page entity from content + embedding. + * + * Creates deterministic `pageId`/`contentHash` from content, a `vectorHash` from + * the raw embedding bytes, and signs the page using the provided key. + */ +export async function buildPage(options: BuildPageOptions): Promise { + const { + content, + embedding, + embeddingOffset, + embeddingDim, + creatorPubKey, + signingKey, + prevPageId = null, + nextPageId = null, + createdAt = new Date().toISOString(), + } = options; + + if (embedding.length !== embeddingDim) { + throw new Error( + `Embedding dimension mismatch: expected ${embeddingDim}, got ${embedding.length}`, + ); + } + + const contentHash = await hashText(content); + const pageId = contentHash; + + // Copy into a new ArrayBuffer-backed view so we never pass a SharedArrayBuffer + // into WebCrypto (and keep TypeScript happy). + const rawVector = new Uint8Array(embedding.byteLength); + rawVector.set(new Uint8Array(embedding.buffer, embedding.byteOffset, embedding.byteLength)); + const vectorHash = await hashBinary(rawVector); + + const unsignedPage = { + pageId, + content, + embeddingOffset, + embeddingDim, + contentHash, + vectorHash, + prevPageId: prevPageId ?? null, + nextPageId: nextPageId ?? null, + creatorPubKey, + createdAt, + } as const; + + // Deterministic canonical representation used for signing. + const canonical = canonicalizePageForSigning(unsignedPage); + const signature = await signData(canonical, signingKey); + + return { + ...unsignedPage, + signature, + }; +} + +function canonicalizePageForSigning(page: Omit): string { + // Keep key order stable for deterministic signing. + return JSON.stringify({ + pageId: page.pageId, + content: page.content, + embeddingOffset: page.embeddingOffset, + embeddingDim: page.embeddingDim, + contentHash: page.contentHash, + vectorHash: page.vectorHash, + prevPageId: page.prevPageId ?? null, + nextPageId: page.nextPageId ?? null, + creatorPubKey: page.creatorPubKey, + createdAt: page.createdAt, + }); +} diff --git a/lib/sharing/CuriosityBroadcaster.ts b/lib/sharing/CuriosityBroadcaster.ts new file mode 100644 index 0000000..ff6e0a3 --- /dev/null +++ b/lib/sharing/CuriosityBroadcaster.ts @@ -0,0 +1,151 @@ +// --------------------------------------------------------------------------- +// CuriosityBroadcaster — broadcast pending probes and handle responses (P2-G0) +// --------------------------------------------------------------------------- +// +// Consumes CuriosityProbe objects queued by KnowledgeGapDetector, serialises +// them for P2P transport, rate-limits broadcasts to prevent spam, and +// delegates incoming graph fragment responses to SubgraphImporter. +// --------------------------------------------------------------------------- + +import { randomUUID } from "../core/crypto/uuid"; +import type { CuriosityProbe, GraphFragment, PeerMessage } from "./types"; + +// --------------------------------------------------------------------------- +// P2P transport abstraction +// --------------------------------------------------------------------------- + +/** + * Minimal P2P transport interface. + * The broadcaster is transport-agnostic — inject any WebRTC/WebSocket + * implementation that satisfies this contract. + */ +export interface P2PTransport { + /** Broadcast a message to all connected peers. */ + broadcast(message: PeerMessage): Promise; + /** Register a listener for incoming messages from peers. */ + onMessage(handler: (message: PeerMessage) => void): void; +} + +// --------------------------------------------------------------------------- +// Response handler +// --------------------------------------------------------------------------- + +export type FragmentHandler = (fragment: GraphFragment) => Promise; + +// --------------------------------------------------------------------------- +// Options +// --------------------------------------------------------------------------- + +export interface CuriosityBroadcasterOptions { + transport: P2PTransport; + /** Local node identifier (used as senderId). */ + nodeId: string; + /** Minimum milliseconds between broadcasts of any probe. Default: 5000. */ + rateLimitMs?: number; + /** Maximum probe queue depth before oldest probes are dropped. Default: 100. */ + maxQueueDepth?: number; +} + +// --------------------------------------------------------------------------- +// CuriosityBroadcaster +// --------------------------------------------------------------------------- + +/** + * Manages the lifecycle of outbound curiosity probes. + * + * Probes are enqueued via `enqueueProbe()`, then broadcast during idle time + * via `flush()`. Rate limiting prevents probe spam. Incoming graph fragment + * responses are dispatched to the registered `onFragment` handler. + */ +export class CuriosityBroadcaster { + private readonly transport: P2PTransport; + private readonly nodeId: string; + private readonly rateLimitMs: number; + private readonly maxQueueDepth: number; + + private pendingProbes: CuriosityProbe[] = []; + private lastBroadcastAt = 0; + private fragmentHandler?: FragmentHandler; + + constructor(options: CuriosityBroadcasterOptions) { + this.transport = options.transport; + this.nodeId = options.nodeId; + this.rateLimitMs = options.rateLimitMs ?? 5_000; + this.maxQueueDepth = options.maxQueueDepth ?? 100; + + // Listen for incoming graph fragment responses + this.transport.onMessage((msg) => { + if (msg.kind === "graph_fragment") { + void this._handleFragment(msg.payload as GraphFragment); + } + }); + } + + /** + * Register a handler that will be called when a graph fragment response + * arrives from a peer. Replaces any previously registered handler. + */ + onFragment(handler: FragmentHandler): void { + this.fragmentHandler = handler; + } + + /** + * Enqueue a CuriosityProbe for broadcast. + * + * If the queue is already at capacity, the oldest probe is dropped to make + * room. A fresh probeId is assigned here if the probe does not already have + * one, ensuring each broadcast can be correlated with its response. + */ + enqueueProbe(probe: Omit & { probeId?: string }): void { + const full: CuriosityProbe = { + ...probe, + probeId: probe.probeId ?? randomUUID(), + }; + + if (this.pendingProbes.length >= this.maxQueueDepth) { + this.pendingProbes.shift(); // drop oldest + } + this.pendingProbes.push(full); + } + + /** + * Flush pending probes to connected peers, respecting the rate limit. + * + * Call this from the IdleScheduler during background passes. Each call + * broadcasts at most one probe; subsequent calls broadcast the next one. + * + * Returns the number of probes broadcast (0 or 1). + */ + async flush(now = Date.now()): Promise { + if (this.pendingProbes.length === 0) return 0; + if (now - this.lastBroadcastAt < this.rateLimitMs) return 0; + + const probe = this.pendingProbes.shift(); + if (!probe) return 0; + + const message: PeerMessage = { + kind: "curiosity_probe", + senderId: this.nodeId, + payload: probe, + }; + + await this.transport.broadcast(message); + this.lastBroadcastAt = now; + return 1; + } + + /** Number of probes waiting to be broadcast. */ + get pendingCount(): number { + return this.pendingProbes.length; + } + + // --------------------------------------------------------------------------- + // Private + // --------------------------------------------------------------------------- + + private async _handleFragment(fragment: GraphFragment): Promise { + if (this.fragmentHandler) { + await this.fragmentHandler(fragment); + } + } +} diff --git a/lib/sharing/EligibilityClassifier.ts b/lib/sharing/EligibilityClassifier.ts new file mode 100644 index 0000000..f27c445 --- /dev/null +++ b/lib/sharing/EligibilityClassifier.ts @@ -0,0 +1,111 @@ +// --------------------------------------------------------------------------- +// EligibilityClassifier — classify pages as share-eligible or blocked (P2-G1) +// --------------------------------------------------------------------------- +// +// Detects identity/PII-bearing content before any graph export operation. +// Emits deterministic eligibility decisions with reason codes for auditability. +// +// Rules: +// - Identity PII: person names with SSN/passport/national ID patterns +// - Credentials: password, API key, secret, token patterns +// - Financial: credit card, IBAN, account number patterns +// - Health: medical record, diagnosis, prescription patterns +// - No public interest: very short or empty content +// --------------------------------------------------------------------------- + +import type { Hash, Page } from "../core/types"; +import type { BlockReason, EligibilityDecision, EligibilityStatus } from "./types"; + +// --------------------------------------------------------------------------- +// PII detection patterns +// --------------------------------------------------------------------------- + +/** Minimum content length (chars) to be considered public-interest. */ +const MIN_PUBLIC_INTEREST_LENGTH = 20; + +const PATTERNS: Array<{ reason: BlockReason; pattern: RegExp }> = [ + { + reason: "pii_credentials", + // Passwords, API keys, tokens, secrets in common formats + pattern: /\b(?:password|passwd|api[_-]?key|secret[_-]?key|auth[_-]?token|access[_-]?token)\s*[:=]\s*\S+/i, + }, + { + reason: "pii_credentials", + // Bearer tokens, basic auth, SSH key headers + pattern: /(?:Bearer\s+[A-Za-z0-9\-._~+/]+=*|-----BEGIN (?:RSA |EC |)PRIVATE KEY-----)/, + }, + { + reason: "pii_financial", + // Credit card: 13-19 digits with optional separators + pattern: /\b(?:4[0-9]{12}(?:[0-9]{3,6})?|5[1-5][0-9]{14}|3[47][0-9]{13}|6(?:011|5[0-9]{2})[0-9]{12,15})\b/, + }, + { + reason: "pii_financial", + // IBAN: up to 34 alphanumeric chars after country code + pattern: /\b[A-Z]{2}[0-9]{2}[A-Z0-9]{4}[0-9]{7}(?:[A-Z0-9]{0,16})?\b/, + }, + { + reason: "pii_identity", + // US Social Security Number + pattern: /\b\d{3}[-\s]\d{2}[-\s]\d{4}\b/, + }, + { + reason: "pii_identity", + // Email addresses (identity signal — may be PII) + pattern: /\b[-a-zA-Z0-9._%+]+@[a-zA-Z0-9.]+\.[a-zA-Z]{2,}\b/i, + }, + { + reason: "pii_health", + // Medical record / health identifiers + pattern: /\b(?:medical[_-]?record|patient[_-]?id|diagnosis|prescription|ICD[-\s]?\d{1,2})\b/i, + }, +]; + +// --------------------------------------------------------------------------- +// Classifier +// --------------------------------------------------------------------------- + +/** + * Classify a single page as share-eligible or blocked. + * + * Scans `page.content` against a set of PII/credential patterns and + * returns a deterministic decision with a reason code when blocked. + */ +export function classifyPage(page: Page): EligibilityDecision { + // Reject trivially short content as not public-interest + if (page.content.trim().length < MIN_PUBLIC_INTEREST_LENGTH) { + return blocked(page.pageId, "no_public_interest"); + } + + for (const { reason, pattern } of PATTERNS) { + if (pattern.test(page.content)) { + return blocked(page.pageId, reason); + } + } + + return { pageId: page.pageId, status: "eligible" }; +} + +/** + * Classify a batch of pages, returning one decision per page. + * + * Results are in the same order as the input array. + */ +export function classifyPages(pages: Page[]): EligibilityDecision[] { + return pages.map((p) => classifyPage(p)); +} + +/** + * Filter a page array down to only share-eligible pages. + */ +export function filterEligible(pages: Page[]): Page[] { + return pages.filter((p) => classifyPage(p).status === "eligible"); +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function blocked(pageId: Hash, reason: BlockReason): EligibilityDecision { + return { pageId, status: "blocked" as EligibilityStatus, reason }; +} diff --git a/lib/sharing/PeerExchange.ts b/lib/sharing/PeerExchange.ts new file mode 100644 index 0000000..5bcea1a --- /dev/null +++ b/lib/sharing/PeerExchange.ts @@ -0,0 +1,131 @@ +// --------------------------------------------------------------------------- +// PeerExchange — opt-in signed subgraph exchange over P2P transport (P2-G3) +// --------------------------------------------------------------------------- +// +// Manages the lifecycle of proactive peer-to-peer graph slice sharing. +// Peers that opt in can receive public-interest graph sections from neighbours. +// All payloads pass eligibility filtering before export and are verified on +// import. Sender identity is never exposed to the receiving peer's queries. +// --------------------------------------------------------------------------- + +import { randomUUID } from "../core/crypto/uuid"; +import type { Hash, MetadataStore, VectorStore } from "../core/types"; +import { exportForExchange } from "./SubgraphExporter"; +import { importSlice } from "./SubgraphImporter"; +import type { P2PTransport } from "./CuriosityBroadcaster"; +import type { ImportResult } from "./SubgraphImporter"; +import type { PeerMessage, SubgraphSlice } from "./types"; + +// --------------------------------------------------------------------------- +// Options +// --------------------------------------------------------------------------- + +export interface PeerExchangeOptions { + transport: P2PTransport; + metadataStore: MetadataStore; + vectorStore: VectorStore; + /** Local node identifier (used as senderId). */ + nodeId: string; + /** + * When true, content hashes on received slices are verified. + * Defaults to false. + */ + verifyContentHashes?: boolean; +} + +export interface ExchangeResult { + sliceId: string; + nodesExported: number; +} + +// --------------------------------------------------------------------------- +// PeerExchange +// --------------------------------------------------------------------------- + +/** + * Orchestrates opt-in signed subgraph exchange with connected peers. + * + * Usage: + * const exchange = new PeerExchange({ transport, metadataStore, vectorStore, nodeId }); + * exchange.onSliceReceived(async (result) => { ... }); + * const result = await exchange.sendSlice(seedPageIds); + */ +export class PeerExchange { + private readonly transport: P2PTransport; + private readonly metadataStore: MetadataStore; + private readonly vectorStore: VectorStore; + private readonly nodeId: string; + private readonly verifyContentHashes: boolean; + private sliceHandler?: (result: ImportResult, slice: SubgraphSlice) => Promise; + + constructor(options: PeerExchangeOptions) { + this.transport = options.transport; + this.metadataStore = options.metadataStore; + this.vectorStore = options.vectorStore; + this.nodeId = options.nodeId; + this.verifyContentHashes = options.verifyContentHashes ?? false; + + this.transport.onMessage((msg) => { + if (msg.kind === "subgraph_slice") { + void this._handleIncoming(msg.payload as SubgraphSlice); + } + }); + } + + /** + * Register a handler called when a slice is received and imported. + * Replaces any previously registered handler. + */ + onSliceReceived(handler: (result: ImportResult, slice: SubgraphSlice) => Promise): void { + this.sliceHandler = handler; + } + + /** + * Export a subgraph slice from the given seed page IDs and broadcast it + * to all connected peers. + * + * Only eligibility-approved nodes are included. Returns null if no eligible + * nodes were found or the export produced an empty slice. + */ + async sendSlice( + seedPageIds: Hash[], + maxNodes = 50, + maxHops = 2, + ): Promise { + const exchangeId = randomUUID(); + + const slice = await exportForExchange(seedPageIds, exchangeId, { + metadataStore: this.metadataStore, + maxNodes, + maxHops, + }); + + if (!slice) return null; + + const message: PeerMessage = { + kind: "subgraph_slice", + senderId: this.nodeId, + payload: slice, + }; + + await this.transport.broadcast(message); + + return { sliceId: slice.sliceId, nodesExported: slice.nodes.length }; + } + + // --------------------------------------------------------------------------- + // Private + // --------------------------------------------------------------------------- + + private async _handleIncoming(slice: SubgraphSlice): Promise { + const result = await importSlice(slice, { + metadataStore: this.metadataStore, + vectorStore: this.vectorStore, + verifyContentHashes: this.verifyContentHashes, + }); + + if (this.sliceHandler) { + await this.sliceHandler(result, slice); + } + } +} diff --git a/lib/sharing/SubgraphExporter.ts b/lib/sharing/SubgraphExporter.ts new file mode 100644 index 0000000..a32db9e --- /dev/null +++ b/lib/sharing/SubgraphExporter.ts @@ -0,0 +1,203 @@ +// --------------------------------------------------------------------------- +// SubgraphExporter — build eligibility-filtered graph slices for sharing (P2-G2) +// --------------------------------------------------------------------------- +// +// Constructs topic-scoped graph slices from pages that pass the eligibility +// classifier. For curiosity responses, the slice is built from a BFS expansion +// around the probe's seed (`m1`), constrained by `maxHops` / `maxNodes`. +// +// Personal metadata fields not needed for discovery are stripped or coarsened +// before export. Node/edge signatures and provenance are preserved. +// --------------------------------------------------------------------------- + +import { randomUUID } from "../core/crypto/uuid"; +import type { Edge, Hash, MetadataStore, SemanticNeighbor, Page } from "../core/types"; +import { filterEligible } from "./EligibilityClassifier"; +import type { CuriosityProbe, SubgraphSlice } from "./types"; + +// --------------------------------------------------------------------------- +// Options +// --------------------------------------------------------------------------- + +export interface ExportOptions { + metadataStore: MetadataStore; + /** Maximum nodes to include in a single slice. Default: 50. */ + maxNodes?: number; + /** Maximum hops to expand from seed nodes. Default: 2. */ + maxHops?: number; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** + * Strip creator public key and signature from a page before export. + * Only content, hashes, and embedding metadata are preserved for discovery. + */ +function coarsenPage(page: Page): Page { + return { + ...page, + creatorPubKey: "", + signature: "", + }; +} + +/** + * Build a provenance map for exported nodes. + * Each node is tagged with the source identifier (probeId or exchangeId). + */ +function buildProvenance(nodeIds: Hash[], sourceId: string): Record { + const prov: Record = {}; + for (const id of nodeIds) { + prov[id] = sourceId; + } + return prov; +} + +// --------------------------------------------------------------------------- +// BFS expansion from seed nodes +// --------------------------------------------------------------------------- + +async function expandSeeds( + seedIds: Hash[], + maxHops: number, + maxNodes: number, + metadataStore: MetadataStore, +): Promise<{ pages: Page[]; edges: Edge[] }> { + const visited = new Set(seedIds); + let frontier = [...seedIds]; + + const collectedPages: Page[] = []; + const edgeMap = new Map(); + + // Load seed pages + for (const id of seedIds) { + const page = await metadataStore.getPage(id); + if (page) collectedPages.push(page); + } + + for (let hop = 0; hop < maxHops && frontier.length > 0; hop++) { + const nextFrontier: Hash[] = []; + + for (const pageId of frontier) { + if (collectedPages.length >= maxNodes) break; + + // Expand via Metroid (semantic) neighbors + const metroidNeighbors: SemanticNeighbor[] = await metadataStore.getSemanticNeighbors(pageId); + for (const n of metroidNeighbors) { + if (!visited.has(n.neighborPageId) && collectedPages.length < maxNodes) { + visited.add(n.neighborPageId); + nextFrontier.push(n.neighborPageId); + const page = await metadataStore.getPage(n.neighborPageId); + if (page) collectedPages.push(page); + } + } + } + + frontier = nextFrontier; + } + + // After BFS completes, collect Hebbian edges among visited nodes using the final visited set + for (const fromPageId of visited) { + const hebbianEdges = await metadataStore.getNeighbors(fromPageId); + for (const e of hebbianEdges) { + if (visited.has(e.toPageId)) { + const key = `${e.fromPageId}\x00${e.toPageId}`; + if (!edgeMap.has(key)) edgeMap.set(key, e); + } + } + } + + return { pages: collectedPages, edges: [...edgeMap.values()] }; +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/** + * Build a subgraph slice for export in response to a CuriosityProbe. + * + * Starts BFS from `m1` in the probe, expands up to `maxHops`, applies + * eligibility filtering, strips personal metadata, and returns a + * signed-provenance SubgraphSlice ready for transmission. + * + * Returns null if no eligible nodes are found. + */ +export async function exportForProbe( + probe: CuriosityProbe, + options: ExportOptions, +): Promise { + const { metadataStore, maxNodes = 50, maxHops = 2 } = options; + + const { pages, edges } = await expandSeeds( + [probe.m1], + maxHops, + maxNodes, + metadataStore, + ); + + const eligiblePages = filterEligible(pages); + if (eligiblePages.length === 0) return null; + + const eligibleIds = new Set(eligiblePages.map((p) => p.pageId)); + const filteredEdges = edges.filter( + (e) => eligibleIds.has(e.fromPageId) && eligibleIds.has(e.toPageId), + ); + + const coarsened = eligiblePages.map(coarsenPage); + const provenance = buildProvenance(coarsened.map((p) => p.pageId), probe.probeId); + + return { + sliceId: randomUUID(), + nodes: coarsened, + edges: filteredEdges, + provenance, + signatures: {}, + timestamp: new Date().toISOString(), + }; +} + +/** + * Build a subgraph slice for proactive opt-in peer exchange. + * + * Starts BFS from `seedPageIds`, applies eligibility filtering, + * and returns a SubgraphSlice tagged with the exchange ID. + * + * Returns null if no eligible nodes are found. + */ +export async function exportForExchange( + seedPageIds: Hash[], + exchangeId: string, + options: ExportOptions, +): Promise { + const { metadataStore, maxNodes = 50, maxHops = 2 } = options; + + const { pages, edges } = await expandSeeds( + seedPageIds, + maxHops, + maxNodes, + metadataStore, + ); + + const eligiblePages = filterEligible(pages); + if (eligiblePages.length === 0) return null; + + const eligibleIds = new Set(eligiblePages.map((p) => p.pageId)); + const filteredEdges = edges.filter( + (e) => eligibleIds.has(e.fromPageId) && eligibleIds.has(e.toPageId), + ); + + const coarsened = eligiblePages.map(coarsenPage); + const provenance = buildProvenance(coarsened.map((p) => p.pageId), exchangeId); + + return { + sliceId: randomUUID(), + nodes: coarsened, + edges: filteredEdges, + provenance, + signatures: {}, + timestamp: new Date().toISOString(), + }; +} diff --git a/lib/sharing/SubgraphImporter.ts b/lib/sharing/SubgraphImporter.ts new file mode 100644 index 0000000..ff20ca3 --- /dev/null +++ b/lib/sharing/SubgraphImporter.ts @@ -0,0 +1,206 @@ +// --------------------------------------------------------------------------- +// SubgraphImporter — safely integrate received graph fragments (P2-G3) +// --------------------------------------------------------------------------- +// +// Verifies schema and (optionally) signatures on incoming graph fragments, +// merges eligible nodes and edges into the local store, and strips sender +// identity metadata so peer identity is not exposed to local queries. +// --------------------------------------------------------------------------- + +import type { Edge, Hash, MetadataStore, Page, VectorStore } from "../core/types"; +import type { GraphFragment, SubgraphSlice } from "./types"; + +// --------------------------------------------------------------------------- +// Options +// --------------------------------------------------------------------------- + +export interface ImportOptions { + metadataStore: MetadataStore; + vectorStore: VectorStore; + /** + * When true, nodes whose pageId does not match SHA-256(content) are + * rejected. Defaults to false for test environments — enable in production. + */ + verifyContentHashes?: boolean; +} + +export interface ImportResult { + nodesImported: number; + edgesImported: number; + rejected: Hash[]; +} + +// --------------------------------------------------------------------------- +// Schema validation helpers +// --------------------------------------------------------------------------- + +function isValidPage(p: unknown): p is Page { + if (typeof p !== "object" || p === null) return false; + const page = p as Partial; + return ( + typeof page.pageId === "string" && page.pageId.length > 0 && + typeof page.content === "string" && + typeof page.embeddingOffset === "number" && + typeof page.embeddingDim === "number" && page.embeddingDim > 0 + ); +} + +function isValidEdge(e: unknown): e is Edge { + if (typeof e !== "object" || e === null) return false; + const edge = e as Partial; + return ( + typeof edge.fromPageId === "string" && edge.fromPageId.length > 0 && + typeof edge.toPageId === "string" && edge.toPageId.length > 0 && + typeof edge.weight === "number" && edge.weight >= 0 + ); +} + +// --------------------------------------------------------------------------- +// Import logic +// --------------------------------------------------------------------------- + +async function computeContentHash(content: string): Promise { + if (!("crypto" in globalThis) || !globalThis.crypto?.subtle) { + throw new Error("SubtleCrypto not available for content hash verification"); + } + + const encoder = new TextEncoder(); + const data = encoder.encode(content); + const digest = await globalThis.crypto.subtle.digest("SHA-256", data); + const bytes = new Uint8Array(digest); + + let hex = ""; + for (const b of bytes) { + hex += b.toString(16).padStart(2, "0"); + } + + return hex; +} + +async function importNodes( + nodes: Page[], + vectorStore: VectorStore, + metadataStore: MetadataStore, + verifyContentHashes: boolean, +): Promise<{ imported: Hash[]; rejected: Hash[] }> { + const imported: Hash[] = []; + const rejected: Hash[] = []; + + for (const raw of nodes) { + if (!isValidPage(raw)) { + if (typeof (raw as Partial).pageId === "string") { + rejected.push((raw as Page).pageId); + } + continue; + } + + // Strip sender identity and discard sender-provided embedding metadata. + // Remote embeddingOffset/embeddingDim refer to the sender's VectorStore and + // are not valid byte offsets in the local store, so we must not persist them. + const page: Page = { + ...raw, + creatorPubKey: "", + signature: "", + // Mark as "no local embedding yet"; downstream code can choose to re-embed. + embeddingOffset: 0, + embeddingDim: 0, + }; + + // Optionally verify that pageId matches SHA-256(content) + if (verifyContentHashes) { + let computedId: string; + try { + computedId = await computeContentHash(page.content); + } catch { + // If we cannot verify hashes, reject the page rather than + // silently accepting unverified content. + rejected.push(page.pageId); + continue; + } + + if (computedId !== page.pageId) { + rejected.push(page.pageId); + continue; + } + } + + // Persist page without trusting remote embedding offsets. + await metadataStore.putPage(page); + imported.push(page.pageId); + } + + return { imported, rejected }; +} + +async function importEdges( + edges: Edge[], + importedNodeIds: Set, + metadataStore: MetadataStore, +): Promise { + const validEdges = edges.filter( + (e) => + isValidEdge(e) && + importedNodeIds.has(e.fromPageId) && + importedNodeIds.has(e.toPageId), + ); + + if (validEdges.length > 0) { + await metadataStore.putEdges(validEdges); + } + + return validEdges.length; +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/** + * Import a GraphFragment received in response to a CuriosityProbe. + * + * Validates schema, strips sender identity metadata, and persists approved + * nodes and edges into the local store. Rejected nodes are returned for + * auditability. + */ +export async function importFragment( + fragment: GraphFragment, + options: ImportOptions, +): Promise { + const { metadataStore, vectorStore, verifyContentHashes = false } = options; + + const { imported, rejected } = await importNodes( + fragment.nodes, + vectorStore, + metadataStore, + verifyContentHashes, + ); + + const importedSet = new Set(imported); + const edgesImported = await importEdges(fragment.edges, importedSet, metadataStore); + + return { nodesImported: imported.length, edgesImported, rejected }; +} + +/** + * Import a SubgraphSlice received via proactive peer exchange. + * + * Applies the same validation and identity stripping as `importFragment`. + */ +export async function importSlice( + slice: SubgraphSlice, + options: ImportOptions, +): Promise { + const { metadataStore, vectorStore, verifyContentHashes = false } = options; + + const { imported, rejected } = await importNodes( + slice.nodes, + vectorStore, + metadataStore, + verifyContentHashes, + ); + + const importedSet = new Set(imported); + const edgesImported = await importEdges(slice.edges, importedSet, metadataStore); + + return { nodesImported: imported.length, edgesImported, rejected }; +} diff --git a/lib/sharing/types.ts b/lib/sharing/types.ts new file mode 100644 index 0000000..600bedd --- /dev/null +++ b/lib/sharing/types.ts @@ -0,0 +1,141 @@ +// --------------------------------------------------------------------------- +// sharing/types.ts — Shared data types for P2P curiosity and subgraph exchange +// --------------------------------------------------------------------------- +// +// All types used across sharing modules are defined here to keep the modules +// decoupled from one another while sharing a single canonical schema. +// --------------------------------------------------------------------------- + +import type { Edge, Hash, Page, Signature } from "../core/types"; + +// --------------------------------------------------------------------------- +// CuriosityProbe — broadcast when a knowledge gap is detected +// --------------------------------------------------------------------------- + +/** + * A P2P curiosity probe broadcast when MetroidBuilder cannot find a valid + * antithesis medoid (m2) for a thesis topic. + * + * Peers receiving a probe MUST verify that `mimeType` and `modelUrn` match + * their local model before attempting to respond. Accepting graph fragments + * from an incompatible model would introduce incommensurable similarity scores. + */ +export interface CuriosityProbe { + /** Unique probe identifier (e.g., UUID or hash of probe content). */ + probeId: string; + + /** The thesis medoid page ID for which antithesis was not found. */ + m1: Hash; + + /** The incomplete Metroid at the boundary of local knowledge. */ + partialMetroid: { + m1: Hash; + m2?: Hash; + /** Serialised centroid embedding as a base-64-encoded Float32Array, optional. */ + centroidB64?: string; + }; + + /** Original query embedding serialised as base-64-encoded Float32Array. */ + queryContextB64: string; + + /** Matryoshka dimensional layer at which antithesis search failed. */ + knowledgeBoundary: number; + + /** + * MIME type of the embedded content (e.g., "text/plain", "image/jpeg"). + * Required: peers must validate content-type commensurability. + */ + mimeType: string; + + /** + * URN identifying the specific embedding model used to produce the vectors + * (e.g., "urn:model:onnx-community/embeddinggemma-300m-ONNX:v1"). + * Required: peers must reject probes with incompatible modelUrn. + */ + modelUrn: string; + + /** ISO 8601 timestamp when this probe was created. */ + timestamp: string; +} + +// --------------------------------------------------------------------------- +// GraphFragment — response payload returned to a curiosity probe +// --------------------------------------------------------------------------- + +/** + * A signed graph fragment returned by a peer in response to a CuriosityProbe. + * Contains nodes and edges relevant to the probe's knowledge boundary. + */ +export interface GraphFragment { + /** Unique fragment identifier. */ + fragmentId: string; + + /** The probe ID this fragment responds to. */ + probeId: string; + + /** Pages included in this fragment (eligibility-filtered). */ + nodes: Page[]; + + /** Hebbian edges among the included nodes. */ + edges: Edge[]; + + /** + * Per-node cryptographic signatures keyed by pageId. + * Recipients verify these before integrating. + */ + signatures: Record; + + /** ISO 8601 timestamp when this fragment was assembled. */ + timestamp: string; +} + +// --------------------------------------------------------------------------- +// Eligibility decisions +// --------------------------------------------------------------------------- + +export type EligibilityStatus = "eligible" | "blocked"; + +export type BlockReason = + | "pii_identity" + | "pii_credentials" + | "pii_financial" + | "pii_health" + | "no_public_interest"; + +/** Deterministic eligibility decision for a single candidate page. */ +export interface EligibilityDecision { + pageId: Hash; + status: EligibilityStatus; + reason?: BlockReason; +} + +// --------------------------------------------------------------------------- +// SubgraphSlice — an exported topic-scoped graph section +// --------------------------------------------------------------------------- + +/** + * A topic-scoped subgraph slice built from eligibility-approved pages. + * Used for both curiosity responses and proactive peer exchange. + */ +export interface SubgraphSlice { + sliceId: string; + nodes: Page[]; + edges: Edge[]; + /** Provenance map: pageId -> source probe or exchange ID. */ + provenance: Record; + /** Signatures map for verification. */ + signatures: Record; + timestamp: string; +} + +// --------------------------------------------------------------------------- +// PeerMessage — top-level P2P transport envelope +// --------------------------------------------------------------------------- + +export type PeerMessageKind = "curiosity_probe" | "graph_fragment" | "subgraph_slice"; + +export interface PeerMessage { + kind: PeerMessageKind; + senderId: string; + payload: CuriosityProbe | GraphFragment | SubgraphSlice; +} diff --git a/lib/storage/IndexedDbMetadataStore.ts b/lib/storage/IndexedDbMetadataStore.ts new file mode 100644 index 0000000..2ffb384 --- /dev/null +++ b/lib/storage/IndexedDbMetadataStore.ts @@ -0,0 +1,614 @@ +import type { + Book, + Edge, + Hash, + HotpathEntry, + MetadataStore, + SemanticNeighbor, + SemanticNeighborSubgraph, + Page, + PageActivity, + Shelf, + Volume, +} from "../core/types"; + +// --------------------------------------------------------------------------- +// Schema constants +// --------------------------------------------------------------------------- + +const DB_VERSION = 3; + +/** Object-store names used across the schema. */ +const STORE = { + pages: "pages", + books: "books", + volumes: "volumes", + shelves: "shelves", + edges: "edges_hebbian", + neighborGraph: "neighbor_graph", + flags: "flags", + pageToBook: "page_to_book", + bookToVolume: "book_to_volume", + volumeToShelf: "volume_to_shelf", + hotpathIndex: "hotpath_index", + pageActivity: "page_activity", +} as const; + +// --------------------------------------------------------------------------- +// Low-level IDB helpers +// --------------------------------------------------------------------------- + +function promisifyTransaction(tx: IDBTransaction): Promise { + return new Promise((resolve, reject) => { + tx.oncomplete = () => resolve(); + tx.onerror = () => reject(tx.error); + tx.onabort = () => reject(tx.error ?? new DOMException("Transaction aborted", "AbortError")); + }); +} + +// --------------------------------------------------------------------------- +// Schema upgrade +// --------------------------------------------------------------------------- + +function applyUpgrade(db: IDBDatabase): void { + // v1 stores + if (!db.objectStoreNames.contains(STORE.pages)) { + db.createObjectStore(STORE.pages, { keyPath: "pageId" }); + } + if (!db.objectStoreNames.contains(STORE.books)) { + db.createObjectStore(STORE.books, { keyPath: "bookId" }); + } + if (!db.objectStoreNames.contains(STORE.volumes)) { + db.createObjectStore(STORE.volumes, { keyPath: "volumeId" }); + } + if (!db.objectStoreNames.contains(STORE.shelves)) { + db.createObjectStore(STORE.shelves, { keyPath: "shelfId" }); + } + + if (!db.objectStoreNames.contains(STORE.edges)) { + const edgeStore = db.createObjectStore(STORE.edges, { + keyPath: ["fromPageId", "toPageId"], + }); + edgeStore.createIndex("by-from", "fromPageId"); + } + + + if (!db.objectStoreNames.contains(STORE.flags)) { + db.createObjectStore(STORE.flags, { keyPath: "volumeId" }); + } + + if (!db.objectStoreNames.contains(STORE.pageToBook)) { + db.createObjectStore(STORE.pageToBook, { keyPath: "pageId" }); + } + if (!db.objectStoreNames.contains(STORE.bookToVolume)) { + db.createObjectStore(STORE.bookToVolume, { keyPath: "bookId" }); + } + if (!db.objectStoreNames.contains(STORE.volumeToShelf)) { + db.createObjectStore(STORE.volumeToShelf, { keyPath: "volumeId" }); + } + + // v2 stores — hotpath index + page activity + if (!db.objectStoreNames.contains(STORE.hotpathIndex)) { + const hp = db.createObjectStore(STORE.hotpathIndex, { keyPath: "entityId" }); + hp.createIndex("by-tier", "tier"); + } + if (!db.objectStoreNames.contains(STORE.pageActivity)) { + db.createObjectStore(STORE.pageActivity, { keyPath: "pageId" }); + } + + // v3 stores — neighbor_graph (replaces the old metroid_neighbors name) + if (!db.objectStoreNames.contains(STORE.neighborGraph)) { + db.createObjectStore(STORE.neighborGraph, { keyPath: "pageId" }); + } +} + +// --------------------------------------------------------------------------- +// IndexedDbMetadataStore +// --------------------------------------------------------------------------- + +/** + * Full MetadataStore implementation backed by IndexedDB. + * + * Reverse-index rows (`page_to_book`, `book_to_volume`, `volume_to_shelf`) are + * maintained atomically inside the same transaction as the owning entity write, + * so they are always consistent with the latest put. + * + * Usage: + * const store = await IndexedDbMetadataStore.open("cortex"); + */ +export class IndexedDbMetadataStore implements MetadataStore { + private constructor(private readonly db: IDBDatabase) {} + + // ------------------------------------------------------------------------- + // Factory + // ------------------------------------------------------------------------- + + static open(dbName: string): Promise { + return new Promise((resolve, reject) => { + const req = indexedDB.open(dbName, DB_VERSION); + + req.onupgradeneeded = (event) => { + applyUpgrade((event.target as IDBOpenDBRequest).result); + }; + + req.onsuccess = () => resolve(new IndexedDbMetadataStore(req.result)); + req.onerror = () => reject(req.error); + }); + } + + // ------------------------------------------------------------------------- + // Page CRUD + // ------------------------------------------------------------------------- + + putPage(page: Page): Promise { + return this._put(STORE.pages, page); + } + + async getPage(pageId: Hash): Promise { + return this._get(STORE.pages, pageId); + } + + /** + * Returns all pages in the store. Used for warm/cold fallbacks in query. + * TODO: Replace with a paginated or indexed scan before production use — + * loading every page into memory is expensive for large corpora. + */ + async getAllPages(): Promise { + return new Promise((resolve, reject) => { + const tx = this.db.transaction(STORE.pages, "readonly"); + const req = tx.objectStore(STORE.pages).getAll(); + req.onsuccess = () => resolve(req.result as Page[]); + req.onerror = () => reject(req.error); + }); + } + + // ------------------------------------------------------------------------- + // Book CRUD + reverse index maintenance + // ------------------------------------------------------------------------- + + putBook(book: Book): Promise { + return new Promise((resolve, reject) => { + const tx = this.db.transaction( + [STORE.books, STORE.pageToBook], + "readwrite", + ); + + // Store the book itself + tx.objectStore(STORE.books).put(book); + + // Update page->book reverse index for every page in this book + const idxStore = tx.objectStore(STORE.pageToBook); + for (const pageId of book.pageIds) { + const getReq = idxStore.get(pageId); + getReq.onsuccess = () => { + const existing: { pageId: Hash; bookIds: Hash[] } | undefined = + getReq.result; + const bookIds = existing?.bookIds ?? []; + if (!bookIds.includes(book.bookId)) { + bookIds.push(book.bookId); + } + idxStore.put({ pageId, bookIds }); + }; + } + + promisifyTransaction(tx).then(resolve).catch(reject); + }); + } + + async getBook(bookId: Hash): Promise { + return this._get(STORE.books, bookId); + } + + // ------------------------------------------------------------------------- + // Volume CRUD + reverse index + // ------------------------------------------------------------------------- + + putVolume(volume: Volume): Promise { + return new Promise((resolve, reject) => { + const tx = this.db.transaction( + [STORE.volumes, STORE.bookToVolume], + "readwrite", + ); + + tx.objectStore(STORE.volumes).put(volume); + + const idxStore = tx.objectStore(STORE.bookToVolume); + for (const bookId of volume.bookIds) { + const getReq = idxStore.get(bookId); + getReq.onsuccess = () => { + const existing: { bookId: Hash; volumeIds: Hash[] } | undefined = + getReq.result; + const volumeIds = existing?.volumeIds ?? []; + if (!volumeIds.includes(volume.volumeId)) { + volumeIds.push(volume.volumeId); + } + idxStore.put({ bookId, volumeIds }); + }; + } + + promisifyTransaction(tx).then(resolve).catch(reject); + }); + } + + async getVolume(volumeId: Hash): Promise { + return this._get(STORE.volumes, volumeId); + } + + async getAllVolumes(): Promise { + return new Promise((resolve, reject) => { + const tx = this.db.transaction(STORE.volumes, "readonly"); + const req = tx.objectStore(STORE.volumes).getAll(); + req.onsuccess = () => resolve(req.result as Volume[]); + req.onerror = () => reject(req.error); + }); + } + + /** + * Delete a volume and clean up its reverse-index entries: + * - Removes the volume from the `bookToVolume` index for each of its books. + * - Deletes the `volumeToShelf` index entry for this volume. + * - Deletes the volume record itself. + * + * Callers should update or remove the volume from any shelf's `volumeIds` + * list before calling this method. + */ + async deleteVolume(volumeId: Hash): Promise { + const volume = await this.getVolume(volumeId); + + return new Promise((resolve, reject) => { + const tx = this.db.transaction( + [STORE.volumes, STORE.bookToVolume, STORE.volumeToShelf], + "readwrite", + ); + + // Remove from bookToVolume reverse index for each book the volume owned + if (volume) { + const bookToVolumeStore = tx.objectStore(STORE.bookToVolume); + for (const bookId of volume.bookIds) { + const getReq = bookToVolumeStore.get(bookId); + getReq.onsuccess = () => { + const existing: { bookId: Hash; volumeIds: Hash[] } | undefined = + getReq.result; + if (!existing) return; + const updatedVolumeIds = existing.volumeIds.filter( + (id) => id !== volumeId, + ); + if (updatedVolumeIds.length === 0) { + bookToVolumeStore.delete(bookId); + } else { + bookToVolumeStore.put({ bookId, volumeIds: updatedVolumeIds }); + } + }; + } + } + + // Remove volumeToShelf reverse index entry + tx.objectStore(STORE.volumeToShelf).delete(volumeId); + + // Delete the volume record itself + tx.objectStore(STORE.volumes).delete(volumeId); + + promisifyTransaction(tx).then(resolve).catch(reject); + }); + } + + // ------------------------------------------------------------------------- + // Shelf CRUD + reverse index + // ------------------------------------------------------------------------- + + putShelf(shelf: Shelf): Promise { + return new Promise((resolve, reject) => { + const tx = this.db.transaction( + [STORE.shelves, STORE.volumeToShelf], + "readwrite", + ); + + tx.objectStore(STORE.shelves).put(shelf); + + const idxStore = tx.objectStore(STORE.volumeToShelf); + for (const volumeId of shelf.volumeIds) { + const getReq = idxStore.get(volumeId); + getReq.onsuccess = () => { + const existing: { volumeId: Hash; shelfIds: Hash[] } | undefined = + getReq.result; + const shelfIds = existing?.shelfIds ?? []; + if (!shelfIds.includes(shelf.shelfId)) { + shelfIds.push(shelf.shelfId); + } + idxStore.put({ volumeId, shelfIds }); + }; + } + + promisifyTransaction(tx).then(resolve).catch(reject); + }); + } + + async getShelf(shelfId: Hash): Promise { + return this._get(STORE.shelves, shelfId); + } + + async getAllShelves(): Promise { + return new Promise((resolve, reject) => { + const tx = this.db.transaction(STORE.shelves, "readonly"); + const req = tx.objectStore(STORE.shelves).getAll(); + req.onsuccess = () => resolve(req.result as Shelf[]); + req.onerror = () => reject(req.error); + }); + } + + // ------------------------------------------------------------------------- + // Hebbian edges + // ------------------------------------------------------------------------- + + putEdges(edges: Edge[]): Promise { + return new Promise((resolve, reject) => { + const tx = this.db.transaction(STORE.edges, "readwrite"); + const store = tx.objectStore(STORE.edges); + for (const edge of edges) { + store.put(edge); + } + promisifyTransaction(tx).then(resolve).catch(reject); + }); + } + + deleteEdge(fromPageId: Hash, toPageId: Hash): Promise { + return new Promise((resolve, reject) => { + const tx = this.db.transaction(STORE.edges, "readwrite"); + tx.objectStore(STORE.edges).delete([fromPageId, toPageId]); + promisifyTransaction(tx).then(resolve).catch(reject); + }); + } + + async getNeighbors(pageId: Hash, limit?: number): Promise { + return new Promise((resolve, reject) => { + const tx = this.db.transaction(STORE.edges, "readonly"); + const idx = tx.objectStore(STORE.edges).index("by-from"); + const req = idx.getAll(IDBKeyRange.only(pageId)); + req.onsuccess = () => { + let rows: Edge[] = req.result; + rows.sort((a, b) => b.weight - a.weight); + if (limit !== undefined) rows = rows.slice(0, limit); + resolve(rows); + }; + req.onerror = () => reject(req.error); + }); + } + + // ------------------------------------------------------------------------- + // Reverse-index helpers + // ------------------------------------------------------------------------- + + async getBooksByPage(pageId: Hash): Promise { + const row = await this._get<{ pageId: Hash; bookIds: Hash[] }>( + STORE.pageToBook, + pageId, + ); + if (!row || row.bookIds.length === 0) return []; + return this._getMany(STORE.books, row.bookIds); + } + + async getVolumesByBook(bookId: Hash): Promise { + const row = await this._get<{ bookId: Hash; volumeIds: Hash[] }>( + STORE.bookToVolume, + bookId, + ); + if (!row || row.volumeIds.length === 0) return []; + return this._getMany(STORE.volumes, row.volumeIds); + } + + async getShelvesByVolume(volumeId: Hash): Promise { + const row = await this._get<{ volumeId: Hash; shelfIds: Hash[] }>( + STORE.volumeToShelf, + volumeId, + ); + if (!row || row.shelfIds.length === 0) return []; + return this._getMany(STORE.shelves, row.shelfIds); + } + + // ------------------------------------------------------------------------- + // Semantic neighbor radius index + // ------------------------------------------------------------------------- + + putSemanticNeighbors(pageId: Hash, neighbors: SemanticNeighbor[]): Promise { + return this._put(STORE.neighborGraph, { pageId, neighbors }); + } + + async getSemanticNeighbors( + pageId: Hash, + maxDegree?: number, + ): Promise { + const row = await this._get<{ pageId: Hash; neighbors: SemanticNeighbor[] }>( + STORE.neighborGraph, + pageId, + ); + if (!row) return []; + const list = row.neighbors; + return maxDegree !== undefined ? list.slice(0, maxDegree) : list; + } + + async getInducedNeighborSubgraph( + seedPageIds: Hash[], + maxHops: number, + ): Promise { + const visited = new Set(seedPageIds); + const nodeSet = new Set(seedPageIds); + const edgeMap = new Map(); + + let frontier = [...seedPageIds]; + + for (let hop = 0; hop < maxHops && frontier.length > 0; hop++) { + const nextFrontier: Hash[] = []; + + for (const pageId of frontier) { + const neighbors = await this.getSemanticNeighbors(pageId); + for (const n of neighbors) { + const key = `${pageId}\x00${n.neighborPageId}`; + if (!edgeMap.has(key)) { + edgeMap.set(key, { + from: pageId, + to: n.neighborPageId, + distance: n.distance, + }); + } + if (!visited.has(n.neighborPageId)) { + visited.add(n.neighborPageId); + nodeSet.add(n.neighborPageId); + nextFrontier.push(n.neighborPageId); + } + } + } + + frontier = nextFrontier; + } + + return { + nodes: [...nodeSet], + edges: [...edgeMap.values()], + }; + } + + // ------------------------------------------------------------------------- + // Dirty-recalc flags + // ------------------------------------------------------------------------- + + async needsNeighborRecalc(volumeId: Hash): Promise { + const row = await this._get<{ volumeId: Hash; needsRecalc: boolean }>( + STORE.flags, + volumeId, + ); + return row?.needsRecalc === true; + } + + flagVolumeForNeighborRecalc(volumeId: Hash): Promise { + return this._put(STORE.flags, { volumeId, needsRecalc: true }); + } + + clearNeighborRecalcFlag(volumeId: Hash): Promise { + return this._put(STORE.flags, { volumeId, needsRecalc: false }); + } + + // ------------------------------------------------------------------------- + // Hotpath index + // ------------------------------------------------------------------------- + + putHotpathEntry(entry: HotpathEntry): Promise { + return this._put(STORE.hotpathIndex, entry); + } + + async getHotpathEntries(tier?: HotpathEntry["tier"]): Promise { + if (tier !== undefined) { + return new Promise((resolve, reject) => { + const tx = this.db.transaction(STORE.hotpathIndex, "readonly"); + const idx = tx.objectStore(STORE.hotpathIndex).index("by-tier"); + const req = idx.getAll(IDBKeyRange.only(tier)); + req.onsuccess = () => resolve(req.result as HotpathEntry[]); + req.onerror = () => reject(req.error); + }); + } + return new Promise((resolve, reject) => { + const tx = this.db.transaction(STORE.hotpathIndex, "readonly"); + const req = tx.objectStore(STORE.hotpathIndex).getAll(); + req.onsuccess = () => resolve(req.result as HotpathEntry[]); + req.onerror = () => reject(req.error); + }); + } + + removeHotpathEntry(entityId: Hash): Promise { + return new Promise((resolve, reject) => { + const tx = this.db.transaction(STORE.hotpathIndex, "readwrite"); + tx.objectStore(STORE.hotpathIndex).delete(entityId); + promisifyTransaction(tx).then(resolve).catch(reject); + }); + } + + async evictWeakest( + tier: HotpathEntry["tier"], + communityId?: string, + ): Promise { + const entries = await this.getHotpathEntries(tier); + const filtered = communityId !== undefined + ? entries.filter((e) => e.communityId === communityId) + : entries; + + if (filtered.length === 0) return; + + // Deterministic: break ties by entityId (smallest wins) + let weakest = filtered[0]; + for (let i = 1; i < filtered.length; i++) { + const e = filtered[i]; + if ( + e.salience < weakest.salience || + (e.salience === weakest.salience && e.entityId < weakest.entityId) + ) { + weakest = e; + } + } + + await this.removeHotpathEntry(weakest.entityId); + } + + async getResidentCount(): Promise { + return new Promise((resolve, reject) => { + const tx = this.db.transaction(STORE.hotpathIndex, "readonly"); + const req = tx.objectStore(STORE.hotpathIndex).count(); + req.onsuccess = () => resolve(req.result); + req.onerror = () => reject(req.error); + }); + } + + // ------------------------------------------------------------------------- + // Page activity + // ------------------------------------------------------------------------- + + putPageActivity(activity: PageActivity): Promise { + return this._put(STORE.pageActivity, activity); + } + + async getPageActivity(pageId: Hash): Promise { + return this._get(STORE.pageActivity, pageId); + } + + // ------------------------------------------------------------------------- + // Private generic helpers + // ------------------------------------------------------------------------- + + private _put(storeName: string, value: unknown): Promise { + return new Promise((resolve, reject) => { + const tx = this.db.transaction(storeName, "readwrite"); + tx.objectStore(storeName).put(value); + promisifyTransaction(tx).then(resolve).catch(reject); + }); + } + + private _get(storeName: string, key: IDBValidKey): Promise { + return new Promise((resolve, reject) => { + const tx = this.db.transaction(storeName, "readonly"); + const req = tx.objectStore(storeName).get(key); + req.onsuccess = () => resolve(req.result as T | undefined); + req.onerror = () => reject(req.error); + }); + } + + private _getMany(storeName: string, keys: IDBValidKey[]): Promise { + return new Promise((resolve, reject) => { + const tx = this.db.transaction(storeName, "readonly"); + const store = tx.objectStore(storeName); + const results: T[] = []; + let pending = keys.length; + + if (pending === 0) { + resolve(results); + return; + } + + keys.forEach((key, i) => { + const req = store.get(key); + req.onsuccess = () => { + if (req.result !== undefined) results[i] = req.result as T; + if (--pending === 0) resolve(results.filter(Boolean)); + }; + req.onerror = () => reject(req.error); + }); + }); + } +} diff --git a/lib/storage/MemoryVectorStore.ts b/lib/storage/MemoryVectorStore.ts new file mode 100644 index 0000000..242701d --- /dev/null +++ b/lib/storage/MemoryVectorStore.ts @@ -0,0 +1,39 @@ +import type { VectorStore } from "../core/types"; +import { FLOAT32_BYTES } from "../core/NumericConstants"; + +/** + * MemoryVectorStore — in-memory implementation of VectorStore. + * + * Byte-offset semantics are identical to OPFSVectorStore so the two + * implementations are interchangeable for testing. + */ +export class MemoryVectorStore implements VectorStore { + private _buf: Uint8Array = new Uint8Array(0); + + async appendVector(vector: Float32Array): Promise { + const byteOffset = this._buf.byteLength; + const incoming = new Uint8Array(vector.buffer, vector.byteOffset, vector.byteLength); + const next = new Uint8Array(byteOffset + incoming.byteLength); + next.set(this._buf); + next.set(incoming, byteOffset); + this._buf = next; + return byteOffset; + } + + async readVector(offset: number, dim: number): Promise { + const byteLen = dim * FLOAT32_BYTES; + return new Float32Array(this._buf.buffer.slice(offset, offset + byteLen)); + } + + async readVectors(offsets: number[], dim: number): Promise { + const byteLen = dim * FLOAT32_BYTES; + return offsets.map( + (offset) => new Float32Array(this._buf.buffer.slice(offset, offset + byteLen)), + ); + } + + /** Total bytes currently stored (useful in tests). */ + get byteLength(): number { + return this._buf.byteLength; + } +} diff --git a/lib/storage/OPFSVectorStore.ts b/lib/storage/OPFSVectorStore.ts new file mode 100644 index 0000000..3f6e597 --- /dev/null +++ b/lib/storage/OPFSVectorStore.ts @@ -0,0 +1,89 @@ +import type { VectorStore } from "../core/types"; +import { FLOAT32_BYTES } from "../core/NumericConstants"; + +/** + * OPFSVectorStore — append-only binary vector file stored in the browser's + * Origin Private File System. + * + * Layout: raw IEEE-754 float32 bytes written sequentially. + * + * Offset semantics: every public method accepts/returns **byte offsets** + * (not vector indices) so callers can store mixed-dimension vectors + * (full embeddings, compressed prototypes, routing codes) in the same file. + * + * Concurrency: for Phase 1 all writes are serialised through a promise chain + * (`_writeQueue`). A dedicated sync-access-handle approach for high-throughput + * ingestion is deferred to Phase 2. + */ +export class OPFSVectorStore implements VectorStore { + private readonly fileName: string; + private _writeQueue: Promise = Promise.resolve(); + + constructor(fileName = "cortex-vectors.bin") { + this.fileName = fileName; + } + + // ------------------------------------------------------------------------- + // VectorStore implementation + // ------------------------------------------------------------------------- + + async appendVector(vector: Float32Array): Promise { + let byteOffset = 0; + + this._writeQueue = this._writeQueue.then(async () => { + const fileHandle = await this._fileHandle(true); + const file = await fileHandle.getFile(); + byteOffset = file.size; + + const writable = await fileHandle.createWritable({ keepExistingData: true }); + await writable.seek(byteOffset); + // Produce a plain ArrayBuffer copy of the exact float bytes. We cast + // to ArrayBuffer (rather than SharedArrayBuffer) because Float32Arrays + // constructed from normal JS sources always back a plain ArrayBuffer, and + // FileSystemWritableFileStream.write() requires ArrayBuffer / ArrayBufferView. + const copy = vector.buffer.slice( + vector.byteOffset, + vector.byteOffset + vector.byteLength, + ) as ArrayBuffer; + await writable.write(copy); + await writable.close(); + }); + + await this._writeQueue; + return byteOffset; + } + + async readVector(offset: number, dim: number): Promise { + const fileHandle = await this._fileHandle(false); + const file = await fileHandle.getFile(); + const slice = file.slice(offset, offset + dim * FLOAT32_BYTES); + const buf = await slice.arrayBuffer(); + return new Float32Array(buf); + } + + async readVectors(offsets: number[], dim: number): Promise { + if (offsets.length === 0) return []; + + const fileHandle = await this._fileHandle(false); + const file = await fileHandle.getFile(); + // Read the entire file once and extract each vector by slice. + const fullBuf = await file.arrayBuffer(); + + return offsets.map((offset) => { + const byteLen = dim * FLOAT32_BYTES; + return new Float32Array(fullBuf.slice(offset, offset + byteLen)); + }); + } + + // ------------------------------------------------------------------------- + // Helpers + // ------------------------------------------------------------------------- + + /** Returns the underlying OPFS file handle, optionally creating the file. */ + private async _fileHandle( + create: boolean, + ): Promise { + const root = await navigator.storage.getDirectory(); + return root.getFileHandle(this.fileName, { create }); + } +} diff --git a/scripts/runtime-harness-server.mjs b/scripts/runtime-harness-server.mjs index c935387..d60850f 100644 --- a/scripts/runtime-harness-server.mjs +++ b/scripts/runtime-harness-server.mjs @@ -9,7 +9,7 @@ import { fileURLToPath, URL } from "node:url"; const HOST = process.env.HARNESS_HOST ?? "127.0.0.1"; const PORT = Number.parseInt(process.env.HARNESS_PORT ?? "4173", 10); const ROOT = path.resolve( - fileURLToPath(new URL("../runtime/harness", import.meta.url)), + fileURLToPath(new URL("../ui/harness", import.meta.url)), ); const MIME_TYPES = { diff --git a/sharing/CuriosityBroadcaster.ts b/sharing/CuriosityBroadcaster.ts index ff6e0a3..ec48636 100644 --- a/sharing/CuriosityBroadcaster.ts +++ b/sharing/CuriosityBroadcaster.ts @@ -1,151 +1 @@ -// --------------------------------------------------------------------------- -// CuriosityBroadcaster — broadcast pending probes and handle responses (P2-G0) -// --------------------------------------------------------------------------- -// -// Consumes CuriosityProbe objects queued by KnowledgeGapDetector, serialises -// them for P2P transport, rate-limits broadcasts to prevent spam, and -// delegates incoming graph fragment responses to SubgraphImporter. -// --------------------------------------------------------------------------- - -import { randomUUID } from "../core/crypto/uuid"; -import type { CuriosityProbe, GraphFragment, PeerMessage } from "./types"; - -// --------------------------------------------------------------------------- -// P2P transport abstraction -// --------------------------------------------------------------------------- - -/** - * Minimal P2P transport interface. - * The broadcaster is transport-agnostic — inject any WebRTC/WebSocket - * implementation that satisfies this contract. - */ -export interface P2PTransport { - /** Broadcast a message to all connected peers. */ - broadcast(message: PeerMessage): Promise; - /** Register a listener for incoming messages from peers. */ - onMessage(handler: (message: PeerMessage) => void): void; -} - -// --------------------------------------------------------------------------- -// Response handler -// --------------------------------------------------------------------------- - -export type FragmentHandler = (fragment: GraphFragment) => Promise; - -// --------------------------------------------------------------------------- -// Options -// --------------------------------------------------------------------------- - -export interface CuriosityBroadcasterOptions { - transport: P2PTransport; - /** Local node identifier (used as senderId). */ - nodeId: string; - /** Minimum milliseconds between broadcasts of any probe. Default: 5000. */ - rateLimitMs?: number; - /** Maximum probe queue depth before oldest probes are dropped. Default: 100. */ - maxQueueDepth?: number; -} - -// --------------------------------------------------------------------------- -// CuriosityBroadcaster -// --------------------------------------------------------------------------- - -/** - * Manages the lifecycle of outbound curiosity probes. - * - * Probes are enqueued via `enqueueProbe()`, then broadcast during idle time - * via `flush()`. Rate limiting prevents probe spam. Incoming graph fragment - * responses are dispatched to the registered `onFragment` handler. - */ -export class CuriosityBroadcaster { - private readonly transport: P2PTransport; - private readonly nodeId: string; - private readonly rateLimitMs: number; - private readonly maxQueueDepth: number; - - private pendingProbes: CuriosityProbe[] = []; - private lastBroadcastAt = 0; - private fragmentHandler?: FragmentHandler; - - constructor(options: CuriosityBroadcasterOptions) { - this.transport = options.transport; - this.nodeId = options.nodeId; - this.rateLimitMs = options.rateLimitMs ?? 5_000; - this.maxQueueDepth = options.maxQueueDepth ?? 100; - - // Listen for incoming graph fragment responses - this.transport.onMessage((msg) => { - if (msg.kind === "graph_fragment") { - void this._handleFragment(msg.payload as GraphFragment); - } - }); - } - - /** - * Register a handler that will be called when a graph fragment response - * arrives from a peer. Replaces any previously registered handler. - */ - onFragment(handler: FragmentHandler): void { - this.fragmentHandler = handler; - } - - /** - * Enqueue a CuriosityProbe for broadcast. - * - * If the queue is already at capacity, the oldest probe is dropped to make - * room. A fresh probeId is assigned here if the probe does not already have - * one, ensuring each broadcast can be correlated with its response. - */ - enqueueProbe(probe: Omit & { probeId?: string }): void { - const full: CuriosityProbe = { - ...probe, - probeId: probe.probeId ?? randomUUID(), - }; - - if (this.pendingProbes.length >= this.maxQueueDepth) { - this.pendingProbes.shift(); // drop oldest - } - this.pendingProbes.push(full); - } - - /** - * Flush pending probes to connected peers, respecting the rate limit. - * - * Call this from the IdleScheduler during background passes. Each call - * broadcasts at most one probe; subsequent calls broadcast the next one. - * - * Returns the number of probes broadcast (0 or 1). - */ - async flush(now = Date.now()): Promise { - if (this.pendingProbes.length === 0) return 0; - if (now - this.lastBroadcastAt < this.rateLimitMs) return 0; - - const probe = this.pendingProbes.shift(); - if (!probe) return 0; - - const message: PeerMessage = { - kind: "curiosity_probe", - senderId: this.nodeId, - payload: probe, - }; - - await this.transport.broadcast(message); - this.lastBroadcastAt = now; - return 1; - } - - /** Number of probes waiting to be broadcast. */ - get pendingCount(): number { - return this.pendingProbes.length; - } - - // --------------------------------------------------------------------------- - // Private - // --------------------------------------------------------------------------- - - private async _handleFragment(fragment: GraphFragment): Promise { - if (this.fragmentHandler) { - await this.fragmentHandler(fragment); - } - } -} +export * from "../lib/sharing/CuriosityBroadcaster"; diff --git a/sharing/EligibilityClassifier.ts b/sharing/EligibilityClassifier.ts index f27c445..dbcd419 100644 --- a/sharing/EligibilityClassifier.ts +++ b/sharing/EligibilityClassifier.ts @@ -1,111 +1 @@ -// --------------------------------------------------------------------------- -// EligibilityClassifier — classify pages as share-eligible or blocked (P2-G1) -// --------------------------------------------------------------------------- -// -// Detects identity/PII-bearing content before any graph export operation. -// Emits deterministic eligibility decisions with reason codes for auditability. -// -// Rules: -// - Identity PII: person names with SSN/passport/national ID patterns -// - Credentials: password, API key, secret, token patterns -// - Financial: credit card, IBAN, account number patterns -// - Health: medical record, diagnosis, prescription patterns -// - No public interest: very short or empty content -// --------------------------------------------------------------------------- - -import type { Hash, Page } from "../core/types"; -import type { BlockReason, EligibilityDecision, EligibilityStatus } from "./types"; - -// --------------------------------------------------------------------------- -// PII detection patterns -// --------------------------------------------------------------------------- - -/** Minimum content length (chars) to be considered public-interest. */ -const MIN_PUBLIC_INTEREST_LENGTH = 20; - -const PATTERNS: Array<{ reason: BlockReason; pattern: RegExp }> = [ - { - reason: "pii_credentials", - // Passwords, API keys, tokens, secrets in common formats - pattern: /\b(?:password|passwd|api[_-]?key|secret[_-]?key|auth[_-]?token|access[_-]?token)\s*[:=]\s*\S+/i, - }, - { - reason: "pii_credentials", - // Bearer tokens, basic auth, SSH key headers - pattern: /(?:Bearer\s+[A-Za-z0-9\-._~+/]+=*|-----BEGIN (?:RSA |EC |)PRIVATE KEY-----)/, - }, - { - reason: "pii_financial", - // Credit card: 13-19 digits with optional separators - pattern: /\b(?:4[0-9]{12}(?:[0-9]{3,6})?|5[1-5][0-9]{14}|3[47][0-9]{13}|6(?:011|5[0-9]{2})[0-9]{12,15})\b/, - }, - { - reason: "pii_financial", - // IBAN: up to 34 alphanumeric chars after country code - pattern: /\b[A-Z]{2}[0-9]{2}[A-Z0-9]{4}[0-9]{7}(?:[A-Z0-9]{0,16})?\b/, - }, - { - reason: "pii_identity", - // US Social Security Number - pattern: /\b\d{3}[-\s]\d{2}[-\s]\d{4}\b/, - }, - { - reason: "pii_identity", - // Email addresses (identity signal — may be PII) - pattern: /\b[-a-zA-Z0-9._%+]+@[a-zA-Z0-9.]+\.[a-zA-Z]{2,}\b/i, - }, - { - reason: "pii_health", - // Medical record / health identifiers - pattern: /\b(?:medical[_-]?record|patient[_-]?id|diagnosis|prescription|ICD[-\s]?\d{1,2})\b/i, - }, -]; - -// --------------------------------------------------------------------------- -// Classifier -// --------------------------------------------------------------------------- - -/** - * Classify a single page as share-eligible or blocked. - * - * Scans `page.content` against a set of PII/credential patterns and - * returns a deterministic decision with a reason code when blocked. - */ -export function classifyPage(page: Page): EligibilityDecision { - // Reject trivially short content as not public-interest - if (page.content.trim().length < MIN_PUBLIC_INTEREST_LENGTH) { - return blocked(page.pageId, "no_public_interest"); - } - - for (const { reason, pattern } of PATTERNS) { - if (pattern.test(page.content)) { - return blocked(page.pageId, reason); - } - } - - return { pageId: page.pageId, status: "eligible" }; -} - -/** - * Classify a batch of pages, returning one decision per page. - * - * Results are in the same order as the input array. - */ -export function classifyPages(pages: Page[]): EligibilityDecision[] { - return pages.map((p) => classifyPage(p)); -} - -/** - * Filter a page array down to only share-eligible pages. - */ -export function filterEligible(pages: Page[]): Page[] { - return pages.filter((p) => classifyPage(p).status === "eligible"); -} - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -function blocked(pageId: Hash, reason: BlockReason): EligibilityDecision { - return { pageId, status: "blocked" as EligibilityStatus, reason }; -} +export * from "../lib/sharing/EligibilityClassifier"; diff --git a/sharing/PeerExchange.ts b/sharing/PeerExchange.ts index 5bcea1a..87aa969 100644 --- a/sharing/PeerExchange.ts +++ b/sharing/PeerExchange.ts @@ -1,131 +1 @@ -// --------------------------------------------------------------------------- -// PeerExchange — opt-in signed subgraph exchange over P2P transport (P2-G3) -// --------------------------------------------------------------------------- -// -// Manages the lifecycle of proactive peer-to-peer graph slice sharing. -// Peers that opt in can receive public-interest graph sections from neighbours. -// All payloads pass eligibility filtering before export and are verified on -// import. Sender identity is never exposed to the receiving peer's queries. -// --------------------------------------------------------------------------- - -import { randomUUID } from "../core/crypto/uuid"; -import type { Hash, MetadataStore, VectorStore } from "../core/types"; -import { exportForExchange } from "./SubgraphExporter"; -import { importSlice } from "./SubgraphImporter"; -import type { P2PTransport } from "./CuriosityBroadcaster"; -import type { ImportResult } from "./SubgraphImporter"; -import type { PeerMessage, SubgraphSlice } from "./types"; - -// --------------------------------------------------------------------------- -// Options -// --------------------------------------------------------------------------- - -export interface PeerExchangeOptions { - transport: P2PTransport; - metadataStore: MetadataStore; - vectorStore: VectorStore; - /** Local node identifier (used as senderId). */ - nodeId: string; - /** - * When true, content hashes on received slices are verified. - * Defaults to false. - */ - verifyContentHashes?: boolean; -} - -export interface ExchangeResult { - sliceId: string; - nodesExported: number; -} - -// --------------------------------------------------------------------------- -// PeerExchange -// --------------------------------------------------------------------------- - -/** - * Orchestrates opt-in signed subgraph exchange with connected peers. - * - * Usage: - * const exchange = new PeerExchange({ transport, metadataStore, vectorStore, nodeId }); - * exchange.onSliceReceived(async (result) => { ... }); - * const result = await exchange.sendSlice(seedPageIds); - */ -export class PeerExchange { - private readonly transport: P2PTransport; - private readonly metadataStore: MetadataStore; - private readonly vectorStore: VectorStore; - private readonly nodeId: string; - private readonly verifyContentHashes: boolean; - private sliceHandler?: (result: ImportResult, slice: SubgraphSlice) => Promise; - - constructor(options: PeerExchangeOptions) { - this.transport = options.transport; - this.metadataStore = options.metadataStore; - this.vectorStore = options.vectorStore; - this.nodeId = options.nodeId; - this.verifyContentHashes = options.verifyContentHashes ?? false; - - this.transport.onMessage((msg) => { - if (msg.kind === "subgraph_slice") { - void this._handleIncoming(msg.payload as SubgraphSlice); - } - }); - } - - /** - * Register a handler called when a slice is received and imported. - * Replaces any previously registered handler. - */ - onSliceReceived(handler: (result: ImportResult, slice: SubgraphSlice) => Promise): void { - this.sliceHandler = handler; - } - - /** - * Export a subgraph slice from the given seed page IDs and broadcast it - * to all connected peers. - * - * Only eligibility-approved nodes are included. Returns null if no eligible - * nodes were found or the export produced an empty slice. - */ - async sendSlice( - seedPageIds: Hash[], - maxNodes = 50, - maxHops = 2, - ): Promise { - const exchangeId = randomUUID(); - - const slice = await exportForExchange(seedPageIds, exchangeId, { - metadataStore: this.metadataStore, - maxNodes, - maxHops, - }); - - if (!slice) return null; - - const message: PeerMessage = { - kind: "subgraph_slice", - senderId: this.nodeId, - payload: slice, - }; - - await this.transport.broadcast(message); - - return { sliceId: slice.sliceId, nodesExported: slice.nodes.length }; - } - - // --------------------------------------------------------------------------- - // Private - // --------------------------------------------------------------------------- - - private async _handleIncoming(slice: SubgraphSlice): Promise { - const result = await importSlice(slice, { - metadataStore: this.metadataStore, - vectorStore: this.vectorStore, - verifyContentHashes: this.verifyContentHashes, - }); - - if (this.sliceHandler) { - await this.sliceHandler(result, slice); - } - } -} +export * from "../lib/sharing/PeerExchange"; diff --git a/sharing/SubgraphExporter.ts b/sharing/SubgraphExporter.ts index a32db9e..0bd8cdc 100644 --- a/sharing/SubgraphExporter.ts +++ b/sharing/SubgraphExporter.ts @@ -1,203 +1 @@ -// --------------------------------------------------------------------------- -// SubgraphExporter — build eligibility-filtered graph slices for sharing (P2-G2) -// --------------------------------------------------------------------------- -// -// Constructs topic-scoped graph slices from pages that pass the eligibility -// classifier. For curiosity responses, the slice is built from a BFS expansion -// around the probe's seed (`m1`), constrained by `maxHops` / `maxNodes`. -// -// Personal metadata fields not needed for discovery are stripped or coarsened -// before export. Node/edge signatures and provenance are preserved. -// --------------------------------------------------------------------------- - -import { randomUUID } from "../core/crypto/uuid"; -import type { Edge, Hash, MetadataStore, SemanticNeighbor, Page } from "../core/types"; -import { filterEligible } from "./EligibilityClassifier"; -import type { CuriosityProbe, SubgraphSlice } from "./types"; - -// --------------------------------------------------------------------------- -// Options -// --------------------------------------------------------------------------- - -export interface ExportOptions { - metadataStore: MetadataStore; - /** Maximum nodes to include in a single slice. Default: 50. */ - maxNodes?: number; - /** Maximum hops to expand from seed nodes. Default: 2. */ - maxHops?: number; -} - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -/** - * Strip creator public key and signature from a page before export. - * Only content, hashes, and embedding metadata are preserved for discovery. - */ -function coarsenPage(page: Page): Page { - return { - ...page, - creatorPubKey: "", - signature: "", - }; -} - -/** - * Build a provenance map for exported nodes. - * Each node is tagged with the source identifier (probeId or exchangeId). - */ -function buildProvenance(nodeIds: Hash[], sourceId: string): Record { - const prov: Record = {}; - for (const id of nodeIds) { - prov[id] = sourceId; - } - return prov; -} - -// --------------------------------------------------------------------------- -// BFS expansion from seed nodes -// --------------------------------------------------------------------------- - -async function expandSeeds( - seedIds: Hash[], - maxHops: number, - maxNodes: number, - metadataStore: MetadataStore, -): Promise<{ pages: Page[]; edges: Edge[] }> { - const visited = new Set(seedIds); - let frontier = [...seedIds]; - - const collectedPages: Page[] = []; - const edgeMap = new Map(); - - // Load seed pages - for (const id of seedIds) { - const page = await metadataStore.getPage(id); - if (page) collectedPages.push(page); - } - - for (let hop = 0; hop < maxHops && frontier.length > 0; hop++) { - const nextFrontier: Hash[] = []; - - for (const pageId of frontier) { - if (collectedPages.length >= maxNodes) break; - - // Expand via Metroid (semantic) neighbors - const metroidNeighbors: SemanticNeighbor[] = await metadataStore.getSemanticNeighbors(pageId); - for (const n of metroidNeighbors) { - if (!visited.has(n.neighborPageId) && collectedPages.length < maxNodes) { - visited.add(n.neighborPageId); - nextFrontier.push(n.neighborPageId); - const page = await metadataStore.getPage(n.neighborPageId); - if (page) collectedPages.push(page); - } - } - } - - frontier = nextFrontier; - } - - // After BFS completes, collect Hebbian edges among visited nodes using the final visited set - for (const fromPageId of visited) { - const hebbianEdges = await metadataStore.getNeighbors(fromPageId); - for (const e of hebbianEdges) { - if (visited.has(e.toPageId)) { - const key = `${e.fromPageId}\x00${e.toPageId}`; - if (!edgeMap.has(key)) edgeMap.set(key, e); - } - } - } - - return { pages: collectedPages, edges: [...edgeMap.values()] }; -} - -// --------------------------------------------------------------------------- -// Public API -// --------------------------------------------------------------------------- - -/** - * Build a subgraph slice for export in response to a CuriosityProbe. - * - * Starts BFS from `m1` in the probe, expands up to `maxHops`, applies - * eligibility filtering, strips personal metadata, and returns a - * signed-provenance SubgraphSlice ready for transmission. - * - * Returns null if no eligible nodes are found. - */ -export async function exportForProbe( - probe: CuriosityProbe, - options: ExportOptions, -): Promise { - const { metadataStore, maxNodes = 50, maxHops = 2 } = options; - - const { pages, edges } = await expandSeeds( - [probe.m1], - maxHops, - maxNodes, - metadataStore, - ); - - const eligiblePages = filterEligible(pages); - if (eligiblePages.length === 0) return null; - - const eligibleIds = new Set(eligiblePages.map((p) => p.pageId)); - const filteredEdges = edges.filter( - (e) => eligibleIds.has(e.fromPageId) && eligibleIds.has(e.toPageId), - ); - - const coarsened = eligiblePages.map(coarsenPage); - const provenance = buildProvenance(coarsened.map((p) => p.pageId), probe.probeId); - - return { - sliceId: randomUUID(), - nodes: coarsened, - edges: filteredEdges, - provenance, - signatures: {}, - timestamp: new Date().toISOString(), - }; -} - -/** - * Build a subgraph slice for proactive opt-in peer exchange. - * - * Starts BFS from `seedPageIds`, applies eligibility filtering, - * and returns a SubgraphSlice tagged with the exchange ID. - * - * Returns null if no eligible nodes are found. - */ -export async function exportForExchange( - seedPageIds: Hash[], - exchangeId: string, - options: ExportOptions, -): Promise { - const { metadataStore, maxNodes = 50, maxHops = 2 } = options; - - const { pages, edges } = await expandSeeds( - seedPageIds, - maxHops, - maxNodes, - metadataStore, - ); - - const eligiblePages = filterEligible(pages); - if (eligiblePages.length === 0) return null; - - const eligibleIds = new Set(eligiblePages.map((p) => p.pageId)); - const filteredEdges = edges.filter( - (e) => eligibleIds.has(e.fromPageId) && eligibleIds.has(e.toPageId), - ); - - const coarsened = eligiblePages.map(coarsenPage); - const provenance = buildProvenance(coarsened.map((p) => p.pageId), exchangeId); - - return { - sliceId: randomUUID(), - nodes: coarsened, - edges: filteredEdges, - provenance, - signatures: {}, - timestamp: new Date().toISOString(), - }; -} +export * from "../lib/sharing/SubgraphExporter"; diff --git a/sharing/SubgraphImporter.ts b/sharing/SubgraphImporter.ts index ff20ca3..27d4bcc 100644 --- a/sharing/SubgraphImporter.ts +++ b/sharing/SubgraphImporter.ts @@ -1,206 +1 @@ -// --------------------------------------------------------------------------- -// SubgraphImporter — safely integrate received graph fragments (P2-G3) -// --------------------------------------------------------------------------- -// -// Verifies schema and (optionally) signatures on incoming graph fragments, -// merges eligible nodes and edges into the local store, and strips sender -// identity metadata so peer identity is not exposed to local queries. -// --------------------------------------------------------------------------- - -import type { Edge, Hash, MetadataStore, Page, VectorStore } from "../core/types"; -import type { GraphFragment, SubgraphSlice } from "./types"; - -// --------------------------------------------------------------------------- -// Options -// --------------------------------------------------------------------------- - -export interface ImportOptions { - metadataStore: MetadataStore; - vectorStore: VectorStore; - /** - * When true, nodes whose pageId does not match SHA-256(content) are - * rejected. Defaults to false for test environments — enable in production. - */ - verifyContentHashes?: boolean; -} - -export interface ImportResult { - nodesImported: number; - edgesImported: number; - rejected: Hash[]; -} - -// --------------------------------------------------------------------------- -// Schema validation helpers -// --------------------------------------------------------------------------- - -function isValidPage(p: unknown): p is Page { - if (typeof p !== "object" || p === null) return false; - const page = p as Partial; - return ( - typeof page.pageId === "string" && page.pageId.length > 0 && - typeof page.content === "string" && - typeof page.embeddingOffset === "number" && - typeof page.embeddingDim === "number" && page.embeddingDim > 0 - ); -} - -function isValidEdge(e: unknown): e is Edge { - if (typeof e !== "object" || e === null) return false; - const edge = e as Partial; - return ( - typeof edge.fromPageId === "string" && edge.fromPageId.length > 0 && - typeof edge.toPageId === "string" && edge.toPageId.length > 0 && - typeof edge.weight === "number" && edge.weight >= 0 - ); -} - -// --------------------------------------------------------------------------- -// Import logic -// --------------------------------------------------------------------------- - -async function computeContentHash(content: string): Promise { - if (!("crypto" in globalThis) || !globalThis.crypto?.subtle) { - throw new Error("SubtleCrypto not available for content hash verification"); - } - - const encoder = new TextEncoder(); - const data = encoder.encode(content); - const digest = await globalThis.crypto.subtle.digest("SHA-256", data); - const bytes = new Uint8Array(digest); - - let hex = ""; - for (const b of bytes) { - hex += b.toString(16).padStart(2, "0"); - } - - return hex; -} - -async function importNodes( - nodes: Page[], - vectorStore: VectorStore, - metadataStore: MetadataStore, - verifyContentHashes: boolean, -): Promise<{ imported: Hash[]; rejected: Hash[] }> { - const imported: Hash[] = []; - const rejected: Hash[] = []; - - for (const raw of nodes) { - if (!isValidPage(raw)) { - if (typeof (raw as Partial).pageId === "string") { - rejected.push((raw as Page).pageId); - } - continue; - } - - // Strip sender identity and discard sender-provided embedding metadata. - // Remote embeddingOffset/embeddingDim refer to the sender's VectorStore and - // are not valid byte offsets in the local store, so we must not persist them. - const page: Page = { - ...raw, - creatorPubKey: "", - signature: "", - // Mark as "no local embedding yet"; downstream code can choose to re-embed. - embeddingOffset: 0, - embeddingDim: 0, - }; - - // Optionally verify that pageId matches SHA-256(content) - if (verifyContentHashes) { - let computedId: string; - try { - computedId = await computeContentHash(page.content); - } catch { - // If we cannot verify hashes, reject the page rather than - // silently accepting unverified content. - rejected.push(page.pageId); - continue; - } - - if (computedId !== page.pageId) { - rejected.push(page.pageId); - continue; - } - } - - // Persist page without trusting remote embedding offsets. - await metadataStore.putPage(page); - imported.push(page.pageId); - } - - return { imported, rejected }; -} - -async function importEdges( - edges: Edge[], - importedNodeIds: Set, - metadataStore: MetadataStore, -): Promise { - const validEdges = edges.filter( - (e) => - isValidEdge(e) && - importedNodeIds.has(e.fromPageId) && - importedNodeIds.has(e.toPageId), - ); - - if (validEdges.length > 0) { - await metadataStore.putEdges(validEdges); - } - - return validEdges.length; -} - -// --------------------------------------------------------------------------- -// Public API -// --------------------------------------------------------------------------- - -/** - * Import a GraphFragment received in response to a CuriosityProbe. - * - * Validates schema, strips sender identity metadata, and persists approved - * nodes and edges into the local store. Rejected nodes are returned for - * auditability. - */ -export async function importFragment( - fragment: GraphFragment, - options: ImportOptions, -): Promise { - const { metadataStore, vectorStore, verifyContentHashes = false } = options; - - const { imported, rejected } = await importNodes( - fragment.nodes, - vectorStore, - metadataStore, - verifyContentHashes, - ); - - const importedSet = new Set(imported); - const edgesImported = await importEdges(fragment.edges, importedSet, metadataStore); - - return { nodesImported: imported.length, edgesImported, rejected }; -} - -/** - * Import a SubgraphSlice received via proactive peer exchange. - * - * Applies the same validation and identity stripping as `importFragment`. - */ -export async function importSlice( - slice: SubgraphSlice, - options: ImportOptions, -): Promise { - const { metadataStore, vectorStore, verifyContentHashes = false } = options; - - const { imported, rejected } = await importNodes( - slice.nodes, - vectorStore, - metadataStore, - verifyContentHashes, - ); - - const importedSet = new Set(imported); - const edgesImported = await importEdges(slice.edges, importedSet, metadataStore); - - return { nodesImported: imported.length, edgesImported, rejected }; -} +export * from "../lib/sharing/SubgraphImporter"; diff --git a/sharing/types.ts b/sharing/types.ts index 600bedd..a4bac48 100644 --- a/sharing/types.ts +++ b/sharing/types.ts @@ -1,141 +1 @@ -// --------------------------------------------------------------------------- -// sharing/types.ts — Shared data types for P2P curiosity and subgraph exchange -// --------------------------------------------------------------------------- -// -// All types used across sharing modules are defined here to keep the modules -// decoupled from one another while sharing a single canonical schema. -// --------------------------------------------------------------------------- - -import type { Edge, Hash, Page, Signature } from "../core/types"; - -// --------------------------------------------------------------------------- -// CuriosityProbe — broadcast when a knowledge gap is detected -// --------------------------------------------------------------------------- - -/** - * A P2P curiosity probe broadcast when MetroidBuilder cannot find a valid - * antithesis medoid (m2) for a thesis topic. - * - * Peers receiving a probe MUST verify that `mimeType` and `modelUrn` match - * their local model before attempting to respond. Accepting graph fragments - * from an incompatible model would introduce incommensurable similarity scores. - */ -export interface CuriosityProbe { - /** Unique probe identifier (e.g., UUID or hash of probe content). */ - probeId: string; - - /** The thesis medoid page ID for which antithesis was not found. */ - m1: Hash; - - /** The incomplete Metroid at the boundary of local knowledge. */ - partialMetroid: { - m1: Hash; - m2?: Hash; - /** Serialised centroid embedding as a base-64-encoded Float32Array, optional. */ - centroidB64?: string; - }; - - /** Original query embedding serialised as base-64-encoded Float32Array. */ - queryContextB64: string; - - /** Matryoshka dimensional layer at which antithesis search failed. */ - knowledgeBoundary: number; - - /** - * MIME type of the embedded content (e.g., "text/plain", "image/jpeg"). - * Required: peers must validate content-type commensurability. - */ - mimeType: string; - - /** - * URN identifying the specific embedding model used to produce the vectors - * (e.g., "urn:model:onnx-community/embeddinggemma-300m-ONNX:v1"). - * Required: peers must reject probes with incompatible modelUrn. - */ - modelUrn: string; - - /** ISO 8601 timestamp when this probe was created. */ - timestamp: string; -} - -// --------------------------------------------------------------------------- -// GraphFragment — response payload returned to a curiosity probe -// --------------------------------------------------------------------------- - -/** - * A signed graph fragment returned by a peer in response to a CuriosityProbe. - * Contains nodes and edges relevant to the probe's knowledge boundary. - */ -export interface GraphFragment { - /** Unique fragment identifier. */ - fragmentId: string; - - /** The probe ID this fragment responds to. */ - probeId: string; - - /** Pages included in this fragment (eligibility-filtered). */ - nodes: Page[]; - - /** Hebbian edges among the included nodes. */ - edges: Edge[]; - - /** - * Per-node cryptographic signatures keyed by pageId. - * Recipients verify these before integrating. - */ - signatures: Record; - - /** ISO 8601 timestamp when this fragment was assembled. */ - timestamp: string; -} - -// --------------------------------------------------------------------------- -// Eligibility decisions -// --------------------------------------------------------------------------- - -export type EligibilityStatus = "eligible" | "blocked"; - -export type BlockReason = - | "pii_identity" - | "pii_credentials" - | "pii_financial" - | "pii_health" - | "no_public_interest"; - -/** Deterministic eligibility decision for a single candidate page. */ -export interface EligibilityDecision { - pageId: Hash; - status: EligibilityStatus; - reason?: BlockReason; -} - -// --------------------------------------------------------------------------- -// SubgraphSlice — an exported topic-scoped graph section -// --------------------------------------------------------------------------- - -/** - * A topic-scoped subgraph slice built from eligibility-approved pages. - * Used for both curiosity responses and proactive peer exchange. - */ -export interface SubgraphSlice { - sliceId: string; - nodes: Page[]; - edges: Edge[]; - /** Provenance map: pageId -> source probe or exchange ID. */ - provenance: Record; - /** Signatures map for verification. */ - signatures: Record; - timestamp: string; -} - -// --------------------------------------------------------------------------- -// PeerMessage — top-level P2P transport envelope -// --------------------------------------------------------------------------- - -export type PeerMessageKind = "curiosity_probe" | "graph_fragment" | "subgraph_slice"; - -export interface PeerMessage { - kind: PeerMessageKind; - senderId: string; - payload: CuriosityProbe | GraphFragment | SubgraphSlice; -} +export * from "../lib/sharing/types"; diff --git a/storage/IndexedDbMetadataStore.ts b/storage/IndexedDbMetadataStore.ts index 2ffb384..0731df4 100644 --- a/storage/IndexedDbMetadataStore.ts +++ b/storage/IndexedDbMetadataStore.ts @@ -1,614 +1 @@ -import type { - Book, - Edge, - Hash, - HotpathEntry, - MetadataStore, - SemanticNeighbor, - SemanticNeighborSubgraph, - Page, - PageActivity, - Shelf, - Volume, -} from "../core/types"; - -// --------------------------------------------------------------------------- -// Schema constants -// --------------------------------------------------------------------------- - -const DB_VERSION = 3; - -/** Object-store names used across the schema. */ -const STORE = { - pages: "pages", - books: "books", - volumes: "volumes", - shelves: "shelves", - edges: "edges_hebbian", - neighborGraph: "neighbor_graph", - flags: "flags", - pageToBook: "page_to_book", - bookToVolume: "book_to_volume", - volumeToShelf: "volume_to_shelf", - hotpathIndex: "hotpath_index", - pageActivity: "page_activity", -} as const; - -// --------------------------------------------------------------------------- -// Low-level IDB helpers -// --------------------------------------------------------------------------- - -function promisifyTransaction(tx: IDBTransaction): Promise { - return new Promise((resolve, reject) => { - tx.oncomplete = () => resolve(); - tx.onerror = () => reject(tx.error); - tx.onabort = () => reject(tx.error ?? new DOMException("Transaction aborted", "AbortError")); - }); -} - -// --------------------------------------------------------------------------- -// Schema upgrade -// --------------------------------------------------------------------------- - -function applyUpgrade(db: IDBDatabase): void { - // v1 stores - if (!db.objectStoreNames.contains(STORE.pages)) { - db.createObjectStore(STORE.pages, { keyPath: "pageId" }); - } - if (!db.objectStoreNames.contains(STORE.books)) { - db.createObjectStore(STORE.books, { keyPath: "bookId" }); - } - if (!db.objectStoreNames.contains(STORE.volumes)) { - db.createObjectStore(STORE.volumes, { keyPath: "volumeId" }); - } - if (!db.objectStoreNames.contains(STORE.shelves)) { - db.createObjectStore(STORE.shelves, { keyPath: "shelfId" }); - } - - if (!db.objectStoreNames.contains(STORE.edges)) { - const edgeStore = db.createObjectStore(STORE.edges, { - keyPath: ["fromPageId", "toPageId"], - }); - edgeStore.createIndex("by-from", "fromPageId"); - } - - - if (!db.objectStoreNames.contains(STORE.flags)) { - db.createObjectStore(STORE.flags, { keyPath: "volumeId" }); - } - - if (!db.objectStoreNames.contains(STORE.pageToBook)) { - db.createObjectStore(STORE.pageToBook, { keyPath: "pageId" }); - } - if (!db.objectStoreNames.contains(STORE.bookToVolume)) { - db.createObjectStore(STORE.bookToVolume, { keyPath: "bookId" }); - } - if (!db.objectStoreNames.contains(STORE.volumeToShelf)) { - db.createObjectStore(STORE.volumeToShelf, { keyPath: "volumeId" }); - } - - // v2 stores — hotpath index + page activity - if (!db.objectStoreNames.contains(STORE.hotpathIndex)) { - const hp = db.createObjectStore(STORE.hotpathIndex, { keyPath: "entityId" }); - hp.createIndex("by-tier", "tier"); - } - if (!db.objectStoreNames.contains(STORE.pageActivity)) { - db.createObjectStore(STORE.pageActivity, { keyPath: "pageId" }); - } - - // v3 stores — neighbor_graph (replaces the old metroid_neighbors name) - if (!db.objectStoreNames.contains(STORE.neighborGraph)) { - db.createObjectStore(STORE.neighborGraph, { keyPath: "pageId" }); - } -} - -// --------------------------------------------------------------------------- -// IndexedDbMetadataStore -// --------------------------------------------------------------------------- - -/** - * Full MetadataStore implementation backed by IndexedDB. - * - * Reverse-index rows (`page_to_book`, `book_to_volume`, `volume_to_shelf`) are - * maintained atomically inside the same transaction as the owning entity write, - * so they are always consistent with the latest put. - * - * Usage: - * const store = await IndexedDbMetadataStore.open("cortex"); - */ -export class IndexedDbMetadataStore implements MetadataStore { - private constructor(private readonly db: IDBDatabase) {} - - // ------------------------------------------------------------------------- - // Factory - // ------------------------------------------------------------------------- - - static open(dbName: string): Promise { - return new Promise((resolve, reject) => { - const req = indexedDB.open(dbName, DB_VERSION); - - req.onupgradeneeded = (event) => { - applyUpgrade((event.target as IDBOpenDBRequest).result); - }; - - req.onsuccess = () => resolve(new IndexedDbMetadataStore(req.result)); - req.onerror = () => reject(req.error); - }); - } - - // ------------------------------------------------------------------------- - // Page CRUD - // ------------------------------------------------------------------------- - - putPage(page: Page): Promise { - return this._put(STORE.pages, page); - } - - async getPage(pageId: Hash): Promise { - return this._get(STORE.pages, pageId); - } - - /** - * Returns all pages in the store. Used for warm/cold fallbacks in query. - * TODO: Replace with a paginated or indexed scan before production use — - * loading every page into memory is expensive for large corpora. - */ - async getAllPages(): Promise { - return new Promise((resolve, reject) => { - const tx = this.db.transaction(STORE.pages, "readonly"); - const req = tx.objectStore(STORE.pages).getAll(); - req.onsuccess = () => resolve(req.result as Page[]); - req.onerror = () => reject(req.error); - }); - } - - // ------------------------------------------------------------------------- - // Book CRUD + reverse index maintenance - // ------------------------------------------------------------------------- - - putBook(book: Book): Promise { - return new Promise((resolve, reject) => { - const tx = this.db.transaction( - [STORE.books, STORE.pageToBook], - "readwrite", - ); - - // Store the book itself - tx.objectStore(STORE.books).put(book); - - // Update page->book reverse index for every page in this book - const idxStore = tx.objectStore(STORE.pageToBook); - for (const pageId of book.pageIds) { - const getReq = idxStore.get(pageId); - getReq.onsuccess = () => { - const existing: { pageId: Hash; bookIds: Hash[] } | undefined = - getReq.result; - const bookIds = existing?.bookIds ?? []; - if (!bookIds.includes(book.bookId)) { - bookIds.push(book.bookId); - } - idxStore.put({ pageId, bookIds }); - }; - } - - promisifyTransaction(tx).then(resolve).catch(reject); - }); - } - - async getBook(bookId: Hash): Promise { - return this._get(STORE.books, bookId); - } - - // ------------------------------------------------------------------------- - // Volume CRUD + reverse index - // ------------------------------------------------------------------------- - - putVolume(volume: Volume): Promise { - return new Promise((resolve, reject) => { - const tx = this.db.transaction( - [STORE.volumes, STORE.bookToVolume], - "readwrite", - ); - - tx.objectStore(STORE.volumes).put(volume); - - const idxStore = tx.objectStore(STORE.bookToVolume); - for (const bookId of volume.bookIds) { - const getReq = idxStore.get(bookId); - getReq.onsuccess = () => { - const existing: { bookId: Hash; volumeIds: Hash[] } | undefined = - getReq.result; - const volumeIds = existing?.volumeIds ?? []; - if (!volumeIds.includes(volume.volumeId)) { - volumeIds.push(volume.volumeId); - } - idxStore.put({ bookId, volumeIds }); - }; - } - - promisifyTransaction(tx).then(resolve).catch(reject); - }); - } - - async getVolume(volumeId: Hash): Promise { - return this._get(STORE.volumes, volumeId); - } - - async getAllVolumes(): Promise { - return new Promise((resolve, reject) => { - const tx = this.db.transaction(STORE.volumes, "readonly"); - const req = tx.objectStore(STORE.volumes).getAll(); - req.onsuccess = () => resolve(req.result as Volume[]); - req.onerror = () => reject(req.error); - }); - } - - /** - * Delete a volume and clean up its reverse-index entries: - * - Removes the volume from the `bookToVolume` index for each of its books. - * - Deletes the `volumeToShelf` index entry for this volume. - * - Deletes the volume record itself. - * - * Callers should update or remove the volume from any shelf's `volumeIds` - * list before calling this method. - */ - async deleteVolume(volumeId: Hash): Promise { - const volume = await this.getVolume(volumeId); - - return new Promise((resolve, reject) => { - const tx = this.db.transaction( - [STORE.volumes, STORE.bookToVolume, STORE.volumeToShelf], - "readwrite", - ); - - // Remove from bookToVolume reverse index for each book the volume owned - if (volume) { - const bookToVolumeStore = tx.objectStore(STORE.bookToVolume); - for (const bookId of volume.bookIds) { - const getReq = bookToVolumeStore.get(bookId); - getReq.onsuccess = () => { - const existing: { bookId: Hash; volumeIds: Hash[] } | undefined = - getReq.result; - if (!existing) return; - const updatedVolumeIds = existing.volumeIds.filter( - (id) => id !== volumeId, - ); - if (updatedVolumeIds.length === 0) { - bookToVolumeStore.delete(bookId); - } else { - bookToVolumeStore.put({ bookId, volumeIds: updatedVolumeIds }); - } - }; - } - } - - // Remove volumeToShelf reverse index entry - tx.objectStore(STORE.volumeToShelf).delete(volumeId); - - // Delete the volume record itself - tx.objectStore(STORE.volumes).delete(volumeId); - - promisifyTransaction(tx).then(resolve).catch(reject); - }); - } - - // ------------------------------------------------------------------------- - // Shelf CRUD + reverse index - // ------------------------------------------------------------------------- - - putShelf(shelf: Shelf): Promise { - return new Promise((resolve, reject) => { - const tx = this.db.transaction( - [STORE.shelves, STORE.volumeToShelf], - "readwrite", - ); - - tx.objectStore(STORE.shelves).put(shelf); - - const idxStore = tx.objectStore(STORE.volumeToShelf); - for (const volumeId of shelf.volumeIds) { - const getReq = idxStore.get(volumeId); - getReq.onsuccess = () => { - const existing: { volumeId: Hash; shelfIds: Hash[] } | undefined = - getReq.result; - const shelfIds = existing?.shelfIds ?? []; - if (!shelfIds.includes(shelf.shelfId)) { - shelfIds.push(shelf.shelfId); - } - idxStore.put({ volumeId, shelfIds }); - }; - } - - promisifyTransaction(tx).then(resolve).catch(reject); - }); - } - - async getShelf(shelfId: Hash): Promise { - return this._get(STORE.shelves, shelfId); - } - - async getAllShelves(): Promise { - return new Promise((resolve, reject) => { - const tx = this.db.transaction(STORE.shelves, "readonly"); - const req = tx.objectStore(STORE.shelves).getAll(); - req.onsuccess = () => resolve(req.result as Shelf[]); - req.onerror = () => reject(req.error); - }); - } - - // ------------------------------------------------------------------------- - // Hebbian edges - // ------------------------------------------------------------------------- - - putEdges(edges: Edge[]): Promise { - return new Promise((resolve, reject) => { - const tx = this.db.transaction(STORE.edges, "readwrite"); - const store = tx.objectStore(STORE.edges); - for (const edge of edges) { - store.put(edge); - } - promisifyTransaction(tx).then(resolve).catch(reject); - }); - } - - deleteEdge(fromPageId: Hash, toPageId: Hash): Promise { - return new Promise((resolve, reject) => { - const tx = this.db.transaction(STORE.edges, "readwrite"); - tx.objectStore(STORE.edges).delete([fromPageId, toPageId]); - promisifyTransaction(tx).then(resolve).catch(reject); - }); - } - - async getNeighbors(pageId: Hash, limit?: number): Promise { - return new Promise((resolve, reject) => { - const tx = this.db.transaction(STORE.edges, "readonly"); - const idx = tx.objectStore(STORE.edges).index("by-from"); - const req = idx.getAll(IDBKeyRange.only(pageId)); - req.onsuccess = () => { - let rows: Edge[] = req.result; - rows.sort((a, b) => b.weight - a.weight); - if (limit !== undefined) rows = rows.slice(0, limit); - resolve(rows); - }; - req.onerror = () => reject(req.error); - }); - } - - // ------------------------------------------------------------------------- - // Reverse-index helpers - // ------------------------------------------------------------------------- - - async getBooksByPage(pageId: Hash): Promise { - const row = await this._get<{ pageId: Hash; bookIds: Hash[] }>( - STORE.pageToBook, - pageId, - ); - if (!row || row.bookIds.length === 0) return []; - return this._getMany(STORE.books, row.bookIds); - } - - async getVolumesByBook(bookId: Hash): Promise { - const row = await this._get<{ bookId: Hash; volumeIds: Hash[] }>( - STORE.bookToVolume, - bookId, - ); - if (!row || row.volumeIds.length === 0) return []; - return this._getMany(STORE.volumes, row.volumeIds); - } - - async getShelvesByVolume(volumeId: Hash): Promise { - const row = await this._get<{ volumeId: Hash; shelfIds: Hash[] }>( - STORE.volumeToShelf, - volumeId, - ); - if (!row || row.shelfIds.length === 0) return []; - return this._getMany(STORE.shelves, row.shelfIds); - } - - // ------------------------------------------------------------------------- - // Semantic neighbor radius index - // ------------------------------------------------------------------------- - - putSemanticNeighbors(pageId: Hash, neighbors: SemanticNeighbor[]): Promise { - return this._put(STORE.neighborGraph, { pageId, neighbors }); - } - - async getSemanticNeighbors( - pageId: Hash, - maxDegree?: number, - ): Promise { - const row = await this._get<{ pageId: Hash; neighbors: SemanticNeighbor[] }>( - STORE.neighborGraph, - pageId, - ); - if (!row) return []; - const list = row.neighbors; - return maxDegree !== undefined ? list.slice(0, maxDegree) : list; - } - - async getInducedNeighborSubgraph( - seedPageIds: Hash[], - maxHops: number, - ): Promise { - const visited = new Set(seedPageIds); - const nodeSet = new Set(seedPageIds); - const edgeMap = new Map(); - - let frontier = [...seedPageIds]; - - for (let hop = 0; hop < maxHops && frontier.length > 0; hop++) { - const nextFrontier: Hash[] = []; - - for (const pageId of frontier) { - const neighbors = await this.getSemanticNeighbors(pageId); - for (const n of neighbors) { - const key = `${pageId}\x00${n.neighborPageId}`; - if (!edgeMap.has(key)) { - edgeMap.set(key, { - from: pageId, - to: n.neighborPageId, - distance: n.distance, - }); - } - if (!visited.has(n.neighborPageId)) { - visited.add(n.neighborPageId); - nodeSet.add(n.neighborPageId); - nextFrontier.push(n.neighborPageId); - } - } - } - - frontier = nextFrontier; - } - - return { - nodes: [...nodeSet], - edges: [...edgeMap.values()], - }; - } - - // ------------------------------------------------------------------------- - // Dirty-recalc flags - // ------------------------------------------------------------------------- - - async needsNeighborRecalc(volumeId: Hash): Promise { - const row = await this._get<{ volumeId: Hash; needsRecalc: boolean }>( - STORE.flags, - volumeId, - ); - return row?.needsRecalc === true; - } - - flagVolumeForNeighborRecalc(volumeId: Hash): Promise { - return this._put(STORE.flags, { volumeId, needsRecalc: true }); - } - - clearNeighborRecalcFlag(volumeId: Hash): Promise { - return this._put(STORE.flags, { volumeId, needsRecalc: false }); - } - - // ------------------------------------------------------------------------- - // Hotpath index - // ------------------------------------------------------------------------- - - putHotpathEntry(entry: HotpathEntry): Promise { - return this._put(STORE.hotpathIndex, entry); - } - - async getHotpathEntries(tier?: HotpathEntry["tier"]): Promise { - if (tier !== undefined) { - return new Promise((resolve, reject) => { - const tx = this.db.transaction(STORE.hotpathIndex, "readonly"); - const idx = tx.objectStore(STORE.hotpathIndex).index("by-tier"); - const req = idx.getAll(IDBKeyRange.only(tier)); - req.onsuccess = () => resolve(req.result as HotpathEntry[]); - req.onerror = () => reject(req.error); - }); - } - return new Promise((resolve, reject) => { - const tx = this.db.transaction(STORE.hotpathIndex, "readonly"); - const req = tx.objectStore(STORE.hotpathIndex).getAll(); - req.onsuccess = () => resolve(req.result as HotpathEntry[]); - req.onerror = () => reject(req.error); - }); - } - - removeHotpathEntry(entityId: Hash): Promise { - return new Promise((resolve, reject) => { - const tx = this.db.transaction(STORE.hotpathIndex, "readwrite"); - tx.objectStore(STORE.hotpathIndex).delete(entityId); - promisifyTransaction(tx).then(resolve).catch(reject); - }); - } - - async evictWeakest( - tier: HotpathEntry["tier"], - communityId?: string, - ): Promise { - const entries = await this.getHotpathEntries(tier); - const filtered = communityId !== undefined - ? entries.filter((e) => e.communityId === communityId) - : entries; - - if (filtered.length === 0) return; - - // Deterministic: break ties by entityId (smallest wins) - let weakest = filtered[0]; - for (let i = 1; i < filtered.length; i++) { - const e = filtered[i]; - if ( - e.salience < weakest.salience || - (e.salience === weakest.salience && e.entityId < weakest.entityId) - ) { - weakest = e; - } - } - - await this.removeHotpathEntry(weakest.entityId); - } - - async getResidentCount(): Promise { - return new Promise((resolve, reject) => { - const tx = this.db.transaction(STORE.hotpathIndex, "readonly"); - const req = tx.objectStore(STORE.hotpathIndex).count(); - req.onsuccess = () => resolve(req.result); - req.onerror = () => reject(req.error); - }); - } - - // ------------------------------------------------------------------------- - // Page activity - // ------------------------------------------------------------------------- - - putPageActivity(activity: PageActivity): Promise { - return this._put(STORE.pageActivity, activity); - } - - async getPageActivity(pageId: Hash): Promise { - return this._get(STORE.pageActivity, pageId); - } - - // ------------------------------------------------------------------------- - // Private generic helpers - // ------------------------------------------------------------------------- - - private _put(storeName: string, value: unknown): Promise { - return new Promise((resolve, reject) => { - const tx = this.db.transaction(storeName, "readwrite"); - tx.objectStore(storeName).put(value); - promisifyTransaction(tx).then(resolve).catch(reject); - }); - } - - private _get(storeName: string, key: IDBValidKey): Promise { - return new Promise((resolve, reject) => { - const tx = this.db.transaction(storeName, "readonly"); - const req = tx.objectStore(storeName).get(key); - req.onsuccess = () => resolve(req.result as T | undefined); - req.onerror = () => reject(req.error); - }); - } - - private _getMany(storeName: string, keys: IDBValidKey[]): Promise { - return new Promise((resolve, reject) => { - const tx = this.db.transaction(storeName, "readonly"); - const store = tx.objectStore(storeName); - const results: T[] = []; - let pending = keys.length; - - if (pending === 0) { - resolve(results); - return; - } - - keys.forEach((key, i) => { - const req = store.get(key); - req.onsuccess = () => { - if (req.result !== undefined) results[i] = req.result as T; - if (--pending === 0) resolve(results.filter(Boolean)); - }; - req.onerror = () => reject(req.error); - }); - }); - } -} +export * from "../lib/storage/IndexedDbMetadataStore"; diff --git a/storage/MemoryVectorStore.ts b/storage/MemoryVectorStore.ts index 242701d..d9e2f86 100644 --- a/storage/MemoryVectorStore.ts +++ b/storage/MemoryVectorStore.ts @@ -1,39 +1 @@ -import type { VectorStore } from "../core/types"; -import { FLOAT32_BYTES } from "../core/NumericConstants"; - -/** - * MemoryVectorStore — in-memory implementation of VectorStore. - * - * Byte-offset semantics are identical to OPFSVectorStore so the two - * implementations are interchangeable for testing. - */ -export class MemoryVectorStore implements VectorStore { - private _buf: Uint8Array = new Uint8Array(0); - - async appendVector(vector: Float32Array): Promise { - const byteOffset = this._buf.byteLength; - const incoming = new Uint8Array(vector.buffer, vector.byteOffset, vector.byteLength); - const next = new Uint8Array(byteOffset + incoming.byteLength); - next.set(this._buf); - next.set(incoming, byteOffset); - this._buf = next; - return byteOffset; - } - - async readVector(offset: number, dim: number): Promise { - const byteLen = dim * FLOAT32_BYTES; - return new Float32Array(this._buf.buffer.slice(offset, offset + byteLen)); - } - - async readVectors(offsets: number[], dim: number): Promise { - const byteLen = dim * FLOAT32_BYTES; - return offsets.map( - (offset) => new Float32Array(this._buf.buffer.slice(offset, offset + byteLen)), - ); - } - - /** Total bytes currently stored (useful in tests). */ - get byteLength(): number { - return this._buf.byteLength; - } -} +export * from "../lib/storage/MemoryVectorStore"; diff --git a/storage/OPFSVectorStore.ts b/storage/OPFSVectorStore.ts index 3f6e597..e7edbd6 100644 --- a/storage/OPFSVectorStore.ts +++ b/storage/OPFSVectorStore.ts @@ -1,89 +1 @@ -import type { VectorStore } from "../core/types"; -import { FLOAT32_BYTES } from "../core/NumericConstants"; - -/** - * OPFSVectorStore — append-only binary vector file stored in the browser's - * Origin Private File System. - * - * Layout: raw IEEE-754 float32 bytes written sequentially. - * - * Offset semantics: every public method accepts/returns **byte offsets** - * (not vector indices) so callers can store mixed-dimension vectors - * (full embeddings, compressed prototypes, routing codes) in the same file. - * - * Concurrency: for Phase 1 all writes are serialised through a promise chain - * (`_writeQueue`). A dedicated sync-access-handle approach for high-throughput - * ingestion is deferred to Phase 2. - */ -export class OPFSVectorStore implements VectorStore { - private readonly fileName: string; - private _writeQueue: Promise = Promise.resolve(); - - constructor(fileName = "cortex-vectors.bin") { - this.fileName = fileName; - } - - // ------------------------------------------------------------------------- - // VectorStore implementation - // ------------------------------------------------------------------------- - - async appendVector(vector: Float32Array): Promise { - let byteOffset = 0; - - this._writeQueue = this._writeQueue.then(async () => { - const fileHandle = await this._fileHandle(true); - const file = await fileHandle.getFile(); - byteOffset = file.size; - - const writable = await fileHandle.createWritable({ keepExistingData: true }); - await writable.seek(byteOffset); - // Produce a plain ArrayBuffer copy of the exact float bytes. We cast - // to ArrayBuffer (rather than SharedArrayBuffer) because Float32Arrays - // constructed from normal JS sources always back a plain ArrayBuffer, and - // FileSystemWritableFileStream.write() requires ArrayBuffer / ArrayBufferView. - const copy = vector.buffer.slice( - vector.byteOffset, - vector.byteOffset + vector.byteLength, - ) as ArrayBuffer; - await writable.write(copy); - await writable.close(); - }); - - await this._writeQueue; - return byteOffset; - } - - async readVector(offset: number, dim: number): Promise { - const fileHandle = await this._fileHandle(false); - const file = await fileHandle.getFile(); - const slice = file.slice(offset, offset + dim * FLOAT32_BYTES); - const buf = await slice.arrayBuffer(); - return new Float32Array(buf); - } - - async readVectors(offsets: number[], dim: number): Promise { - if (offsets.length === 0) return []; - - const fileHandle = await this._fileHandle(false); - const file = await fileHandle.getFile(); - // Read the entire file once and extract each vector by slice. - const fullBuf = await file.arrayBuffer(); - - return offsets.map((offset) => { - const byteLen = dim * FLOAT32_BYTES; - return new Float32Array(fullBuf.slice(offset, offset + byteLen)); - }); - } - - // ------------------------------------------------------------------------- - // Helpers - // ------------------------------------------------------------------------- - - /** Returns the underlying OPFS file handle, optionally creating the file. */ - private async _fileHandle( - create: boolean, - ): Promise { - const root = await navigator.storage.getDirectory(); - return root.getFileHandle(this.fileName, { create }); - } -} +export * from "../lib/storage/OPFSVectorStore"; diff --git a/tsconfig.json b/tsconfig.json index 18867a4..9324519 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -13,10 +13,8 @@ "include": [ "*.ts", "*.d.ts", - "core/**/*.ts", - "storage/**/*.ts", - "embeddings/**/*.ts", - "hippocampus/**/*.ts", + "lib/**/*.ts", + "lib/**/*.d.ts", "tests/**/*.ts" ], "exclude": ["dist", "node_modules"] diff --git a/runtime/harness/index.html b/ui/harness/index.html similarity index 100% rename from runtime/harness/index.html rename to ui/harness/index.html From 3a097d151d11001041104d8b449495b6158a53a2 Mon Sep 17 00:00:00 2001 From: devlux76 Date: Sat, 14 Mar 2026 03:10:48 -0600 Subject: [PATCH 2/4] fix: update test results to reflect failure status and record failed test IDs --- test-results/.last-run.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test-results/.last-run.json b/test-results/.last-run.json index cbcc1fb..ab6358d 100644 --- a/test-results/.last-run.json +++ b/test-results/.last-run.json @@ -1,4 +1,6 @@ { - "status": "passed", - "failedTests": [] + "status": "failed", + "failedTests": [ + "ab4f1da33ff38cedbbb8-db758bf5015d8c294550" + ] } \ No newline at end of file From b734f0757065d1920dfeaf897034aeb7afa2d9db Mon Sep 17 00:00:00 2001 From: devlux76 Date: Sat, 14 Mar 2026 03:24:24 -0600 Subject: [PATCH 3/4] fix: add unzip package to Dockerfile dependencies --- docker/electron-debug/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/electron-debug/Dockerfile b/docker/electron-debug/Dockerfile index e485dac..dc20b28 100644 --- a/docker/electron-debug/Dockerfile +++ b/docker/electron-debug/Dockerfile @@ -28,6 +28,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libxshmfence1 \ libxss1 \ libxtst6 \ + unzip \ xauth \ xvfb \ && rm -rf /var/lib/apt/lists/* From 003ccadf99637045a65712b5db5ee2ef6b3ab8e0 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sat, 14 Mar 2026 03:35:17 -0600 Subject: [PATCH 4/4] =?UTF-8?q?fix:=20ensure=20WebGL=E2=86=92WASM=20fallba?= =?UTF-8?q?ck=20catches=20synchronous=20throws=20(#86)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Initial plan * fix: wrap WebGlVectorBackend.create() in .then() to capture synchronous throws for WASM fallback Co-authored-by: devlux76 <86517969+devlux76@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: devlux76 <86517969+devlux76@users.noreply.github.com> --- lib/CreateVectorBackend.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/CreateVectorBackend.ts b/lib/CreateVectorBackend.ts index 532f9b0..b5ed091 100644 --- a/lib/CreateVectorBackend.ts +++ b/lib/CreateVectorBackend.ts @@ -15,9 +15,9 @@ export async function createVectorBackend( ); } if (kind === "webgl") { - return Promise.resolve(WebGlVectorBackend.create()).catch(() => - WasmVectorBackend.create(wasmBytes) - ); + return Promise.resolve() + .then(() => WebGlVectorBackend.create()) + .catch(() => WasmVectorBackend.create(wasmBytes)); } if (kind === "webnn") { return WebNnVectorBackend.create(wasmBytes).catch(() =>