diff --git a/packages/contexto/README.md b/packages/contexto/README.md index a2762af..fc83001 100644 --- a/packages/contexto/README.md +++ b/packages/contexto/README.md @@ -121,6 +121,40 @@ For the deeper technical reasoning: | Property | Type | Required | Description | | --- | --- | --- | --- | | `apiKey` | string | Yes | Your Contexto API key | +| `contextEnabled` | boolean | No | Enable or disable context retrieval (default: `true`) | +| `maxContextChars` | number | No | Maximum characters to inject as retrieved context | +| `compactThreshold` | number | No | Fraction of token budget that triggers compaction (default: `0.50`) | +| `compactionStrategy` | string | No | `"default"` or `"sliding-window"` (default: `"default"`) | +| `rlmEnabled` | boolean | No | Enable RLM tools for processing large contexts (default: `false`) | + +## Large Context Processing (RLM) + +When a user sends a message that exceeds 50% of the available token budget — a PDF, a spreadsheet, a massive log dump — the standard approach of stuffing it into the prompt breaks down. Contexto includes optional support for **Recursive Language Model (RLM)** processing to handle these cases. + +When enabled, Contexto automatically detects oversized inputs, offloads them to an in-memory buffer, and gives the agent a set of six tools to explore, search, and reason over the content iteratively — without flooding the context window. + +### Enabling RLM + +Set `rlmEnabled` to `true` in your plugin config: + +```bash +openclaw config set plugins.entries.contexto.config.rlmEnabled true +``` + +When disabled (the default), the plugin behaves exactly as before — no RLM tools are registered and no additional dependencies are loaded. + +### How It Works + +1. During context assembly, if the user's message exceeds 50% of the token budget, the content is moved to an in-memory **ContextBuffer** and the message is replaced with a brief instruction. +2. The agent receives six RLM tools: **rlm_overview**, **rlm_peek**, **rlm_grep**, **rlm_slice**, **rlm_query**, and **rlm_repl** — covering structural exploration, pattern search, targeted extraction, sub-LLM reasoning, and sandboxed scripting. +3. The agent iteratively explores and synthesizes an answer using these tools, keeping token usage bounded regardless of input size. +4. Once complete, the synthesized result is ingested into the mindmap as an episode, making it available for future recall just like any other conversation context. + +RLM can also be invoked explicitly by the user, regardless of message size. + +Sub-LLM calls are routed through [pi-ai](https://docs.openclaw.ai/pi) via OpenRouter's auto-routing, which automatically selects an appropriate model. No additional API keys or provider SDKs are needed beyond what OpenClaw already manages. + +The RLM tools are provided by the [`@ekai/rlm`](../rlm/) package, which can also be used standalone outside of Contexto. See its [README](../rlm/README.md) for full tool documentation. ## Community diff --git a/packages/contexto/openclaw.plugin.json b/packages/contexto/openclaw.plugin.json index a740537..b6093ce 100644 --- a/packages/contexto/openclaw.plugin.json +++ b/packages/contexto/openclaw.plugin.json @@ -15,6 +15,11 @@ "maxContextChars": { "type": "number", "description": "Maximum characters of context to inject (default: 2000)" + }, + "rlmEnabled": { + "type": "boolean", + "default": false, + "description": "Enable Recursive Language Model (RLM) tools for processing large contexts that exceed the token budget" } } } diff --git a/packages/contexto/package.json b/packages/contexto/package.json index 42c9ffc..d824aa9 100644 --- a/packages/contexto/package.json +++ b/packages/contexto/package.json @@ -31,8 +31,17 @@ "scripts": { "build": "tsc --noEmit" }, + "dependencies": { + "@ekai/rlm": "workspace:*" + }, "peerDependencies": { - "openclaw": "*" + "openclaw": "*", + "@mariozechner/pi-ai": "*" + }, + "peerDependenciesMeta": { + "@mariozechner/pi-ai": { + "optional": true + } }, "devDependencies": { "@types/node": "^20.10.0", diff --git a/packages/contexto/src/engine/base.ts b/packages/contexto/src/engine/base.ts index 2cf1925..a046ce7 100644 --- a/packages/contexto/src/engine/base.ts +++ b/packages/contexto/src/engine/base.ts @@ -4,9 +4,9 @@ import type { IngestResult, IngestBatchResult, SubagentSpawnPreparation, } from 'openclaw/plugin-sdk'; import type { ContextoBackend, Logger, BaseConfig } from '../types.js'; -import { stripMetadataEnvelope, formatSearchResults, assembleContextMessages } from '../helpers.js'; +import { stripMetadataEnvelope, formatSearchResults, assembleContextMessages, buildPayload } from '../helpers.js'; import type { - CompactionState, + CompactionState, PendingContext, BootstrapParams, IngestParams, IngestBatchParams, AfterTurnParams, AssembleParams, CompactParams, SubagentSpawnParams, SubagentEndedParams, @@ -15,6 +15,7 @@ import type { const DEFAULT_MAX_CONTEXT_CHARS = 2000; const DEFAULT_MAX_RESULTS = 7; const DEFAULT_MIN_SCORE = 0.45; +const RLM_CONTEXT_THRESHOLD = 0.5; /** * Abstract base class for context engine implementations. @@ -74,6 +75,37 @@ export abstract class AbstractContextEngine implements ContextEngine { return { messages, estimatedTokens: 0 }; } + // --- RLM: detect large user context --- + if (this.config.rlmEnabled && tokenBudget && lastMsg?.role === 'user') { + const userText = this.extractMessageText(lastMsg); + if (userText) { + const estimatedTokens = Math.ceil(userText.length / 4); + const threshold = Math.floor(tokenBudget * RLM_CONTEXT_THRESHOLD); + + if (estimatedTokens > threshold) { + const sessionKey = (params as any).sessionKey ?? (params as any).sessionId ?? 'default'; + this.state.pendingLargeContext.set(sessionKey, { + content: userText, + tokenEstimate: estimatedTokens, + }); + this.logger.info(`[contexto] RLM: large context detected (${userText.length} chars, ~${estimatedTokens} tokens, threshold: ${threshold}). Stored pending context for session ${sessionKey}`); + + // Replace the large user message with an instruction + const replacement = `[Large context provided — ${userText.length} chars, ~${estimatedTokens} tokens. Use the rlm_query tool to analyze it.]`; + const modifiedMessages = [ + ...messages.slice(0, -1), + { ...lastMsg, content: replacement }, + ]; + + return { + messages: modifiedMessages, + estimatedTokens: Math.ceil(replacement.length / 4), + systemPromptAddition: 'The user has provided a large context that exceeds the context window. Use the RLM tools (rlm_overview, rlm_peek, rlm_grep, rlm_slice, rlm_query, rlm_repl) to analyze it. Start with rlm_overview to understand the structure, then use other tools to answer the user\'s question.', + }; + } + } + } + const query = params.prompt ? stripMetadataEnvelope(params.prompt) : undefined; if (!query) { return { messages, estimatedTokens: 0 }; @@ -126,11 +158,108 @@ export abstract class AbstractContextEngine implements ContextEngine { return assembleContextMessages(context, messages); } - async prepareSubagentSpawn(_params: SubagentSpawnParams): Promise { + async prepareSubagentSpawn(params: SubagentSpawnParams): Promise { + const childSessionKey = (params as any).childSessionKey; + if (!childSessionKey) return undefined; + + // Check if there's a pending large context for the current session + // The parent session key is stored when assemble() detects large content + for (const [sessionKey, pending] of this.state.pendingLargeContext.entries()) { + // Map child session to parent so onSubagentEnded can find the context + this.state.activeRlmSessions.set(childSessionKey, sessionKey); + this.logger.info(`[contexto] prepareSubagentSpawn: mapped child ${childSessionKey} → parent ${sessionKey} (${pending.tokenEstimate} est. tokens)`); + + return { + rollback: () => { + this.state.activeRlmSessions.delete(childSessionKey); + this.logger.info(`[contexto] prepareSubagentSpawn rollback: removed child ${childSessionKey}`); + }, + }; + } + return undefined; } - async onSubagentEnded(_params: SubagentEndedParams): Promise {} + async onSubagentEnded(params: SubagentEndedParams): Promise { + const childSessionKey = (params as any).childSessionKey; + const result = (params as any).result; + if (!childSessionKey) return; + + const parentSessionKey = this.state.activeRlmSessions.get(childSessionKey); + if (!parentSessionKey) return; + + // Clean up session mapping + this.state.activeRlmSessions.delete(childSessionKey); + + const pending = this.state.pendingLargeContext.get(parentSessionKey); + if (!pending) return; + + // Clean up pending context + this.state.pendingLargeContext.delete(parentSessionKey); + + // Extract the subagent's answer + const answer = typeof result === 'string' ? result : this.extractSubagentAnswer(result); + if (!answer) { + this.logger.warn(`[contexto] onSubagentEnded: no answer from subagent ${childSessionKey}`); + return; + } + + // Ingest the processed result into the mindmap for future recall + const payload = buildPayload('rlm-summary', 'processed', parentSessionKey, { + charCount: pending.content.length, + tokenEstimate: pending.tokenEstimate, + }, undefined, { + userMessage: { role: 'user', content: `[Large context: ${pending.content.length} chars, ~${pending.tokenEstimate} tokens]` }, + assistantMessages: [{ role: 'assistant', content: answer }], + }); + + try { + await this.backend.ingest(payload); + this.logger.info(`[contexto] onSubagentEnded: ingested RLM summary (${answer.length} chars) for session ${parentSessionKey}`); + } catch (err) { + this.logger.warn(`[contexto] onSubagentEnded: failed to ingest RLM summary — ${err}`); + } + } + + // --- Helpers --- + + /** Extract text from a message with string or ContentBlock[] content. */ + protected extractMessageText(msg: any): string | undefined { + if (!msg) return undefined; + if (typeof msg.content === 'string') return msg.content; + if (Array.isArray(msg.content)) { + const texts = msg.content + .filter((b: any) => b.type === 'text' && typeof b.text === 'string') + .map((b: any) => b.text); + return texts.length > 0 ? texts.join('\n') : undefined; + } + return undefined; + } + + /** Get pending large context for a session key. */ + getPendingContext(sessionKey: string): PendingContext | undefined { + return this.state.pendingLargeContext.get(sessionKey); + } + + /** Clear pending large context for a session key. */ + clearPendingContext(sessionKey: string): void { + this.state.pendingLargeContext.delete(sessionKey); + } + + /** Extract the last assistant text from subagent result. */ + private extractSubagentAnswer(result: any): string | undefined { + if (!result) return undefined; + // Handle array of messages + const messages = Array.isArray(result) ? result : result?.messages; + if (!Array.isArray(messages)) return typeof result === 'string' ? result : undefined; + + for (let i = messages.length - 1; i >= 0; i--) { + const msg = messages[i]; + const text = this.extractMessageText(msg); + if (msg?.role === 'assistant' && text) return text; + } + return undefined; + } // --- Template method with apiKey guard --- diff --git a/packages/contexto/src/engine/types.ts b/packages/contexto/src/engine/types.ts index 9cda002..962b361 100644 --- a/packages/contexto/src/engine/types.ts +++ b/packages/contexto/src/engine/types.ts @@ -16,6 +16,12 @@ export type CompactParams = MethodParams<'compact'>; export type SubagentSpawnParams = MethodParams<'prepareSubagentSpawn'>; export type SubagentEndedParams = MethodParams<'onSubagentEnded'>; +/** Large context pending RLM processing. */ +export interface PendingContext { + content: string; + tokenEstimate: number; +} + // Internal state — not part of the SDK contract export interface CompactionState { bufferedMessages: WebhookPayload[]; @@ -23,6 +29,10 @@ export interface CompactionState { lastSessionKey: string; cachedTokenBudget: number | undefined; injectedItemIds: Set; + /** Large contexts awaiting RLM subagent processing, keyed by sessionKey. */ + pendingLargeContext: Map; + /** Active RLM subagent sessions, keyed by childSessionKey → parentSessionKey. */ + activeRlmSessions: Map; } export function createCompactionState(): CompactionState { @@ -32,5 +42,7 @@ export function createCompactionState(): CompactionState { lastSessionKey: '', cachedTokenBudget: undefined, injectedItemIds: new Set(), + pendingLargeContext: new Map(), + activeRlmSessions: new Map(), }; } diff --git a/packages/contexto/src/engine/utils.ts b/packages/contexto/src/engine/utils.ts index 0fdd9e4..7c2fa66 100644 --- a/packages/contexto/src/engine/utils.ts +++ b/packages/contexto/src/engine/utils.ts @@ -78,8 +78,9 @@ export function selectMessagesToEvict( }; } -/** Extract the firstKeptEntryId from the first message in an array (if available). */ -export function getFirstKeptEntryId(messages: any[]): string | undefined { - const first = messages.length > 0 ? messages[0] : null; - return first?.id ?? first?.entryId ?? undefined; +/** Extract the firstKeptEntryId from the user message in the first kept episode payload. */ +export function getFirstKeptEntryId(kept: WebhookPayload[]): string | undefined { + if (kept.length === 0) return undefined; + const data = kept[0].data as Record | undefined; + return data?.userMessage?.id ?? undefined; } diff --git a/packages/contexto/src/index.ts b/packages/contexto/src/index.ts index 642c969..9bba92a 100644 --- a/packages/contexto/src/index.ts +++ b/packages/contexto/src/index.ts @@ -1,6 +1,7 @@ import type { PluginConfig } from './types.js'; import { RemoteBackend } from './client.js'; import { createContextEngine } from './engine/index.js'; +import type { AbstractContextEngine } from './engine/base.js'; // Public API — use ContextoBackend to implement a custom (e.g. local) backend export type { ContextoBackend, SearchResult, WebhookPayload, Logger } from './types.js'; @@ -20,15 +21,18 @@ export default { maxContextChars: { type: 'number' }, compactThreshold: { type: 'number', default: 0.50 }, compactionStrategy: { type: 'string', default: 'default' }, + rlmEnabled: { type: 'boolean', default: false }, }, }, register(api: any) { const strategy = api.pluginConfig?.compactionStrategy ?? 'default'; + const base = { apiKey: api.pluginConfig?.apiKey, contextEnabled: api.pluginConfig?.contextEnabled ?? true, maxContextChars: api.pluginConfig?.maxContextChars, + rlmEnabled: api.pluginConfig?.rlmEnabled ?? false, }; const config: PluginConfig = strategy === 'default' @@ -48,10 +52,201 @@ export default { const backend = new RemoteBackend(config, logger); - const engine = createContextEngine(config, backend, logger); + const engine = createContextEngine(config, backend, logger) as AbstractContextEngine; api.registerContextEngine('contexto', () => engine); - logger.info(`[contexto] Plugin registered (contextEnabled: ${config.contextEnabled})`); + // --- RLM tool registration --- + if (config.rlmEnabled) { + registerRlmTools(api, engine, config, logger); + } + + logger.info(`[contexto] Plugin registered (contextEnabled: ${config.contextEnabled}, rlm: ${config.rlmEnabled})`); }, }; + +/** + * Register RLM tools with OpenClaw's tool registry. + * Tools are available to all agents; handlers resolve the ContextBuffer + * from the engine's pending context state. + */ +function registerRlmTools(api: any, engine: AbstractContextEngine, config: PluginConfig, logger: any) { + // Lazy imports to avoid loading @ekai/rlm when RLM is not configured + let rlmImports: any; + let completionProvider: any; + + const ensureImports = async () => { + if (rlmImports) return rlmImports; + rlmImports = await import('@ekai/rlm'); + return rlmImports; + }; + + const RLM_PROVIDER = 'openrouter'; + const RLM_MODEL_ID = 'openrouter/auto'; + + const ensureProvider = async () => { + if (completionProvider) return completionProvider; + const { createPiAiCompletionProvider } = await import('./rlm/adapter.js'); + const apiKey = await api.runtime?.modelAuth?.resolveApiKeyForProvider?.(RLM_PROVIDER) + ?? config.apiKey; + completionProvider = createPiAiCompletionProvider({ + provider: RLM_PROVIDER, + modelId: RLM_MODEL_ID, + apiKey, + }); + return completionProvider; + }; + + // Session-keyed ContextBuffer instances + const buffers = new Map(); + + const getOrCreateBuffer = async (sessionKey: string): Promise => { + if (buffers.has(sessionKey)) return buffers.get(sessionKey)!; + + const pending = engine.getPendingContext(sessionKey); + if (!pending) return null; + + const rlm = await ensureImports(); + const buffer = new rlm.ContextBuffer(pending.content); + buffers.set(sessionKey, buffer); + return buffer; + }; + + // Tool definitions — registered eagerly so the agent sees them in tool lists + const toolDefs = [ + { + name: 'rlm_overview', + description: 'Get structural overview of the loaded context — size, line count, detected sections, and a preview of the beginning', + parameters: { type: 'object', properties: {} }, + }, + { + name: 'rlm_peek', + description: 'View lines from the context at a given offset', + parameters: { + type: 'object', + properties: { + offset: { type: 'number', description: 'Line offset from start (0-indexed)' }, + length: { type: 'number', description: 'Number of lines to return (default: 50)' }, + }, + required: ['offset'], + }, + }, + { + name: 'rlm_grep', + description: 'Search the context for a pattern (substring or regex)', + parameters: { + type: 'object', + properties: { + pattern: { type: 'string', description: 'Search query or regex pattern' }, + regex: { type: 'boolean', description: 'Treat pattern as regex (default: false)' }, + limit: { type: 'number', description: 'Max results (default: 20)' }, + }, + required: ['pattern'], + }, + }, + { + name: 'rlm_slice', + description: 'Extract a contiguous range of lines from the context', + parameters: { + type: 'object', + properties: { + start: { type: 'number', description: 'Start line (inclusive, 0-indexed)' }, + end: { type: 'number', description: 'End line (exclusive)' }, + }, + required: ['start', 'end'], + }, + }, + { + name: 'rlm_query', + description: 'Ask a question about a portion of the context — dispatches to a sub-LLM with the relevant chunk', + parameters: { + type: 'object', + properties: { + question: { type: 'string', description: 'Question to answer from context' }, + start: { type: 'number', description: 'Start line of chunk to analyze (default: 0)' }, + end: { type: 'number', description: 'End line of chunk (default: entire context up to budget)' }, + }, + required: ['question'], + }, + }, + { + name: 'rlm_repl', + description: 'Run JavaScript in a sandboxed REPL with access to the full context, all retrieval functions (peek, grep, slice, llm_query), and variable persistence (store/get). Use FINAL(answer) or FINAL_VAR(name) to return results.', + parameters: { + type: 'object', + properties: { + code: { type: 'string', description: 'JavaScript code to execute in the sandbox' }, + }, + required: ['code'], + }, + }, + ]; + + // Handler map — each tool resolves the buffer from the session context + const handlers: Record Promise> = { + rlm_overview: async (_params, ctx) => { + const buffer = await getOrCreateBuffer(ctx?.sessionKey ?? 'default'); + if (!buffer) return { content: 'No large context loaded for this session.' }; + const rlm = await ensureImports(); + const handler = rlm.createOverviewHandler(buffer); + return { content: JSON.stringify(await handler(_params)) }; + }, + rlm_peek: async (params, ctx) => { + const buffer = await getOrCreateBuffer(ctx?.sessionKey ?? 'default'); + if (!buffer) return { content: 'No large context loaded for this session.' }; + const rlm = await ensureImports(); + const handler = rlm.createPeekHandler(buffer); + return { content: await handler(params) }; + }, + rlm_grep: async (params, ctx) => { + const buffer = await getOrCreateBuffer(ctx?.sessionKey ?? 'default'); + if (!buffer) return { content: 'No large context loaded for this session.' }; + const rlm = await ensureImports(); + const handler = rlm.createGrepHandler(buffer); + return { content: JSON.stringify(await handler(params)) }; + }, + rlm_slice: async (params, ctx) => { + const buffer = await getOrCreateBuffer(ctx?.sessionKey ?? 'default'); + if (!buffer) return { content: 'No large context loaded for this session.' }; + const rlm = await ensureImports(); + const handler = rlm.createSliceHandler(buffer); + return { content: await handler(params) }; + }, + rlm_query: async (params, ctx) => { + const buffer = await getOrCreateBuffer(ctx?.sessionKey ?? 'default'); + if (!buffer) return { content: 'No large context loaded for this session.' }; + const rlm = await ensureImports(); + const provider = await ensureProvider(); + const handler = rlm.createQueryHandler(buffer, provider); + return { content: await handler(params) }; + }, + rlm_repl: async (params, ctx) => { + const buffer = await getOrCreateBuffer(ctx?.sessionKey ?? 'default'); + if (!buffer) return { content: 'No large context loaded for this session.' }; + const rlm = await ensureImports(); + const provider = await ensureProvider(); + const handler = rlm.createReplHandler(buffer, provider); + return { content: JSON.stringify(await handler(params)) }; + }, + }; + + // Register each tool with OpenClaw + for (const def of toolDefs) { + api.registerTool(def.name, { + ...def, + execute: async (params: any, context?: any) => { + const sessionKey = context?.sessionKey ?? 'default'; + logger.info(`[contexto:rlm] Tool ${def.name} called for session ${sessionKey}`); + try { + return await handlers[def.name](params, context); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + logger.warn(`[contexto:rlm] Tool ${def.name} failed: ${msg}`); + return { content: `Error: ${msg}` }; + } + }, + }); + } + + logger.info(`[contexto:rlm] Registered ${toolDefs.length} RLM tools (provider: ${RLM_PROVIDER}, model: ${RLM_MODEL_ID})`); +} diff --git a/packages/contexto/src/rlm/adapter.ts b/packages/contexto/src/rlm/adapter.ts new file mode 100644 index 0000000..2bda6aa --- /dev/null +++ b/packages/contexto/src/rlm/adapter.ts @@ -0,0 +1,56 @@ +import { complete, getModel } from '@mariozechner/pi-ai'; +import type { Api, Model } from '@mariozechner/pi-ai'; +import type { CompletionProvider, Message } from '@ekai/rlm'; + +export interface PiAiProviderConfig { + provider: string; + modelId: string; + apiKey: string; +} + +/** + * Creates a CompletionProvider backed by pi-ai's complete() function. + * Uses OpenClaw's model registry and auth — no external API clients. + */ +export function createPiAiCompletionProvider(config: PiAiProviderConfig): CompletionProvider { + const model = getModel(config.provider as any, config.modelId as any) as Model; + + return { + async completion(messages: Message[]): Promise { + let systemPrompt: string | undefined; + const piMessages: Array<{ role: 'user'; content: string; timestamp: number }> = []; + + for (const m of messages) { + if (m.role === 'system') { + systemPrompt = systemPrompt ? `${systemPrompt}\n${m.content}` : m.content; + } else { + piMessages.push({ + role: 'user', + content: m.content, + timestamp: Date.now(), + }); + } + } + + const result = await complete( + model, + { systemPrompt, messages: piMessages as any }, + { apiKey: config.apiKey }, + ); + + const textParts = result.content + .filter((block: any) => block.type === 'text') + .map((block: any) => block.text); + + if (textParts.length === 0) { + throw new Error('No text content in LLM response'); + } + + return textParts.join(''); + }, + + async completionStr(prompt: string): Promise { + return this.completion([{ role: 'user', content: prompt }]); + }, + }; +} diff --git a/packages/contexto/src/types.ts b/packages/contexto/src/types.ts index 57dfd98..c729a73 100644 --- a/packages/contexto/src/types.ts +++ b/packages/contexto/src/types.ts @@ -4,6 +4,7 @@ export interface BaseConfig { maxContextChars?: number; minScore?: number; filter?: Record; + rlmEnabled?: boolean; } export interface DefaultConfig extends BaseConfig { diff --git a/packages/rlm/README.md b/packages/rlm/README.md new file mode 100644 index 0000000..f3616d9 --- /dev/null +++ b/packages/rlm/README.md @@ -0,0 +1,189 @@ +# @ekai/rlm + +A toolkit for reasoning over large contexts that exceed a language model's practical token budget. Built for LLM agent loops, `@ekai/rlm` provides structured tools that let an agent explore, search, and query over massive inputs — documents, logs, codebases, data exports — without flooding the context window. + +Inspired by the [Recursive Language Model](https://arxiv.org/abs/2502.07814) paper, adapted from a monolithic engine into a modular, tools-first architecture that composes naturally with any agent framework. + +## Why + +Large language models are powerful reasoners, but they have finite context windows. When a user provides a 200-page PDF, a sprawling codebase, or a multi-megabyte log file, the naive approach — stuffing everything into the prompt — either truncates the content, degrades quality, or simply fails. + +`@ekai/rlm` solves this by keeping the full content in an efficient in-memory buffer and exposing it through six purpose-built tools. The agent decides what to look at, when, and how deeply — much like a human skimming a document, searching for keywords, and reading sections of interest closely. + +The result is bounded token usage regardless of input size, with no loss of reasoning coverage. + +## How It Works + +The core abstraction is the **ContextBuffer** — an in-memory, line-indexed representation of the input text. Content is loaded once and never passed directly into any LLM prompt. Instead, the agent interacts with it through tools that return small, targeted slices. + +A typical agent session flows like this: + +1. The agent calls **rlm_overview** to understand the structure — how many lines, what sections exist, a brief preview of the beginning. +2. Based on the overview, it uses **rlm_grep** to locate relevant sections by keyword or pattern. +3. It reads specific regions with **rlm_peek** or **rlm_slice** to examine the content in detail. +4. For questions that require synthesis, it uses **rlm_query** to delegate a focused sub-LLM call over a bounded chunk. +5. For complex multi-step analysis, the agent writes and executes JavaScript in **rlm_repl** — a sandboxed environment with access to all retrieval functions and persistent variables across iterations. + +The agent controls the decomposition strategy. Simple questions might need only a grep and a slice. Complex analysis might involve dozens of REPL iterations with intermediate sub-LLM calls. The tools support both. + +## Tools + +### rlm_overview + +Returns a structural summary of the loaded content: total character and line counts, a list of detected sections (identified from markdown headers and page breaks), and a short preview of the opening lines. This is typically the first tool an agent calls to orient itself within a large document. + +### rlm_peek + +Displays a window of lines starting at a given offset. The default window is 50 lines. This is the primary tool for sequential reading — the agent can page through a document by advancing the offset, similar to scrolling through a file. + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `offset` | Yes | Line number to start from (0-indexed) | +| `length` | No | Number of lines to return (default: 50) | + +### rlm_grep + +Searches the full content for a pattern and returns matching lines along with their surrounding context. Supports both plain substring matching and regular expressions. Results are capped to prevent overwhelming the agent with too many matches. + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `pattern` | Yes | Search term or regex pattern | +| `regex` | No | Interpret pattern as a regular expression (default: false) | +| `limit` | No | Maximum number of results (default: 20) | + +### rlm_slice + +Extracts a contiguous block of lines by range. Unlike peek, which is designed for browsing, slice is intended for targeted extraction — pulling out a specific function, a table, or a section the agent has already located. + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `start` | Yes | Start line, inclusive (0-indexed) | +| `end` | Yes | End line, exclusive | + +### rlm_query + +Sends a question along with a bounded chunk of the context to a sub-LLM call. The agent specifies which portion of the content to include, and the tool handles chunking, prompt construction, and response extraction. This is the primary tool for semantic reasoning over content that the agent has identified as relevant. + +The sub-LLM call is made through the injected **CompletionProvider**, keeping the package decoupled from any specific LLM vendor or SDK. + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `question` | Yes | The question to answer from the context | +| `start` | No | Start line of the chunk to analyze (default: 0) | +| `end` | No | End line of the chunk (default: full content, up to token budget) | + +### rlm_repl + +The most powerful tool in the set. It provides a sandboxed JavaScript execution environment (built on Node.js `vm`) with access to the full context, all retrieval functions, and sub-LLM calls. Variables persist across invocations, enabling multi-turn analysis workflows. + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `code` | Yes | JavaScript code to execute | + +The sandbox exposes the following built-in functions and variables: + +- **`context`** — the raw text content as a string +- **`peek(offset, length)`** — same as the rlm_peek tool +- **`grep(pattern, opts)`** — same as rlm_grep +- **`slice(start, end)`** — same as rlm_slice +- **`llm_query(prompt)`** — asynchronous sub-LLM call +- **`store(key, value)` / `get(key)`** — key-value storage that persists across REPL invocations +- **`FINAL(answer)` / `FINAL_VAR(name)`** — declare the final result to return to the agent +- **`len(x)` / `chunk(text, size)`** — utility helpers + +The sandbox enforces strict security constraints: 30-second execution timeout, a maximum of 20 iterations per session, and 20,000-character output truncation. Dangerous constructs — `eval`, `Function`, `require`, `import`, `process`, `fetch`, and prototype chain access — are blocked at the code validation level before execution. + +## Document Parsing + +Raw documents must be converted to plain text before loading into the ContextBuffer. The package includes parsers for common formats: + +| Format | MIME Types | Peer Dependency | +|--------|-----------|-----------------| +| PDF | `application/pdf` | `pdf-parse` (optional) | +| Excel | `application/vnd.ms-excel`, `application/vnd.openxmlformats-officedocument.spreadsheetml.sheet` | `xlsx` (optional) | +| Text | `text/plain`, `text/markdown`, `text/csv`, `application/json` | None (built-in) | + +The `parseDocument` function dispatches to the appropriate parser based on MIME type and returns an array of chunks with metadata (page numbers, sheet names, etc.). These chunks are typically concatenated into a single string for the ContextBuffer. + +PDF and Excel support require their respective peer dependencies to be installed. If a parser's dependency is missing at runtime, it will throw a clear error explaining what to install. + +## CompletionProvider + +The `rlm_query` and `rlm_repl` tools need access to an LLM for sub-queries. Rather than bundling provider-specific SDKs, `@ekai/rlm` defines a minimal `CompletionProvider` contract: a `completion` method that accepts a message array and returns a string, and a convenience `completionStr` method for single-prompt calls. + +This design keeps the package standalone and provider-agnostic. Any LLM backend — OpenAI, Anthropic, a local model, or a custom gateway — can be used by implementing two functions. + +When used within the [Contexto](https://github.com/ekailabs/contexto) plugin, the provider is automatically wired to [pi-ai](https://docs.openclaw.ai/pi), OpenClaw's built-in LLM abstraction. + +## Architecture + +``` + ┌─────────────────────────────────────┐ + │ Large Input │ + │ (PDF, Excel, logs, code, text...) │ + └──────────────┬──────────────────────┘ + │ + parseDocument() + │ + ▼ + ┌─────────────────────────────────────┐ + │ ContextBuffer │ + │ In-memory, line-indexed, immutable │ + └──────────────┬──────────────────────┘ + │ + Agent selects tools + │ + ┌────────────────────┼────────────────────┐ + │ │ │ + Exploration Retrieval Reasoning + ┌───────────┐ ┌───────────┐ ┌───────────┐ + │ overview │ │ grep │ │ query │ + │ peek │ │ slice │ │ repl │ + └───────────┘ └───────────┘ └───────────┘ + │ + sub-LLM calls + (CompletionProvider) + │ + ▼ + Synthesized Answer +``` + +The content never enters the agent's context window directly. Token usage stays bounded regardless of input size, while the agent retains full reasoning coverage over the entire document. + +## Usage with Contexto + +When used as part of the [Contexto](https://github.com/ekailabs/contexto) plugin for [OpenClaw](https://docs.openclaw.ai), RLM can be enabled through the plugin configuration by setting `rlmEnabled` to `true`: + +```json +{ + "plugins": { + "contexto": { + "apiKey": "your-contexto-api-key", + "rlmEnabled": true + } + } +} +``` + +When enabled, the plugin automatically registers all six RLM tools with the agent and wires the CompletionProvider to pi-ai via OpenRouter's auto-routing. When disabled (the default), the plugin operates normally without RLM — no tools are registered and no additional dependencies are loaded. + +Once enabled, RLM activates automatically when a user message exceeds 50% of the configured token budget. The large content is offloaded to a ContextBuffer, the original message is replaced with a brief instruction telling the agent to use the RLM tools, and the agent proceeds to explore and reason over the content iteratively. After the agent finishes, the synthesized result is ingested into the mindmap for future recall. + +RLM can also be invoked explicitly by the user regardless of message size. + +## Installation + +```bash +npm install @ekai/rlm +``` + +For document parsing support: + +```bash +npm install pdf-parse # optional — enables PDF parsing +npm install xlsx # optional — enables Excel parsing +``` + +## License + +[Apache-2.0](../../LICENSE) diff --git a/packages/rlm/package.json b/packages/rlm/package.json new file mode 100644 index 0000000..130b664 --- /dev/null +++ b/packages/rlm/package.json @@ -0,0 +1,35 @@ +{ + "name": "@ekai/rlm", + "version": "0.1.0", + "description": "Recursive Language Model tools for processing large contexts", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "files": [ + "dist" + ], + "scripts": { + "build": "tsc -p tsconfig.json", + "prepublishOnly": "npm run build", + "type-check": "tsc --noEmit" + }, + "dependencies": {}, + "devDependencies": { + "@types/node": "^20.10.0", + "typescript": "^5.3.2" + }, + "peerDependencies": { + "pdf-parse": ">=1.1.0", + "xlsx": ">=0.18.0" + }, + "peerDependenciesMeta": { + "pdf-parse": { "optional": true }, + "xlsx": { "optional": true } + } +} diff --git a/packages/rlm/src/context/context-buffer.ts b/packages/rlm/src/context/context-buffer.ts new file mode 100644 index 0000000..5e0b7ed --- /dev/null +++ b/packages/rlm/src/context/context-buffer.ts @@ -0,0 +1,154 @@ +import type { Section, OverviewResult, ContextChunk, GrepResult } from './types.js'; + +const PREVIEW_LINES = 20; +const DEFAULT_GREP_LIMIT = 20; +const GREP_CONTEXT_LINES = 2; + +// Markdown-style header patterns for section detection +const HEADER_RE = /^(#{1,6})\s+(.+)/; +const PAGE_BREAK_RE = /^-{3,}$|^={3,}$|^\f/; + +/** + * ContextBuffer — holds large text and provides structured access. + * + * All RLM tools delegate to this. The buffer is ephemeral — it exists + * only for the subagent's lifetime. No persistence needed. + */ +export class ContextBuffer { + private content: string; + private lines: string[]; + private sections: Section[]; + private meta?: { filename?: string; mimeType?: string }; + + constructor(content: string, metadata?: { filename?: string; mimeType?: string }) { + this.content = content; + this.lines = content.split('\n'); + this.meta = metadata; + this.sections = this.detectSections(); + } + + get lineCount(): number { + return this.lines.length; + } + + get charCount(): number { + return this.content.length; + } + + /** Get the raw content string. */ + getRawContent(): string { + return this.content; + } + + /** Structural overview: line count, char count, detected sections, preview. */ + overview(): OverviewResult { + return { + charCount: this.content.length, + lineCount: this.lines.length, + sections: this.sections, + preview: this.lines.slice(0, PREVIEW_LINES).join('\n'), + metadata: this.meta, + }; + } + + /** View lines at offset from start. */ + peek(offset: number, length: number = 50): string { + const start = Math.max(0, offset); + const end = Math.min(this.lines.length, start + length); + return this.lines.slice(start, end).join('\n'); + } + + /** Search by substring or regex, returns matching lines with context. */ + grep(pattern: string, opts?: { regex?: boolean; limit?: number }): GrepResult[] { + const limit = opts?.limit ?? DEFAULT_GREP_LIMIT; + const results: GrepResult[] = []; + + let matcher: (line: string) => boolean; + if (opts?.regex) { + try { + const re = new RegExp(pattern, 'i'); + matcher = (line) => re.test(line); + } catch { + // Invalid regex — fall back to substring + matcher = (line) => line.toLowerCase().includes(pattern.toLowerCase()); + } + } else { + const lower = pattern.toLowerCase(); + matcher = (line) => line.toLowerCase().includes(lower); + } + + for (let i = 0; i < this.lines.length && results.length < limit; i++) { + if (matcher(this.lines[i])) { + const ctxStart = Math.max(0, i - GREP_CONTEXT_LINES); + const ctxEnd = Math.min(this.lines.length, i + GREP_CONTEXT_LINES + 1); + results.push({ + lineNumber: i, + line: this.lines[i], + context: this.lines.slice(ctxStart, ctxEnd), + }); + } + } + + return results; + } + + /** Extract contiguous line range. */ + slice(start: number, end: number): string { + const s = Math.max(0, start); + const e = Math.min(this.lines.length, end); + return this.lines.slice(s, e).join('\n'); + } + + /** Get a chunk suitable for sub-LLM query (respects char budget). */ + getChunk(start: number, maxChars: number): ContextChunk { + const s = Math.max(0, start); + let charCount = 0; + let endLine = s; + + for (let i = s; i < this.lines.length; i++) { + const lineLen = this.lines[i].length + 1; // +1 for newline + if (charCount + lineLen > maxChars && i > s) break; + charCount += lineLen; + endLine = i + 1; + } + + return { + content: this.lines.slice(s, endLine).join('\n'), + startLine: s, + endLine, + charCount, + }; + } + + /** Detect structural sections (markdown headers, page breaks). */ + private detectSections(): Section[] { + const sections: Section[] = []; + let currentSection: Section | null = null; + + for (let i = 0; i < this.lines.length; i++) { + const headerMatch = this.lines[i].match(HEADER_RE); + if (headerMatch) { + if (currentSection) { + currentSection.endLine = i - 1; + sections.push(currentSection); + } + currentSection = { + title: headerMatch[2].trim(), + startLine: i, + endLine: this.lines.length - 1, + level: headerMatch[1].length, + }; + } else if (PAGE_BREAK_RE.test(this.lines[i]) && currentSection) { + currentSection.endLine = i - 1; + sections.push(currentSection); + currentSection = null; + } + } + + if (currentSection) { + sections.push(currentSection); + } + + return sections; + } +} diff --git a/packages/rlm/src/context/types.ts b/packages/rlm/src/context/types.ts new file mode 100644 index 0000000..d929b8c --- /dev/null +++ b/packages/rlm/src/context/types.ts @@ -0,0 +1,31 @@ +/** A detected structural section in the context (e.g., markdown header, page break). */ +export interface Section { + title: string; + startLine: number; + endLine: number; + level: number; +} + +/** Structural overview of the loaded context. */ +export interface OverviewResult { + charCount: number; + lineCount: number; + sections: Section[]; + preview: string; + metadata?: { filename?: string; mimeType?: string }; +} + +/** A chunk extracted from the context with position metadata. */ +export interface ContextChunk { + content: string; + startLine: number; + endLine: number; + charCount: number; +} + +/** A grep match with line number and surrounding context. */ +export interface GrepResult { + lineNumber: number; + line: string; + context: string[]; +} diff --git a/packages/rlm/src/document/excel.ts b/packages/rlm/src/document/excel.ts new file mode 100644 index 0000000..ceac0b3 --- /dev/null +++ b/packages/rlm/src/document/excel.ts @@ -0,0 +1,41 @@ +import type { DocumentParser, DocumentChunk } from './types.js'; + +const EXCEL_MIME_TYPES = [ + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + 'application/vnd.ms-excel', +]; + +export class ExcelParser implements DocumentParser { + canParse(mimeType: string): boolean { + return EXCEL_MIME_TYPES.includes(mimeType); + } + + async parse(buffer: Buffer, filename?: string): Promise { + let XLSX: any; + try { + // @ts-ignore — optional peer dependency + XLSX = await import('xlsx'); + } catch { + throw new Error('xlsx is not installed. Install it with: npm install xlsx'); + } + + const workbook = XLSX.read(buffer, { type: 'buffer' }); + const source = filename ?? 'document.xlsx'; + const chunks: DocumentChunk[] = []; + + for (let i = 0; i < workbook.SheetNames.length; i++) { + const sheetName = workbook.SheetNames[i]; + const sheet = workbook.Sheets[sheetName]; + const csv = XLSX.utils.sheet_to_csv(sheet); + + if (csv.trim()) { + chunks.push({ + content: `Sheet: ${sheetName}\n${csv}`, + metadata: { source, sheet: sheetName, index: i }, + }); + } + } + + return chunks; + } +} diff --git a/packages/rlm/src/document/index.ts b/packages/rlm/src/document/index.ts new file mode 100644 index 0000000..abb61c2 --- /dev/null +++ b/packages/rlm/src/document/index.ts @@ -0,0 +1,31 @@ +import type { DocumentParser, DocumentChunk } from './types.js'; +import { PdfParser } from './pdf.js'; +import { ExcelParser } from './excel.js'; +import { TextParser } from './text.js'; + +const parsers: DocumentParser[] = [ + new PdfParser(), + new ExcelParser(), + new TextParser(), +]; + +/** + * Parse a document buffer into text, dispatching to the right parser based on MIME type. + * Returns concatenated text from all chunks, ready to be loaded into a ContextBuffer. + */ +export async function parseDocument( + buffer: Buffer, + mimeType: string, + filename?: string, +): Promise { + const parser = parsers.find(p => p.canParse(mimeType)); + if (!parser) { + // Fall back to treating as plain text + return buffer.toString('utf-8'); + } + + const chunks = await parser.parse(buffer, filename); + return chunks.map(c => c.content).join('\n\n'); +} + +export type { DocumentParser, DocumentChunk }; diff --git a/packages/rlm/src/document/pdf.ts b/packages/rlm/src/document/pdf.ts new file mode 100644 index 0000000..e319bcf --- /dev/null +++ b/packages/rlm/src/document/pdf.ts @@ -0,0 +1,37 @@ +import type { DocumentParser, DocumentChunk } from './types.js'; + +const PDF_MIME_TYPES = ['application/pdf']; + +export class PdfParser implements DocumentParser { + canParse(mimeType: string): boolean { + return PDF_MIME_TYPES.includes(mimeType); + } + + async parse(buffer: Buffer, filename?: string): Promise { + let pdfParse: any; + try { + // @ts-ignore — optional peer dependency + pdfParse = (await import('pdf-parse')).default; + } catch { + throw new Error('pdf-parse is not installed. Install it with: npm install pdf-parse'); + } + + const data = await pdfParse(buffer); + const source = filename ?? 'document.pdf'; + + // Split by form feed (page break) if present, otherwise treat as single page + const pages = data.text.split('\f').filter((p: string) => p.trim()); + + if (pages.length <= 1) { + return [{ + content: data.text, + metadata: { source, page: 1, index: 0 }, + }]; + } + + return pages.map((page: string, i: number) => ({ + content: page.trim(), + metadata: { source, page: i + 1, index: i }, + })); + } +} diff --git a/packages/rlm/src/document/text.ts b/packages/rlm/src/document/text.ts new file mode 100644 index 0000000..f2ab442 --- /dev/null +++ b/packages/rlm/src/document/text.ts @@ -0,0 +1,25 @@ +import type { DocumentParser, DocumentChunk } from './types.js'; + +const TEXT_MIME_TYPES = [ + 'text/plain', + 'text/markdown', + 'text/csv', + 'text/tab-separated-values', + 'application/json', +]; + +export class TextParser implements DocumentParser { + canParse(mimeType: string): boolean { + return TEXT_MIME_TYPES.includes(mimeType) || mimeType.startsWith('text/'); + } + + async parse(buffer: Buffer, filename?: string): Promise { + const content = buffer.toString('utf-8'); + const source = filename ?? 'document.txt'; + + return [{ + content, + metadata: { source, index: 0 }, + }]; + } +} diff --git a/packages/rlm/src/document/types.ts b/packages/rlm/src/document/types.ts new file mode 100644 index 0000000..6557ba5 --- /dev/null +++ b/packages/rlm/src/document/types.ts @@ -0,0 +1,14 @@ +export interface DocumentChunk { + content: string; + metadata: { + source: string; + page?: number; + sheet?: string; + index: number; + }; +} + +export interface DocumentParser { + canParse(mimeType: string): boolean; + parse(buffer: Buffer, filename?: string): Promise; +} diff --git a/packages/rlm/src/index.ts b/packages/rlm/src/index.ts new file mode 100644 index 0000000..ab3e3b5 --- /dev/null +++ b/packages/rlm/src/index.ts @@ -0,0 +1,31 @@ +// Core +export { ContextBuffer } from './context/context-buffer.js'; + +// Tools — definitions + handler factories +export { + RLM_OVERVIEW_DEFINITION, + RLM_PEEK_DEFINITION, + RLM_GREP_DEFINITION, + RLM_SLICE_DEFINITION, + RLM_QUERY_DEFINITION, + RLM_REPL_DEFINITION, + ALL_RLM_DEFINITIONS, +} from './tools/definitions.js'; +export { createOverviewHandler } from './tools/rlm-overview.js'; +export { createPeekHandler } from './tools/rlm-peek.js'; +export { createGrepHandler } from './tools/rlm-grep.js'; +export { createSliceHandler } from './tools/rlm-slice.js'; +export { createQueryHandler } from './tools/rlm-query.js'; +export { createReplHandler } from './tools/rlm-repl.js'; + +// REPL internals (for advanced use) +export { REPLSandbox } from './repl/sandbox.js'; +export type { REPLResult } from './repl/sandbox.js'; + +// Document parsing +export { parseDocument } from './document/index.js'; + +// Types +export type { CompletionProvider, Message, Context } from './types.js'; +export type { ContextChunk, OverviewResult, GrepResult, Section } from './context/types.js'; +export type { DocumentParser, DocumentChunk } from './document/types.js'; diff --git a/packages/rlm/src/repl/builtins.ts b/packages/rlm/src/repl/builtins.ts new file mode 100644 index 0000000..8642e54 --- /dev/null +++ b/packages/rlm/src/repl/builtins.ts @@ -0,0 +1,56 @@ +import type { ContextBuffer } from '../context/context-buffer.js'; +import type { CompletionProvider } from '../types.js'; + +/** + * Build the set of functions available inside the REPL sandbox. + * These bridge from the vm isolate back to the host ContextBuffer and CompletionProvider. + */ +export interface REPLBuiltins { + peek: (offset: number, length?: number) => string; + grep: (pattern: string, opts?: { regex?: boolean; limit?: number }) => object[]; + slice: (start: number, end: number) => string; + llm_query: (prompt: string) => Promise; + len: (x: unknown) => number; + chunk: (text: string, size: number) => string[]; +} + +export function createBuiltins(buffer: ContextBuffer, provider: CompletionProvider): REPLBuiltins { + return { + peek(offset: number, length: number = 50): string { + return buffer.peek(offset, length); + }, + + grep(pattern: string, opts?: { regex?: boolean; limit?: number }): object[] { + return buffer.grep(pattern, opts).map(r => ({ + lineNumber: r.lineNumber, + line: r.line, + })); + }, + + slice(start: number, end: number): string { + return buffer.slice(start, end); + }, + + async llm_query(prompt: string): Promise { + try { + return await provider.completionStr(prompt); + } catch (e) { + return `Error making LLM query: ${e}`; + } + }, + + len(x: unknown): number { + if (typeof x === 'string') return x.length; + if (Array.isArray(x)) return x.length; + return 0; + }, + + chunk(text: string, size: number): string[] { + const chunks: string[] = []; + for (let i = 0; i < text.length; i += size) { + chunks.push(text.slice(i, i + size)); + } + return chunks; + }, + }; +} diff --git a/packages/rlm/src/repl/code-validator.ts b/packages/rlm/src/repl/code-validator.ts new file mode 100644 index 0000000..81ff146 --- /dev/null +++ b/packages/rlm/src/repl/code-validator.ts @@ -0,0 +1,54 @@ +/** + * Validates model-generated code before it reaches the vm sandbox. + * Rejects known escape patterns and dangerous constructs. + */ + +const BLOCKED_PATTERNS = [ + // Prototype chain escapes + /\.__proto__/, + /\.constructor\.constructor/, + /\[['"]constructor['"]\]/, + /Object\.getPrototypeOf/, + /Reflect\./, + + // Module system access + /\brequire\s*\(/, + /\bimport\s*\(/, + /\bimport\s+/, + + // Process/system access + /\bprocess\./, + /\bglobalThis\b/, + /\bFunction\s*\(/, + + // Filesystem + /\bfs\b/, + new RegExp('\\b' + 'child' + '_' + 'process' + '\\b'), + /\bpath\b\./, + + // Network + /\bfetch\s*\(/, + /\bXMLHttpRequest/, + /\bWebSocket/, + + // Timers (blocked — sandbox should not set intervals) + /\bsetInterval\s*\(/, +]; + +export interface ValidationResult { + valid: boolean; + reason?: string; +} + +export function validateCode(code: string): ValidationResult { + for (const pattern of BLOCKED_PATTERNS) { + if (pattern.test(code)) { + return { + valid: false, + reason: `Blocked pattern detected: ${pattern.source}`, + }; + } + } + + return { valid: true }; +} diff --git a/packages/rlm/src/repl/sandbox.ts b/packages/rlm/src/repl/sandbox.ts new file mode 100644 index 0000000..b53bb8f --- /dev/null +++ b/packages/rlm/src/repl/sandbox.ts @@ -0,0 +1,192 @@ +import vm from 'node:vm'; +import type { REPLBuiltins } from './builtins.js'; +import { validateCode } from './code-validator.js'; + +const DEFAULT_TIMEOUT_MS = 30_000; +const MAX_OUTPUT_CHARS = 20_000; +const MAX_ITERATIONS = 20; + +/** Result from a single REPL code execution. */ +export interface REPLResult { + output: string; + error?: string; + finalAnswer?: string; + variables: Record; + executionTimeMs: number; +} + +/** + * REPL sandbox using Node.js vm module. + * + * Provides the agent with a JavaScript execution environment that has access + * to the context buffer (via builtins), sub-LLM calls (via llm_query), + * and variable persistence across calls (via store/get). + * + * Note: Node's vm module is NOT a security sandbox. The code validator + * provides defense-in-depth, not absolute isolation. + */ +export class REPLSandbox { + private context: vm.Context | null = null; + private builtins: REPLBuiltins; + private contextContent: string; + private output: string[] = []; + private variables = new Map(); + private iterationCount = 0; + private finalAnswer: string | null = null; + private timeoutMs: number; + + constructor( + builtins: REPLBuiltins, + contextContent: string, + opts?: { timeoutMs?: number }, + ) { + this.builtins = builtins; + this.contextContent = contextContent; + this.timeoutMs = opts?.timeoutMs ?? DEFAULT_TIMEOUT_MS; + } + + async initialize(): Promise { + const self = this; + + const sandbox: Record = { + // The loaded context + context: this.contextContent, + + // Console + print: (...args: unknown[]) => { + const line = args.map(a => typeof a === 'string' ? a : JSON.stringify(a)).join(' '); + self.output.push(line); + }, + console: { + log: (...args: unknown[]) => { + const line = args.map(a => typeof a === 'string' ? a : JSON.stringify(a)).join(' '); + self.output.push(line); + }, + }, + + // Variable persistence across REPL calls + store: (name: string, value: unknown) => { + self.variables.set(name, typeof value === 'string' ? value : JSON.stringify(value)); + }, + get: (name: string) => { + const v = self.variables.get(name); + if (v === undefined) return null; + try { return JSON.parse(v); } catch { return v; } + }, + + // Completion signals + FINAL: (value: unknown) => { + self.finalAnswer = typeof value === 'string' ? value : JSON.stringify(value); + }, + FINAL_VAR: (name: string) => { + const val = self.variables.get(name); + if (val !== undefined) { + self.finalAnswer = val; + return val; + } + return `Error: Variable '${name}' not found. Available: ${[...self.variables.keys()].join(', ')}`; + }, + + // Context buffer builtins + peek: (offset: number, length?: number) => self.builtins.peek(offset, length), + grep: (pattern: string, opts?: object) => self.builtins.grep(pattern, opts as any), + slice: (start: number, end: number) => self.builtins.slice(start, end), + llm_query: async (prompt: string) => self.builtins.llm_query(prompt), + + // Utility + len: self.builtins.len, + chunk: self.builtins.chunk, + + // Standard JS builtins + JSON, Math, Date, Array, Object, String, Number, Boolean, + Map, Set, RegExp, Promise, + parseInt, parseFloat, isNaN, isFinite, + encodeURIComponent, decodeURIComponent, encodeURI, decodeURI, + Error, TypeError, RangeError, SyntaxError, + undefined, NaN, Infinity, + + // Blocked + setTimeout: undefined, + setInterval: undefined, + }; + + this.context = vm.createContext(sandbox, { + codeGeneration: { strings: false, wasm: false }, + }); + } + + async runCode(code: string): Promise { + if (!this.context) { + return { + output: 'Error: Sandbox not initialized. Call initialize() first.', + error: 'Sandbox not initialized', + variables: Object.fromEntries(this.variables), + executionTimeMs: 0, + }; + } + + if (this.iterationCount >= MAX_ITERATIONS) { + return { + output: `Error: Maximum iterations (${MAX_ITERATIONS}) reached.`, + error: 'Max iterations exceeded', + variables: Object.fromEntries(this.variables), + executionTimeMs: 0, + }; + } + + const validation = validateCode(code); + if (!validation.valid) { + return { + output: `Code rejected: ${validation.reason}`, + error: validation.reason, + variables: Object.fromEntries(this.variables), + executionTimeMs: 0, + }; + } + + this.iterationCount++; + this.output = []; + this.finalAnswer = null; + const startTime = Date.now(); + + try { + const wrappedCode = `(async () => { ${code} })()`; + const script = new vm.Script(wrappedCode); + const promise = script.runInContext(this.context, { timeout: this.timeoutMs }); + await promise; + + const rawOutput = this.output.join('\n'); + const output = rawOutput.length > MAX_OUTPUT_CHARS + ? rawOutput.slice(0, MAX_OUTPUT_CHARS) + `\n... [truncated, ${rawOutput.length - MAX_OUTPUT_CHARS} chars omitted]` + : rawOutput; + + return { + output, + finalAnswer: this.finalAnswer ?? undefined, + variables: Object.fromEntries(this.variables), + executionTimeMs: Date.now() - startTime, + }; + } catch (err) { + const message = `${err}`; + + return { + output: this.output.join('\n'), + error: message, + variables: Object.fromEntries(this.variables), + executionTimeMs: Date.now() - startTime, + }; + } + } + + getFinalAnswer(): string | null { + return this.finalAnswer; + } + + getIterationCount(): number { + return this.iterationCount; + } + + dispose(): void { + this.context = null; + } +} diff --git a/packages/rlm/src/tools/definitions.ts b/packages/rlm/src/tools/definitions.ts new file mode 100644 index 0000000..731830c --- /dev/null +++ b/packages/rlm/src/tools/definitions.ts @@ -0,0 +1,83 @@ +export const RLM_OVERVIEW_DEFINITION = { + name: 'rlm_overview', + description: 'Get structural overview of the loaded context — size, line count, detected sections, and a preview of the beginning', + parameters: { + type: 'object' as const, + properties: {}, + }, +}; + +export const RLM_PEEK_DEFINITION = { + name: 'rlm_peek', + description: 'View lines from the context at a given offset', + parameters: { + type: 'object' as const, + properties: { + offset: { type: 'number', description: 'Line offset from start (0-indexed)' }, + length: { type: 'number', description: 'Number of lines to return (default: 50)' }, + }, + required: ['offset'], + }, +}; + +export const RLM_GREP_DEFINITION = { + name: 'rlm_grep', + description: 'Search the context for a pattern (substring or regex)', + parameters: { + type: 'object' as const, + properties: { + pattern: { type: 'string', description: 'Search query or regex pattern' }, + regex: { type: 'boolean', description: 'Treat pattern as regex (default: false)' }, + limit: { type: 'number', description: 'Max results (default: 20)' }, + }, + required: ['pattern'], + }, +}; + +export const RLM_SLICE_DEFINITION = { + name: 'rlm_slice', + description: 'Extract a contiguous range of lines from the context', + parameters: { + type: 'object' as const, + properties: { + start: { type: 'number', description: 'Start line (inclusive, 0-indexed)' }, + end: { type: 'number', description: 'End line (exclusive)' }, + }, + required: ['start', 'end'], + }, +}; + +export const RLM_QUERY_DEFINITION = { + name: 'rlm_query', + description: 'Ask a question about a portion of the context — dispatches to a cheap sub-LLM with the relevant chunk', + parameters: { + type: 'object' as const, + properties: { + question: { type: 'string', description: 'Question to answer from context' }, + start: { type: 'number', description: 'Start line of chunk to analyze (default: 0)' }, + end: { type: 'number', description: 'End line of chunk (default: entire context up to budget)' }, + }, + required: ['question'], + }, +}; + +export const RLM_REPL_DEFINITION = { + name: 'rlm_repl', + description: 'Run JavaScript in a sandboxed REPL with access to the full context, all retrieval functions (peek, grep, slice, llm_query), and variable persistence (store/get). Use FINAL(answer) or FINAL_VAR(name) to return results. The context variable holds the loaded text.', + parameters: { + type: 'object' as const, + properties: { + code: { type: 'string', description: 'JavaScript code to execute in the sandbox' }, + }, + required: ['code'], + }, +}; + +export const ALL_RLM_DEFINITIONS = [ + RLM_OVERVIEW_DEFINITION, + RLM_PEEK_DEFINITION, + RLM_GREP_DEFINITION, + RLM_SLICE_DEFINITION, + RLM_QUERY_DEFINITION, + RLM_REPL_DEFINITION, +]; diff --git a/packages/rlm/src/tools/rlm-grep.ts b/packages/rlm/src/tools/rlm-grep.ts new file mode 100644 index 0000000..ffd86ab --- /dev/null +++ b/packages/rlm/src/tools/rlm-grep.ts @@ -0,0 +1,19 @@ +import type { ContextBuffer } from '../context/context-buffer.js'; + +export function createGrepHandler(buffer: ContextBuffer) { + return async (params: Record) => { + const pattern = params.pattern as string; + const regex = (params.regex as boolean) ?? false; + const limit = (params.limit as number) ?? 20; + + const results = buffer.grep(pattern, { regex, limit }); + return { + matchCount: results.length, + matches: results.map(r => ({ + lineNumber: r.lineNumber, + line: r.line, + context: r.context.join('\n'), + })), + }; + }; +} diff --git a/packages/rlm/src/tools/rlm-overview.ts b/packages/rlm/src/tools/rlm-overview.ts new file mode 100644 index 0000000..15da8b4 --- /dev/null +++ b/packages/rlm/src/tools/rlm-overview.ts @@ -0,0 +1,18 @@ +import type { ContextBuffer } from '../context/context-buffer.js'; + +export function createOverviewHandler(buffer: ContextBuffer) { + return async (_params: Record) => { + const overview = buffer.overview(); + return { + charCount: overview.charCount, + lineCount: overview.lineCount, + sections: overview.sections.map(s => ({ + title: s.title, + startLine: s.startLine, + level: s.level, + })), + preview: overview.preview, + metadata: overview.metadata, + }; + }; +} diff --git a/packages/rlm/src/tools/rlm-peek.ts b/packages/rlm/src/tools/rlm-peek.ts new file mode 100644 index 0000000..a962430 --- /dev/null +++ b/packages/rlm/src/tools/rlm-peek.ts @@ -0,0 +1,15 @@ +import type { ContextBuffer } from '../context/context-buffer.js'; + +export function createPeekHandler(buffer: ContextBuffer) { + return async (params: Record) => { + const offset = (params.offset as number) ?? 0; + const length = (params.length as number) ?? 50; + const text = buffer.peek(offset, length); + return { + text, + fromLine: offset, + toLine: Math.min(offset + length, buffer.lineCount), + totalLines: buffer.lineCount, + }; + }; +} diff --git a/packages/rlm/src/tools/rlm-query.ts b/packages/rlm/src/tools/rlm-query.ts new file mode 100644 index 0000000..cc10d99 --- /dev/null +++ b/packages/rlm/src/tools/rlm-query.ts @@ -0,0 +1,31 @@ +import type { ContextBuffer } from '../context/context-buffer.js'; +import type { CompletionProvider } from '../types.js'; + +const MAX_CHUNK_CHARS = 500_000; + +export function createQueryHandler(buffer: ContextBuffer, provider: CompletionProvider) { + return async (params: Record) => { + const question = params.question as string; + const start = (params.start as number) ?? 0; + const end = (params.end as number) ?? buffer.lineCount; + + const chunk = buffer.getChunk(start, MAX_CHUNK_CHARS); + const sliced = start === 0 && end >= buffer.lineCount + ? chunk.content + : buffer.slice(start, end); + + const context = sliced.length > MAX_CHUNK_CHARS + ? sliced.slice(0, MAX_CHUNK_CHARS) + '\n... [truncated]' + : sliced; + + const answer = await provider.completionStr( + `Answer this question based on the context below. Be thorough and cite specific details from the context.\n\nQuestion: ${question}\n\nContext (lines ${start}-${end}):\n${context}` + ); + + return { + answer, + linesAnalyzed: { start, end: Math.min(end, buffer.lineCount) }, + chunkChars: context.length, + }; + }; +} diff --git a/packages/rlm/src/tools/rlm-repl.ts b/packages/rlm/src/tools/rlm-repl.ts new file mode 100644 index 0000000..0e79509 --- /dev/null +++ b/packages/rlm/src/tools/rlm-repl.ts @@ -0,0 +1,44 @@ +import type { ContextBuffer } from '../context/context-buffer.js'; +import type { CompletionProvider } from '../types.js'; +import { REPLSandbox } from '../repl/sandbox.js'; +import { createBuiltins } from '../repl/builtins.js'; + +/** + * Creates a handler for the rlm_repl tool. + * Runs JavaScript code in the REPL sandbox with access to context and all builtins. + * Sandbox persists across calls within a session for variable persistence. + */ +export function createReplHandler(buffer: ContextBuffer, provider: CompletionProvider) { + let sandbox: REPLSandbox | null = null; + + return async (params: Record) => { + const code = params.code as string; + if (!code || typeof code !== 'string') { + return { output: 'Error: code parameter is required (string)', error: 'missing code' }; + } + + // Lazy init sandbox + if (!sandbox) { + const builtins = createBuiltins(buffer, provider); + sandbox = new REPLSandbox(builtins, buffer.getRawContent(), { timeoutMs: 30_000 }); + await sandbox.initialize(); + } + + const result = await sandbox.runCode(code); + + const parts: string[] = []; + if (result.output) parts.push(result.output); + if (result.error) parts.push(`Error: ${result.error}`); + if (result.finalAnswer) parts.push(`FINAL: ${result.finalAnswer}`); + + const vars = Object.keys(result.variables); + if (vars.length > 0) parts.push(`Variables: ${vars.join(', ')}`); + + return { + output: parts.join('\n') || 'No output', + finalAnswer: result.finalAnswer, + executionTimeMs: result.executionTimeMs, + iteration: sandbox.getIterationCount(), + }; + }; +} diff --git a/packages/rlm/src/tools/rlm-slice.ts b/packages/rlm/src/tools/rlm-slice.ts new file mode 100644 index 0000000..946073b --- /dev/null +++ b/packages/rlm/src/tools/rlm-slice.ts @@ -0,0 +1,15 @@ +import type { ContextBuffer } from '../context/context-buffer.js'; + +export function createSliceHandler(buffer: ContextBuffer) { + return async (params: Record) => { + const start = params.start as number; + const end = params.end as number; + const text = buffer.slice(start, end); + return { + text, + fromLine: Math.max(0, start), + toLine: Math.min(end, buffer.lineCount), + charCount: text.length, + }; + }; +} diff --git a/packages/rlm/src/types.ts b/packages/rlm/src/types.ts new file mode 100644 index 0000000..2758026 --- /dev/null +++ b/packages/rlm/src/types.ts @@ -0,0 +1,19 @@ +/** Chat message for LLM interactions. */ +export interface Message { + role: 'system' | 'user' | 'assistant'; + content: string; +} + +/** Context that can be passed to the RLM tools — any large text or structured data. */ +export type Context = string | Record | unknown[]; + +/** + * Abstraction over LLM completion — injected by the consumer. + * Wire this to pi-ai, OpenAI, Anthropic, or any LLM SDK. + */ +export interface CompletionProvider { + /** Chat completion from a list of messages. */ + completion(messages: Message[]): Promise; + /** Simple completion from a single string prompt. */ + completionStr(prompt: string): Promise; +} diff --git a/packages/rlm/tsconfig.json b/packages/rlm/tsconfig.json new file mode 100644 index 0000000..1e2b559 --- /dev/null +++ b/packages/rlm/tsconfig.json @@ -0,0 +1,22 @@ +{ + "compilerOptions": { + "target": "ESNext", + "module": "Node16", + "moduleResolution": "node16", + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "lib": ["ES2022"], + "types": ["node"], + "strict": false, + "skipLibCheck": true, + "noEmitOnError": false, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "outDir": "./dist", + "declaration": true, + "declarationMap": true, + "sourceMap": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 63a8c86..6a27fbf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -41,6 +41,12 @@ importers: packages/contexto: dependencies: + '@ekai/rlm': + specifier: workspace:* + version: link:../rlm + '@mariozechner/pi-ai': + specifier: '*' + version: 0.65.0(@modelcontextprotocol/sdk@1.29.0(zod@4.3.6))(ws@8.20.0)(zod@4.3.6) openclaw: specifier: '*' version: 2026.4.5(@napi-rs/canvas@0.1.97) @@ -133,6 +139,22 @@ importers: specifier: ^3.0.0 version: 3.2.4(@types/node@25.5.0)(jiti@2.6.1)(lightningcss@1.32.0)(tsx@4.21.0)(yaml@2.8.3) + packages/rlm: + dependencies: + pdf-parse: + specifier: '>=1.1.0' + version: 2.4.5 + xlsx: + specifier: '>=0.18.0' + version: 0.18.5 + devDependencies: + '@types/node': + specifier: ^20.10.0 + version: 20.19.37 + typescript: + specifier: ^5.3.2 + version: 5.9.3 + packages/ui/dashboard: dependencies: '@types/dagre': @@ -970,30 +992,61 @@ packages: resolution: {integrity: sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==} engines: {node: '>=14.0.0'} + '@napi-rs/canvas-android-arm64@0.1.80': + resolution: {integrity: sha512-sk7xhN/MoXeuExlggf91pNziBxLPVUqF2CAVnB57KLG/pz7+U5TKG8eXdc3pm0d7Od0WreB6ZKLj37sX9muGOQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + '@napi-rs/canvas-android-arm64@0.1.97': resolution: {integrity: sha512-V1c/WVw+NzH8vk7ZK/O8/nyBSCQimU8sfMsB/9qeSvdkGKNU7+mxy/bIF0gTgeBFmHpj30S4E9WHMSrxXGQuVQ==} engines: {node: '>= 10'} cpu: [arm64] os: [android] + '@napi-rs/canvas-darwin-arm64@0.1.80': + resolution: {integrity: sha512-O64APRTXRUiAz0P8gErkfEr3lipLJgM6pjATwavZ22ebhjYl/SUbpgM0xcWPQBNMP1n29afAC/Us5PX1vg+JNQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + '@napi-rs/canvas-darwin-arm64@0.1.97': resolution: {integrity: sha512-ok+SCEF4YejcxuJ9Rm+WWunHHpf2HmiPxfz6z1a/NFQECGXtsY7A4B8XocK1LmT1D7P174MzwPF9Wy3AUAwEPw==} engines: {node: '>= 10'} cpu: [arm64] os: [darwin] + '@napi-rs/canvas-darwin-x64@0.1.80': + resolution: {integrity: sha512-FqqSU7qFce0Cp3pwnTjVkKjjOtxMqRe6lmINxpIZYaZNnVI0H5FtsaraZJ36SiTHNjZlUB69/HhxNDT1Aaa9vA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + '@napi-rs/canvas-darwin-x64@0.1.97': resolution: {integrity: sha512-PUP6e6/UGlclUvAQNnuXCcnkpdUou6VYZfQOQxExLp86epOylmiwLkqXIvpFmjoTEDmPmXrI+coL/9EFU1gKPA==} engines: {node: '>= 10'} cpu: [x64] os: [darwin] + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.80': + resolution: {integrity: sha512-eyWz0ddBDQc7/JbAtY4OtZ5SpK8tR4JsCYEZjCE3dI8pqoWUC8oMwYSBGCYfsx2w47cQgQCgMVRVTFiiO38hHQ==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.97': resolution: {integrity: sha512-XyXH2L/cic8eTNtbrXCcvqHtMX/nEOxN18+7rMrAM2XtLYC/EB5s0wnO1FsLMWmK+04ZSLN9FBGipo7kpIkcOw==} engines: {node: '>= 10'} cpu: [arm] os: [linux] + '@napi-rs/canvas-linux-arm64-gnu@0.1.80': + resolution: {integrity: sha512-qwA63t8A86bnxhuA/GwOkK3jvb+XTQaTiVML0vAWoHyoZYTjNs7BzoOONDgTnNtr8/yHrq64XXzUoLqDzU+Uuw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [glibc] + '@napi-rs/canvas-linux-arm64-gnu@0.1.97': resolution: {integrity: sha512-Kuq/M3djq0K8ktgz6nPlK7Ne5d4uWeDxPpyKWOjWDK2RIOhHVtLtyLiJw2fuldw7Vn4mhw05EZXCEr4Q76rs9w==} engines: {node: '>= 10'} @@ -1001,6 +1054,13 @@ packages: os: [linux] libc: [glibc] + '@napi-rs/canvas-linux-arm64-musl@0.1.80': + resolution: {integrity: sha512-1XbCOz/ymhj24lFaIXtWnwv/6eFHXDrjP0jYkc6iHQ9q8oXKzUX1Lc6bu+wuGiLhGh2GS/2JlfORC5ZcXimRcg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [musl] + '@napi-rs/canvas-linux-arm64-musl@0.1.97': resolution: {integrity: sha512-kKmSkQVnWeqg7qdsiXvYxKhAFuHz3tkBjW/zyQv5YKUPhotpaVhpBGv5LqCngzyuRV85SXoe+OFj+Tv0a0QXkQ==} engines: {node: '>= 10'} @@ -1008,6 +1068,13 @@ packages: os: [linux] libc: [musl] + '@napi-rs/canvas-linux-riscv64-gnu@0.1.80': + resolution: {integrity: sha512-XTzR125w5ZMs0lJcxRlS1K3P5RaZ9RmUsPtd1uGt+EfDyYMu4c6SEROYsxyatbbu/2+lPe7MPHOO/0a0x7L/gw==} + engines: {node: '>= 10'} + cpu: [riscv64] + os: [linux] + libc: [glibc] + '@napi-rs/canvas-linux-riscv64-gnu@0.1.97': resolution: {integrity: sha512-Jc7I3A51jnEOIAXeLsN/M/+Z28LUeakcsXs07FLq9prXc0eYOtVwsDEv913Gr+06IRo34gJJVgT0TXvmz+N2VA==} engines: {node: '>= 10'} @@ -1015,6 +1082,13 @@ packages: os: [linux] libc: [glibc] + '@napi-rs/canvas-linux-x64-gnu@0.1.80': + resolution: {integrity: sha512-BeXAmhKg1kX3UCrJsYbdQd3hIMDH/K6HnP/pG2LuITaXhXBiNdh//TVVVVCBbJzVQaV5gK/4ZOCMrQW9mvuTqA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [glibc] + '@napi-rs/canvas-linux-x64-gnu@0.1.97': resolution: {integrity: sha512-iDUBe7AilfuBSRbSa8/IGX38Mf+iCSBqoVKLSQ5XaY2JLOaqz1TVyPFEyIck7wT6mRQhQt5sN6ogfjIDfi74tg==} engines: {node: '>= 10'} @@ -1022,6 +1096,13 @@ packages: os: [linux] libc: [glibc] + '@napi-rs/canvas-linux-x64-musl@0.1.80': + resolution: {integrity: sha512-x0XvZWdHbkgdgucJsRxprX/4o4sEed7qo9rCQA9ugiS9qE2QvP0RIiEugtZhfLH3cyI+jIRFJHV4Fuz+1BHHMg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [musl] + '@napi-rs/canvas-linux-x64-musl@0.1.97': resolution: {integrity: sha512-AKLFd/v0Z5fvgqBDqhvqtAdx+fHMJ5t9JcUNKq4FIZ5WH+iegGm8HPdj00NFlCSnm83Fp3Ln8I2f7uq1aIiWaA==} engines: {node: '>= 10'} @@ -1035,12 +1116,22 @@ packages: cpu: [arm64] os: [win32] + '@napi-rs/canvas-win32-x64-msvc@0.1.80': + resolution: {integrity: sha512-Z8jPsM6df5V8B1HrCHB05+bDiCxjE9QA//3YrkKIdVDEwn5RKaqOxCJDRJkl48cJbylcrJbW4HxZbTte8juuPg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + '@napi-rs/canvas-win32-x64-msvc@0.1.97': resolution: {integrity: sha512-sWtD2EE3fV0IzN+iiQUqr/Q1SwqWhs2O1FKItFlxtdDkikpEj5g7DKQpY3x55H/MAOnL8iomnlk3mcEeGiUMoQ==} engines: {node: '>= 10'} cpu: [x64] os: [win32] + '@napi-rs/canvas@0.1.80': + resolution: {integrity: sha512-DxuT1ClnIPts1kQx8FBmkk4BQDTfI5kIzywAaMjQSXfNnra5UFU9PwurXrl+Je3bJ6BGsp/zmshVVFbCmyI+ww==} + engines: {node: '>= 10'} + '@napi-rs/canvas@0.1.97': resolution: {integrity: sha512-8cFniXvrIEnVwuNSRCW9wirRZbHvrD3JVujdS2P5n5xiJZNZMOZcfOvJ1pb66c7jXMKHHglJEDVJGbm8XWFcXQ==} engines: {node: '>= 10'} @@ -2183,6 +2274,10 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + adler-32@1.3.1: + resolution: {integrity: sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==} + engines: {node: '>=0.8'} + agent-base@7.1.4: resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} engines: {node: '>= 14'} @@ -2347,6 +2442,7 @@ packages: basic-ftp@5.2.0: resolution: {integrity: sha512-VoMINM2rqJwJgfdHq6RiUudKt2BV+FY5ZFezP/ypmwayk68+NzzAQy4XXLlqsGD4MCzq3DrmNFD/uUmBJuGoXw==} engines: {node: '>=10.0.0'} + deprecated: Security vulnerability fixed in 5.2.1, please upgrade before-after-hook@4.0.0: resolution: {integrity: sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ==} @@ -2455,6 +2551,10 @@ packages: caniuse-lite@1.0.30001781: resolution: {integrity: sha512-RdwNCyMsNBftLjW6w01z8bKEvT6e/5tpPVEgtn22TiLGlstHOVecsX2KHFkD5e/vRnIE4EGzpuIODb3mtswtkw==} + cfb@1.2.2: + resolution: {integrity: sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==} + engines: {node: '>=0.8'} + chai@5.3.3: resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==} engines: {node: '>=18'} @@ -2528,6 +2628,10 @@ packages: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} + codepage@1.15.0: + resolution: {integrity: sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==} + engines: {node: '>=0.8'} + color-convert@1.9.3: resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} @@ -2632,6 +2736,11 @@ packages: typescript: optional: true + crc-32@1.2.2: + resolution: {integrity: sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==} + engines: {node: '>=0.8'} + hasBin: true + croner@10.0.1: resolution: {integrity: sha512-ixNtAJndqh173VQ4KodSdJEI6nuioBWI0V1ITNKhZZsO0pEMoDxz539T4FTTbSZ/xIOSuDnzxLVRqBVSvPNE2g==} engines: {node: '>=18.0'} @@ -3341,6 +3450,10 @@ packages: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} + frac@1.1.2: + resolution: {integrity: sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==} + engines: {node: '>=0.8'} + fresh@0.5.2: resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} engines: {node: '>= 0.6'} @@ -4790,6 +4903,15 @@ packages: resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} engines: {node: '>= 14.16'} + pdf-parse@2.4.5: + resolution: {integrity: sha512-mHU89HGh7v+4u2ubfnevJ03lmPgQ5WU4CxAVmTSh/sxVTEDYd1er/dKS/A6vg77NX47KTEoihq8jZBLr8Cxuwg==} + engines: {node: '>=20.16.0 <21 || >=22.3.0'} + hasBin: true + + pdfjs-dist@5.4.296: + resolution: {integrity: sha512-DlOzet0HO7OEnmUmB6wWGJrrdvbyJKftI1bhMitK7O2N8W2gc757yyYBbINy9IDafXAV9wmKr9t7xsTaNKRG5Q==} + engines: {node: '>=20.16.0 || >=22.3.0'} + pdfjs-dist@5.6.205: resolution: {integrity: sha512-tlUj+2IDa7G1SbvBNN74UHRLJybZDWYom+k6p5KIZl7huBvsA4APi6mKL+zCxd3tLjN5hOOEE9Tv7VdzO88pfg==} engines: {node: '>=20.19.0 || >=22.13.0 || >=24'} @@ -5352,6 +5474,10 @@ packages: sqlite-vec@0.1.9: resolution: {integrity: sha512-L7XJWRIBNvR9O5+vh1FQ+IGkh/3D2AzVksW5gdtk28m78Hy8skFD0pqReKH1Yp0/BUKRGcffgKvyO/EON5JXpA==} + ssf@0.11.2: + resolution: {integrity: sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==} + engines: {node: '>=0.8'} + stable-hash@0.0.5: resolution: {integrity: sha512-+L3ccpzibovGXFK+Ap/f8LOS0ahMrHTf3xu7mMLSpEGU0EO9ucaysSylKo9eRDFNhWve/y275iPmIZ4z39a9iA==} @@ -5923,10 +6049,18 @@ packages: engines: {node: '>=8'} hasBin: true + wmf@1.0.2: + resolution: {integrity: sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==} + engines: {node: '>=0.8'} + word-wrap@1.2.5: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} + word@0.3.0: + resolution: {integrity: sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==} + engines: {node: '>=0.8'} + wordwrap@1.0.0: resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==} @@ -5953,6 +6087,11 @@ packages: utf-8-validate: optional: true + xlsx@0.18.5: + resolution: {integrity: sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==} + engines: {node: '>=0.8'} + hasBin: true + xtend@4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} @@ -7070,39 +7209,82 @@ snapshots: '@mozilla/readability@0.6.0': {} + '@napi-rs/canvas-android-arm64@0.1.80': + optional: true + '@napi-rs/canvas-android-arm64@0.1.97': optional: true + '@napi-rs/canvas-darwin-arm64@0.1.80': + optional: true + '@napi-rs/canvas-darwin-arm64@0.1.97': optional: true + '@napi-rs/canvas-darwin-x64@0.1.80': + optional: true + '@napi-rs/canvas-darwin-x64@0.1.97': optional: true + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.80': + optional: true + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.97': optional: true + '@napi-rs/canvas-linux-arm64-gnu@0.1.80': + optional: true + '@napi-rs/canvas-linux-arm64-gnu@0.1.97': optional: true + '@napi-rs/canvas-linux-arm64-musl@0.1.80': + optional: true + '@napi-rs/canvas-linux-arm64-musl@0.1.97': optional: true + '@napi-rs/canvas-linux-riscv64-gnu@0.1.80': + optional: true + '@napi-rs/canvas-linux-riscv64-gnu@0.1.97': optional: true + '@napi-rs/canvas-linux-x64-gnu@0.1.80': + optional: true + '@napi-rs/canvas-linux-x64-gnu@0.1.97': optional: true + '@napi-rs/canvas-linux-x64-musl@0.1.80': + optional: true + '@napi-rs/canvas-linux-x64-musl@0.1.97': optional: true '@napi-rs/canvas-win32-arm64-msvc@0.1.97': optional: true + '@napi-rs/canvas-win32-x64-msvc@0.1.80': + optional: true + '@napi-rs/canvas-win32-x64-msvc@0.1.97': optional: true + '@napi-rs/canvas@0.1.80': + optionalDependencies: + '@napi-rs/canvas-android-arm64': 0.1.80 + '@napi-rs/canvas-darwin-arm64': 0.1.80 + '@napi-rs/canvas-darwin-x64': 0.1.80 + '@napi-rs/canvas-linux-arm-gnueabihf': 0.1.80 + '@napi-rs/canvas-linux-arm64-gnu': 0.1.80 + '@napi-rs/canvas-linux-arm64-musl': 0.1.80 + '@napi-rs/canvas-linux-riscv64-gnu': 0.1.80 + '@napi-rs/canvas-linux-x64-gnu': 0.1.80 + '@napi-rs/canvas-linux-x64-musl': 0.1.80 + '@napi-rs/canvas-win32-x64-msvc': 0.1.80 + '@napi-rs/canvas@0.1.97': optionalDependencies: '@napi-rs/canvas-android-arm64': 0.1.97 @@ -8379,6 +8561,8 @@ snapshots: acorn@8.16.0: {} + adler-32@1.3.1: {} + agent-base@7.1.4: {} aggregate-error@3.1.0: @@ -8687,6 +8871,11 @@ snapshots: caniuse-lite@1.0.30001781: {} + cfb@1.2.2: + dependencies: + adler-32: 1.3.1 + crc-32: 1.2.2 + chai@5.3.3: dependencies: assertion-error: 2.0.1 @@ -8765,6 +8954,8 @@ snapshots: clsx@2.1.1: {} + codepage@1.15.0: {} + color-convert@1.9.3: dependencies: color-name: 1.1.3 @@ -8871,6 +9062,8 @@ snapshots: optionalDependencies: typescript: 5.9.3 + crc-32@1.2.2: {} + croner@10.0.1: {} cross-spawn@7.0.6: @@ -9821,6 +10014,8 @@ snapshots: forwarded@0.2.0: {} + frac@1.1.2: {} + fresh@0.5.2: {} fresh@2.0.0: {} @@ -11245,6 +11440,15 @@ snapshots: pathval@2.0.1: {} + pdf-parse@2.4.5: + dependencies: + '@napi-rs/canvas': 0.1.80 + pdfjs-dist: 5.4.296 + + pdfjs-dist@5.4.296: + optionalDependencies: + '@napi-rs/canvas': 0.1.97 + pdfjs-dist@5.6.205: optionalDependencies: '@napi-rs/canvas': 0.1.97 @@ -11987,6 +12191,10 @@ snapshots: sqlite-vec-linux-x64: 0.1.9 sqlite-vec-windows-x64: 0.1.9 + ssf@0.11.2: + dependencies: + frac: 1.1.2 + stable-hash@0.0.5: {} stackback@0.0.2: {} @@ -12603,8 +12811,12 @@ snapshots: siginfo: 2.0.0 stackback: 0.0.2 + wmf@1.0.2: {} + word-wrap@1.2.5: {} + word@0.3.0: {} + wordwrap@1.0.0: {} wrap-ansi@7.0.0: @@ -12623,6 +12835,16 @@ snapshots: ws@8.20.0: {} + xlsx@0.18.5: + dependencies: + adler-32: 1.3.1 + cfb: 1.2.2 + codepage: 1.15.0 + crc-32: 1.2.2 + ssf: 0.11.2 + wmf: 1.0.2 + word: 0.3.0 + xtend@4.0.2: {} y18n@5.0.8: {}