diff --git a/cli.ts b/cli.ts index 946cfb1..e230b03 100644 --- a/cli.ts +++ b/cli.ts @@ -510,6 +510,7 @@ export function registerMemoryCLI(program: Command, context: CLIContext): void { .option("--dry-run", "Show what would be re-embedded without writing") .option("--skip-existing", "Skip entries whose id already exists in the target DB") .option("--force", "Allow using the same source-db as the target dbPath (DANGEROUS)") + .option("--storage-options ", "Storage options for LanceDB connection (JSON string, e.g. '{\"aws_access_key_id\":\"...\"}')") .action(async (options) => { try { if (!context.embedder) { @@ -526,6 +527,23 @@ export function registerMemoryCLI(program: Command, context: CLIContext): void { const skipExisting = options.skipExisting === true; const force = options.force === true; + let storageOptions: Record = {}; + if (options.storageOptions) { + try { + const parsed = JSON.parse(options.storageOptions as string); + if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) { + for (const [key, value] of Object.entries(parsed)) { + if (typeof value === "string") { + storageOptions[key] = value; + } + } + } + } catch { + console.error("Invalid JSON in --storage-options"); + process.exit(1); + } + } + // Safety: prevent accidental in-place re-embedding let sourceReal = sourceDbPath; let targetReal = context.store.dbPath; @@ -542,7 +560,7 @@ export function registerMemoryCLI(program: Command, context: CLIContext): void { } const lancedb = await loadLanceDB(); - const db = await lancedb.connect(sourceDbPath); + const db = await lancedb.connect(sourceDbPath, { storageOptions }); const table = await db.openTable("memories"); let query = table diff --git a/index.ts b/index.ts index 48d7fb9..791af02 100644 --- a/index.ts +++ b/index.ts @@ -156,6 +156,7 @@ interface PluginConfig { dedupeErrorSignals?: boolean; }; mdMirror?: { enabled?: boolean; dir?: string }; + storageOptions?: Record; } type ReflectionThinkLevel = "off" | "minimal" | "low" | "medium" | "high"; @@ -1585,17 +1586,26 @@ const memoryLanceDBProPlugin = { // Parse and validate configuration const config = parsePluginConfig(api.pluginConfig); - const resolvedDbPath = api.resolvePath(config.dbPath || getDefaultDbPath()); + const dbPath = config.dbPath || getDefaultDbPath(); + const isCloud = /^[a-z][a-z0-9+.-]*:\/\//i.test(dbPath.trim()); - // Pre-flight: validate storage path (symlink resolution, mkdir, write check). - // Runs synchronously and logs warnings; does NOT block gateway startup. - try { - validateStoragePath(resolvedDbPath); - } catch (err) { - api.logger.warn( - `memory-lancedb-pro: storage path issue — ${String(err)}\n` + - ` The plugin will still attempt to start, but writes may fail.`, - ); + let resolvedDbPath: string; + if (isCloud) { + resolvedDbPath = dbPath; + } else { + // Local path: resolve and validate + resolvedDbPath = api.resolvePath(dbPath); + + // Pre-flight: validate storage path (symlink resolution, mkdir, write check). + // Runs synchronously and logs warnings; does NOT block gateway startup. + try { + validateStoragePath(resolvedDbPath); + } catch (err) { + api.logger.warn( + `memory-lancedb-pro: storage path issue — ${String(err)}\n` + + ` The plugin will still attempt to start, but writes may fail.`, + ); + } } const vectorDim = getVectorDimensions( @@ -1604,7 +1614,7 @@ const memoryLanceDBProPlugin = { ); // Initialize core components - const store = new MemoryStore({ dbPath: resolvedDbPath, vectorDim }); + const store = new MemoryStore({ dbPath: resolvedDbPath, vectorDim, storageOptions: config.storageOptions }); const embedder = createEmbedder({ provider: "openai-compatible", apiKey: config.embedding.apiKey, @@ -3330,6 +3340,23 @@ export function parsePluginConfig(value: unknown): PluginConfig { : undefined, } : undefined, + storageOptions: + typeof cfg.storageOptions === "object" && cfg.storageOptions !== null && !Array.isArray(cfg.storageOptions) + ? (() => { + const opts = cfg.storageOptions as Record; + const resolved: Record = {}; + for (const [key, value] of Object.entries(opts)) { + if (typeof value !== "string") { + throw new Error( + `storageOptions[${key}] is invalid: expected string, got ${typeof value}` + ); + } + // Resolve environment variables in value + resolved[key] = resolveEnvVars(value); + } + return resolved; + })() + : undefined, }; } diff --git a/openclaw.plugin.json b/openclaw.plugin.json index 6f408db..a8d4341 100644 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -451,6 +451,13 @@ "description": "Fallback directory for Markdown mirror files when agent workspace is unknown" } } + }, + "storageOptions": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "Additional storage options for LanceDB connection configuration" } }, "required": [ @@ -780,6 +787,12 @@ "help": "Define which scopes each agent can access", "advanced": true }, + "storageOptions": { + "label": "Storage Options", + "help": "Additional storage options for LanceDB connection configuration (e.g., cloud provider credentials)", + "advanced": true, + "sensitive": true + }, "enableManagementTools": { "label": "Management Tools", "help": "Enable management/debug tools such as memory_list, memory_stats, and governance-oriented self-improvement review/extract actions.", diff --git a/src/store.ts b/src/store.ts index 764fa55..6519323 100644 --- a/src/store.ts +++ b/src/store.ts @@ -181,8 +181,11 @@ export class MemoryStore { private initPromise: Promise | null = null; private ftsIndexCreated = false; private updateQueue: Promise = Promise.resolve(); + private readonly storageOptions: Record; - constructor(private readonly config: StoreConfig) {} + constructor(private readonly config: StoreConfig) { + this.storageOptions = config.storageOptions || {}; + } get dbPath(): string { return this.config.dbPath; @@ -208,7 +211,10 @@ export class MemoryStore { let db: LanceDB.Connection; try { - db = await lancedb.connect(this.config.dbPath); + const connectOpts = Object.keys(this.storageOptions).length > 0 + ? { storageOptions: this.storageOptions } + : undefined; + db = await lancedb.connect(this.config.dbPath, connectOpts); } catch (err: any) { const code = err.code || ""; const message = err.message || String(err); diff --git a/test/plugin-config.test.mjs b/test/plugin-config.test.mjs new file mode 100644 index 0000000..ca3b04f --- /dev/null +++ b/test/plugin-config.test.mjs @@ -0,0 +1,289 @@ +import { describe, it, beforeEach, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import jitiFactory from "jiti"; + +const testDir = path.dirname(fileURLToPath(import.meta.url)); +const pluginSdkStubPath = path.resolve(testDir, "helpers", "openclaw-plugin-sdk-stub.mjs"); +const jiti = jitiFactory(import.meta.url, { + interopDefault: true, + alias: { + "openclaw/plugin-sdk": pluginSdkStubPath, + }, +}); +const { parsePluginConfig } = jiti("../index.ts"); + +function baseConfig() { + return { + embedding: { + apiKey: "test-api-key", + }, + }; +} + +describe("parsePluginConfig: storageOptions validation", () => { + describe("non-string values in storageOptions are rejected", () => { + it("rejects number value in storageOptions", () => { + assert.throws( + () => + parsePluginConfig({ + ...baseConfig(), + storageOptions: { + aws_access_key_id: "valid-string", + timeout: 30, + }, + }), + /storageOptions\[timeout\] is invalid: expected string, got number/, + ); + }); + + it("rejects boolean value in storageOptions", () => { + assert.throws( + () => + parsePluginConfig({ + ...baseConfig(), + storageOptions: { + aws_access_key_id: "valid-string", + enable_ssl: true, + }, + }), + /storageOptions\[enable_ssl\] is invalid: expected string, got boolean/, + ); + }); + + it("rejects null value in storageOptions", () => { + assert.throws( + () => + parsePluginConfig({ + ...baseConfig(), + storageOptions: { + aws_access_key_id: null, + }, + }), + /storageOptions\[aws_access_key_id\] is invalid: expected string, got object/, + ); + }); + + it("rejects nested object value in storageOptions", () => { + assert.throws( + () => + parsePluginConfig({ + ...baseConfig(), + storageOptions: { + credentials: { key: "value" }, + }, + }), + /storageOptions\[credentials\] is invalid: expected string, got object/, + ); + }); + + it("rejects array value in storageOptions", () => { + assert.throws( + () => + parsePluginConfig({ + ...baseConfig(), + storageOptions: { + endpoints: ["host1", "host2"], + }, + }), + /storageOptions\[endpoints\] is invalid: expected string, got object/, + ); + }); + + it("accepts valid string-only storageOptions", () => { + const parsed = parsePluginConfig({ + ...baseConfig(), + storageOptions: { + aws_access_key_id: "my-key", + aws_secret_access_key: "my-secret", + region: "us-east-1", + }, + }); + assert.deepEqual(parsed.storageOptions, { + aws_access_key_id: "my-key", + aws_secret_access_key: "my-secret", + region: "us-east-1", + }); + }); + }); + + describe("storageOptions ${ENV_VAR} placeholder resolution", () => { + const originalEnv = {}; + + beforeEach(() => { + originalEnv.AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID; + originalEnv.AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY; + originalEnv.MY_CUSTOM_REGION = process.env.MY_CUSTOM_REGION; + originalEnv.EMPTY_VAR = process.env.EMPTY_VAR; + originalEnv.HOST = process.env.HOST; + originalEnv.PORT = process.env.PORT; + originalEnv.DYNAMIC_VALUE = process.env.DYNAMIC_VALUE; + }); + + afterEach(() => { + for (const [key, value] of Object.entries(originalEnv)) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } + }); + + it("resolves single env var placeholder", () => { + process.env.AWS_ACCESS_KEY_ID = "resolved-key-123"; + const parsed = parsePluginConfig({ + ...baseConfig(), + storageOptions: { + aws_access_key_id: "${AWS_ACCESS_KEY_ID}", + }, + }); + assert.equal(parsed.storageOptions?.aws_access_key_id, "resolved-key-123"); + }); + + it("resolves multiple env var placeholders in same value", () => { + process.env.HOST = "example.com"; + process.env.PORT = "8080"; + const parsed = parsePluginConfig({ + ...baseConfig(), + storageOptions: { + endpoint: "https://${HOST}:${PORT}", + }, + }); + assert.equal(parsed.storageOptions?.endpoint, "https://example.com:8080"); + }); + + it("resolves multiple storageOptions with different env vars", () => { + process.env.AWS_ACCESS_KEY_ID = "my-access-key"; + process.env.AWS_SECRET_ACCESS_KEY = "my-secret-key"; + process.env.MY_CUSTOM_REGION = "eu-west-1"; + + const parsed = parsePluginConfig({ + ...baseConfig(), + storageOptions: { + aws_access_key_id: "${AWS_ACCESS_KEY_ID}", + aws_secret_access_key: "${AWS_SECRET_ACCESS_KEY}", + region: "${MY_CUSTOM_REGION}", + }, + }); + + assert.deepEqual(parsed.storageOptions, { + aws_access_key_id: "my-access-key", + aws_secret_access_key: "my-secret-key", + region: "eu-west-1", + }); + }); + + it("throws when env var is not set", () => { + delete process.env.UNDEFINED_VAR; + assert.throws( + () => + parsePluginConfig({ + ...baseConfig(), + storageOptions: { + key: "${UNDEFINED_VAR}", + }, + }), + /Environment variable UNDEFINED_VAR is not set/, + ); + }); + + it("handles mixed literal and env var values", () => { + process.env.DYNAMIC_VALUE = "dynamic"; + const parsed = parsePluginConfig({ + ...baseConfig(), + storageOptions: { + static_key: "static-value", + dynamic_key: "${DYNAMIC_VALUE}", + mixed_key: "prefix-${DYNAMIC_VALUE}-suffix", + }, + }); + assert.deepEqual(parsed.storageOptions, { + static_key: "static-value", + dynamic_key: "dynamic", + mixed_key: "prefix-dynamic-suffix", + }); + }); + + it("throws when env var is empty string (treated as not set)", () => { + process.env.EMPTY_VAR = ""; + assert.throws( + () => + parsePluginConfig({ + ...baseConfig(), + storageOptions: { + empty_key: "${EMPTY_VAR}", + }, + }), + /Environment variable EMPTY_VAR is not set/, + ); + }); + }); +}); + +describe("parsePluginConfig: cloud path detection", () => { + const cloudPathPatterns = [ + "s3://my-bucket/lancedb-data", + "gs://my-gcs-bucket/data", + "az://my-azure-container/data", + "abfs://my-container/data", + "s3+https://bucket.s3.amazonaws.com/path", + "gcs://project-id/bucket/path", + "tos://my-tos-bucket/path", + "hdfs://namenode:8020/path/to/db", + "file:///absolute/path/to/db", + ]; + + const localPathPatterns = [ + "./relative/path", + "../parent/path", + "memory-data", + "/absolute/local/path", + "~/home/path", + "data/memory", + ]; + + it("identifies cloud paths correctly", () => { + const cloudRegex = /^[a-z][a-z0-9+.-]*:\/\//i; + for (const cloudPath of cloudPathPatterns) { + assert.match( + cloudPath, + cloudRegex, + `Expected "${cloudPath}" to be identified as cloud path`, + ); + } + }); + + it("identifies local paths correctly (not matching cloud pattern)", () => { + const cloudRegex = /^[a-z][a-z0-9+.-]*:\/\//i; + for (const localPath of localPathPatterns) { + assert.doesNotMatch( + localPath, + cloudRegex, + `Expected "${localPath}" to NOT be identified as cloud path`, + ); + } + }); + + it("cloud path detection is case-insensitive for scheme", () => { + const cloudRegex = /^[a-z][a-z0-9+.-]*:\/\//i; + assert.match("S3://bucket/path", cloudRegex); + assert.match("GS://bucket/path", cloudRegex); + assert.match("AZ://container/path", cloudRegex); + }); + + it("validates scheme format (must start with letter)", () => { + const cloudRegex = /^[a-z][a-z0-9+.-]*:\/\//i; + assert.doesNotMatch("3s://bucket/path", cloudRegex); + assert.doesNotMatch("+invalid://path", cloudRegex); + assert.doesNotMatch("://no-scheme", cloudRegex); + }); + + it("allows valid characters in scheme (letters, digits, plus, dot, hyphen)", () => { + const cloudRegex = /^[a-z][a-z0-9+.-]*:\/\//i; + assert.match("s3+https://bucket/path", cloudRegex); + assert.match("my-custom.scheme://path", cloudRegex); + assert.match("scheme-v2://path", cloudRegex); + }); +});