diff --git a/package.json b/package.json index 0f5dcf3..d3b028a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@thinkfleet/memory-sdk", - "version": "0.4.1", + "version": "0.5.0", "description": "TypeScript SDK for app.memmesh.ai — admin + project memory CRUD, semantic search, feedback, and Lattice behavioral patterns", "type": "module", "main": "./dist/index.cjs", diff --git a/src/index.ts b/src/index.ts index 9b53130..fba1f61 100644 --- a/src/index.ts +++ b/src/index.ts @@ -170,6 +170,8 @@ export type { ObserveAttachmentRequest, ObserveVoiceRequest, ObserveDocumentRequest, + IngestMediaRequest, + IngestMediaResult, ConsolidateRequest, ConsolidateResult, ReflectRequest, diff --git a/src/resources/memory.ts b/src/resources/memory.ts index 2fb5143..e78fd53 100644 --- a/src/resources/memory.ts +++ b/src/resources/memory.ts @@ -12,6 +12,8 @@ import { type MemorySearchRequest, type MemorySearchResult, type MemoryStats, + type IngestMediaRequest, + type IngestMediaResult, type ConsolidateRequest, type ConsolidateResult, type BackfillEmbeddingsRequest, @@ -251,6 +253,35 @@ export class MemoryResource { ) } + /** + * Ingest a media item — image, audio, or document — as memories. The engine + * extracts text (vision / transcription / OCR via LiteLLM) and runs it + * through the full observe pipeline, so the result is real memories with + * graph wiring and embeddings, not just a stored file. Requires multimodal + * to be enabled on the engine. + * + * ```ts + * const res = await tf.memory.ingestMedia({ + * media: fs.readFileSync('receipt.png'), + * mimeType: 'image/png', + * source: 'receipt.png', + * }) + * console.log(res.extractedText, res.saved.length) + * ``` + */ + async ingestMedia( + body: IngestMediaRequest, + options?: RequestOptions, + ): Promise { + const { media, ...rest } = body + const dataBase64 = typeof media === 'string' ? media : toBase64(media) + return this.http.post( + '/memory/media', + { ...rest, dataBase64 }, + options, + ) + } + /** * List the current user's memories across all scopes. */ diff --git a/src/types/memory.ts b/src/types/memory.ts index 7cf75b1..c049ecd 100644 --- a/src/types/memory.ts +++ b/src/types/memory.ts @@ -122,6 +122,39 @@ export interface ObserveVoiceRequest audio: Uint8Array | string } +/** + * Ingest a media item (image / audio / document). The engine extracts text via + * LiteLLM, then runs it through the full Observe pipeline — so the returned + * memories get extraction, graph wiring, and embedding like any observation. + * Requires multimodal to be enabled on the engine. + */ +export interface IngestMediaRequest { + /** Bytes — Uint8Array / Buffer — or a pre-encoded base64 string. */ + media: Uint8Array | string + /** Standard mime: `image/png`, `audio/mpeg`, `application/pdf`, `text/*`. */ + mimeType: string + /** Optional attribution + threading (same as observe). */ + userId?: string + agentId?: string + sessionId?: string + /** Optional source (filename / URL), recorded on the memory's provenance. */ + source?: string +} + +/** Result of ingesting one media item. */ +export interface IngestMediaResult { + /** Memories extracted from the media. */ + saved: MemoryItem[] + /** Extraction candidates before dedupe. `saved.length <= candidateCount`. */ + candidateCount: number + /** The text the model extracted from the media. */ + extractedText: string + /** Classified modality: `image` | `audio` | `document`. */ + modality: string + /** Durable location of the retained media (empty if not stored). */ + blobUri: string +} + /** Document-flavored convenience type — uses `document` instead of `image`. */ export interface ObserveDocumentRequest extends Omit {