feat: memory.ingestMedia() — multimodal ingestion (v0.5.0)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@thinkfleet/memory-sdk",
-  "version": "0.4.1",
+  "version": "0.5.0",
   "description": "TypeScript SDK for app.memmesh.ai — admin + project memory CRUD, semantic search, feedback, and Lattice behavioral patterns",
   "type": "module",
   "main": "./dist/index.cjs",

src/index.ts

@@ -170,6 +170,8 @@ export type {
   ObserveAttachmentRequest,
   ObserveVoiceRequest,
   ObserveDocumentRequest,
+  IngestMediaRequest,
+  IngestMediaResult,
   ConsolidateRequest,
   ConsolidateResult,
   ReflectRequest,

src/resources/memory.ts

@@ -12,6 +12,8 @@ import {
   type MemorySearchRequest,
   type MemorySearchResult,
   type MemoryStats,
+  type IngestMediaRequest,
+  type IngestMediaResult,
   type ConsolidateRequest,
   type ConsolidateResult,
   type BackfillEmbeddingsRequest,
@@ -251,6 +253,35 @@ export class MemoryResource {
     )
   }
 
+  /**
+   * Ingest a media item — image, audio, or document — as memories. The engine
+   * extracts text (vision / transcription / OCR via LiteLLM) and runs it
+   * through the full observe pipeline, so the result is real memories with
+   * graph wiring and embeddings, not just a stored file. Requires multimodal
+   * to be enabled on the engine.
+   *
+   * ```ts
+   * const res = await tf.memory.ingestMedia({
+   *   media: fs.readFileSync('receipt.png'),
+   *   mimeType: 'image/png',
+   *   source: 'receipt.png',
+   * })
+   * console.log(res.extractedText, res.saved.length)
+   * ```
+   */
+  async ingestMedia(
+    body: IngestMediaRequest,
+    options?: RequestOptions,
+  ): Promise<IngestMediaResult> {
+    const { media, ...rest } = body
+    const dataBase64 = typeof media === 'string' ? media : toBase64(media)
+    return this.http.post<IngestMediaResult>(
+      '/memory/media',
+      { ...rest, dataBase64 },
+      options,
+    )
+  }
+
   /**
    * List the current user's memories across all scopes.
    */

src/types/memory.ts

@@ -122,6 +122,39 @@ export interface ObserveVoiceRequest
   audio: Uint8Array | string
 }
 
+/**
+ * Ingest a media item (image / audio / document). The engine extracts text via
+ * LiteLLM, then runs it through the full Observe pipeline — so the returned
+ * memories get extraction, graph wiring, and embedding like any observation.
+ * Requires multimodal to be enabled on the engine.
+ */
+export interface IngestMediaRequest {
+  /** Bytes — Uint8Array / Buffer — or a pre-encoded base64 string. */
+  media: Uint8Array | string
+  /** Standard mime: `image/png`, `audio/mpeg`, `application/pdf`, `text/*`. */
+  mimeType: string
+  /** Optional attribution + threading (same as observe). */
+  userId?: string
+  agentId?: string
+  sessionId?: string
+  /** Optional source (filename / URL), recorded on the memory's provenance. */
+  source?: string
+}
+
+/** Result of ingesting one media item. */
+export interface IngestMediaResult {
+  /** Memories extracted from the media. */
+  saved: MemoryItem[]
+  /** Extraction candidates before dedupe. `saved.length <= candidateCount`. */
+  candidateCount: number
+  /** The text the model extracted from the media. */
+  extractedText: string
+  /** Classified modality: `image` | `audio` | `document`. */
+  modality: string
+  /** Durable location of the retained media (empty if not stored). */
+  blobUri: string
+}
+
 /** Document-flavored convenience type — uses `document` instead of `image`. */
 export interface ObserveDocumentRequest
   extends Omit<ObserveAttachmentRequest, 'image'> {