From 6a5e1022f350afa3b6ca480740ce2876da2915de Mon Sep 17 00:00:00 2001
From: rrader2890 <ryan@scheduleventures.com>
Date: Wed, 8 Jul 2026 11:00:27 -0400
Subject: [PATCH] feat: memory.ingestMedia() for multimodal ingestion (v0.5.0)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds ingestMedia(media, mimeType, ...) — POSTs base64 media to
/memory/media, which the shell proxies to the engine's IngestMedia RPC
(LiteLLM extraction -> Observe pipeline). Returns the created memories +
extracted text + modality + blob uri.

- src/types/memory.ts: IngestMediaRequest/IngestMediaResult.
- src/resources/memory.ts: MemoryResource.ingestMedia (reuses toBase64).
- src/index.ts: export the new types.
- bump 0.4.1 -> 0.5.0.

typecheck + build clean; proto-drift check passes (ingestMedia covered).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 package.json            |  2 +-
 src/index.ts            |  2 ++
 src/resources/memory.ts | 31 +++++++++++++++++++++++++++++++
 src/types/memory.ts     | 33 +++++++++++++++++++++++++++++++++
 4 files changed, 67 insertions(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 0f5dcf3..d3b028a 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@thinkfleet/memory-sdk",
-  "version": "0.4.1",
+  "version": "0.5.0",
   "description": "TypeScript SDK for app.memmesh.ai — admin + project memory CRUD, semantic search, feedback, and Lattice behavioral patterns",
   "type": "module",
   "main": "./dist/index.cjs",
diff --git a/src/index.ts b/src/index.ts
index 9b53130..fba1f61 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -170,6 +170,8 @@ export type {
   ObserveAttachmentRequest,
   ObserveVoiceRequest,
   ObserveDocumentRequest,
+  IngestMediaRequest,
+  IngestMediaResult,
   ConsolidateRequest,
   ConsolidateResult,
   ReflectRequest,
diff --git a/src/resources/memory.ts b/src/resources/memory.ts
index 2fb5143..e78fd53 100644
--- a/src/resources/memory.ts
+++ b/src/resources/memory.ts
@@ -12,6 +12,8 @@ import {
   type MemorySearchRequest,
   type MemorySearchResult,
   type MemoryStats,
+  type IngestMediaRequest,
+  type IngestMediaResult,
   type ConsolidateRequest,
   type ConsolidateResult,
   type BackfillEmbeddingsRequest,
@@ -251,6 +253,35 @@ export class MemoryResource {
     )
   }
 
+  /**
+   * Ingest a media item — image, audio, or document — as memories. The engine
+   * extracts text (vision / transcription / OCR via LiteLLM) and runs it
+   * through the full observe pipeline, so the result is real memories with
+   * graph wiring and embeddings, not just a stored file. Requires multimodal
+   * to be enabled on the engine.
+   *
+   * ```ts
+   * const res = await tf.memory.ingestMedia({
+   *   media: fs.readFileSync('receipt.png'),
+   *   mimeType: 'image/png',
+   *   source: 'receipt.png',
+   * })
+   * console.log(res.extractedText, res.saved.length)
+   * ```
+   */
+  async ingestMedia(
+    body: IngestMediaRequest,
+    options?: RequestOptions,
+  ): Promise<IngestMediaResult> {
+    const { media, ...rest } = body
+    const dataBase64 = typeof media === 'string' ? media : toBase64(media)
+    return this.http.post<IngestMediaResult>(
+      '/memory/media',
+      { ...rest, dataBase64 },
+      options,
+    )
+  }
+
   /**
    * List the current user's memories across all scopes.
    */
diff --git a/src/types/memory.ts b/src/types/memory.ts
index 7cf75b1..c049ecd 100644
--- a/src/types/memory.ts
+++ b/src/types/memory.ts
@@ -122,6 +122,39 @@ export interface ObserveVoiceRequest
   audio: Uint8Array | string
 }
 
+/**
+ * Ingest a media item (image / audio / document). The engine extracts text via
+ * LiteLLM, then runs it through the full Observe pipeline — so the returned
+ * memories get extraction, graph wiring, and embedding like any observation.
+ * Requires multimodal to be enabled on the engine.
+ */
+export interface IngestMediaRequest {
+  /** Bytes — Uint8Array / Buffer — or a pre-encoded base64 string. */
+  media: Uint8Array | string
+  /** Standard mime: `image/png`, `audio/mpeg`, `application/pdf`, `text/*`. */
+  mimeType: string
+  /** Optional attribution + threading (same as observe). */
+  userId?: string
+  agentId?: string
+  sessionId?: string
+  /** Optional source (filename / URL), recorded on the memory's provenance. */
+  source?: string
+}
+
+/** Result of ingesting one media item. */
+export interface IngestMediaResult {
+  /** Memories extracted from the media. */
+  saved: MemoryItem[]
+  /** Extraction candidates before dedupe. `saved.length <= candidateCount`. */
+  candidateCount: number
+  /** The text the model extracted from the media. */
+  extractedText: string
+  /** Classified modality: `image` | `audio` | `document`. */
+  modality: string
+  /** Durable location of the retained media (empty if not stored). */
+  blobUri: string
+}
+
 /** Document-flavored convenience type — uses `document` instead of `image`. */
 export interface ObserveDocumentRequest
   extends Omit<ObserveAttachmentRequest, 'image'> {