From 73406ea26a58412861b96817e99c20aa39b9bca2 Mon Sep 17 00:00:00 2001
From: RissRIce <jsdavid278@gmail.com>
Date: Sun, 9 Aug 2026 18:03:42 -0600
Subject: [PATCH] Enforce OpenContext HTTP byte limit while streaming

---
 .../opencontext/src/adapters/http.test.ts     | 33 +++++++
 packages/opencontext/src/adapters/http.ts     | 93 ++++++++++++-------
 2 files changed, 95 insertions(+), 31 deletions(-)
 create mode 100644 packages/opencontext/src/adapters/http.test.ts

diff --git a/packages/opencontext/src/adapters/http.test.ts b/packages/opencontext/src/adapters/http.test.ts
new file mode 100644
index 0000000..8995e75
--- /dev/null
+++ b/packages/opencontext/src/adapters/http.test.ts
@@ -0,0 +1,33 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { httpAdapter } from "./http.js";
+
+afterEach(() => {
+  vi.unstubAllGlobals();
+});
+
+describe("http adapter response limits", () => {
+  it("enforces the byte limit for multibyte responses without a content-length header", async () => {
+    const content = "é".repeat(3 * 1024 * 1024);
+    vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response(content)));
+
+    await expect(
+      httpAdapter.load("https://example.com/context.md", {
+        dir: process.cwd(),
+        offline: false,
+        config: {}
+      })
+    ).rejects.toThrow(/response exceeds the 5242880 byte limit/);
+  });
+
+  it("decodes responses within the byte limit", async () => {
+    vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("café")));
+
+    await expect(
+      httpAdapter.load("https://example.com/context.md", {
+        dir: process.cwd(),
+        offline: false,
+        config: {}
+      })
+    ).resolves.toMatchObject({ content: "café", trust: "untrusted" });
+  });
+});
diff --git a/packages/opencontext/src/adapters/http.ts b/packages/opencontext/src/adapters/http.ts
index e900547..a8e7114 100644
--- a/packages/opencontext/src/adapters/http.ts
+++ b/packages/opencontext/src/adapters/http.ts
@@ -20,6 +20,38 @@ export class OfflineError extends Error {
 const DEFAULT_TIMEOUT_MS = 10_000;
 const MAX_BYTES = 5 * 1024 * 1024;
 
+async function readBody(response: Response, uri: string): Promise<string> {
+  if (!response.body) return "";
+
+  const reader = response.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let totalBytes = 0;
+
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+
+      totalBytes += value.byteLength;
+      if (totalBytes > MAX_BYTES) {
+        await reader.cancel().catch(() => undefined);
+        throw new Error(`Refusing to load ${uri}: response exceeds the ${MAX_BYTES} byte limit.`);
+      }
+      chunks.push(value);
+    }
+  } finally {
+    reader.releaseLock();
+  }
+
+  const bytes = new Uint8Array(totalBytes);
+  let offset = 0;
+  for (const chunk of chunks) {
+    bytes.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return new TextDecoder().decode(bytes);
+}
+
 export const httpAdapter: Adapter = {
   name: "http",
   schemes: ["http", "https"],
@@ -39,41 +71,40 @@ export const httpAdapter: Adapter = {
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), timeoutMs);
 
-    let response: Response;
     try {
-      response = await fetch(url, {
-        signal: controller.signal,
-        // Redirects can move a request to a host the author never named, so the
-        // final URL is reported back rather than followed silently.
-        redirect: "follow",
-        headers: { accept: "text/markdown, text/plain, application/json;q=0.9, */*;q=0.8" }
-      });
-    } catch (error) {
-      throw new Error(`Failed to fetch ${uri}: ${(error as Error).message}`);
-    } finally {
-      clearTimeout(timer);
-    }
+      let response: Response;
+      try {
+        response = await fetch(url, {
+          signal: controller.signal,
+          // Redirects can move a request to a host the author never named, so the
+          // final URL is reported back rather than followed silently.
+          redirect: "follow",
+          headers: { accept: "text/markdown, text/plain, application/json;q=0.9, */*;q=0.8" }
+        });
+      } catch (error) {
+        throw new Error(`Failed to fetch ${uri}: ${(error as Error).message}`);
+      }
 
-    if (!response.ok) {
-      throw new Error(`Failed to fetch ${uri}: HTTP ${response.status} ${response.statusText}`);
-    }
+      if (!response.ok) {
+        throw new Error(`Failed to fetch ${uri}: HTTP ${response.status} ${response.statusText}`);
+      }
 
-    const declaredLength = Number(response.headers.get("content-length") ?? "0");
-    if (declaredLength > MAX_BYTES) {
-      throw new Error(`Refusing to load ${uri}: ${declaredLength} bytes exceeds the ${MAX_BYTES} byte limit.`);
-    }
+      const declaredLength = Number(response.headers.get("content-length") ?? "0");
+      if (declaredLength > MAX_BYTES) {
+        throw new Error(`Refusing to load ${uri}: ${declaredLength} bytes exceeds the ${MAX_BYTES} byte limit.`);
+      }
 
-    const content = await response.text();
-    if (content.length > MAX_BYTES) {
-      throw new Error(`Refusing to load ${uri}: response exceeds the ${MAX_BYTES} byte limit.`);
-    }
+      const content = await readBody(response, uri);
 
-    return {
-      content,
-      contentType: (response.headers.get("content-type") ?? "text/plain").split(";")[0]!.trim(),
-      digest: sha256Uri(content),
-      retrievedAt: new Date().toISOString(),
-      trust: (ctx.config.trust as AdapterResult["trust"]) ?? "untrusted"
-    };
+      return {
+        content,
+        contentType: (response.headers.get("content-type") ?? "text/plain").split(";")[0]!.trim(),
+        digest: sha256Uri(content),
+        retrievedAt: new Date().toISOString(),
+        trust: (ctx.config.trust as AdapterResult["trust"]) ?? "untrusted"
+      };
+    } finally {
+      clearTimeout(timer);
+    }
   }
 };