diff --git a/src/responses/state.ts b/src/responses/state.ts index 2e7c2abb6..d2028ffb4 100644 --- a/src/responses/state.ts +++ b/src/responses/state.ts @@ -724,6 +724,45 @@ function inputItems(input: unknown): unknown[] { return [input]; } +/** + * Canonical identity used by replay-overlap detection. Volatile fields that differ between a + * stored response item and the client's later input resend (`id`, `status`, sequence numbers) + * are ignored; the remaining shape is what identifies "the same history item". + */ +function canonicalReplayValue(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalReplayValue); + if (value && typeof value === "object") { + // Null prototype so an own JSON `__proto__` key survives as a serializable property + // instead of being treated as a prototype assignment. + const out: Record = Object.create(null); + for (const key of Object.keys(value as Record).sort()) { + out[key] = canonicalReplayValue((value as Record)[key]); + } + return out; + } + return value; +} + +function canonicalReplayItemKey(item: unknown): string | undefined { + if (!item || typeof item !== "object" || Array.isArray(item)) return undefined; + const { id: _id, status: _status, sequence_number: _sequenceNumber, ...rest } = item as Record; + // Sort every retained key (including nested objects and arrays) so equivalent items + // produce the same canonical string regardless of the original property order. + return JSON.stringify(canonicalReplayValue(rest)); +} + +/** Longest leading run of stored history items already present at the start of the request input. */ +function replayedPrefixOverlap(stored: unknown[], requestInput: unknown[]): number { + let n = 0; + while (n < stored.length && n < requestInput.length) { + const left = canonicalReplayItemKey(stored[n]); + const right = canonicalReplayItemKey(requestInput[n]); + if (left === undefined || left !== right) break; + n++; + } + return n; +} + function pruneResponses(at = now()): void { for (const [id, state] of states) { if (at - state.createdAt > RESPONSE_TTL_MS) deleteEntry(id); @@ -840,6 +879,13 @@ function materializeEntry( return { ok: true, state }; } +/** + * Expand a chained /v1/responses request's `previous_response_id` into the full stored + * history when the request carries only a delta, and never duplicate history the request + * already carries (stateless upstreams force full-body resends). Returns a new body so + * callers can tell expansion happened; an overlap-only request keeps its own input + * untouched and marks the leading stored-length items as the replay prefix. + */ export function expandPreviousResponseInput(body: unknown): unknown { if (!body || typeof body !== "object" || Array.isArray(body)) return body; const request = body as Record; @@ -854,11 +900,29 @@ export function expandPreviousResponseInput(body: unknown): unknown { replayFailures.set(request, materialized.failure); return body; } + const storedItems = materialized.state.items; + const requestItems = inputItems(request.input); + // A chained turn may already carry the full conversation (stateless upstreams such as + // DeepSeek force the client to resend it every turn). Prepending the stored history to a + // full-body request duplicates it, and remembering that duplicated body makes the bloat + // sticky across turns: 1x -> 2x -> 3x -> ... (observed 1,333,682 input tokens on + // 2026-08-10, ~10x the real ~127k conversation). Detect the overlap: ONLY a complete + // canonical stored-prefix overlap keeps the request untouched. Request length is not proof + // of a full resend — a genuine delta can be as long as the stored history, and returning it + // unchanged would drop the required prefix. Delta turns prepend the stored history and + // append the ENTIRE request input: request items are never dropped, because a repeated + // `context_compaction` marker or an identical message is a new occurrence that must survive. + const overlap = replayedPrefixOverlap(storedItems, requestItems); + if (overlap >= storedItems.length) { + const full = { ...request }; + replayedInputPrefixLengths.set(full, Math.min(storedItems.length, requestItems.length)); + return full; + } const expanded = { ...request, - input: [...materialized.state.items, ...inputItems(request.input)], + input: [...storedItems, ...requestItems], }; - replayedInputPrefixLengths.set(expanded, materialized.state.items.length); + replayedInputPrefixLengths.set(expanded, storedItems.length); return expanded; } diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 3b8904cd0..29552a9c2 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -41,6 +41,7 @@ import { import { isInjectionDebugEnabled } from "../../lib/debug-settings"; import { injectionDebugLog } from "../../lib/injection-debug-log"; import { resolveClientRetryAfter } from "../../lib/retry-after"; +import { estimateTokens } from "../../lib/token-estimate"; import { enrichOpenCodeZenRateLimitMessage } from "../../providers/opencode-zen-rate-limit"; import { modelInList, namespacedToolName } from "../../types"; import type { AdapterEvent, OcxConfig, OcxParsedRequest, OcxProviderConfig, OcxProviderContinuationState, OcxUsage } from "../../types"; @@ -1621,6 +1622,38 @@ async function handleResponsesInner( ); } + // Input-size guard: refuse to forward an input that exceeds the model's advertised context + // window. The client compacts well before this limit, so an oversized body means abnormal + // duplication (observed: a 4x replay expansion pushed a ~400k-token conversation to 1.6M). + // Forwarding it on Windows balloons bun RSS and can native-crash the whole proxy (upstream + // Bun memory bug, issue #314), taking every active thread down at once. Fail one request + // cleanly instead. Reuse the model/CJK-aware estimate that already drives usage and compact + // decisions; summing parts avoids materializing another copy of a multi-megabyte request. + const advertisedWindow = route.provider.modelContextWindows?.[route.modelId]; + if (typeof advertisedWindow === "number" && advertisedWindow > 0) { + let estimatedInputTokens = 0; + for (const msg of parsed.context.messages) { + const content = msg.content; + if (typeof content === "string") { + estimatedInputTokens += estimateTokens(content, route.modelId); + } else if (Array.isArray(content)) { + for (const part of content) { + if (part && typeof part === "object" && typeof (part as { text?: unknown }).text === "string") { + estimatedInputTokens += estimateTokens((part as { text: string }).text, route.modelId); + } + } + } + } + if (estimatedInputTokens > advertisedWindow) { + return formatErrorResponse( + 413, + "request_too_large", + `input (≈${estimatedInputTokens} tokens) exceeds ${route.modelId} context window (${advertisedWindow} tokens); refusing to forward`, + { code: "input_context_window_exceeded" }, + ); + } + } + // Captured before normalization: whether the CLIENT asked for SSE. The // transport-neutral upstream-streaming policy below may force a bounded JSON // upstream for reliability (#875); the answer must then be reframed to SSE diff --git a/tests/responses-input-guard.test.ts b/tests/responses-input-guard.test.ts new file mode 100644 index 000000000..bf93ea106 --- /dev/null +++ b/tests/responses-input-guard.test.ts @@ -0,0 +1,110 @@ +/** + * Regression coverage for the responses input-size guard: a request whose input + * exceeds the model's advertised context window must be rejected with a clean 413 + * instead of being forwarded (forwarding a ~1.6M-token duplication on Windows + * ballooned bun RSS and native-crashed the whole proxy, issue #314). + */ +import { afterEach, beforeEach, describe, expect, setDefaultTimeout, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { handleResponses } from "../src/server/responses"; +import type { OcxConfig } from "../src/types"; +import type { RequestLogContext } from "../src/server/request-log"; + +setDefaultTimeout(30_000); + +const originalFetch = globalThis.fetch; +let testDir: string; +let previousOpencodexHome: string | undefined; +let previousCodexHome: string | undefined; + +beforeEach(() => { + testDir = mkdtempSync(join(tmpdir(), "ocx-input-guard-")); + previousOpencodexHome = process.env.OPENCODEX_HOME; + previousCodexHome = process.env.CODEX_HOME; + process.env.OPENCODEX_HOME = testDir; + process.env.CODEX_HOME = testDir; +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + rmSync(testDir, { recursive: true, force: true }); + if (previousOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousOpencodexHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; +}); + +function deepseekConfig(): OcxConfig { + return { + port: 0, + defaultProvider: "deepseek", + providers: { + deepseek: { + adapter: "openai-responses", + baseUrl: "https://api.deepseek.com", + responsesPath: "/responses", + authMode: "key", + apiKey: "sk-test", + models: ["deepseek-v4-flash"], + modelContextWindows: { "deepseek-v4-flash": 1_000_000 }, + }, + }, + } as OcxConfig; +} + +async function postResponses(config: OcxConfig, body: Record): Promise { + return handleResponses( + new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }), + config, + { model: "", provider: "" } as RequestLogContext, + ); +} + +describe("responses input-size guard", () => { + test("rejects an input above the advertised context window without calling upstream", async () => { + let upstreamCalls = 0; + globalThis.fetch = (async () => { + upstreamCalls += 1; + return Response.json({ + id: "resp_x", + object: "response", + status: "completed", + output: [], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + }); + }) as typeof fetch; + // The shared DeepSeek estimator uses 3.5 chars/token, so this is above the 1M window. + const bigText = "a".repeat(4_200_000); + const res = await postResponses(deepseekConfig(), { + model: "deepseek/deepseek-v4-flash", + input: [{ role: "user", content: [{ type: "input_text", text: bigText }] }], + }); + expect(res.status).toBe(413); + expect(upstreamCalls).toBe(0); + }); + + test("forwards an input within the window", async () => { + let upstreamCalls = 0; + globalThis.fetch = (async () => { + upstreamCalls += 1; + return Response.json({ + id: "resp_x", + object: "response", + status: "completed", + output: [], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + }); + }) as typeof fetch; + const res = await postResponses(deepseekConfig(), { + model: "deepseek/deepseek-v4-flash", + input: [{ role: "user", content: [{ type: "input_text", text: "hello" }] }], + }); + expect(upstreamCalls).toBe(1); + }); +}); diff --git a/tests/responses-replay-overlap.test.ts b/tests/responses-replay-overlap.test.ts new file mode 100644 index 000000000..c2ff7613d --- /dev/null +++ b/tests/responses-replay-overlap.test.ts @@ -0,0 +1,304 @@ +/** + * Regression coverage for previous_response_id expansion overlapping a request + * that already carries the full conversation (stateless upstreams such as + * DeepSeek force the client to resend full history every turn). The old + * unconditional prepend compounded the stored history each turn: 1x -> 2x -> + * 3x -> ... (observed 1,333,682 input tokens, ~10x the real ~127k conversation, + * on 2026-08-10). Full-body chained turns must stay 1x, while genuine delta + * turns must still expand to the stored history plus their delta. + */ +import { afterEach, beforeEach, describe, expect, setDefaultTimeout, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + clearResponseStateForTests, + expandPreviousResponseInput, + rememberResponseState, +} from "../src/responses/state"; +import { handleResponses } from "../src/server/responses"; +import type { OcxConfig } from "../src/types"; +import type { RequestLogContext } from "../src/server/request-log"; + +setDefaultTimeout(30_000); + +const originalFetch = globalThis.fetch; +let testDir: string; +let previousOpencodexHome: string | undefined; +let previousCodexHome: string | undefined; + +beforeEach(() => { + testDir = mkdtempSync(join(tmpdir(), "ocx-replay-overlap-")); + previousOpencodexHome = process.env.OPENCODEX_HOME; + previousCodexHome = process.env.CODEX_HOME; + process.env.OPENCODEX_HOME = testDir; + process.env.CODEX_HOME = testDir; + clearResponseStateForTests(); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + clearResponseStateForTests(); + rmSync(testDir, { recursive: true, force: true }); + if (previousOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousOpencodexHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; +}); + +const userItem = (text: string): Record => ({ + type: "message", + role: "user", + content: [{ type: "input_text", text }], +}); + +const assistantInputItem = (text: string): Record => ({ + type: "message", + role: "assistant", + content: [{ type: "output_text", text }], +}); + +const assistantOutputItem = (text: string): Record => ({ + type: "message", + role: "assistant", + id: `msg_${text}`, + status: "completed", + content: [{ type: "output_text", text }], +}); + +const MODEL = "deepseek/deepseek-v4-flash"; + +describe("previous_response_id replay overlap", () => { + test("full-history chained turns stay 1x across four turns", () => { + const base = Array.from({ length: 20 }, (_, i) => userItem(`base ${i}`)); + const conversation = [...base, userItem("turn 1")]; + let respId = "resp_0"; + rememberResponseState( + { model: MODEL, input: conversation }, + { id: respId, status: "completed", output: [assistantOutputItem("a1")] }, + undefined, + { force: true }, + ); + conversation.push(assistantInputItem("a1")); + + for (let i = 2; i <= 5; i++) { + conversation.push(userItem(`turn ${i}`)); + const next = { model: MODEL, previous_response_id: respId, input: [...conversation] }; + const expanded = expandPreviousResponseInput(next); + expect((expanded.input as unknown[]).length).toBe(conversation.length); + expect(expanded.input).toEqual(conversation); + respId = `resp_${i}`; + rememberResponseState( + expanded, + { id: respId, status: "completed", output: [assistantOutputItem(`a${i}`)] }, + undefined, + { force: true }, + ); + conversation.push(assistantInputItem(`a${i}`)); + } + }); + + test("a genuine delta turn still expands to stored history plus its delta", () => { + const base = Array.from({ length: 20 }, (_, i) => userItem(`base ${i}`)); + rememberResponseState( + { model: MODEL, input: base }, + { id: "resp_delta", status: "completed", output: [assistantOutputItem("a1")] }, + undefined, + { force: true }, + ); + const expanded = expandPreviousResponseInput({ + model: MODEL, + previous_response_id: "resp_delta", + input: [userItem("delta")], + }); + expect((expanded.input as unknown[]).length).toBe(base.length + 2); + expect((expanded.input as unknown[]).slice(0, base.length)).toEqual(base); + expect((expanded.input as unknown[]).at(-1)).toEqual(userItem("delta")); + }); + + test("stored output-shaped items canonical-match the client input resend", () => { + rememberResponseState( + { model: MODEL, input: [userItem("hello")] }, + { id: "resp_shape", status: "completed", output: [assistantOutputItem("hi")] }, + undefined, + { force: true }, + ); + // The client resend carries the assistant reply as an input item without id/status. + const full = [userItem("hello"), assistantInputItem("hi"), userItem("next")]; + const expanded = expandPreviousResponseInput({ + model: MODEL, + previous_response_id: "resp_shape", + input: full, + }); + expect(expanded.input).toEqual(full); + }); + + test("canonical keys ignore retained property order", () => { + const storedItem = { + type: "message", + role: "assistant", + id: "msg_x", + status: "completed", + content: [{ type: "output_text", text: "hi" }], + }; + const resendItem = { + role: "assistant", + content: [{ text: "hi", type: "output_text" }], + type: "message", + }; + rememberResponseState( + { model: MODEL, input: [userItem("hello")] }, + { id: "resp_order", status: "completed", output: [storedItem] }, + undefined, + { force: true }, + ); + const full = [userItem("hello"), resendItem, userItem("next")]; + const expanded = expandPreviousResponseInput({ + model: MODEL, + previous_response_id: "resp_order", + input: full, + }); + expect(expanded.input).toEqual(full); + }); + + test("partial prefix keeps stored history and never drops request items", () => { + const stored = [userItem("u1"), assistantInputItem("a1"), userItem("u2"), assistantInputItem("a2")]; + rememberResponseState( + { model: MODEL, input: stored }, + { id: "resp_partial", status: "completed", output: [] }, + undefined, + { force: true }, + ); + const request = [userItem("u1"), userItem("X"), userItem("D")]; + const expanded = expandPreviousResponseInput({ + model: MODEL, + previous_response_id: "resp_partial", + input: request, + }); + const input = expanded.input as unknown[]; + expect(input.slice(0, stored.length)).toEqual(stored); + // The request's own leading item is a NEW occurrence (it can be a repeated marker or an + // identical message), so it must survive even though it also matches the stored prefix. + expect(input.slice(stored.length)).toEqual([userItem("u1"), userItem("X"), userItem("D")]); + }); + + test("a delta as long as the stored history still expands to stored plus every delta item", () => { + const stored = [userItem("u1"), assistantInputItem("a1"), userItem("u2")]; + rememberResponseState( + { model: MODEL, input: stored }, + { id: "resp_long_delta", status: "completed", output: [] }, + undefined, + { force: true }, + ); + // Four new items >= stored's three: request length must NOT be treated as a full resend. + const request = [userItem("n1"), userItem("n2"), userItem("n3"), userItem("n4")]; + const expanded = expandPreviousResponseInput({ + model: MODEL, + previous_response_id: "resp_long_delta", + input: request, + }); + const input = expanded.input as unknown[]; + expect(input.slice(0, stored.length)).toEqual(stored); + expect(input.slice(stored.length)).toEqual(request); + }); +}); + +function statelessDeepseekConfig(): OcxConfig { + return { + port: 0, + defaultProvider: "deepseek", + providers: { + deepseek: { + adapter: "openai-responses", + baseUrl: "https://api.deepseek.com", + responsesPath: "/responses", + authMode: "key", + apiKey: "sk-test", + models: ["deepseek-v4-flash"], + statelessResponses: true, + modelContextWindows: { "deepseek-v4-flash": 1_000_000 }, + }, + }, + } as OcxConfig; +} + +async function postResponses(config: OcxConfig, body: Record): Promise { + return handleResponses( + new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }), + config, + { model: "", provider: "" } as RequestLogContext, + ); +} + +describe("stateless DeepSeek end-to-end replay", () => { + test("four full-history chained turns reach upstream at 1x every time", async () => { + const upstreamBodies: unknown[][] = []; + let nextId = 1; + globalThis.fetch = (async (_url: unknown, init?: { body?: string }) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { input?: unknown[] }; + upstreamBodies.push(body.input ?? []); + const n = nextId++; + return Response.json({ + id: `resp_${n}`, + object: "response", + status: "completed", + model: "deepseek-v4-flash", + output: [assistantOutputItem(`a${n}`)], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + }); + }) as typeof fetch; + + const config = statelessDeepseekConfig(); + const conversation = [...Array.from({ length: 20 }, (_, i) => userItem(`base ${i}`)), userItem("turn 1")]; + let respId = ""; + for (let i = 1; i <= 4; i++) { + if (i > 1) conversation.push(userItem(`turn ${i}`)); + const res = await postResponses(config, { + model: MODEL, + ...(respId ? { previous_response_id: respId } : {}), + input: [...conversation], + }); + expect(res.status).toBe(200); + expect(upstreamBodies.at(-1)?.length).toBe(conversation.length); + conversation.push(assistantInputItem(`a${i}`)); + respId = `resp_${i}`; + } + expect(upstreamBodies).toHaveLength(4); + }); + + test("a delta continuation still expands to the full conversation upstream", async () => { + const upstreamBodies: unknown[][] = []; + let nextId = 1; + globalThis.fetch = (async (_url: unknown, init?: { body?: string }) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { input?: unknown[] }; + upstreamBodies.push(body.input ?? []); + const n = nextId++; + return Response.json({ + id: `resp_${n}`, + object: "response", + status: "completed", + model: "deepseek-v4-flash", + output: [assistantOutputItem(`a${n}`)], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + }); + }) as typeof fetch; + + const config = statelessDeepseekConfig(); + const conversation = Array.from({ length: 20 }, (_, i) => userItem(`base ${i}`)); + await postResponses(config, { model: MODEL, input: [...conversation] }); + expect(upstreamBodies[0]!.length).toBe(conversation.length); + conversation.push(assistantInputItem("a1")); + + const res = await postResponses(config, { + model: MODEL, + previous_response_id: "resp_1", + input: [userItem("delta")], + }); + expect(res.status).toBe(200); + expect(upstreamBodies[1]!.length).toBe(conversation.length + 1); + }); +});