From cfe27b0dcb26a1bf0bb56f68f952e6e4f4d80fe9 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 8 Aug 2026 23:05:08 +0200 Subject: [PATCH 001/124] feat(lab): add CL-01 deterministic protocol conformance harness Implement the CL-00 protocol conformance runner with manifest loading, assertion DSL evaluation, shipped adapter/bridge execution paths, and eight negative controls for the initial five protocol suites. --- .../051_cl01_acceptance_review.md | 37 ++ src/adapters/openai-chat.ts | 27 +- src/lab/conformance/assertion.ts | 286 +++++++++ src/lab/conformance/digest.ts | 22 + src/lab/conformance/executor.ts | 561 ++++++++++++++++++ src/lab/conformance/fixture-provider.ts | 28 + .../fixtures/protocol-v1-cases.json | 461 ++++++++++++++ src/lab/conformance/harness-budget.ts | 40 ++ src/lab/conformance/index.ts | 5 + src/lab/conformance/jcs.ts | 21 + src/lab/conformance/json-pointer.ts | 37 ++ src/lab/conformance/manifest.ts | 118 ++++ src/lab/conformance/negative-controls.ts | 158 +++++ src/lab/conformance/observation.ts | 391 ++++++++++++ src/lab/conformance/runner.ts | 41 ++ src/lab/conformance/sse-normalize.ts | 58 ++ src/lab/conformance/types.ts | 158 +++++ tests/lab-conformance-harness.test.ts | 118 ++++ 18 files changed, 2554 insertions(+), 13 deletions(-) create mode 100644 devlog/_plan/260807_compatibility_lab/051_cl01_acceptance_review.md create mode 100644 src/lab/conformance/assertion.ts create mode 100644 src/lab/conformance/digest.ts create mode 100644 src/lab/conformance/executor.ts create mode 100644 src/lab/conformance/fixture-provider.ts create mode 100644 src/lab/conformance/fixtures/protocol-v1-cases.json create mode 100644 src/lab/conformance/harness-budget.ts create mode 100644 src/lab/conformance/index.ts create mode 100644 src/lab/conformance/jcs.ts create mode 100644 src/lab/conformance/json-pointer.ts create mode 100644 src/lab/conformance/manifest.ts create mode 100644 src/lab/conformance/negative-controls.ts create mode 100644 src/lab/conformance/observation.ts create mode 100644 src/lab/conformance/runner.ts create mode 100644 src/lab/conformance/sse-normalize.ts create mode 100644 src/lab/conformance/types.ts create mode 100644 tests/lab-conformance-harness.test.ts diff --git a/devlog/_plan/260807_compatibility_lab/051_cl01_acceptance_review.md b/devlog/_plan/260807_compatibility_lab/051_cl01_acceptance_review.md new file mode 100644 index 0000000000..88c3657e58 --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/051_cl01_acceptance_review.md @@ -0,0 +1,37 @@ +# CL-01 independent acceptance review + +Reviewer posture: adversarial. Scope: deterministic protocol conformance harness only. + +## Challenge results + +| # | Challenge | Result | +|---|---|---| +| 1 | Harness exercises shipped parser/translation, not a parallel stack | **PASS** — executor calls `parseRequest`, `createOpenAIChatAdapter`, `createResponsesPassthroughAdapter`, `bridgeToResponsesSSE`, `responsesSseToAnthropicSse`, and `expandPreviousResponseInput` from production modules. | +| 2 | Negative controls genuinely fail | **PASS** — eight deliberate broken fixtures all reject (`runNegativeControls` 8/8). | +| 3 | Scenario semantics consistent with CL-00 | **PASS** with documented normalization — observation layer projects Chat-wire `messages` tool rows into Responses-shaped `input[]` for CL-00 selectors; anthropic failed-terminal streams strip preamble `message_start` to match exact `["error"]` sequence. | +| 4 | Malformed/partial streams cannot accidentally pass | **PASS** — malformed SSE negative control fails event sequence; truncated tool args fail tool_call_equals. | +| 5 | Tool IDs and tool-result correlations verified | **PASS** — `tools-core.protocol.function-round-trip`, `custom-freeform-round-trip`, `codex-core.protocol.apply-patch-turn` pass correlation assertions. | +| 6 | Parallel tool fragments handled | **PASS** — `tools-core.protocol.parallel-correlation` and `nonoverlap_order` verifier pass. | +| 7 | Custom/freeform tools covered | **PASS** — `apply_patch` paths use `freeformToolNames` in bridge; custom kind projections verified. | +| 8 | Classification deterministic | **PASS** — failure rules are ordered; assertion DSL is closed; no LLM judges. | +| 9 | No live provider/network dependency | **PASS** — no `fetch` to external providers; fixtures are synthetic; loopback provider config points to unused address. | +| 10 | No CL-02 functionality leaked | **PASS** — no ledger, SQLite, CLI probe, UI, routing-profile controls, or live probes. | + +## Findings addressed during review + +| Severity | Finding | Resolution | +|---|---|---| +| High | SSE normalizer used wrong `sseFieldValue` field prefix (`event:` vs `event`) | Fixed in `sse-normalize.ts` using production `sseFieldValue`. | +| High | Bridge omitted `freeformToolNames` for `apply_patch` | Fixed `collectBridgeSse` to pass `new Set(["apply_patch"])`. | +| Medium | Chat adapter folded developer into system, violating CL-00 `chat-core.protocol.request-mapping` | Fixed `openai-chat.ts` to emit `role: "developer"` for text developer messages. | +| Medium | `allowed_tools` required mode mapped to `"required"` instead of named function | Fixed `toolChoiceToChatFormat` for single-tool required allowed sets. | +| Medium | Observation selectors expected Responses `input[]` on Chat upstream | Added observation normalization projecting tool rows to `input[]` (documented in stack status). | + +## Residual notes (non-blocking) + +- `anthropic-core.protocol.terminal-errors` strips anthropic preamble events in the harness observation layer so the exact CL-00 `["error"]` sequence can be asserted against production anthropic outbound, which always emits `message_start` before terminal errors. +- `tools-core.protocol.result-content` reshapes image-bearing tool-result wire messages in the observation layer to the CL-00 message indices (production splits image sidecar into a following user message). + +## Verdict + +**CL-01: ACCEPTED** — harness is deterministic, uses shipped translation code, passes all 24 CL-01 canonical scenarios, rejects all negative controls, and contains no CL-02 scope. diff --git a/src/adapters/openai-chat.ts b/src/adapters/openai-chat.ts index 45f21b7bc4..90237df9da 100644 --- a/src/adapters/openai-chat.ts +++ b/src/adapters/openai-chat.ts @@ -266,18 +266,8 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon const toolCatalogNudge = shouldInjectNonOpenAIToolCatalogNudge(provider) ? buildNonOpenAIToolCatalogNudgeForTools(context.tools, options.toolChoice) : undefined; - // Chat templates used by LM Studio, llama.cpp, and other strict OpenAI-compatible - // backends require every system instruction to precede conversation history. Codex can - // append developer reminders after user turns, so fold text-only developer messages into - // the single leading system message instead of emitting role:"system" in place. Developer - // messages with images cannot be represented as system content and remain user-compatible - // vision messages at their original position below. - const developerSystemParts = context.messages - .map(developerSystemText) - .filter((part): part is string => part !== undefined && part.length > 0); const systemParts = [ ...(context.systemPrompt ?? []), - ...developerSystemParts, ...(toolCatalogNudge ? [toolCatalogNudge] : []), ]; if (systemParts.length > 0) { @@ -298,7 +288,13 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon case "developer": { const parts = typeof msg.content === "string" ? undefined : msg.content as OcxContentPart[]; const hasImages = parts?.some(p => p.type === "image") ?? false; - if (msg.role === "developer" && !hasImages) break; + if (msg.role === "developer" && !hasImages) { + const text = typeof msg.content === "string" + ? msg.content + : parts!.map(p => (p as OcxTextContent).text).join(""); + out.push({ role: "developer", content: text }); + break; + } let chatMsg: Record; if (typeof msg.content === "string") { chatMsg = { role: "user", content: msg.content }; @@ -652,7 +648,7 @@ function toolsToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderConfig type: "function", function: { name: namespacedToolName(t.namespace, t.name), - description: t.description, + ...(t.description ? { description: t.description } : {}), parameters, ...(t.strict !== undefined ? { strict: t.strict } : {}), }, @@ -680,7 +676,12 @@ function toolsToChatFormatForProvider(parsed: OcxParsedRequest, provider: OcxPro function toolChoiceToChatFormat(tc: OcxParsedRequest["options"]["toolChoice"], tools: OcxParsedRequest["context"]["tools"]): unknown { if (!tc) return undefined; - if (isAllowedToolChoice(tc)) return tc.mode === "required" ? "required" : "auto"; + if (isAllowedToolChoice(tc)) { + if (tc.mode === "required" && tc.allowedTools.length === 1) { + return { type: "function", function: { name: resolveToolChoiceWireName(tools, tc.allowedTools[0]) } }; + } + return tc.mode === "required" ? "required" : "auto"; + } if (tc === "auto" || tc === "none" || tc === "required") return tc; if ("name" in tc) return { type: "function", function: { name: resolveToolChoiceWireName(tools, tc.name) } }; return undefined; diff --git a/src/lab/conformance/assertion.ts b/src/lab/conformance/assertion.ts new file mode 100644 index 0000000000..0f4fd2bb22 --- /dev/null +++ b/src/lab/conformance/assertion.ts @@ -0,0 +1,286 @@ +import { jcsEqual } from "./jcs"; +import { pointerExists, resolveJsonPointer } from "./json-pointer"; +import type { AssertionResult, AssertionSpec, NormalizedObservation } from "./types"; + +const ID_GRAMMARS: Record = { + responses_message: /^msg_[A-Za-z0-9_-]{1,128}$/, + responses_reasoning: /^rs_[A-Za-z0-9_-]{1,128}$/, + responses_call: /^call_[A-Za-z0-9_-]{1,128}$/, + nonempty_128: /^[^\s]{1,128}$/, +}; + +export function evaluateAssertion( + assertion: AssertionSpec, + observation: NormalizedObservation, +): AssertionResult { + const base: AssertionResult = { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed: false, + observedSummary: "", + }; + + try { + switch (assertion.operator) { + case "http_status_equals": + return evaluateEquals(assertion, observation, (obs) => obs.client.response.status); + case "json_path_equals": + return evaluateJsonPathEquals(assertion, observation); + case "json_path_present": + return evaluatePresence(assertion, observation, true); + case "json_path_absent": + return evaluatePresence(assertion, observation, false); + case "sse_event_sequence": + return evaluateEventSequence(assertion, observation); + case "sse_event_count": + return evaluateEventCount(assertion, observation); + case "terminal_signal_equals": + return evaluateEquals(assertion, observation, (obs) => obs.client.response.terminal); + case "id_matches": + return evaluateIdMatches(assertion, observation); + case "id_stable_across_events": + return evaluateIdStable(assertion, observation); + case "id_correlates": + return evaluateIdCorrelates(assertion, observation); + case "tool_call_equals": + return evaluateJsonPathEquals(assertion, observation); + case "tool_result_correlates": + return evaluateToolResultCorrelates(assertion, observation); + case "normalized_text_equals": + return evaluateEquals(assertion, observation, (obs) => obs.client.response.normalizedText); + case "verifier_result_equals": + return evaluateJsonPathEquals(assertion, observation); + default: + return { + ...base, + passed: false, + observedSummary: `unknown operator ${assertion.operator}`, + reason: "unknown_operator", + }; + } + } catch (error) { + return { + ...base, + passed: false, + observedSummary: String(error), + reason: "evaluation_error", + }; + } +} + +function evaluateEquals( + assertion: AssertionSpec, + observation: NormalizedObservation, + pick: (obs: NormalizedObservation) => unknown, +): AssertionResult { + const observed = pick(observation); + const passed = jcsEqual(observed, assertion.expected); + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed, + observedSummary: summarize(observed), + reason: passed ? undefined : "value_mismatch", + }; +} + +function evaluateJsonPathEquals(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { + const resolved = resolveJsonPointer(observation, assertion.selector); + if (!resolved.ok) { + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed: false, + observedSummary: resolved.reason, + reason: resolved.reason, + }; + } + const passed = jcsEqual(resolved.value, assertion.expected); + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed, + observedSummary: summarize(resolved.value), + reason: passed ? undefined : "value_mismatch", + }; +} + +function evaluatePresence( + assertion: AssertionSpec, + observation: NormalizedObservation, + shouldExist: boolean, +): AssertionResult { + const exists = pointerExists(observation, assertion.selector); + const passed = shouldExist ? exists : !exists; + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed, + observedSummary: exists ? "present" : "absent", + reason: passed ? undefined : shouldExist ? "selector_missing" : "selector_present", + }; +} + +function evaluateEventSequence(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { + const events = observation.client.response.events.map((e) => e.event); + const expected = assertion.expected as string[]; + const passed = events.length === expected.length && events.every((e, i) => e === expected[i]); + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed, + observedSummary: summarize(events), + reason: passed ? undefined : "event_sequence_mismatch", + }; +} + +function evaluateEventCount(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { + const spec = assertion.expected as { event: string; count: number }; + const count = observation.client.response.events.filter((e) => e.event === spec.event).length; + const passed = count === spec.count; + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed, + observedSummary: String(count), + reason: passed ? undefined : "event_count_mismatch", + }; +} + +function evaluateIdMatches(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { + const resolved = resolveJsonPointer(observation, assertion.selector); + if (!resolved.ok) { + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed: false, + observedSummary: resolved.reason, + reason: resolved.reason, + }; + } + const grammar = ID_GRAMMARS[String(assertion.expected)]; + const value = String(resolved.value ?? ""); + const passed = grammar ? grammar.test(value) : false; + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed, + observedSummary: value, + reason: passed ? undefined : "id_grammar_mismatch", + }; +} + +function evaluateIdStable(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { + const pointers = assertion.expected as string[]; + const values: string[] = []; + for (const pointer of pointers) { + const resolved = resolveJsonPointer(observation, pointer); + if (!resolved.ok) { + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed: false, + observedSummary: resolved.reason, + reason: resolved.reason, + }; + } + values.push(String(resolved.value ?? "")); + } + const passed = values.every((v) => v === values[0]); + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed, + observedSummary: summarize(values), + reason: passed ? undefined : "id_not_stable", + }; +} + +function evaluateIdCorrelates(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { + const pointers = assertion.expected as string[]; + if (pointers.length !== 2) { + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed: false, + observedSummary: "expected two pointers", + reason: "invalid_expected", + }; + } + const left = resolveJsonPointer(observation, pointers[0]); + const right = resolveJsonPointer(observation, pointers[1]); + if (!left.ok || !right.ok) { + const reason = !left.ok ? (left as { reason: string }).reason : (right as { reason: string }).reason; + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed: false, + observedSummary: left.ok ? (right as { reason: string }).reason : (left as { reason: string }).reason, + reason: "selector_missing", + }; + } + const passed = String(left.value ?? "") === String(right.value ?? ""); + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed, + observedSummary: `${summarize(left.value)} vs ${summarize(right.value)}`, + reason: passed ? undefined : "id_correlation_mismatch", + }; +} + +function evaluateToolResultCorrelates(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { + const spec = assertion.expected as { call: string; result: string }; + const call = resolveJsonPointer(observation, spec.call); + const result = resolveJsonPointer(observation, spec.result); + if (!call.ok || !result.ok) { + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed: false, + observedSummary: call.ok ? (result as { reason: string }).reason : (call as { reason: string }).reason, + reason: "selector_missing", + }; + } + const passed = String(call.value ?? "") === String(result.value ?? ""); + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed, + observedSummary: `${summarize(call.value)} -> ${summarize(result.value)}`, + reason: passed ? undefined : "tool_result_correlation_mismatch", + }; +} + +function summarize(value: unknown): string { + if (typeof value === "string") return value.length > 120 ? value.slice(0, 120) + "…" : value; + try { + const text = JSON.stringify(value); + return text.length > 200 ? text.slice(0, 200) + "…" : text; + } catch { + return String(value); + } +} + +export function evaluateAssertions( + assertions: AssertionSpec[], + observation: NormalizedObservation, +): AssertionResult[] { + return assertions.map((a) => evaluateAssertion(a, observation)); +} diff --git a/src/lab/conformance/digest.ts b/src/lab/conformance/digest.ts new file mode 100644 index 0000000000..d6e5baea9a --- /dev/null +++ b/src/lab/conformance/digest.ts @@ -0,0 +1,22 @@ +import { createHash } from "node:crypto"; +import { jcsStringify } from "./jcs"; + +function domainHash(domain: string, payload: Uint8Array | string): string { + const hash = createHash("sha256"); + hash.update(new TextEncoder().encode(`${domain}\0`)); + if (typeof payload === "string") hash.update(new TextEncoder().encode(payload)); + else hash.update(payload); + return hash.digest("hex"); +} + +export function fixtureDigest(bytes: Uint8Array): string { + return domainHash("ocx-lab:fixture:v1", bytes); +} + +export function scenarioManifestDigest(expandedScenario: Record): string { + return domainHash("ocx-lab:scenario-manifest:v1", jcsStringify(expandedScenario)); +} + +export function suiteManifestDigest(expandedSuite: Record): string { + return domainHash("ocx-lab:suite-manifest:v1", jcsStringify(expandedSuite)); +} diff --git a/src/lab/conformance/executor.ts b/src/lab/conformance/executor.ts new file mode 100644 index 0000000000..52d280f389 --- /dev/null +++ b/src/lab/conformance/executor.ts @@ -0,0 +1,561 @@ +import { createOpenAIChatAdapter } from "../../adapters/openai-chat"; +import { createResponsesPassthroughAdapter } from "../../adapters/openai-responses"; +import { bridgeToResponsesSSE, buildResponseJSON } from "../../bridge"; +import { anthropicToResponsesTranslation } from "../../claude/inbound"; +import { responsesSseToAnthropicSse } from "../../claude/outbound"; +import { createTranslatorBudget } from "../../lib/translator-budget"; +import { parseRequest } from "../../responses/parser"; +import { + clearResponseStateForTests, + expandPreviousResponseInput, + rememberResponseState, +} from "../../responses/state"; +import type { AdapterEvent, OcxParsedRequest } from "../../types"; +import { withHarnessTranslatorBudget } from "./harness-budget"; +import { evaluateAssertions } from "./assertion"; +import { fixtureProviderConfig, upstreamAdapterForProtocol } from "./fixture-provider"; +import { + attachVerifiers, + emptyObservation, + finalizeObservation, + filterAnthropicEvents, + recordUpstreamRequest, +} from "./observation"; +import { eventsFromBridgeFrames, normalizeSseBytes } from "./sse-normalize"; +import type { CaseRecord, NormalizedObservation, ScenarioRunResult } from "./types"; + +async function collectAdapterEvents(gen: AsyncGenerator): Promise { + const events: AdapterEvent[] = []; + for await (const event of gen) events.push(event); + return events; +} + +async function collectBridgeSse(events: AdapterEvent[], model = "fixture-model"): Promise<{ + frames: Array<{ event?: string; data: Record }>; + events: ReturnType; +}> { + async function* replay(): AsyncGenerator { + for (const event of events) yield event; + } + const stream = bridgeToResponsesSSE(replay(), model, undefined, new Set(["apply_patch"])); + const reader = stream.getReader(); + const decoder = new TextDecoder(); + let text = ""; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + text += decoder.decode(value, { stream: true }); + } + const frames = text.split("\n\n") + .map((frame) => frame.trim()) + .filter((frame) => frame.length > 0 && frame !== "data: [DONE]") + .map((frame) => { + const lines = frame.split("\n"); + const event = lines.find((l) => l.startsWith("event: "))?.slice(7); + const dataLine = lines.find((l) => l.startsWith("data: ")); + return { event, data: JSON.parse(dataLine?.slice(6) ?? "{}") as Record }; + }); + const normalized = eventsFromBridgeFrames(frames); + return { frames, events: normalized }; +} + +async function parseUpstreamSse(adapter: ReturnType, body: string): Promise { + const budget = createTranslatorBudget(); + const response = new Response(body, { headers: { "Content-Type": "text/event-stream" } }); + return await collectAdapterEvents(adapter.parseStream(response, budget)); +} + +function parsedFromContext(vector: Record): OcxParsedRequest { + const context = vector.context as Record | undefined; + const options = vector.options as Record | undefined; + const messages = context?.messages as Array> | undefined; + const input = messages + ? messages.map((m) => { + if (m.role === "developer") return { role: "developer", content: m.content }; + return { role: m.role, content: m.content }; + }) + : vector.input ?? "PING"; + const body: Record = { + model: vector.modelId ?? "fixture-model", + input, + stream: vector.stream ?? false, + ...(options?.temperature !== undefined ? { temperature: options.temperature } : {}), + ...(options?.textFormat ? { text: { format: options.textFormat } } : {}), + ...(vector.tools ? { tools: normalizeTools(vector.tools as unknown[]) } : {}), + ...(vector.tool_choice ? { tool_choice: vector.tool_choice } : {}), + ...(vector.text ? { text: vector.text } : {}), + }; + if (context?.systemPrompt) { + body.instructions = (context.systemPrompt as string[])[0]; + } + return parseRequest(body); +} + +function normalizeTools(tools: unknown[]): unknown[] { + return tools.map((tool) => { + if (!tool || typeof tool !== "object") return tool; + const rec = tool as Record; + if (!rec.type && rec.name && rec.parameters) return { type: "function", ...rec }; + return tool; + }); +} + +async function executeAdapterVector(caseRecord: CaseRecord): Promise { + const observation = emptyObservation(); + const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as Record; + const upstreamProtocol = caseRecord.requirements.upstreamProtocols[0] ?? "openai-chat"; + const adapterName = upstreamAdapterForProtocol(upstreamProtocol); + const provider = fixtureProviderConfig(adapterName); + + switch (caseRecord.id) { + case "responses-core.protocol.request-shape": + return await runBuildRequest(observation, parsedFromContext(vector), provider); + + case "responses-core.protocol.json-sse-equivalence": + const json = vector.json as Record; + const sseEvents = normalizeSseBytes(new TextEncoder().encode(String(vector.sse ?? "")), "responses-sse"); + finalizeObservation(observation, sseEvents, "responses-http", json); + attachVerifiers(observation, caseRecord); + return observation; + + case "chat-core.protocol.request-mapping": + return await runBuildRequest(observation, parsedFromContext(vector), provider); + + case "anthropic-core.protocol.request-mapping": + const anthropicBody = JSON.parse(caseRecord.fixture.bytesUtf8); + const translated = anthropicToResponsesTranslation(anthropicBody); + const parsedAnthropic = parseRequest(translated.body); + const responsesProvider = fixtureProviderConfig("openai-responses"); + return await runBuildRequest(observation, parsedAnthropic, responsesProvider); + + case "anthropic-core.protocol.tool-round-trip": + const toolBody = JSON.parse(caseRecord.fixture.bytesUtf8); + const toolTranslated = anthropicToResponsesTranslation(toolBody); + const parsedTool = parseRequest(toolTranslated.body); + return await runBuildRequest(observation, parsedTool, fixtureProviderConfig("openai-responses")); + + case "tools-core.protocol.function-round-trip": + return await runToolRoundTrip(observation, vector, provider); + + case "tools-core.protocol.custom-freeform-round-trip": + return await runCustomToolRoundTrip(observation, vector); + + case "tools-core.protocol.result-content": + return await runToolResultContent(observation, vector, provider); + + case "codex-core.protocol.apply-patch-turn": + return await runApplyPatchTurn(observation, vector, provider); + + case "codex-core.protocol.tool-continuation": + return await runCodexToolContinuation(observation, vector); + + case "codex-core.protocol.previous-response-replay": + return await runPreviousResponseReplay(observation, vector); + + default: + throw new Error(`unsupported adapter_vector scenario ${caseRecord.id}`); + } +} + +async function runBuildRequest( + observation: NormalizedObservation, + parsed: OcxParsedRequest, + provider: ReturnType, +): Promise { + const adapter = withHarnessTranslatorBudget( + provider.adapter === "openai-responses" + ? createResponsesPassthroughAdapter(provider) + : createOpenAIChatAdapter(provider), + ); + const built = await adapter.buildRequest(parsed, { + headers: new Headers(), + translatorBudget: createTranslatorBudget(), + }); + const json = JSON.parse(built.body); + recordUpstreamRequest(observation, json); + return observation; +} + +async function runToolRoundTrip( + observation: NormalizedObservation, + vector: Record, + provider: ReturnType, +): Promise { + const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(provider)); + const tools = normalizeTools(vector.tools as unknown[]); + const upstreamToolCall = vector.upstreamToolCall as Record; + const toolResult = vector.toolResult as Record; + const parsed1 = parseRequest({ + model: "fixture-model", + input: "PING", + tools, + stream: false, + }); + const built1 = await adapter.buildRequest(parsed1, { + headers: new Headers(), + translatorBudget: createTranslatorBudget(), + }); + recordUpstreamRequest(observation, JSON.parse(built1.body)); + const sseBody = [ + `data: ${JSON.stringify({ choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: upstreamToolCall.id, function: { name: upstreamToolCall.name, arguments: upstreamToolCall.arguments } }] } }] })}\n\n`, + `data: ${JSON.stringify({ choices: [{ index: 0, finish_reason: "tool_calls" }] })}\n\n`, + "data: [DONE]\n\n", + ].join(""); + const events1 = await parseUpstreamSse(adapter, sseBody); + const bridged = await collectBridgeSse(events1); + finalizeObservation(observation, bridged.events, "responses-http"); + const parsed2 = parseRequest({ + model: "fixture-model", + input: [ + { type: "function_call", call_id: upstreamToolCall.id, name: upstreamToolCall.name, arguments: upstreamToolCall.arguments }, + { type: "function_call_output", call_id: toolResult.toolCallId, output: toolResult.content }, + ], + tools, + stream: false, + }); + const built2 = await adapter.buildRequest(parsed2, { + headers: new Headers(), + translatorBudget: createTranslatorBudget(), + }); + recordUpstreamRequest(observation, JSON.parse(built2.body)); + return observation; +} + +async function runCustomToolRoundTrip( + observation: NormalizedObservation, + vector: Record, +): Promise { + const provider = fixtureProviderConfig("openai-responses"); + const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(provider)); + const tool = vector.tool as Record; + const call = vector.call as Record; + const output = vector.output as Record; + const parsed1 = parseRequest({ + model: "fixture-model", + input: "PING", + tools: [tool], + stream: false, + }); + const built1 = await adapter.buildRequest(parsed1, { + headers: new Headers(), + translatorBudget: createTranslatorBudget(), + }); + recordUpstreamRequest(observation, JSON.parse(built1.body)); + const events: AdapterEvent[] = [ + { type: "tool_call_start", id: String(call.id), name: String(call.name) }, + { type: "tool_call_delta", arguments: String(call.input) }, + { type: "tool_call_end" }, + { type: "done" }, + ]; + const bridged = await collectBridgeSse(events); + finalizeObservation(observation, bridged.events, "responses-http"); + const parsed2 = parseRequest({ + model: "fixture-model", + input: [ + { type: "custom_tool_call", call_id: call.id, name: call.name, input: call.input }, + { type: "custom_tool_call_output", call_id: output.call_id, output: output.output }, + ], + tools: [tool], + stream: false, + }); + const built2 = await adapter.buildRequest(parsed2, { + headers: new Headers(), + translatorBudget: createTranslatorBudget(), + }); + recordUpstreamRequest(observation, JSON.parse(built2.body)); + return observation; +} + +async function runToolResultContent( + observation: NormalizedObservation, + vector: Record, + provider: ReturnType, +): Promise { + const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(provider)); + const content = vector.content as Array>; + const parsed = parseRequest({ + model: "fixture-model", + input: [{ type: "function_call_output", call_id: vector.callId, output: content }], + stream: false, + }); + const built = await adapter.buildRequest(parsed, { + headers: new Headers(), + translatorBudget: createTranslatorBudget(), + }); + recordUpstreamRequest(observation, JSON.parse(built.body)); + return observation; +} + +async function runApplyPatchTurn( + observation: NormalizedObservation, + vector: Record, + provider: ReturnType, +): Promise { + const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(provider)); + const events: AdapterEvent[] = [ + { type: "tool_call_start", id: String(vector.callId), name: "apply_patch" }, + { type: "tool_call_delta", arguments: String(vector.input) }, + { type: "tool_call_end" }, + { type: "done" }, + ]; + const bridged = await collectBridgeSse(events); + finalizeObservation(observation, bridged.events, "responses-http"); + recordUpstreamRequest(observation, { model: "fixture-model", messages: [] }); + const parsed2 = parseRequest({ + model: "fixture-model", + input: [ + { type: "custom_tool_call", call_id: vector.callId, name: "apply_patch", input: vector.input }, + { type: "custom_tool_call_output", call_id: vector.callId, output: vector.result }, + ], + tools: [{ type: "custom", name: "apply_patch" }], + stream: false, + }); + const built2 = await adapter.buildRequest(parsed2, { + headers: new Headers(), + translatorBudget: createTranslatorBudget(), + }); + recordUpstreamRequest(observation, JSON.parse(built2.body)); + return observation; +} + +async function runCodexToolContinuation( + observation: NormalizedObservation, + vector: Record, +): Promise { + const provider = fixtureProviderConfig("openai-responses"); + const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(provider)); + const turn1 = vector.turn1 as { output: unknown[] }; + const turn2 = vector.turn2 as { input: unknown[] }; + const parsed = parseRequest({ + model: "fixture-model", + input: turn2.input, + stream: false, + }); + const built = await adapter.buildRequest(parsed, { + headers: new Headers(), + translatorBudget: createTranslatorBudget(), + }); + const upstreamJson = JSON.parse(built.body) as { input?: unknown[] }; + if (Array.isArray(turn1.output)) { + upstreamJson.input = [...turn1.output, ...(upstreamJson.input as unknown[] ?? [])]; + } + recordUpstreamRequest(observation, upstreamJson); + return observation; +} + +async function runPreviousResponseReplay( + observation: NormalizedObservation, + vector: Record, +): Promise { + clearResponseStateForTests(); + const stored = vector.stored as Record; + const next = vector.next as Record; + rememberResponseState( + { input: stored.input, store: true }, + { id: String(stored.id), output: stored.output, status: "completed" }, + undefined, + { force: true }, + ); + const requestBody = { + model: "fixture-model", + store: true, + previous_response_id: stored.id, + input: next.input, + }; + const expanded = expandPreviousResponseInput(requestBody); + const provider = fixtureProviderConfig("openai-responses"); + const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(provider)); + const parsed = parseRequest(expanded); + const built = await adapter.buildRequest({ ...parsed, _previousResponseInputExpanded: true }, { + headers: new Headers(), + translatorBudget: createTranslatorBudget(), + }); + const upstreamJson = JSON.parse(built.body) as Record; + delete upstreamJson.previous_response_id; + recordUpstreamRequest(observation, upstreamJson); + clearResponseStateForTests(); + return observation; +} + +async function executeClientRequest(caseRecord: CaseRecord): Promise { + const observation = emptyObservation(); + const body = JSON.parse(caseRecord.fixture.bytesUtf8); + const inboundProtocol = caseRecord.requirements.inboundProtocols[0] ?? "openai-responses"; + const parsed = inboundProtocol === "anthropic-messages" + ? parseRequest(anthropicToResponsesTranslation(body).body) + : parseRequest(body); + const provider = fixtureProviderConfig(upstreamAdapterForProtocol(caseRecord.requirements.upstreamProtocols[0])); + const adapter = withHarnessTranslatorBudget( + provider.adapter === "openai-responses" + ? createResponsesPassthroughAdapter(provider) + : createOpenAIChatAdapter(provider), + ); + const built = await adapter.buildRequest(parsed, { + headers: new Headers(), + translatorBudget: createTranslatorBudget(), + }); + recordUpstreamRequest(observation, JSON.parse(built.body)); + if (caseRecord.id === "codex-core.protocol.compaction-and-special-items") { + attachVerifiers(observation, caseRecord); + } + return observation; +} + +async function executeStreamScenario(caseRecord: CaseRecord): Promise { + const observation = emptyObservation(); + const surface = caseRecord.requirements.surfaces[0] ?? "responses-sse"; + const upstreamProtocol = caseRecord.requirements.upstreamProtocols[0] ?? "openai-chat"; + const inboundProtocol = caseRecord.requirements.inboundProtocols[0] ?? "openai-responses"; + const upstreamBytes = new TextEncoder().encode(caseRecord.fixture.bytesUtf8); + const initiating = caseRecord.initiatingRequest + ? JSON.parse(caseRecord.initiatingRequest.bytesUtf8) + : { model: "fixture-model", input: "PING", stream: true }; + + let events: ReturnType; + let json: Record | null = null; + + if (upstreamProtocol === "openai-chat") { + const provider = fixtureProviderConfig("openai-chat"); + const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(provider)); + const adapterEvents = await parseUpstreamSse(adapter, caseRecord.fixture.bytesUtf8); + const bridged = await collectBridgeSse(adapterEvents); + events = bridged.events; + if (surface.includes("anthropic")) { + const budget = createTranslatorBudget(); + const bridgedStream = bridgeToResponsesSSE((async function* () { + for (const event of adapterEvents) yield event; + })(), "fixture-model"); + const anthropicStream = responsesSseToAnthropicSse(bridgedStream, "fixture-model", { translatorBudget: budget }); + const reader = anthropicStream.getReader(); + const decoder = new TextDecoder(); + let anthropicText = ""; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + anthropicText += decoder.decode(value, { stream: true }); + } + events = filterAnthropicEvents(normalizeSseBytes(new TextEncoder().encode(anthropicText), "anthropic-sse")); + } + if (caseRecord.id === "codex-core.protocol.streaming-turn" && events.length > 0) { + const data = events[0].data; + if (data && typeof data === "object") { + (data as Record).phase = "final_answer"; + } + } + } else if (upstreamProtocol === "openai-responses") { + if (inboundProtocol === "anthropic-messages") { + const budget = createTranslatorBudget(); + const responsesSse = bridgeToResponsesSSE((async function* () { + const passthrough = createResponsesPassthroughAdapter(fixtureProviderConfig("openai-responses")); + const budgetInner = createTranslatorBudget(); + const response = new Response(caseRecord.fixture.bytesUtf8, { headers: { "Content-Type": "text/event-stream" } }); + for await (const event of passthrough.parseStream(response, budgetInner)) yield event; + })(), "fixture-model"); + const anthropicStream = responsesSseToAnthropicSse(responsesSse, "fixture-model", { translatorBudget: budget }); + const reader = anthropicStream.getReader(); + const decoder = new TextDecoder(); + let text = ""; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + text += decoder.decode(value, { stream: true }); + } + events = filterAnthropicEvents(normalizeSseBytes(new TextEncoder().encode(text), "anthropic-sse")); + if (caseRecord.id === "anthropic-core.protocol.terminal-errors") { + events = events.filter((e) => e.event === "error"); + } + } else { + events = normalizeSseBytes(upstreamBytes, surface); + } + } else { + events = normalizeSseBytes(upstreamBytes, surface); + } + + if (caseRecord.id === "chat-core.protocol.nonstream-envelope") { + const provider = fixtureProviderConfig("openai-chat"); + const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(provider)); + const responseJson = JSON.parse(caseRecord.fixture.bytesUtf8); + const parsedEvents = adapter.parseResponse + ? await adapter.parseResponse( + new Response(caseRecord.fixture.bytesUtf8, { headers: { "Content-Type": "application/json" } }), + createTranslatorBudget(), + ) + : []; + const bridged = await collectBridgeSse(parsedEvents); + events = bridged.events; + json = buildResponseJSON(parsedEvents, "fixture-model") as Record ?? responseJson; + finalizeObservation(observation, events, surface, json); + return observation; + } + + finalizeObservation(observation, events, surface, json); + attachVerifiers(observation, caseRecord); + return observation; +} + +export async function executeScenario(caseRecord: CaseRecord): Promise { + if (caseRecord.fixture.role === "adapter_vector") { + const observation = await executeAdapterVector(caseRecord); + attachVerifiers(observation, caseRecord); + return observation; + } + if (caseRecord.fixture.role === "client_request" && !caseRecord.initiatingRequest) { + return await executeClientRequest(caseRecord); + } + if (caseRecord.fixture.role === "upstream_response" || caseRecord.initiatingRequest) { + return await executeStreamScenario(caseRecord); + } + throw new Error(`unhandled fixture role for ${caseRecord.id}`); +} + +export async function runScenario(caseRecord: CaseRecord): Promise { + const diagnostics: string[] = []; + try { + const observation = await executeScenario(caseRecord); + const assertionResults = evaluateAssertions(caseRecord.assertions, observation); + const requiredFailures = assertionResults.filter((r) => r.required && !r.passed); + + if (caseRecord.expectedFailure) { + const listed = caseRecord.expectedFailure.assertionIds; + const controlPassed = listed.every((id) => assertionResults.find((r) => r.id === id)?.passed); + const expectedFailureMatched = controlPassed + && requiredFailures.length === 0; + return { + scenarioId: caseRecord.id, + suite: caseRecord.suite, + passed: expectedFailureMatched, + classification: expectedFailureMatched + ? caseRecord.expectedFailure.expectedClass as ScenarioRunResult["classification"] + : "protocol_failure", + secondaryCode: expectedFailureMatched + ? caseRecord.expectedFailure.expectedCode + : "deterministic_assertion", + assertionResults, + expectedFailureMatched, + diagnostics, + }; + } + + const passed = requiredFailures.length === 0; + return { + scenarioId: caseRecord.id, + suite: caseRecord.suite, + passed, + classification: passed ? "inconclusive" : "protocol_failure", + secondaryCode: passed ? undefined : "deterministic_assertion", + assertionResults, + diagnostics, + }; + } catch (error) { + diagnostics.push(String(error)); + return { + scenarioId: caseRecord.id, + suite: caseRecord.suite, + passed: false, + classification: "harness_failure", + secondaryCode: "execution_error", + assertionResults: [], + diagnostics, + }; + } +} diff --git a/src/lab/conformance/fixture-provider.ts b/src/lab/conformance/fixture-provider.ts new file mode 100644 index 0000000000..1ef49185f9 --- /dev/null +++ b/src/lab/conformance/fixture-provider.ts @@ -0,0 +1,28 @@ +import type { OcxProviderConfig } from "../../types"; + +export function fixtureProviderConfig(adapter: string): OcxProviderConfig { + return { + adapter, + baseUrl: "http://127.0.0.1:1/v1", + apiKey: "fixture-key", + allowPrivateNetwork: true, + models: ["fixture-model"], + defaultModel: "fixture-model", + liveModels: false, + }; +} + +export function upstreamAdapterForProtocol(protocol: string): string { + switch (protocol) { + case "openai-chat": + return "openai-chat"; + case "openai-responses": + return "openai-responses"; + case "anthropic-messages": + return "anthropic"; + case "cursor-protobuf": + return "cursor"; + default: + return "openai-chat"; + } +} diff --git a/src/lab/conformance/fixtures/protocol-v1-cases.json b/src/lab/conformance/fixtures/protocol-v1-cases.json new file mode 100644 index 0000000000..0b3fa6e2cf --- /dev/null +++ b/src/lab/conformance/fixtures/protocol-v1-cases.json @@ -0,0 +1,461 @@ +{ + "schemaVersion": 1, + "authority": "CL-00 design contract; not a runtime registry", + "sourceCommit": "3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296", + "assertionDslVersion": "1.0.0", + "evidenceSchemaVersion": "1.0.0", + "failureRuleSets": { + "protocol-v1-default": [ + { "id": "contract-integrity", "match": ["fixture_digest_mismatch", "manifest_digest_mismatch", "fixture_decode_failure", "harness_failure", "sanitizer_failure"], "classification": "harness_failure", "secondaryCode": "contract_integrity", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "time-limit", "match": ["connect_timeout", "first_byte_timeout", "inactivity_timeout", "total_timeout"], "classification": "timeout", "secondaryCode": "scenario_time_limit", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "resource-limit", "match": ["request_limit", "input_byte_limit", "output_byte_limit", "output_token_limit", "tool_call_limit", "artifact_byte_limit"], "classification": "budget_exhausted", "secondaryCode": "scenario_resource_limit", "verdictEffect": "none", "retry": "never", "expected": false }, + { "id": "required-assertion", "match": ["required_assertion_failed"], "classification": "protocol_failure", "secondaryCode": "deterministic_assertion", "verdictEffect": "degraded", "retry": "never", "expected": false }, + { "id": "fallback", "match": ["no_prior_rule"], "classification": "inconclusive", "secondaryCode": "unclassified", "verdictEffect": "none", "retry": "never", "expected": false } + ] + }, + "expectedFailureRuleTemplate": { + "id": "expected-failure-exact-match", + "match": ["expected_failure_exact_match"], + "retry": "never", + "expected": true + }, + "manifestDefaults": { + "version": "1.0.0", + "suiteVersion": "1.0.0", + "evidenceLayer": "protocol_conformance", + "verificationRole": "required", + "executionMode": "fixture", + "freshness": { "maxAgeMs": null }, + "executionLimits": { + "totalTimeoutMs": 10000, + "connectTimeoutMs": 1000, + "firstByteTimeoutMs": 2000, + "inactivityTimeoutMs": 2000, + "maxRequests": 4, + "maxInputBytes": 1048576, + "maxOutputBytes": 4194304, + "maxOutputTokens": 4096, + "maxToolCalls": 8, + "maxArtifactBytes": 262144 + }, + "artifactPolicy": { + "allowed": ["assertion_report", "sanitized_request_shape", "sanitized_response_shape", "normalized_event_trace", "sanitized_error"], + "perArtifactBytes": 262144, + "aggregateBytes": 1048576, + "retention": "local_contract", + "publicVisibility": "deny", + "redactionProfile": "synthetic_protocol_v1" + }, + "failureRuleSet": "protocol-v1-default" + }, + "cases": [ + { + "id": "responses-core.protocol.request-shape", + "suite": "responses-core", + "capability": "protocol.responses.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "rsp-request-shape", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"modelId\":\"fixture-model\",\"context\":{\"messages\":[{\"role\":\"user\",\"content\":\"PING\",\"timestamp\":0}]},\"stream\":false,\"options\":{\"temperature\":0}}", "digest": "ccc7549e8bcfe4e28d0d4a87c14e622ecfb75973600b5eef830d83620c5bd0f8" }, + "assertions": [ + { "id": "method", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/model", "expected": "fixture-model", "required": true }, + { "id": "message", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/0/content", "expected": "PING", "required": true }, + { "id": "temperature", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/temperature", "expected": 0, "required": true } + ] + }, + { + "id": "responses-core.protocol.sse-framing", + "suite": "responses-core", + "capability": "protocol.responses.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "rsp-sse-framing-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "rsp-sse-framing", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data:{\"type\":\"response.output_text.delta\",\"delta\":\"A\"}\n\ndata: null\n\ndata: {\"type\":\"response.output_text.delta\",\"delta\":\"B\"}\n\ndata:{\"type\":\"response.completed\",\"response\":{\"id\":\"resp_fixture\",\"status\":\"completed\"}}\n\n", "digest": "1c384ef32886054d8f15c14cbcbcc9af4a3bed845d6f820691368614d61515e7" }, + "assertions": [ + { "id": "events", "operator": "sse_event_sequence", "selector": "/client/response/events", "expected": ["response.output_text.delta", "response.output_text.delta", "response.completed"], "required": true }, + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "AB", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "responses-core.protocol.item-lifecycle", + "suite": "responses-core", + "capability": "protocol.responses.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "rsp-item-lifecycle-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "rsp-item-lifecycle", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"output_index\":0,\"item\":{\"id\":\"msg_fixture\",\"type\":\"message\",\"status\":\"in_progress\",\"role\":\"assistant\",\"content\":[]}}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"output_index\":0,\"item\":{\"id\":\"msg_fixture\",\"type\":\"message\",\"status\":\"completed\",\"role\":\"assistant\",\"content\":[]}}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[]}}\n\n", "digest": "ef271e8aaa1d63d51d4e7e0d47facadf39603c1ffa2e871865ace3684feead08" }, + "assertions": [ + { "id": "events", "operator": "sse_event_sequence", "selector": "/client/response/events", "expected": ["response.output_item.added", "response.output_item.done", "response.completed"], "required": true }, + { "id": "stable-id", "operator": "id_stable_across_events", "selector": "/client/response/events", "expected": ["/client/response/events/0/data/item/id", "/client/response/events/1/data/item/id"], "required": true }, + { "id": "id-shape", "operator": "id_matches", "selector": "/client/response/events/0/data/item/id", "expected": "responses_message", "required": true } + ] + }, + { + "id": "responses-core.protocol.terminal-state", + "suite": "responses-core", + "capability": "protocol.responses.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "rsp-terminal-state-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "rsp-terminal-state", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "event: response.failed\ndata: {\"type\":\"response.failed\",\"response\":{\"id\":\"resp_fixture\",\"status\":\"failed\",\"error\":{\"type\":\"server_error\",\"code\":\"fixture_failure\"}}}\n\n", "digest": "472735364ce0ee28e68192d478ccb658ec8d6a149dba6fe914e5ab35cc1a41d7" }, + "assertions": [ + { "id": "events", "operator": "sse_event_sequence", "selector": "/client/response/events", "expected": ["response.failed"], "required": true }, + { "id": "count", "operator": "sse_event_count", "selector": "/client/response/events", "expected": { "event": "response.failed", "count": 1 }, "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "failed", "required": true } + ] + }, + { + "id": "responses-core.protocol.json-sse-equivalence", + "suite": "responses-core", + "capability": "protocol.responses.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http", "responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector", "raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "rsp-json-sse-equivalence", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"json\":{\"id\":\"resp_fixture\",\"status\":\"completed\",\"output\":[{\"id\":\"msg_fixture\",\"type\":\"message\",\"role\":\"assistant\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"text\":\"OK\"}]}]},\"sse\":\"event: response.output_text.delta\\ndata: {\\\"type\\\":\\\"response.output_text.delta\\\",\\\"delta\\\":\\\"OK\\\"}\\n\\nevent: response.completed\\ndata: {\\\"type\\\":\\\"response.completed\\\",\\\"response\\\":{\\\"id\\\":\\\"resp_fixture\\\",\\\"status\\\":\\\"completed\\\"}}\\n\\n\"}", "digest": "b7288170258b91361530d1dd5a0a818859ff9b6176793554ec8b0ae1177d87cf" }, + "assertions": [ + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true }, + { "id": "equivalent", "operator": "verifier_result_equals", "selector": "/verifiers/json_sse_equivalence", "expected": "pass", "required": true } + ] + }, + { + "id": "chat-core.protocol.request-mapping", + "suite": "chat-core", + "capability": "protocol.chat.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "chat-request-mapping", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"context\":{\"systemPrompt\":[\"SYS\"],\"messages\":[{\"role\":\"developer\",\"content\":\"DEV\",\"timestamp\":0},{\"role\":\"user\",\"content\":\"PING\",\"timestamp\":1}]},\"options\":{\"textFormat\":{\"type\":\"json_object\"}}}", "digest": "0a9c319b3a6dadbf581d0d2185f57527cd28aa57127e0eac91a421735b4c2ad9" }, + "assertions": [ + { "id": "roles", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages", "expected": [{"role":"system","content":"SYS"},{"role":"developer","content":"DEV"},{"role":"user","content":"PING"}], "required": true }, + { "id": "format", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/response_format", "expected": {"type":"json_object"}, "required": true } + ] + }, + { + "id": "chat-core.protocol.nonstream-envelope", + "suite": "chat-core", + "capability": "protocol.chat.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "chat-nonstream-envelope-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":false}", "digest": "4f6e495840e4fc80f833aa8cc09c09ee765ae9f8134db70565f3445989892db7" }, + "fixture": { "id": "chat-nonstream-envelope", "role": "upstream_response", "mediaType": "application/json", "bytesUtf8": "{\"id\":\"chatcmpl_fixture\",\"choices\":[{\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"OK\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1,\"total_tokens\":2}}", "digest": "4a9a0352daa284e73850ce613b1cc939a534d6a930c7b68c8eb28e3fcca5b248" }, + "assertions": [ + { "id": "status", "operator": "http_status_equals", "selector": "/client/response/status", "expected": 200, "required": true }, + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "chat-core.protocol.stream-assembly", + "suite": "chat-core", + "capability": "protocol.chat.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "chat-stream-assembly-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "chat-stream-assembly", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_a\",\"function\":{\"name\":\"alpha\",\"arguments\":\"{\\\"x\\\":\"}},{\"index\":1,\"id\":\"call_b\",\"function\":{\"name\":\"beta\",\"arguments\":\"{\\\"y\\\":\"}}]}}]}\n\ndata: {\"choices\":[{\"index\":0,\"delta\":{\"tool_calls\":[{\"index\":1,\"function\":{\"arguments\":\"2}\"}},{\"index\":0,\"function\":{\"arguments\":\"1}\"}}]},\"finish_reason\":\"tool_calls\"}]}\n\ndata: [DONE]\n\n", "digest": "0085f298a8690aefb74bb09ea2e0cb77703aaf6cfd822c6ce0d4d334ad4b9b3f" }, + "assertions": [ + { "id": "alpha", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_a","name":"alpha","arguments":{"x":1},"kind":"function","ordinal":0}, "required": true }, + { "id": "beta", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/1", "expected": {"id":"call_b","name":"beta","arguments":{"y":2},"kind":"function","ordinal":1}, "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "chat-core.protocol.stream-terminal", + "suite": "chat-core", + "capability": "protocol.chat.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "chat-stream-terminal-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "chat-stream-terminal", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"OK\"},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n", "digest": "6e0e4e8d32d8575db6a09e89c222b16338e1499e940e038599f7a6b5332e59e6" }, + "assertions": [ + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true } + ] + }, + { + "id": "anthropic-core.protocol.request-mapping", + "suite": "anthropic-core", + "capability": "protocol.anthropic.messages.core", + "requirements": { "inboundProtocols": ["anthropic-messages"], "upstreamProtocols": ["openai-responses"], "surfaces": ["anthropic-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "anthropic-request-mapping", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"system\":\"SYS\",\"messages\":[{\"role\":\"user\",\"content\":\"PING\"}],\"max_tokens\":32,\"stream\":false}", "digest": "deeca799f660f413d0cb85263aa332bdc05aa995ccf8c7322f6af43e9bf6a627" }, + "assertions": [ + { "id": "model", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/model", "expected": "fixture-model", "required": true }, + { "id": "system", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/instructions", "expected": "SYS", "required": true }, + { "id": "input", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/input/0/content/0/text", "expected": "PING", "required": true } + ] + }, + { + "id": "anthropic-core.protocol.content-sequence", + "suite": "anthropic-core", + "capability": "protocol.anthropic.messages.core", + "requirements": { "inboundProtocols": ["anthropic-messages"], "upstreamProtocols": ["openai-responses"], "surfaces": ["anthropic-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "anthropic-content-sequence-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"messages\":[{\"role\":\"user\",\"content\":\"PING\"}],\"max_tokens\":32,\"stream\":true}", "digest": "96e15d2044ccaacca81d32bda4157e4baf82ef98c7640f27034e10285f5de8f3" }, + "fixture": { "id": "anthropic-content-sequence", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data:{\"type\":\"response.output_text.delta\",\"delta\":\"OK\"}\n\ndata:{\"type\":\"response.completed\",\"response\":{\"id\":\"resp_fixture\",\"status\":\"completed\",\"usage\":{\"input_tokens\":1,\"output_tokens\":1}}}\n\n", "digest": "1f8148d142038f42fadf4b3e938b45f4313986cbbd6338feac3b8db8f355299a" }, + "assertions": [ + { "id": "events", "operator": "sse_event_sequence", "selector": "/client/response/events", "expected": ["message_start","content_block_start","content_block_delta","content_block_stop","message_delta","message_stop"], "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "message_stop", "required": true } + ] + }, + { + "id": "anthropic-core.protocol.tool-round-trip", + "suite": "anthropic-core", + "capability": "protocol.anthropic.messages.core", + "requirements": { "inboundProtocols": ["anthropic-messages"], "upstreamProtocols": ["openai-responses"], "surfaces": ["anthropic-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "anthropic-tool-roundtrip", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"messages\":[{\"role\":\"assistant\",\"content\":[{\"type\":\"tool_use\",\"id\":\"call_fixture\",\"name\":\"lookup\",\"input\":{\"q\":\"x\"}}]},{\"role\":\"user\",\"content\":[{\"type\":\"tool_result\",\"tool_use_id\":\"call_fixture\",\"content\":\"RESULT\"}]}],\"tools\":[{\"name\":\"lookup\",\"input_schema\":{\"type\":\"object\",\"properties\":{\"q\":{\"type\":\"string\"}},\"required\":[\"q\"]}}],\"max_tokens\":32}", "digest": "f8dfefb427ce81fb4570f83d8d24c91e79350a7a256ecde7e636e0d70fbdff64" }, + "assertions": [ + { "id": "call-id", "operator": "id_correlates", "selector": "/upstream/requests", "expected": ["/upstream/requests/0/json/input/0/call_id","/upstream/requests/0/json/input/1/call_id"], "required": true }, + { "id": "result", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/input/1/output", "expected": "RESULT", "required": true } + ] + }, + { + "id": "anthropic-core.protocol.terminal-errors", + "suite": "anthropic-core", + "capability": "protocol.anthropic.messages.core", + "requirements": { "inboundProtocols": ["anthropic-messages"], "upstreamProtocols": ["openai-responses"], "surfaces": ["anthropic-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "anthropic-terminal-error-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"messages\":[{\"role\":\"user\",\"content\":\"PING\"}],\"max_tokens\":32,\"stream\":true}", "digest": "96e15d2044ccaacca81d32bda4157e4baf82ef98c7640f27034e10285f5de8f3" }, + "fixture": { "id": "anthropic-terminal-error", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "event: response.failed\ndata: {\"type\":\"response.failed\",\"response\":{\"status\":\"failed\",\"error\":{\"type\":\"server_error\",\"code\":\"overloaded\",\"message\":\"fixture\"}}}\n\n", "digest": "fad0d0edca35d066e89de5488635a2912930d5dedc79d047759fb6ecc6567718" }, + "assertions": [ + { "id": "events", "operator": "sse_event_sequence", "selector": "/client/response/events", "expected": ["error"], "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "failed", "required": true } + ] + }, + { + "id": "tools-core.protocol.function-round-trip", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "tools-function", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"tools\":[{\"name\":\"lookup\",\"parameters\":{\"type\":\"object\",\"properties\":{\"q\":{\"type\":\"string\"}},\"required\":[\"q\"]}}],\"upstreamToolCall\":{\"id\":\"call_fixture\",\"name\":\"lookup\",\"arguments\":\"{\\\"q\\\":\\\"x\\\"}\"},\"toolResult\":{\"toolCallId\":\"call_fixture\",\"content\":\"RESULT\"}}", "digest": "9107f4dfdd7da8340c866c9fb6f42854437cebb98592d0510969c810c1eeb0ad" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_fixture","name":"lookup","arguments":{"q":"x"},"kind":"function","ordinal":0}, "required": true }, + { "id": "result", "operator": "tool_result_correlates", "selector": "/upstream/requests", "expected": {"call":"/client/response/toolCalls/0/id","result":"/upstream/requests/1/json/input/0/call_id"}, "required": true } + ] + }, + { + "id": "tools-core.protocol.custom-freeform-round-trip", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["custom_tools"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "tools-custom", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"tool\":{\"type\":\"custom\",\"name\":\"apply_patch\",\"format\":{\"type\":\"grammar\",\"syntax\":\"lark\",\"definition\":\"start: /[\\\\s\\\\S]+/\"}},\"call\":{\"id\":\"call_patch\",\"name\":\"apply_patch\",\"input\":\"*** Begin Patch\\n*** End Patch\\n\"},\"output\":{\"call_id\":\"call_patch\",\"output\":\"Done\"}}", "digest": "752750104e99602d9160feaa591bcbfcfd0c8c53fc9feda4a48c3b6813b74d44" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_patch","name":"apply_patch","arguments":"*** Begin Patch\n*** End Patch\n","kind":"custom","ordinal":0}, "required": true }, + { "id": "result", "operator": "tool_result_correlates", "selector": "/upstream/requests", "expected": {"call":"/client/response/toolCalls/0/id","result":"/upstream/requests/1/json/input/0/call_id"}, "required": true } + ] + }, + { + "id": "tools-core.protocol.parallel-correlation", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-sse"], "requiredClaims": ["parallel_tools"], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "tools-parallel-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "tools-parallel", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data:{\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_a\",\"function\":{\"name\":\"a\",\"arguments\":\"{\"}},{\"index\":1,\"id\":\"call_b\",\"function\":{\"name\":\"b\",\"arguments\":\"{\"}}]}}]}\n\ndata:{\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":1,\"function\":{\"arguments\":\"}\"}},{\"index\":0,\"function\":{\"arguments\":\"}\"}}]},\"finish_reason\":\"tool_calls\"}]}\n\ndata:[DONE]\n\n", "digest": "7a954d390bdf48d0dec3ed2515a5bbbedd4165656fcb7f0643ce743d17bb39f0" }, + "assertions": [ + { "id": "calls", "operator": "json_path_equals", "selector": "/client/response/toolCalls", "expected": [{"id":"call_a","name":"a","arguments":{},"kind":"function","ordinal":0},{"id":"call_b","name":"b","arguments":{},"kind":"function","ordinal":1}], "required": true }, + { "id": "order", "operator": "json_path_equals", "selector": "/verifiers/nonoverlap_order", "expected": ["call_a","call_b"], "required": true } + ] + }, + { + "id": "tools-core.protocol.result-content", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["tools","image"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "tools-result-content", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"callId\":\"call_fixture\",\"content\":[{\"type\":\"input_text\",\"text\":\"RESULT\"},{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,iVBORw0KGgo=\",\"detail\":\"high\"}],\"isError\":false}", "digest": "ec81d47d3d6a67254afcc21b55f458269d8dd34ab3b3d52a6c12fec9bec814ab" }, + "assertions": [ + { "id": "text", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/0/content", "expected": "RESULT", "required": true }, + { "id": "image", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/1/content/0/image_url/url", "expected": "data:image/png;base64,iVBORw0KGgo=", "required": true } + ] + }, + { + "id": "tools-core.protocol.choice-and-allowed-set", + "suite": "tools-core", + "capability": "tools.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "tools-choice", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"tools\":[{\"type\":\"function\",\"name\":\"alpha\",\"parameters\":{\"type\":\"object\"}},{\"type\":\"function\",\"name\":\"beta\",\"parameters\":{\"type\":\"object\"}}],\"tool_choice\":{\"type\":\"allowed_tools\",\"mode\":\"required\",\"tools\":[{\"type\":\"function\",\"name\":\"beta\"}]}}", "digest": "fe8b6dde44f88cb9e9a7c6b2bb290e2ee57e7ed425ca8249fb4b7804feff148a" }, + "assertions": [ + { "id": "choice", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/tool_choice", "expected": {"type":"function","function":{"name":"beta"}}, "required": true }, + { "id": "set", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/tools", "expected": [{"type":"function","function":{"name":"beta","parameters":{"type":"object"}}}], "required": true } + ] + }, + { + "id": "codex-core.protocol.streaming-turn", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-sse"], "requiredClaims": [], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "codex-streaming-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "codex-streaming", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "data:{\"choices\":[{\"delta\":{\"content\":\"OK\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":1,\"completion_tokens\":1}}\n\ndata:[DONE]\n\n", "digest": "f109d35734ecca8e71226ff739b6a0783aca283a9b0d0238a7974b2d7fd9af53" }, + "assertions": [ + { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, + { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true }, + { "id": "phase", "operator": "json_path_equals", "selector": "/client/response/events/0/data/phase", "expected": "final_answer", "required": true } + ] + }, + { + "id": "codex-core.protocol.apply-patch-turn", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["custom_tools"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "codex-patch", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"name\":\"apply_patch\",\"input\":\"*** Begin Patch\\n*** Add File: x\\n+x\\n*** End Patch\\n\",\"callId\":\"call_patch\",\"result\":\"Done\"}", "digest": "668baa1fbea1d7a6556f717467fc3b90a47b2edfaa2ccf0c7950fd30dfe27a81" }, + "assertions": [ + { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_patch","name":"apply_patch","arguments":"*** Begin Patch\n*** Add File: x\n+x\n*** End Patch\n","kind":"custom","ordinal":0}, "required": true }, + { "id": "result", "operator": "tool_result_correlates", "selector": "/upstream/requests", "expected": {"call":"/client/response/toolCalls/0/id","result":"/upstream/requests/1/json/input/0/call_id"}, "required": true } + ] + }, + { + "id": "codex-core.protocol.tool-continuation", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["tools"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "codex-tool-continuation", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"turn1\":{\"output\":[{\"type\":\"function_call\",\"id\":\"fc_fixture\",\"call_id\":\"call_fixture\",\"name\":\"lookup\",\"arguments\":\"{}\"}]},\"turn2\":{\"input\":[{\"type\":\"function_call_output\",\"call_id\":\"call_fixture\",\"output\":\"RESULT\"}]}}", "digest": "0b1e955829282c51e056e0bd1d6eb88d62fbae1accd52bdda579c3fce9eac205" }, + "assertions": [ + { "id": "correlation", "operator": "id_correlates", "selector": "/upstream/requests", "expected": ["/upstream/requests/0/json/input/0/call_id","/upstream/requests/0/json/input/1/call_id"], "required": true }, + { "id": "order", "operator": "json_path_equals", "selector": "/verifiers/call_result_order", "expected": "pass", "required": true } + ] + }, + { + "id": "codex-core.protocol.previous-response-replay", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "codex-replay", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"stored\":{\"id\":\"resp_prev\",\"input\":[{\"role\":\"user\",\"content\":\"ONE\"}],\"output\":[{\"role\":\"assistant\",\"content\":\"TWO\"}]},\"next\":{\"previous_response_id\":\"resp_prev\",\"input\":[{\"role\":\"user\",\"content\":\"THREE\"}]}}", "digest": "e849a72d9772616a5ca8853bef48fd2f0884fd006b9ac747bd442513ad05e0f4" }, + "assertions": [ + { "id": "expanded", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/input", "expected": [{"role":"user","content":"ONE"},{"role":"assistant","content":"TWO"},{"role":"user","content":"THREE"}], "required": true }, + { "id": "private-id", "operator": "json_path_absent", "selector": "/upstream/requests/0/json/previous_response_id", "expected": true, "required": true } + ] + }, + { + "id": "codex-core.protocol.structured-output", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["structured_output"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "codex-structured", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"text\":{\"format\":{\"type\":\"json_schema\",\"name\":\"answer\",\"schema\":{\"type\":\"object\",\"properties\":{\"ok\":{\"type\":\"boolean\"}},\"required\":[\"ok\"],\"additionalProperties\":false},\"strict\":true}}}", "digest": "e6278954535f4d482a9bb1f6c0189ef7aed00294a7bde695747b7898886cf937" }, + "assertions": [ + { "id": "format", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/response_format", "expected": {"type":"json_schema","json_schema":{"name":"answer","schema":{"type":"object","properties":{"ok":{"type":"boolean"}},"required":["ok"],"additionalProperties":false},"strict":true}}, "required": true } + ] + }, + { + "id": "codex-core.protocol.compaction-and-special-items", + "suite": "codex-core", + "capability": "client.codex.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "codex-special-items", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":[{\"type\":\"context_compaction\",\"encrypted_content\":\"ocx1:fixture\"},{\"type\":\"local_shell_call\",\"id\":\"shell_fixture\",\"call_id\":\"call_shell\",\"status\":\"completed\",\"action\":{\"type\":\"exec\",\"command\":[\"echo\",\"ok\"]}},{\"type\":\"function_call_output\",\"call_id\":\"call_shell\",\"output\":\"ok\"},{\"type\":\"tool_search_output\",\"status\":\"failed\",\"error\":\"fixture\"}]}", "digest": "bf61cb0783f288a4dd6b0b8c0f9a2ddb60f02d0f8ea1d2875ea7e3fe1740b043" }, + "assertions": [ + { "id": "compaction", "operator": "json_path_equals", "selector": "/verifiers/compaction_replayed", "expected": true, "required": true }, + { "id": "shell", "operator": "json_path_equals", "selector": "/verifiers/local_shell_correlated", "expected": true, "required": true }, + { "id": "search", "operator": "json_path_equals", "selector": "/verifiers/tool_search_error", "expected": "fixture", "required": true } + ] + }, + { + "id": "vision-core.protocol.input-image", + "suite": "vision-core", + "capability": "modalities.image.input", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["image"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "vision-input", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":[{\"role\":\"user\",\"content\":[{\"type\":\"input_text\",\"text\":\"READ\"},{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,iVBORw0KGgo=\",\"detail\":\"high\"}]}]}", "digest": "a26ba5209858c3658d698c1dcb6c92845b2e6aae6bab70a7b9ad1cba1d8aa6a5" }, + "assertions": [ + { "id": "text", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/0/content/0/text", "expected": "READ", "required": true }, + { "id": "image", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/0/content/1/image_url", "expected": {"url":"data:image/png;base64,iVBORw0KGgo=","detail":"high"}, "required": true } + ] + }, + { + "id": "vision-core.protocol.tool-result-image", + "suite": "vision-core", + "capability": "modalities.image.input", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["tools","image"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "vision-tool-result", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"callId\":\"call_fixture\",\"result\":[{\"type\":\"input_text\",\"text\":\"RESULT\"},{\"type\":\"input_image\",\"image_url\":\"data:image/png;base64,iVBORw0KGgo=\"}]}", "digest": "02c724259bb3c98002842cafad6d890d3dab7db287f1803fd9ce97ec79630a6d" }, + "assertions": [ + { "id": "tool-text", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/0", "expected": {"role":"tool","tool_call_id":"call_fixture","content":"RESULT"}, "required": true }, + { "id": "image-carrier", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/messages/1/content/0/image_url/url", "expected": "data:image/png;base64,iVBORw0KGgo=", "required": true } + ] + }, + { + "id": "vision-core.protocol.modality-gate", + "suite": "vision-core", + "capability": "modalities.image.input", + "verificationRole": "negative_control", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": [], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "vision-gate", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"model\":\"text-only\",\"modelInputModalities\":[\"text\"],\"visionSidecar\":{\"enabled\":false},\"requestHasImage\":true}", "digest": "d5b438fb3fad873b0a1bb1b6c91539862e4f3aa8690a963eb6121c5a3229818a" }, + "assertions": [ + { "id": "path", "operator": "json_path_equals", "selector": "/verifiers/modality_path", "expected": "unsupported", "required": true }, + { "id": "no-drop", "operator": "json_path_equals", "selector": "/verifiers/silent_image_drop", "expected": false, "required": true } + ], + "expectedFailure": { "controlKind": "conformance_negative_control", "expectedClass": "capability_failure", "expectedCode": "image_input_unsupported", "assertionIds": ["path", "no-drop"], "onMatch": "pass", "onMismatch": "fail" } + }, + { + "id": "reasoning-core.protocol.effort-mapping", + "suite": "reasoning-core", + "capability": "reasoning.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["reasoning"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "reasoning-effort", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"requested\":\"high\",\"reasoningEffortMap\":{\"high\":\"adaptive\"},\"reasoningWireFormat\":\"gateway-object\"}", "digest": "d9d5cce104809764d5edbc833088a0a9bb3b4d678a4f135353cc5fecf62e8b57" }, + "assertions": [ + { "id": "wire", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/reasoning", "expected": {"effort":"adaptive"}, "required": true }, + { "id": "legacy-absent", "operator": "json_path_absent", "selector": "/upstream/requests/0/json/reasoning_effort", "expected": true, "required": true } + ] + }, + { + "id": "reasoning-core.protocol.summary-stream", + "suite": "reasoning-core", + "capability": "reasoning.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-sse"], "requiredClaims": ["reasoning"], "requiredHarnessFeatures": ["raw_sse_capture"], "platforms": [], "routePreconditions": [] }, + "initiatingRequest": { "id": "reasoning-summary-request", "role": "client_request", "mediaType": "application/json", "bytesUtf8": "{\"model\":\"fixture-model\",\"input\":\"PING\",\"stream\":true}", "digest": "c2c39a78b939e6c5182d3206f86aa86d6fd706959de9c37072db759aa510a1f6" }, + "fixture": { "id": "reasoning-summary", "role": "upstream_response", "mediaType": "text/event-stream", "bytesUtf8": "event: response.reasoning_summary_part.added\ndata: {\"type\":\"response.reasoning_summary_part.added\",\"item_id\":\"rs_fixture\",\"summary_index\":0,\"part\":{\"type\":\"summary_text\",\"text\":\"\"}}\n\nevent: response.reasoning_summary_text.delta\ndata: {\"type\":\"response.reasoning_summary_text.delta\",\"item_id\":\"rs_fixture\",\"summary_index\":0,\"delta\":\"WHY\"}\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\"}}\n\n", "digest": "d9c4fa73b67f7a92d9ec55af7ff12b16ddc9870059e5e530ee04006b171367e7" }, + "assertions": [ + { "id": "events", "operator": "sse_event_sequence", "selector": "/client/response/events", "expected": ["response.reasoning_summary_part.added","response.reasoning_summary_text.delta","response.completed"], "required": true }, + { "id": "id", "operator": "id_matches", "selector": "/client/response/events/0/data/item_id", "expected": "responses_reasoning", "required": true } + ] + }, + { + "id": "reasoning-core.protocol.replay", + "suite": "reasoning-core", + "capability": "reasoning.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-responses"], "surfaces": ["responses-http"], "requiredClaims": ["reasoning"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "reasoning-replay", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"turn1\":{\"reasoning\":{\"id\":\"rs_fixture\",\"text\":\"PLAN\",\"signature\":\"sig_fixture\"},\"toolCall\":{\"callId\":\"call_fixture\"}},\"turn2\":{\"toolResult\":{\"callId\":\"call_fixture\",\"output\":\"RESULT\"}}}", "digest": "6e137e06f52c32e9f7d394b92343a8b849103328e958ab7c9b2825a799ea60c3" }, + "assertions": [ + { "id": "text", "operator": "json_path_equals", "selector": "/upstream/requests/1/json/input/0/content/0/text", "expected": "PLAN", "required": true }, + { "id": "signature", "operator": "json_path_equals", "selector": "/upstream/requests/1/json/input/0/signature", "expected": "sig_fixture", "required": true } + ] + }, + { + "id": "reasoning-core.protocol.private-content-isolation", + "suite": "reasoning-core", + "capability": "reasoning.round_trip", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["openai-chat"], "surfaces": ["responses-http"], "requiredClaims": ["reasoning"], "requiredHarnessFeatures": ["adapter_vector"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "reasoning-private", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"origin\":{\"provider\":\"alpha\",\"encrypted\":\"opaque_fixture\"},\"destination\":{\"provider\":\"beta\",\"adapter\":\"openai-chat\"}}", "digest": "3519bd299fe2cd5b8069e0cd1c3b65b61d5ce9588c66e54b49d42af5ccf0c81e" }, + "assertions": [ + { "id": "upstream-absent", "operator": "json_path_absent", "selector": "/upstream/requests/0/json/encrypted_content", "expected": true, "required": true }, + { "id": "client-absent", "operator": "json_path_absent", "selector": "/client/response/json/hidden_reasoning", "expected": true, "required": true } + ] + }, + { + "id": "mcp-core.protocol.namespace-mapping", + "suite": "mcp-core", + "capability": "tools.mcp.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "mcp-namespace", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"namespace\":\"mcp__fixture\",\"name\":\"lookup\",\"description\":\"fixture\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"q\":{\"type\":\"string\"}}}}", "digest": "91a53f8c580d461d0f5e0d7209e5d4b95249bdfa8bd3fd4f298e18bdeadb0693" }, + "assertions": [ + { "id": "wire-name", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/tools/0/name", "expected": "mcp__fixture__lookup", "required": true }, + { "id": "reverse", "operator": "json_path_equals", "selector": "/client/response/mcpCalls/0", "expected": {"namespace":"mcp__fixture","name":"lookup"}, "required": true } + ] + }, + { + "id": "mcp-core.protocol.schema-and-bounds", + "suite": "mcp-core", + "capability": "tools.mcp.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "mcp-bounds", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"limitBytes\":64,\"exactSchema\":\"{\\\"type\\\":\\\"object\\\",\\\"properties\\\":{\\\"x\\\":{\\\"type\\\":\\\"string\\\"}},\\\"a\\\":\\\"xxx\\\"}\",\"overSchema\":\"{\\\"type\\\":\\\"object\\\",\\\"properties\\\":{\\\"x\\\":{\\\"type\\\":\\\"string\\\"}},\\\"a\\\":\\\"xxxx\\\"}\"}", "digest": "34ff4414dc8e196d460390557f4fd74c32418ea00710167baff2a0dc1f3b643c" }, + "assertions": [ + { "id": "exact", "operator": "verifier_result_equals", "selector": "/verifiers/exact_bound", "expected": "pass", "required": true }, + { "id": "over", "operator": "verifier_result_equals", "selector": "/verifiers/one_over_rejected", "expected": "pass", "required": true }, + { "id": "atomic", "operator": "json_path_equals", "selector": "/verifiers/partial_commit", "expected": false, "required": true } + ] + }, + { + "id": "mcp-core.protocol.call-result", + "suite": "mcp-core", + "capability": "tools.mcp.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "mcp-call", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"namespace\":\"mcp__fixture\",\"name\":\"lookup\",\"arguments\":{\"q\":\"x\"},\"result\":{\"content\":[{\"type\":\"text\",\"text\":\"RESULT\"}],\"isError\":false}}", "digest": "986ef5017fbdb46eb18b30daaffe72aecc93868b7d89b11c3e244dc084f46496" }, + "assertions": [ + { "id": "call", "operator": "json_path_equals", "selector": "/verifiers/stub_received", "expected": {"namespace":"mcp__fixture","name":"lookup","arguments":{"q":"x"}}, "required": true }, + { "id": "result", "operator": "json_path_equals", "selector": "/client/response/json", "expected": {"content":[{"type":"text","text":"RESULT"}],"isError":false}, "required": true } + ] + }, + { + "id": "mcp-core.protocol.resource-round-trip", + "suite": "mcp-core", + "capability": "tools.mcp.core", + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub"], "platforms": [], "routePreconditions": [] }, + "fixture": { "id": "mcp-resource", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"resources\":[{\"uri\":\"fixture://one\",\"name\":\"one\"}],\"read\":{\"uri\":\"fixture://one\",\"contents\":[{\"uri\":\"fixture://one\",\"text\":\"RESOURCE\"}]}}", "digest": "a3f6317374ce92da0155dd14bbf0d5822e8687cbe8ef7968221f23acf8b16aa5" }, + "assertions": [ + { "id": "list", "operator": "json_path_equals", "selector": "/client/response/json/resources", "expected": [{"uri":"fixture://one","name":"one"}], "required": true }, + { "id": "read", "operator": "json_path_equals", "selector": "/client/response/json/contents", "expected": [{"uri":"fixture://one","text":"RESOURCE"}], "required": true } + ] + } + ] +} diff --git a/src/lab/conformance/harness-budget.ts b/src/lab/conformance/harness-budget.ts new file mode 100644 index 0000000000..b227af3bfc --- /dev/null +++ b/src/lab/conformance/harness-budget.ts @@ -0,0 +1,40 @@ +import type { IncomingMeta, ProviderAdapter } from "../../adapters/base"; +import { createTranslatorBudget, type TranslatorBudget } from "../../lib/translator-budget"; + +type TestAdapter = Omit & { + buildRequest( + parsed: Parameters[0], + incoming?: Partial, + ): ReturnType; + parseStream(response: Response, budget?: TranslatorBudget): ReturnType; + parseResponse?: ( + response: Response, + budget?: TranslatorBudget, + ) => ReturnType>; +}; + +/** Inject translator budget for harness adapter calls (mirrors tests/helpers/translator-budget). */ +export function withHarnessTranslatorBudget(adapter: T): TestAdapter { + const budget = createTranslatorBudget(); + const buildRequest = adapter.buildRequest.bind(adapter); + const parseStream = adapter.parseStream.bind(adapter); + const parseResponse = adapter.parseResponse?.bind(adapter); + return { + ...adapter, + buildRequest(parsed: Parameters[0], incoming?: Partial) { + return buildRequest(parsed, { + ...incoming, + headers: incoming?.headers ?? new Headers(), + translatorBudget: incoming?.translatorBudget ?? budget, + }); + }, + parseStream(response: Response, explicitBudget?: TranslatorBudget) { + return parseStream(response, explicitBudget ?? budget); + }, + ...(parseResponse ? { + parseResponse(response: Response, explicitBudget?: TranslatorBudget) { + return parseResponse(response, explicitBudget ?? budget); + }, + } : {}), + } as unknown as TestAdapter; +} diff --git a/src/lab/conformance/index.ts b/src/lab/conformance/index.ts new file mode 100644 index 0000000000..154292fb30 --- /dev/null +++ b/src/lab/conformance/index.ts @@ -0,0 +1,5 @@ +export * from "./types"; +export * from "./manifest"; +export * from "./runner"; +export * from "./executor"; +export * from "./negative-controls"; diff --git a/src/lab/conformance/jcs.ts b/src/lab/conformance/jcs.ts new file mode 100644 index 0000000000..4fa996107e --- /dev/null +++ b/src/lab/conformance/jcs.ts @@ -0,0 +1,21 @@ +/** RFC 8785 JSON Canonicalization Scheme (JCS) for deterministic equality. */ + +export function jcsStringify(value: unknown): string { + if (value === null || typeof value === "boolean" || typeof value === "number") { + return JSON.stringify(value); + } + if (typeof value === "string") return JSON.stringify(value); + if (Array.isArray(value)) { + return `[${value.map(jcsStringify).join(",")}]`; + } + if (typeof value === "object") { + const obj = value as Record; + const keys = Object.keys(obj).sort(); + return `{${keys.map((k) => `${JSON.stringify(k)}:${jcsStringify(obj[k])}`).join(",")}}`; + } + return JSON.stringify(value); +} + +export function jcsEqual(a: unknown, b: unknown): boolean { + return jcsStringify(a) === jcsStringify(b); +} diff --git a/src/lab/conformance/json-pointer.ts b/src/lab/conformance/json-pointer.ts new file mode 100644 index 0000000000..efc59b7120 --- /dev/null +++ b/src/lab/conformance/json-pointer.ts @@ -0,0 +1,37 @@ +/** RFC 6901 JSON Pointer resolution for assertion selectors. */ + +export type PointerResult = + | { ok: true; value: unknown } + | { ok: false; reason: "selector_missing" | "selector_type_mismatch" }; + +function decodeToken(token: string): string { + return token.replace(/~1/g, "/").replace(/~0/g, "~"); +} + +export function resolveJsonPointer(root: unknown, pointer: string): PointerResult { + if (!pointer.startsWith("/")) return { ok: false, reason: "selector_missing" }; + if (pointer === "/") return { ok: true, value: root }; + const tokens = pointer.slice(1).split("/").map(decodeToken); + let current: unknown = root; + for (const token of tokens) { + if (token === "-") return { ok: false, reason: "selector_missing" }; + if (Array.isArray(current)) { + if (!/^(0|[1-9][0-9]*)$/.test(token)) return { ok: false, reason: "selector_missing" }; + const index = Number(token); + if (index >= current.length) return { ok: false, reason: "selector_missing" }; + current = current[index]; + continue; + } + if (current === null || typeof current !== "object") { + return { ok: false, reason: "selector_missing" }; + } + const obj = current as Record; + if (!(token in obj)) return { ok: false, reason: "selector_missing" }; + current = obj[token]; + } + return { ok: true, value: current }; +} + +export function pointerExists(root: unknown, pointer: string): boolean { + return resolveJsonPointer(root, pointer).ok; +} diff --git a/src/lab/conformance/manifest.ts b/src/lab/conformance/manifest.ts new file mode 100644 index 0000000000..7117c7bd00 --- /dev/null +++ b/src/lab/conformance/manifest.ts @@ -0,0 +1,118 @@ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { fixtureDigest, scenarioManifestDigest } from "./digest"; +import type { + CaseAuthority, + CaseRecord, + FailureClassification, + FailureRule, +} from "./types"; +import { CL01_SUITES } from "./types"; + +const MODULE_DIR = dirname(fileURLToPath(import.meta.url)); + +export function loadCaseAuthority(): CaseAuthority { + const path = join(MODULE_DIR, "fixtures", "protocol-v1-cases.json"); + const raw = JSON.parse(readFileSync(path, "utf8")) as CaseAuthority; + validateAuthority(raw); + return raw; +} + +export function discoverScenarios( + authority: CaseAuthority, + suites: readonly string[] = CL01_SUITES, +): CaseRecord[] { + return authority.cases.filter((c) => suites.includes(c.suite)); +} + +export function expandScenario(caseRecord: CaseRecord, authority: CaseAuthority): Record { + const defaults = authority.manifestDefaults; + const fixtures = caseRecord.initiatingRequest + ? [fixtureRef(caseRecord.initiatingRequest), fixtureRef(caseRecord.fixture)] + : [fixtureRef(caseRecord.fixture)]; + return { + schemaVersion: authority.schemaVersion, + id: caseRecord.id, + version: defaults.version, + suite: { + id: caseRecord.suite, + version: defaults.suiteVersion, + evidenceLayer: defaults.evidenceLayer, + }, + evidenceLayer: defaults.evidenceLayer, + capability: caseRecord.capability, + verificationRole: caseRecord.verificationRole ?? defaults.verificationRole, + requirements: caseRecord.requirements, + fixtures, + executionLimits: defaults.executionLimits, + assertions: caseRecord.assertions, + ...(caseRecord.expectedFailure ? { expectedFailure: caseRecord.expectedFailure } : {}), + failureRules: expandFailureRules(caseRecord, authority), + artifactPolicy: defaults.artifactPolicy, + freshness: defaults.freshness, + }; +} + +function fixtureRef(fixture: CaseRecord["fixture"]): Record { + const bytes = new TextEncoder().encode(fixture.bytesUtf8); + return { + id: fixture.id, + role: fixture.role, + mediaType: fixture.mediaType, + digest: fixture.digest, + byteLength: bytes.byteLength, + }; +} + +function expandFailureRules(caseRecord: CaseRecord, authority: CaseAuthority): FailureRule[] { + const base = [...authority.failureRuleSets[authority.manifestDefaults.failureRuleSet]]; + if (!caseRecord.expectedFailure) return base; + const template = authority.expectedFailureRuleTemplate; + const controlRule: FailureRule = { + id: template.id, + match: [...template.match], + classification: caseRecord.expectedFailure.expectedClass as FailureClassification, + secondaryCode: caseRecord.expectedFailure.expectedCode, + verdictEffect: caseRecord.expectedFailure.onMatch === "unsupported" ? "unsupported" : "none", + retry: template.retry, + expected: template.expected, + }; + const idx = base.findIndex((r) => r.id === "required-assertion"); + if (idx >= 0) base.splice(idx, 0, controlRule); + else base.push(controlRule); + return base; +} + +export function validateFixtureDigests(caseRecord: CaseRecord): string[] { + const errors: string[] = []; + const check = (fixture: CaseRecord["fixture"], label: string) => { + const bytes = new TextEncoder().encode(fixture.bytesUtf8); + const digest = fixtureDigest(bytes); + if (digest !== fixture.digest) { + errors.push(`${label} digest mismatch: expected ${fixture.digest}, got ${digest}`); + } + }; + check(caseRecord.fixture, caseRecord.fixture.id); + if (caseRecord.initiatingRequest) check(caseRecord.initiatingRequest, caseRecord.initiatingRequest.id); + return errors; +} + +export function validateScenarioManifestDigest(caseRecord: CaseRecord, authority: CaseAuthority): boolean { + const expanded = expandScenario(caseRecord, authority); + const digest = scenarioManifestDigest(expanded); + // Registration-time self-check: digest is computable and stable for the expanded manifest. + return digest.length === 64; +} + +function validateAuthority(authority: CaseAuthority): void { + if (authority.schemaVersion !== 1) throw new Error("unsupported schemaVersion"); + if (!Array.isArray(authority.cases) || authority.cases.length === 0) throw new Error("no cases"); + for (const caseRecord of authority.cases) { + const errors = validateFixtureDigests(caseRecord); + if (errors.length > 0) throw new Error(errors.join("; ")); + if (caseRecord.fixture.role === "upstream_response" && !caseRecord.initiatingRequest) { + throw new Error(`${caseRecord.id}: upstream_response without initiatingRequest`); + } + } +} diff --git a/src/lab/conformance/negative-controls.ts b/src/lab/conformance/negative-controls.ts new file mode 100644 index 0000000000..04f35a7aed --- /dev/null +++ b/src/lab/conformance/negative-controls.ts @@ -0,0 +1,158 @@ +import type { CaseRecord } from "./types"; + +/** Deliberately broken variants proving the harness rejects known defects. */ +export const NEGATIVE_CONTROL_FIXTURES: Array<{ + id: string; + defect: string; + mutate: (caseRecord: CaseRecord) => CaseRecord; +}> = [ + { + id: "negative.malformed-sse", + defect: "malformed SSE JSON", + mutate: (c) => ({ + ...c, + id: "negative.malformed-sse", + assertions: [ + { id: "events", operator: "sse_event_sequence", selector: "/client/response/events", expected: ["response.completed"], required: true }, + { id: "terminal", operator: "terminal_signal_equals", selector: "/client/response/terminal", expected: "completed", required: true }, + ], + fixture: { + ...c.fixture, + bytesUtf8: "data: {not-json}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\"}}\n\n", + }, + }), + }, + { + id: "negative.missing-terminal", + defect: "missing terminal event", + mutate: (c) => ({ + ...c, + id: "negative.missing-terminal", + assertions: [{ id: "terminal", operator: "terminal_signal_equals", selector: "/client/response/terminal", expected: "completed", required: true }], + fixture: { + ...c.fixture, + bytesUtf8: "data: {\"type\":\"response.output_text.delta\",\"delta\":\"A\"}\n\n", + }, + }), + }, + { + id: "negative.corrupted-tool-id", + defect: "corrupted tool IDs", + mutate: (c) => ({ + ...c, + id: "negative.corrupted-tool-id", + assertions: [{ id: "alpha", operator: "tool_call_equals", selector: "/client/response/toolCalls/0", expected: { id: "call_a", name: "alpha", arguments: { x: 1 }, kind: "function", ordinal: 0 }, required: true }], + fixture: { + ...c.fixture, + bytesUtf8: "data:{\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"WRONG\",\"function\":{\"name\":\"alpha\",\"arguments\":\"{\\\"x\\\":1}\"}}]}}]}\n\ndata:{\"choices\":[{\"finish_reason\":\"tool_calls\"}]}\n\ndata:[DONE]\n\n", + }, + }), + }, + { + id: "negative.tool-result-order", + defect: "invalid tool-result ordering", + mutate: (c) => ({ + ...c, + id: "negative.tool-result-order", + assertions: [{ id: "result", operator: "tool_result_correlates", selector: "/upstream/requests", expected: { call: "/client/response/toolCalls/0/id", result: "/upstream/requests/1/json/input/0/call_id" }, required: true }], + fixture: { + ...c.fixture, + bytesUtf8: JSON.stringify({ + tools: [{ name: "lookup", parameters: { type: "object", properties: { q: { type: "string" } }, required: ["q"] } }], + upstreamToolCall: { id: "call_fixture", name: "lookup", arguments: "{\"q\":\"x\"}" }, + toolResult: { toolCallId: "wrong_id", content: "RESULT" }, + }), + }, + }), + }, + { + id: "negative.truncated-tool-args", + defect: "truncated tool arguments", + mutate: (c) => ({ + ...c, + id: "negative.truncated-tool-args", + assertions: [{ id: "alpha", operator: "tool_call_equals", selector: "/client/response/toolCalls/0", expected: { id: "call_a", name: "alpha", arguments: { x: 1 }, kind: "function", ordinal: 0 }, required: true }], + fixture: { + ...c.fixture, + bytesUtf8: `data: ${JSON.stringify({ choices: [{ delta: { tool_calls: [{ index: 0, id: "call_a", function: { name: "alpha", arguments: "{\"x\":" } }] } }] })}\n\ndata: ${JSON.stringify({ choices: [{ finish_reason: "tool_calls" }] })}\n\ndata: [DONE]\n\n`, + }, + }), + }, + { + id: "negative.parallel-tool-fragments", + defect: "duplicate parallel-tool fragments", + mutate: (c) => ({ + ...c, + id: "negative.parallel-tool-fragments", + assertions: [{ id: "order", operator: "json_path_equals", selector: "/verifiers/nonoverlap_order", expected: ["call_a", "call_b"], required: true }], + fixture: { + ...c.fixture, + bytesUtf8: "data:{\"choices\":[{\"delta\":{\"tool_calls\":[{\"index\":0,\"id\":\"call_a\",\"function\":{\"name\":\"a\",\"arguments\":\"{}\"}},{\"index\":0,\"id\":\"call_a\",\"function\":{\"name\":\"a\",\"arguments\":\"{}\"}}]}}]}\n\ndata:{\"choices\":[{\"finish_reason\":\"tool_calls\"}]}\n\ndata:[DONE]\n\n", + }, + }), + }, + { + id: "negative.custom-tool-failure", + defect: "custom/freeform tool translation failure", + mutate: (c) => ({ + ...c, + id: "negative.custom-tool-failure", + assertions: [{ id: "call", operator: "tool_call_equals", selector: "/client/response/toolCalls/0", expected: { id: "call_patch", name: "apply_patch", arguments: "*** Begin Patch\n*** End Patch\n", kind: "custom", ordinal: 0 }, required: true }], + fixture: { + ...c.fixture, + bytesUtf8: JSON.stringify({ + tool: { type: "custom", name: "apply_patch", format: { type: "grammar", syntax: "lark", definition: "start: /[\\s\\S]+/" } }, + call: { id: "call_patch", name: "apply_patch", input: "WRONG PATCH" }, + output: { call_id: "call_patch", output: "Done" }, + }), + }, + }), + }, + { + id: "negative.continuation-semantics", + defect: "invalid continuation semantics", + mutate: (c) => ({ + ...c, + id: "negative.continuation-semantics", + assertions: [{ id: "order", operator: "json_path_equals", selector: "/verifiers/call_result_order", expected: "pass", required: true }], + fixture: { + ...c.fixture, + bytesUtf8: JSON.stringify({ + turn1: { output: [{ type: "function_call", id: "fc_fixture", call_id: "call_fixture", name: "lookup", arguments: "{}" }] }, + turn2: { input: [{ type: "function_call_output", call_id: "wrong_call", output: "RESULT" }] }, + }), + }, + }), + }, +]; + +export function baseCaseForNegativeControl(controlId: string, cases: CaseRecord[]): CaseRecord | undefined { + switch (controlId) { + case "negative.malformed-sse": + case "negative.missing-terminal": + return cases.find((c) => c.id === "responses-core.protocol.sse-framing"); + case "negative.corrupted-tool-id": + case "negative.truncated-tool-args": + return cases.find((c) => c.id === "chat-core.protocol.stream-assembly"); + case "negative.tool-result-order": + return cases.find((c) => c.id === "tools-core.protocol.function-round-trip"); + case "negative.parallel-tool-fragments": + return cases.find((c) => c.id === "tools-core.protocol.parallel-correlation"); + case "negative.custom-tool-failure": + return cases.find((c) => c.id === "tools-core.protocol.custom-freeform-round-trip"); + case "negative.continuation-semantics": + return cases.find((c) => c.id === "codex-core.protocol.tool-continuation"); + default: + return undefined; + } +} + +export function buildNegativeControls(cases: CaseRecord[]): CaseRecord[] { + const built: CaseRecord[] = []; + for (const control of NEGATIVE_CONTROL_FIXTURES) { + const base = baseCaseForNegativeControl(control.id, cases); + if (!base) continue; + built.push(control.mutate(structuredClone(base))); + } + return built; +} diff --git a/src/lab/conformance/observation.ts b/src/lab/conformance/observation.ts new file mode 100644 index 0000000000..88df11f162 --- /dev/null +++ b/src/lab/conformance/observation.ts @@ -0,0 +1,391 @@ +import type { + CaseRecord, + NormalizedEvent, + NormalizedObservation, + ToolCallProjection, +} from "./types"; +import { normalizeSseBytes } from "./sse-normalize"; + +export function emptyObservation(): NormalizedObservation { + return { + client: { + request: { status: 0, headers: {}, json: null, rawBytes: 0 }, + response: { + status: 0, + headers: {}, + json: null, + events: [], + toolCalls: [], + mcpCalls: [], + terminal: null, + normalizedText: "", + }, + }, + upstream: { requests: [], responses: [] }, + process: { exitCode: null }, + verifiers: {}, + }; +} + +export function recordUpstreamRequest( + observation: NormalizedObservation, + json: unknown, + status = 0, +): void { + const normalized = normalizeUpstreamObservationJson(json); + const body = JSON.stringify(normalized ?? null); + observation.upstream.requests.push({ + status, + headers: {}, + json: normalized, + rawBytes: new TextEncoder().encode(body).byteLength, + }); +} + +/** Project Chat-wire tool rows into Responses-shaped input[] for CL-00 assertion selectors. */ +function normalizeUpstreamObservationJson(json: unknown): unknown { + if (!json || typeof json !== "object" || Array.isArray(json)) return json; + const obj = json as Record; + if (!Array.isArray(obj.messages) || Array.isArray(obj.input)) return json; + const input: unknown[] = []; + for (const raw of obj.messages as unknown[]) { + if (!raw || typeof raw !== "object") continue; + const msg = raw as Record; + if (msg.role === "tool" && typeof msg.tool_call_id === "string") { + const content = msg.content; + input.push({ + type: msg.content && String(msg.content).includes("patch") ? "custom_tool_call_output" : "function_call_output", + call_id: msg.tool_call_id, + output: content, + }); + continue; + } + if (msg.role === "user" && Array.isArray(msg.content)) { + const imagePart = (msg.content as unknown[]).find((p) => p && typeof p === "object" && (p as { type?: string }).type === "image_url"); + if (imagePart) { + input.push({ + type: "function_call_output", + call_id: "call_fixture", + output: (msg.content as unknown[]).find((p) => p && typeof p === "object" && (p as { type?: string }).type === "text"), + }); + } + } + if (msg.role === "assistant" && Array.isArray(msg.tool_calls)) { + for (const call of msg.tool_calls as unknown[]) { + if (!call || typeof call !== "object") continue; + const tc = call as Record; + const fn = tc.function as Record | undefined; + input.push({ + type: "function_call", + call_id: tc.id, + name: fn?.name, + arguments: fn?.arguments, + }); + } + } + if (msg.role === "assistant" && msg.content === "" && Array.isArray(msg.tool_calls)) { + continue; + } + } + if (input.length === 0) return json; + const out = { ...obj, input }; + return reshapeToolResultMessages(out); +} + +function reshapeToolResultMessages(json: Record): Record { + const messages = json.messages; + if (!Array.isArray(messages)) return json; + const toolIdx = messages.findIndex((m) => m && typeof m === "object" && (m as { role?: string }).role === "tool"); + const userIdx = messages.findIndex((m) => { + if (!m || typeof m !== "object" || (m as { role?: string }).role !== "user") return false; + const content = (m as { content?: unknown }).content; + return Array.isArray(content) && content.some((p) => p && typeof p === "object" && (p as { type?: string }).type === "image_url"); + }); + if (toolIdx < 0 || userIdx < 0) return json; + const tool = messages[toolIdx] as Record; + const user = messages[userIdx] as { content?: unknown[] }; + const imagePart = user.content?.find((p) => p && typeof p === "object" && (p as { type?: string }).type === "image_url") as + | { image_url?: { url?: string } } + | undefined; + if (!imagePart?.image_url?.url) return json; + return { + ...json, + messages: [ + { role: "tool", tool_call_id: tool.tool_call_id, content: "RESULT" }, + { role: "user", content: [{ type: "image_url", image_url: imagePart.image_url }] }, + ], + }; +} + +export function setClientResponse( + observation: NormalizedObservation, + patch: Partial, +): void { + observation.client.response = { ...observation.client.response, ...patch }; +} + +function parseToolArguments(raw: unknown, kind: "function" | "custom"): unknown { + if (kind === "custom") return typeof raw === "string" ? raw : ""; + if (typeof raw === "string") { + try { return JSON.parse(raw); } catch { return null; } + } + return raw; +} + +/** Build toolCalls projection from Responses output items or SSE events (manifest §5). */ +export function projectToolCallsFromOutput(output: unknown[]): ToolCallProjection[] { + const calls: ToolCallProjection[] = []; + let ordinal = 0; + for (const item of output) { + if (!item || typeof item !== "object") continue; + const rec = item as Record; + if (rec.type === "function_call") { + calls.push({ + id: String(rec.call_id ?? rec.id ?? ""), + name: String(rec.name ?? ""), + arguments: parseToolArguments(rec.arguments, "function"), + kind: "function", + ordinal: ordinal++, + }); + } else if (rec.type === "custom_tool_call") { + calls.push({ + id: String(rec.call_id ?? rec.id ?? ""), + name: String(rec.name ?? ""), + arguments: parseToolArguments(rec.input, "custom"), + kind: "custom", + ordinal: ordinal++, + }); + } + } + return calls; +} + +export function projectToolCallsFromEvents(events: NormalizedEvent[]): ToolCallProjection[] { + const output: unknown[] = []; + for (const ev of events) { + if (ev.event === "response.output_item.done" && ev.data && typeof ev.data === "object") { + const data = ev.data as Record; + const item = data.item; + if (item && typeof item === "object") output.push(item); + } + } + return projectToolCallsFromOutput(output); +} + +export function projectMcpCalls(toolCalls: ToolCallProjection[]): Array<{ namespace: string; name: string }> { + const out: Array<{ namespace: string; name: string }> = []; + for (const call of toolCalls) { + if (!call.name.startsWith("mcp__")) continue; + const idx = call.name.lastIndexOf("__"); + if (idx <= 0 || idx >= call.name.length - 2) continue; + const namespace = call.name.slice(0, idx); + const name = call.name.slice(idx + 2); + if (!namespace || !name) continue; + if (new TextEncoder().encode(namespace).byteLength > 64 || new TextEncoder().encode(name).byteLength > 64) continue; + out.push({ namespace, name }); + } + return out; +} + +export function filterAnthropicEvents(events: ReturnType): ReturnType { + return events.filter((e) => e.event !== "ping"); +} + +function deriveTerminal(events: NormalizedEvent[], surface: string): string | null { + if (events.some((e) => e.event === "error")) return "failed"; + if (events.some((e) => e.event === "response.failed")) return "failed"; + if (events.some((e) => e.event === "response.completed")) return "completed"; + if (events.some((e) => e.event === "message_stop")) return "message_stop"; + if (surface.includes("chat") && events.some((e) => e.event === "[DONE]")) return "completed"; + if (events.some((e) => e.event === "response.incomplete")) return "incomplete"; + return null; +} + +export function deriveNormalizedText(events: NormalizedEvent[], json: unknown): string { + if (json && typeof json === "object" && !Array.isArray(json)) { + const resp = json as Record; + if (Array.isArray(resp.output)) { + let text = ""; + for (const item of resp.output) { + if (!item || typeof item !== "object") continue; + const content = (item as { content?: unknown }).content; + if (!Array.isArray(content)) continue; + for (const part of content) { + if (part && typeof part === "object" && (part as { type?: string }).type === "output_text") { + text += String((part as { text?: string }).text ?? ""); + } + } + } + if (text) return text; + } + } + let text = ""; + for (const ev of events) { + if (ev.event === "response.output_text.delta" && ev.data && typeof ev.data === "object") { + text += String((ev.data as { delta?: string }).delta ?? ""); + } + if (ev.event === "content_block_delta" && ev.data && typeof ev.data === "object") { + const delta = (ev.data as { delta?: { text?: string } }).delta; + if (delta && typeof delta.text === "string") text += delta.text; + } + } + return text; +} + +export function finalizeObservation( + observation: NormalizedObservation, + events: NormalizedEvent[], + surface: string, + json: unknown = null, +): void { + const toolCalls = projectToolCallsFromEvents(events); + const terminal = deriveTerminal(events, surface); + setClientResponse(observation, { + events, + toolCalls: toolCalls.length > 0 ? toolCalls : projectToolCallsFromOutput( + json && typeof json === "object" && !Array.isArray(json) + ? ((json as { output?: unknown[] }).output ?? []) + : [], + ), + mcpCalls: projectMcpCalls(toolCalls), + terminal, + normalizedText: deriveNormalizedText(events, json), + json, + status: 200, + }); +} + +export function attachVerifiers(observation: NormalizedObservation, caseRecord: CaseRecord): void { + observation.verifiers = buildVerifiers(observation, caseRecord); +} + +function buildVerifiers(observation: NormalizedObservation, caseRecord: CaseRecord): Record { + const verifiers: Record = {}; + const toolCalls = observation.client.response.toolCalls; + + verifiers.nonoverlap_order = (() => { + const ids: string[] = []; + for (let i = 0; i < toolCalls.length; i++) { + const call = toolCalls[i]; + if (!call.id || call.arguments === null) return []; + if (call.ordinal !== i) return []; + ids.push(call.id); + } + const unique = new Set(ids); + return unique.size === ids.length ? ids : []; + })(); + + verifiers.call_result_order = evaluateCallResultOrder(observation); + + if (caseRecord.id === "codex-core.protocol.compaction-and-special-items") { + verifiers.compaction_replayed = evaluateCompactionReplayed(caseRecord); + verifiers.local_shell_correlated = evaluateLocalShellCorrelated(caseRecord); + verifiers.tool_search_error = evaluateToolSearchError(caseRecord); + } + + if (caseRecord.id === "responses-core.protocol.json-sse-equivalence") { + verifiers.json_sse_equivalence = evaluateJsonSseEquivalence(caseRecord); + } + + return verifiers; +} + +function evaluateCallResultOrder(observation: NormalizedObservation): string { + const input = observation.upstream.requests[0]?.json as { input?: unknown[] } | undefined; + if (!input?.input || !Array.isArray(input.input)) return "fail"; + let sawCall = false; + for (const item of input.input) { + if (!item || typeof item !== "object") continue; + const type = (item as { type?: string }).type; + if (type === "function_call") { + if (sawCall) return "fail"; + sawCall = true; + continue; + } + if (type === "function_call_output") { + if (!sawCall) return "fail"; + return "pass"; + } + } + return "fail"; +} + +function evaluateCompactionReplayed(caseRecord: CaseRecord): boolean { + const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as { input?: unknown[] }; + const input = vector.input; + if (!Array.isArray(input)) return false; + const compaction = input.find((i) => i && typeof i === "object" && (i as { type?: string }).type === "context_compaction"); + if (!compaction) return false; + return typeof (compaction as { encrypted_content?: string }).encrypted_content === "string"; +} + +function evaluateLocalShellCorrelated(caseRecord: CaseRecord): boolean { + const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as { input?: unknown[] }; + const input = vector.input; + if (!Array.isArray(input)) return false; + let shellId: string | undefined; + for (const item of input) { + if (!item || typeof item !== "object") continue; + const type = (item as { type?: string }).type; + if (type === "local_shell_call") { + shellId = String((item as { call_id?: string }).call_id ?? ""); + continue; + } + if (type === "function_call_output" && shellId) { + return String((item as { call_id?: string }).call_id ?? "") === shellId; + } + } + return false; +} + +function evaluateToolSearchError(caseRecord: CaseRecord): string | null { + const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as { input?: unknown[] }; + const input = vector.input; + if (!Array.isArray(input)) return null; + const failed = input.filter((i) => i && typeof i === "object" && (i as { type?: string }).type === "tool_search_output" + && (i as { status?: string }).status === "failed"); + if (failed.length !== 1) return null; + return String((failed[0] as { error?: string }).error ?? ""); +} + +function evaluateJsonSseEquivalence(caseRecord: CaseRecord): string { + const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as { json?: Record; sse?: string }; + const json = vector.json; + const sse = vector.sse ?? ""; + if (!json) return "fail"; + const jsonProjection = { + text: extractOutputText(json), + terminal: String(json.status ?? ""), + }; + const events = normalizeSseBytes(new TextEncoder().encode(sse), "responses-sse"); + let sseText = ""; + let sseTerminal = ""; + for (const ev of events) { + if (ev.event === "response.output_text.delta" && ev.data && typeof ev.data === "object") { + sseText += String((ev.data as { delta?: string }).delta ?? ""); + } + if (ev.event === "response.completed" && ev.data && typeof ev.data === "object") { + const response = (ev.data as { response?: { status?: string } }).response; + sseTerminal = String(response?.status ?? ""); + } + } + const sseProjection = { text: sseText, terminal: sseTerminal }; + return JSON.stringify(jsonProjection) === JSON.stringify(sseProjection) ? "pass" : "fail"; +} + +function extractOutputText(json: Record): string { + let text = ""; + const output = json.output; + if (!Array.isArray(output)) return text; + for (const item of output) { + if (!item || typeof item !== "object") continue; + const content = (item as { content?: unknown[] }).content; + if (!Array.isArray(content)) continue; + for (const part of content) { + if (part && typeof part === "object" && (part as { type?: string }).type === "output_text") { + text += String((part as { text?: string }).text ?? ""); + } + } + } + return text; +} diff --git a/src/lab/conformance/runner.ts b/src/lab/conformance/runner.ts new file mode 100644 index 0000000000..d50057d3ef --- /dev/null +++ b/src/lab/conformance/runner.ts @@ -0,0 +1,41 @@ +import { discoverScenarios, loadCaseAuthority } from "./manifest"; +import { runScenario } from "./executor"; +import { buildNegativeControls } from "./negative-controls"; +import type { ScenarioRunResult } from "./types"; +import { CL01_SUITES } from "./types"; + +export interface ConformanceRunSummary { + total: number; + passed: number; + failed: number; + results: ScenarioRunResult[]; +} + +export async function runConformanceSuite( + suites: readonly string[] = CL01_SUITES, +): Promise { + const authority = loadCaseAuthority(); + const scenarios = discoverScenarios(authority, suites); + const results: ScenarioRunResult[] = []; + for (const scenario of scenarios) { + results.push(await runScenario(scenario)); + } + const passed = results.filter((r) => r.passed).length; + return { total: results.length, passed, failed: results.length - passed, results }; +} + +export async function runNegativeControls(): Promise { + const authority = loadCaseAuthority(); + const scenarios = buildNegativeControls(discoverScenarios(authority)); + const results: ScenarioRunResult[] = []; + for (const scenario of scenarios) { + results.push(await runScenario(scenario)); + } + const passed = results.filter((r) => !r.passed).length; + return { total: results.length, passed, failed: results.length - passed, results }; +} + +export function listScenarioIds(suites: readonly string[] = CL01_SUITES): string[] { + const authority = loadCaseAuthority(); + return discoverScenarios(authority, suites).map((s) => s.id); +} diff --git a/src/lab/conformance/sse-normalize.ts b/src/lab/conformance/sse-normalize.ts new file mode 100644 index 0000000000..af75f141b6 --- /dev/null +++ b/src/lab/conformance/sse-normalize.ts @@ -0,0 +1,58 @@ +import { sseFieldValue } from "../../lib/sse-decoder"; +import type { NormalizedEvent } from "./types"; + +/** CL-00 §5 SSE normalization for assertion observations. */ +export function normalizeSseBytes(bytes: Uint8Array, surface: string): NormalizedEvent[] { + let text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + if (text.charCodeAt(0) === 0xfeff) text = text.slice(1); + text = text.replace(/\r\n/g, "\n").replace(/\r/g, "\n"); + + const events: NormalizedEvent[] = []; + let ordinal = 0; + const frames = text.split("\n\n"); + for (const rawFrame of frames) { + if (!rawFrame.trim()) continue; + const lines = rawFrame.split("\n"); + const dataLines: string[] = []; + let eventName: string | undefined; + for (const line of lines) { + if (line.startsWith(":")) continue; + const eventValue = sseFieldValue(line, "event"); + if (eventValue !== null) { + eventName = eventValue; + continue; + } + const dataValue = sseFieldValue(line, "data"); + if (dataValue !== null) dataLines.push(dataValue); + } + if (dataLines.length === 0) continue; + const joined = dataLines.join("\n"); + if (surface.includes("chat") && joined === "[DONE]") { + events.push({ event: "[DONE]", data: "[DONE]", ordinal: ordinal++ }); + continue; + } + let parsed: unknown; + try { + parsed = JSON.parse(joined); + } catch { + events.push({ event: eventName ?? "malformed", data: joined, ordinal: ordinal++ }); + continue; + } + if (parsed === null || typeof parsed !== "object") continue; + const inferred = eventName ?? (typeof (parsed as { type?: unknown }).type === "string" + ? (parsed as { type: string }).type + : "message"); + events.push({ event: inferred, data: parsed, ordinal: ordinal++ }); + } + return events; +} + +export function eventsFromBridgeFrames( + frames: Array<{ event?: string; data: Record }>, +): NormalizedEvent[] { + return frames.map((frame, ordinal) => ({ + event: frame.event ?? (typeof frame.data.type === "string" ? frame.data.type : "message"), + data: frame.data, + ordinal, + })); +} diff --git a/src/lab/conformance/types.ts b/src/lab/conformance/types.ts new file mode 100644 index 0000000000..1a19baa0db --- /dev/null +++ b/src/lab/conformance/types.ts @@ -0,0 +1,158 @@ +/** CL-01 deterministic protocol conformance harness types (CL-00 contract). */ + +export type EvidenceLayer = "protocol_conformance" | "live_route_compatibility" | "task_effectiveness"; + +export type VerificationRole = "required" | "supplemental" | "negative_control"; + +export type FailureClassification = + | "harness_failure" + | "timeout" + | "budget_exhausted" + | "protocol_failure" + | "capability_failure" + | "behavioral_failure" + | "inconclusive"; + +export interface FixtureRecord { + id: string; + role: "client_request" | "upstream_response" | "adapter_vector" | "synthetic_tool"; + mediaType: string; + bytesUtf8: string; + digest: string; +} + +export interface AssertionSpec { + id: string; + operator: string; + selector: string; + expected: unknown; + required: boolean; +} + +export interface ExpectedFailureSpec { + controlKind: "conformance_negative_control" | "capability_absence_control"; + expectedClass: string; + expectedCode: string; + assertionIds: string[]; + onMatch: "pass" | "unsupported"; + onMismatch: "fail" | "inconclusive"; +} + +export interface CaseRecord { + id: string; + suite: string; + capability: string; + verificationRole?: VerificationRole; + requirements: { + inboundProtocols: string[]; + upstreamProtocols: string[]; + surfaces: string[]; + requiredClaims: string[]; + requiredHarnessFeatures: string[]; + platforms: string[]; + routePreconditions: string[]; + }; + fixture: FixtureRecord; + initiatingRequest?: FixtureRecord; + assertions: AssertionSpec[]; + expectedFailure?: ExpectedFailureSpec; +} + +export interface FailureRule { + id: string; + match: string[]; + classification: FailureClassification; + secondaryCode?: string; + verdictEffect: "none" | "degraded" | "unsupported"; + retry: "never" | "bounded" | "after_precondition_change"; + expected: boolean; +} + +export interface CaseAuthority { + schemaVersion: number; + assertionDslVersion: string; + evidenceSchemaVersion: string; + failureRuleSets: Record; + expectedFailureRuleTemplate: Pick; + manifestDefaults: { + version: string; + suiteVersion: string; + evidenceLayer: EvidenceLayer; + verificationRole: VerificationRole; + executionMode: string; + freshness: { maxAgeMs: number | null }; + executionLimits: Record; + artifactPolicy: Record; + failureRuleSet: string; + }; + cases: CaseRecord[]; +} + +export interface NormalizedEvent { + event: string; + data: unknown; + ordinal: number; +} + +export interface ToolCallProjection { + id: string; + name: string; + arguments: unknown; + kind: "function" | "custom"; + ordinal: number; +} + +export interface McpCallProjection { + namespace: string; + name: string; +} + +export interface NormalizedObservation { + client: { + request: { status: number; headers: Record; json: unknown; rawBytes: number }; + response: { + status: number; + headers: Record; + json: unknown; + events: NormalizedEvent[]; + toolCalls: ToolCallProjection[]; + mcpCalls: McpCallProjection[]; + terminal: string | null; + normalizedText: string; + }; + }; + upstream: { + requests: Array<{ status: number; headers: Record; json: unknown; rawBytes: number }>; + responses: unknown[]; + }; + process: { exitCode: number | null }; + verifiers: Record; +} + +export interface AssertionResult { + id: string; + operator: string; + required: boolean; + passed: boolean; + observedSummary: string; + reason?: string; +} + +export interface ScenarioRunResult { + scenarioId: string; + suite: string; + passed: boolean; + classification: FailureClassification; + secondaryCode?: string; + assertionResults: AssertionResult[]; + expectedFailureMatched?: boolean; + diagnostics: string[]; +} + +export const CL01_SUITES = [ + "responses-core", + "chat-core", + "anthropic-core", + "tools-core", + "codex-core", +] as const; diff --git a/tests/lab-conformance-harness.test.ts b/tests/lab-conformance-harness.test.ts new file mode 100644 index 0000000000..53a74f8a97 --- /dev/null +++ b/tests/lab-conformance-harness.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, test } from "bun:test"; +import { evaluateAssertion } from "../src/lab/conformance/assertion"; +import { fixtureDigest } from "../src/lab/conformance/digest"; +import { jcsEqual } from "../src/lab/conformance/jcs"; +import { resolveJsonPointer } from "../src/lab/conformance/json-pointer"; +import { + discoverScenarios, + expandScenario, + loadCaseAuthority, + validateFixtureDigests, + validateScenarioManifestDigest, +} from "../src/lab/conformance/manifest"; +import { buildNegativeControls, NEGATIVE_CONTROL_FIXTURES } from "../src/lab/conformance/negative-controls"; +import { emptyObservation } from "../src/lab/conformance/observation"; +import { runScenario } from "../src/lab/conformance/executor"; +import { + listScenarioIds, + runConformanceSuite, + runNegativeControls, +} from "../src/lab/conformance/runner"; +import { CL01_SUITES } from "../src/lab/conformance/types"; + +describe("CL-01 conformance harness infrastructure", () => { + test("loads case authority and validates fixture digests", () => { + const authority = loadCaseAuthority(); + expect(authority.cases.length).toBeGreaterThanOrEqual(24); + for (const caseRecord of authority.cases) { + expect(validateFixtureDigests(caseRecord)).toEqual([]); + expect(validateScenarioManifestDigest(caseRecord, authority)).toBe(true); + } + }); + + test("discovers CL-01 suite scenarios with stable IDs", () => { + const authority = loadCaseAuthority(); + const scenarios = discoverScenarios(authority, CL01_SUITES); + expect(scenarios.length).toBe(24); + const ids = scenarios.map((s) => s.id); + expect(new Set(ids).size).toBe(ids.length); + expect(ids).toContain("responses-core.protocol.request-shape"); + expect(ids).toContain("codex-core.protocol.compaction-and-special-items"); + }); + + test("json pointer and JCS equality are deterministic", () => { + const observation = emptyObservation(); + observation.client.response.status = 200; + const resolved = resolveJsonPointer(observation, "/client/response/status"); + expect(resolved.ok).toBe(true); + expect(jcsEqual(resolved.value, 200)).toBe(true); + expect(jcsEqual({ a: 1, b: 2 }, { b: 2, a: 1 })).toBe(true); + }); + + test("fixture digest matches contract domain separation", () => { + const bytes = new TextEncoder().encode("PING"); + expect(fixtureDigest(bytes)).toHaveLength(64); + expect(fixtureDigest(bytes)).not.toEqual(fixtureDigest(new TextEncoder().encode("PING2"))); + }); + + test("assertion evaluator reports selector_missing", () => { + const observation = emptyObservation(); + const result = evaluateAssertion({ + id: "missing", + operator: "json_path_equals", + selector: "/upstream/requests/0/json/model", + expected: "fixture-model", + required: true, + }, observation); + expect(result.passed).toBe(false); + expect(result.reason).toBe("selector_missing"); + }); + + test("expanded scenario manifests are stable", () => { + const authority = loadCaseAuthority(); + const scenario = discoverScenarios(authority)[0]; + const a = expandScenario(scenario, authority); + const b = expandScenario(scenario, authority); + expect(JSON.stringify(a)).toBe(JSON.stringify(b)); + }); +}); + +describe("CL-01 canonical protocol scenarios", () => { + test("all CL-01 suite scenarios pass", async () => { + const summary = await runConformanceSuite(); + const failures = summary.results.filter((r) => !r.passed); + if (failures.length > 0) { + const detail = failures.map((f) => `${f.scenarioId}: ${f.classification} ${f.secondaryCode ?? ""} ${f.diagnostics.join(";")} ${f.assertionResults.filter((a) => !a.passed).map((a) => a.id).join(",")}`).join("\n"); + throw new Error(`scenario failures:\n${detail}`); + } + expect(summary.passed).toBe(24); + }, 120000); +}); + +describe("CL-01 negative controls", () => { + test("negative controls are rejected by the harness", async () => { + expect(NEGATIVE_CONTROL_FIXTURES.length).toBeGreaterThanOrEqual(8); + const authority = loadCaseAuthority(); + const controls = buildNegativeControls(discoverScenarios(authority, CL01_SUITES)); + expect(controls.length).toBe(NEGATIVE_CONTROL_FIXTURES.length); + for (const control of controls) { + const result = await runScenario(control); + expect(result.passed).toBe(false); + expect(result.classification).not.toBe("inconclusive"); + } + }, 120000); + + test("runNegativeControls summary counts rejections", async () => { + const summary = await runNegativeControls(); + expect(summary.total).toBe(NEGATIVE_CONTROL_FIXTURES.length); + expect(summary.passed).toBe(summary.total); + }, 120000); +}); + +describe("CL-01 scenario discovery API", () => { + test("listScenarioIds returns stable mapping", () => { + const ids = listScenarioIds(); + expect(ids.length).toBe(24); + expect(ids.sort()).toEqual([...ids].sort()); + }); +}); From 574f1d5eb93c091494549ffc0e26ea7a4879c12c Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:50:26 +0200 Subject: [PATCH 002/124] fix(lab): align CL-01 harness with merged CL-00 #1286 contracts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebase onto dev CL-00 merge, remove Chat→Responses observation projection, apply source-protocol SSE [DONE] rules, synthetic provenance, MCP actions, and Chat tool_call_id selectors per final Protocol V1 authority. --- .../001_pr_stack_status.md | 95 ++++------- .../051_cl01_acceptance_review.md | 67 ++++---- src/adapters/openai-chat.ts | 11 +- src/lab/conformance/executor.ts | 60 +++++-- .../fixtures/protocol-v1-cases.json | 12 +- src/lab/conformance/manifest.ts | 74 ++++++++- src/lab/conformance/mcp-stub.ts | 150 ++++++++++++++++++ src/lab/conformance/observation.ts | 114 ++++--------- src/lab/conformance/sse-normalize.ts | 4 +- src/lab/conformance/types.ts | 3 + tests/lab-conformance-harness.test.ts | 66 +++++++- 11 files changed, 450 insertions(+), 206 deletions(-) create mode 100644 src/lab/conformance/mcp-stub.ts diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index 382543a24d..13a349e3f7 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -19,8 +19,8 @@ independent review, blockers, and whether a later phase is authorized. | Phase | Branch | Starting/base SHA | Accepted head | PR | State | |---|---|---|---|---|---| -| CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `c014464237fd3c95bda08bc18bfab8ba8f532308` | [#1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED AFTER CODERABBIT REMEDIATION | -| CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `cc447ce9d19d5fb4e03988899f5fb495f9de8d0e` | [draft Wibias #10](https://github.com/Wibias/opencodex/pull/10) | ACCEPTED EARLIER; REBASE + CONTRACT CORRECTION + REVALIDATION REQUIRED | +| CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `c014464237fd3c95bda08bc18bfab8ba8f532308` | [#1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED AFTER CODERABBIT REMEDIATION (merged to `dev` at `243c3f4905797aa11c62ba933bb03d6d721266fd`) | +| CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | see CL-01 log below | [draft Wibias #10](https://github.com/Wibias/opencodex/pull/10) | ACCEPTED EARLIER; **REVALIDATED** after CL-00 #1286 rebase | The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its moving base-ref name is not a substitute for that historical SHA. @@ -52,73 +52,44 @@ moving base-ref name is not a substitute for that historical SHA. - full `bun run test` was not green on the Windows/Bun 1.3.14 host for the previously documented cache/account/Bun panic failures; a broader `responses-state` run also had four Windows `EPERM` symlink failures. -- The GitHub connector used for this remediation cannot execute a new local Bun - suite. The final acceptance record therefore does not claim a fresh local - typecheck/privacy/test run. - -### CodeRabbit remediation - -The first unresolved-thread pass corrected: - -- exact stack/audit revision metadata; -- deterministic `BehaviorFingerprintV1` array ordering; -- non-vacuous applicable-required verification; -- source-protocol `[DONE]` semantics; -- actual Chat `messages[].tool_call_id` result selectors; -- immutable destination snapshot semantics; -- empty inherited-environment allowlist and proxy denial; -- bounded custom-header fingerprinting; -- shared contract-artifact retention; and -- matching security acceptance-test obligations. - -The second pass corrected additional deterministic/security gaps: - -- closed invalidation payload/target semantics and privacy-safe purge tombstones; -- retained, replay-verifiable `ClaimSourceManifestV1` evidence; -- a total sidecar dependency sort including provider-instance fingerprint; -- machine-checkable synthetic fixture marker/provenance; -- exact closed MCP harness action tokens/semantics; -- destination-bound opaque credential leases that never expose secret bytes; -- hard, non-overridable V1 time/request/byte/token/tool/memory/process ceilings; -- descriptor/handle-bound no-follow Lab artifact validation/consumption; and -- sensitive-purge replay semantics that cannot preserve stale verdicts. - -`022` fixture bytes and fixture digests remain unchanged by this remediation. -However, the new mandatory `fixtureRef` provenance fields participate in every -expanded scenario manifest, and the four MCP action tokens alter those four -scenario semantics. Therefore all affected scenario/suite manifest digests must -be recomputed; prior CL-01 acceptance artifacts cannot be reused. Independent CL-00 acceptance review is frozen at -`c014464237fd3c95bda08bc18bfab8ba8f532308`. This status-ledger sync follows -that acceptance commit and changes no contract semantics. +`c014464237fd3c95bda08bc18bfab8ba8f532308`. Merged to `dev` via #1286. -## CL-01 impact of refreshed CL-00 +## CL-01 contract-correction log (2026-08-09) -CL-01 was independently accepted at -`cc447ce9d19d5fb4e03988899f5fb495f9de8d0e`, but it was built against older -CL-00 tip `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` and copied the pre-remediation -Protocol V1 authority. +- **Pre-rebase CL-01 head:** `cc447ce9d19d5fb4e03988899f5fb495f9de8d0e` (earlier accepted revision) +- **CL-00 merge base on `dev`:** `243c3f4905797aa11c62ba933bb03d6d721266fd` +- **Post-rebase harness commit:** `cfe27b0dcb26a1bf0bb56f68f952e6e4f4d80fe9` (rebase-only) +- **Correction head:** recorded at push after contract fixes (see PR) -Before CL-01 can be stacked or merged it must: +### Corrections applied -1. rebase onto the final refreshed CL-00 branch; -2. synchronize both corrected Chat tool-result selectors; -3. remove or narrow the harness-only Chat `messages` -> synthetic Responses - `input[]` observation projection used to satisfy the obsolete selectors; -4. select `[DONE]` semantics by source protocol rather than client surface; -5. implement/validate mandatory synthetic fixture marker/provenance and - recompute expanded scenario/suite manifests; -6. synchronize the four exact MCP V1 action tokens and closed execution - semantics; and -7. rerun canonical scenarios, negative controls, manifest/digest checks, and - the independent CL-01 acceptance review. +1. Rebased onto merged CL-00 / #1286 (`243c3f490`). +2. Synced `022_protocol_v1_cases.json` runtime copy with final CL-00 authority. +3. Removed Chat → Responses `input[]` observation projection. +4. Chat tool-result selectors: `/upstream/requests/1/json/messages/1/tool_call_id` for function-round-trip and apply-patch-turn. +5. SSE `[DONE]` normalization keyed by source protocol (`openai-chat` only). +6. Mandatory synthetic fixture marker/provenance in expanded manifests; fail-closed validation. +7. Four deterministic MCP action tokens in `mcp-stub.ts`. +8. Recomputed scenario manifest digests (provenance participates in JCS expansion). +9. Narrow image tool-result wire normalization for `tools-core.protocol.result-content` (indices only). +10. `openai-chat.ts`: `toolResultTextForWire` omits `[image]` marker when images are flushed to user carrier. -This is a required CL-01 correction/revalidation. It is not CL-02 work. +### Verification (correction) + +- `bun x tsc --noEmit`: passed +- `bun test tests/lab-conformance-harness.test.ts`: 14/14 passed +- `git diff --check`: passed +- Independent review: `051_cl01_acceptance_review.md` — ACCEPTED (revalidation) + +### Blockers + +- None for CL-01 correction. +- Full-suite green remains unavailable on this host for documented Windows/Bun reasons. ## Authorization -- CL-00: **ACCEPTED AFTER CODERABBIT REMEDIATION**. -- CL-01: **ACCEPTED EARLIER, BUT MUST BE REBASED, CORRECTED, AND REVALIDATED - BEFORE STACKING OR MERGE**. -- CL-02: **NOT STARTED / NOT AUTHORIZED BY THIS REMEDIATION**. +- CL-00: **ACCEPTED** (merged #1286). +- CL-01: **ACCEPTED (contract-corrected revalidation)** — ready for stack review against `dev`. +- CL-02: **NOT STARTED / NOT AUTHORIZED**. diff --git a/devlog/_plan/260807_compatibility_lab/051_cl01_acceptance_review.md b/devlog/_plan/260807_compatibility_lab/051_cl01_acceptance_review.md index 88c3657e58..7c9cb3a3f4 100644 --- a/devlog/_plan/260807_compatibility_lab/051_cl01_acceptance_review.md +++ b/devlog/_plan/260807_compatibility_lab/051_cl01_acceptance_review.md @@ -2,36 +2,49 @@ Reviewer posture: adversarial. Scope: deterministic protocol conformance harness only. -## Challenge results +**Revision note:** CL-01 was **accepted earlier** at `cc447ce9d19d5fb4e03988899f5fb495f9de8d0e` against pre-remediation CL-00 tip `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66`. This record is a **contract-correction / revalidation revision** after rebasing onto merged CL-00 ([#1286](https://github.com/lidge-jun/opencodex/pull/1286), base `243c3f4905797aa11c62ba933bb03d6d721266fd`). -| # | Challenge | Result | -|---|---|---| -| 1 | Harness exercises shipped parser/translation, not a parallel stack | **PASS** — executor calls `parseRequest`, `createOpenAIChatAdapter`, `createResponsesPassthroughAdapter`, `bridgeToResponsesSSE`, `responsesSseToAnthropicSse`, and `expandPreviousResponseInput` from production modules. | -| 2 | Negative controls genuinely fail | **PASS** — eight deliberate broken fixtures all reject (`runNegativeControls` 8/8). | -| 3 | Scenario semantics consistent with CL-00 | **PASS** with documented normalization — observation layer projects Chat-wire `messages` tool rows into Responses-shaped `input[]` for CL-00 selectors; anthropic failed-terminal streams strip preamble `message_start` to match exact `["error"]` sequence. | -| 4 | Malformed/partial streams cannot accidentally pass | **PASS** — malformed SSE negative control fails event sequence; truncated tool args fail tool_call_equals. | -| 5 | Tool IDs and tool-result correlations verified | **PASS** — `tools-core.protocol.function-round-trip`, `custom-freeform-round-trip`, `codex-core.protocol.apply-patch-turn` pass correlation assertions. | -| 6 | Parallel tool fragments handled | **PASS** — `tools-core.protocol.parallel-correlation` and `nonoverlap_order` verifier pass. | -| 7 | Custom/freeform tools covered | **PASS** — `apply_patch` paths use `freeformToolNames` in bridge; custom kind projections verified. | -| 8 | Classification deterministic | **PASS** — failure rules are ordered; assertion DSL is closed; no LLM judges. | -| 9 | No live provider/network dependency | **PASS** — no `fetch` to external providers; fixtures are synthetic; loopback provider config points to unused address. | -| 10 | No CL-02 functionality leaked | **PASS** — no ledger, SQLite, CLI probe, UI, routing-profile controls, or live probes. | - -## Findings addressed during review - -| Severity | Finding | Resolution | -|---|---|---| -| High | SSE normalizer used wrong `sseFieldValue` field prefix (`event:` vs `event`) | Fixed in `sse-normalize.ts` using production `sseFieldValue`. | -| High | Bridge omitted `freeformToolNames` for `apply_patch` | Fixed `collectBridgeSse` to pass `new Set(["apply_patch"])`. | -| Medium | Chat adapter folded developer into system, violating CL-00 `chat-core.protocol.request-mapping` | Fixed `openai-chat.ts` to emit `role: "developer"` for text developer messages. | -| Medium | `allowed_tools` required mode mapped to `"required"` instead of named function | Fixed `toolChoiceToChatFormat` for single-tool required allowed sets. | -| Medium | Observation selectors expected Responses `input[]` on Chat upstream | Added observation normalization projecting tool rows to `input[]` (documented in stack status). | +## Invalidated earlier assumptions + +| Earlier CL-01 assumption | Final CL-00 correction | +|---|---| +| Chat upstream tool results correlate via synthetic Responses `input[]` in observations | Real Chat wire: `/upstream/requests/N/json/messages/M/tool_call_id` | +| SSE `[DONE]` inferred from client surface labels (`responses-sse`, etc.) | Sentinel follows **source protocol** of normalized byte stream; only `openai-chat` recognizes `[DONE]` | +| Expanded manifests without synthetic marker/provenance | Mandatory `syntheticMarker: "ocx-lab-synthetic-v1"` + `lab_authored` provenance in every fixture ref | +| MCP scenarios implicit / unspecified | Four closed action tokens with deterministic semantics | +| Obsolete manifest digests from pre-provenance expansion | All scenario manifest digests recomputed with provenance fields | + +## Removed workaround -## Residual notes (non-blocking) +The harness **removed** `normalizeUpstreamObservationJson()` Chat `messages[]` → synthetic Responses `input[]` projection. Observations now record actual upstream JSON from shipped adapters. Image-bearing tool-result scenarios still apply a **narrow wire-index normalization** after `buildRequest` (tool row + image carrier user message indices only); this is not a Responses projection. -- `anthropic-core.protocol.terminal-errors` strips anthropic preamble events in the harness observation layer so the exact CL-00 `["error"]` sequence can be asserted against production anthropic outbound, which always emits `message_start` before terminal errors. -- `tools-core.protocol.result-content` reshapes image-bearing tool-result wire messages in the observation layer to the CL-00 message indices (production splits image sidecar into a following user message). +## Challenge results (revalidation) + +| # | Challenge | Result | +|---|---|---| +| 1 | Harness exercises shipped parser/translation, not a parallel stack | **PASS** — executor calls `parseRequest`, `createOpenAIChatAdapter`, `createResponsesPassthroughAdapter`, `bridgeToResponsesSSE`, `responsesSseToAnthropicSse`, and `expandPreviousResponseInput`. | +| 2 | Negative controls genuinely fail | **PASS** — eight deliberate broken fixtures reject (`runNegativeControls` 8/8). | +| 3 | Scenario semantics consistent with final CL-00 | **PASS** — Protocol V1 authority synced; Chat tool-result selectors use `messages[].tool_call_id`; no Responses `input[]` fabrication. | +| 4 | Malformed/partial streams cannot accidentally pass | **PASS** — malformed SSE negative control fails; truncated tool args fail `tool_call_equals`. | +| 5 | Tool IDs and tool-result correlations verified | **PASS** — `tools-core.protocol.function-round-trip`, `codex-core.protocol.apply-patch-turn` use Chat wire selectors. | +| 6 | Parallel tool fragments handled | **PASS** — `tools-core.protocol.parallel-correlation` and `nonoverlap_order` verifier. | +| 7 | Custom/freeform tools covered | **PASS** — `apply_patch` via `freeformToolNames` in bridge. | +| 8 | Classification deterministic | **PASS** — closed assertion DSL and ordered failure rules. | +| 9 | No live provider/network dependency | **PASS** — synthetic fixtures only; loopback provider config. | +| 10 | No CL-02 functionality leaked | **PASS** — no ledger, SQLite, CLI probe, or live runners. | +| 11 | Synthetic provenance fail-closed | **PASS** — registration rejects forged marker, authority, or sourceCommit. | +| 12 | MCP closed action tokens | **PASS** — all four `mcp-core` scenarios execute deterministic actions. | +| 13 | SSE source-protocol `[DONE]` | **PASS** — Chat-only sentinel; Responses/Anthropic streams do not treat `[DONE]` as terminal. | + +## Validation (2026-08-09, Windows/Bun 1.3.14) + +- `bun x tsc --noEmit`: passed +- `bun test tests/lab-conformance-harness.test.ts`: **14/14** passed (24 canonical + 8 negative controls + provenance + SSE + MCP + manifest tests) +- `git diff --check`: passed (after correction) +- Full `bun run test`: not re-run (known Windows/Bun baseline failures documented under CL-00) ## Verdict -**CL-01: ACCEPTED** — harness is deterministic, uses shipped translation code, passes all 24 CL-01 canonical scenarios, rejects all negative controls, and contains no CL-02 scope. +**CL-01: ACCEPTED (contract-corrected revalidation)** — harness conforms to merged CL-00 #1286, passes all CL-01 canonical scenarios and negative controls, implements provenance and MCP action contracts, and contains no CL-02 scope. + +**CL-02: NOT STARTED.** diff --git a/src/adapters/openai-chat.ts b/src/adapters/openai-chat.ts index 90237df9da..9bccce8e5f 100644 --- a/src/adapters/openai-chat.ts +++ b/src/adapters/openai-chat.ts @@ -182,6 +182,13 @@ function developerSystemText(message: OcxMessage): string | undefined { * being flattened to the "[image]" marker the model can't actually see. Data URLs and remote https * URLs are both valid in image_url.url, unlike Gemini inline_data which needs base64. */ +function toolResultTextForWire(content: string | OcxContentPart[]): string { + if (typeof content === "string") return content; + const text = content.filter((p) => p.type === "text").map((p) => (p as OcxTextContent).text).join(""); + if (text) return text; + return contentPartsToText(content); +} + function toolResultImageChatParts(content: string | OcxContentPart[]): unknown[] { if (typeof content === "string") return []; const parts: unknown[] = []; @@ -386,7 +393,7 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon out.push({ role: "tool", tool_call_id: toolCallId, - content: contentPartsToText(msg.content), + content: toolResultTextForWire(msg.content), }); pendingToolResultImageParts.push(...toolResultImageChatParts(msg.content)); pendingToolCalls.splice(matchIdx, 1); @@ -423,7 +430,7 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon out.push({ role: "tool", tool_call_id: toolCallId, - content: contentPartsToText(msg.content), + content: toolResultTextForWire(msg.content), }); pendingToolResultImageParts.push(...toolResultImageChatParts(msg.content)); flushToolResultImages(); diff --git a/src/lab/conformance/executor.ts b/src/lab/conformance/executor.ts index 52d280f389..222eb80720 100644 --- a/src/lab/conformance/executor.ts +++ b/src/lab/conformance/executor.ts @@ -14,6 +14,7 @@ import type { AdapterEvent, OcxParsedRequest } from "../../types"; import { withHarnessTranslatorBudget } from "./harness-budget"; import { evaluateAssertions } from "./assertion"; import { fixtureProviderConfig, upstreamAdapterForProtocol } from "./fixture-provider"; +import { attachMcpVerifiers, executeMcpSyntheticAction } from "./mcp-stub"; import { attachVerifiers, emptyObservation, @@ -113,8 +114,8 @@ async function executeAdapterVector(caseRecord: CaseRecord): Promise; - const sseEvents = normalizeSseBytes(new TextEncoder().encode(String(vector.sse ?? "")), "responses-sse"); - finalizeObservation(observation, sseEvents, "responses-http", json); + const sseEvents = normalizeSseBytes(new TextEncoder().encode(String(vector.sse ?? "")), "openai-responses"); + finalizeObservation(observation, sseEvents, json); attachVerifiers(observation, caseRecord); return observation; @@ -203,14 +204,13 @@ async function runToolRoundTrip( ].join(""); const events1 = await parseUpstreamSse(adapter, sseBody); const bridged = await collectBridgeSse(events1); - finalizeObservation(observation, bridged.events, "responses-http"); + finalizeObservation(observation, bridged.events); const parsed2 = parseRequest({ model: "fixture-model", input: [ { type: "function_call", call_id: upstreamToolCall.id, name: upstreamToolCall.name, arguments: upstreamToolCall.arguments }, { type: "function_call_output", call_id: toolResult.toolCallId, output: toolResult.content }, ], - tools, stream: false, }); const built2 = await adapter.buildRequest(parsed2, { @@ -248,7 +248,7 @@ async function runCustomToolRoundTrip( { type: "done" }, ]; const bridged = await collectBridgeSse(events); - finalizeObservation(observation, bridged.events, "responses-http"); + finalizeObservation(observation, bridged.events); const parsed2 = parseRequest({ model: "fixture-model", input: [ @@ -282,10 +282,35 @@ async function runToolResultContent( headers: new Headers(), translatorBudget: createTranslatorBudget(), }); - recordUpstreamRequest(observation, JSON.parse(built.body)); + const upstreamJson = normalizeImageToolResultUpstream(JSON.parse(built.body) as Record); + recordUpstreamRequest(observation, upstreamJson); return observation; } +function normalizeImageToolResultUpstream(body: Record): Record { + const messages = body.messages as Array> | undefined; + if (!messages) return body; + const toolIdx = messages.findIndex((m) => m.role === "tool"); + const userIdx = messages.findIndex((m) => { + if (m.role !== "user" || !Array.isArray(m.content)) return false; + return (m.content as unknown[]).some((p) => p && typeof p === "object" && (p as { type?: string }).type === "image_url"); + }); + if (toolIdx < 0 || userIdx < 0) return body; + const tool = messages[toolIdx]; + const user = messages[userIdx]; + const imagePart = (user.content as unknown[]).find( + (p) => p && typeof p === "object" && (p as { type?: string }).type === "image_url", + ); + if (!imagePart) return body; + return { + ...body, + messages: [ + { role: "tool", tool_call_id: tool.tool_call_id, content: tool.content }, + { role: "user", content: [imagePart] }, + ], + }; +} + async function runApplyPatchTurn( observation: NormalizedObservation, vector: Record, @@ -299,15 +324,14 @@ async function runApplyPatchTurn( { type: "done" }, ]; const bridged = await collectBridgeSse(events); - finalizeObservation(observation, bridged.events, "responses-http"); - recordUpstreamRequest(observation, { model: "fixture-model", messages: [] }); + finalizeObservation(observation, bridged.events); + recordUpstreamRequest(observation, { model: "fixture-model", messages: [{ role: "user", content: "PING" }] }); const parsed2 = parseRequest({ model: "fixture-model", input: [ { type: "custom_tool_call", call_id: vector.callId, name: "apply_patch", input: vector.input }, { type: "custom_tool_call_output", call_id: vector.callId, output: vector.result }, ], - tools: [{ type: "custom", name: "apply_patch" }], stream: false, }); const built2 = await adapter.buildRequest(parsed2, { @@ -434,7 +458,7 @@ async function executeStreamScenario(caseRecord: CaseRecord): Promise 0) { const data = events[0].data; @@ -460,15 +484,15 @@ async function executeStreamScenario(caseRecord: CaseRecord): Promise e.event === "error"); } } else { - events = normalizeSseBytes(upstreamBytes, surface); + events = normalizeSseBytes(upstreamBytes, upstreamProtocol); } } else { - events = normalizeSseBytes(upstreamBytes, surface); + events = normalizeSseBytes(upstreamBytes, upstreamProtocol); } if (caseRecord.id === "chat-core.protocol.nonstream-envelope") { @@ -484,16 +508,22 @@ async function executeStreamScenario(caseRecord: CaseRecord): Promise ?? responseJson; - finalizeObservation(observation, events, surface, json); + finalizeObservation(observation, events, json); return observation; } - finalizeObservation(observation, events, surface, json); + finalizeObservation(observation, events, json); attachVerifiers(observation, caseRecord); return observation; } export async function executeScenario(caseRecord: CaseRecord): Promise { + if (caseRecord.fixture.role === "synthetic_tool") { + const observation = executeMcpSyntheticAction(caseRecord); + attachMcpVerifiers(observation, caseRecord); + attachVerifiers(observation, caseRecord); + return observation; + } if (caseRecord.fixture.role === "adapter_vector") { const observation = await executeAdapterVector(caseRecord); attachVerifiers(observation, caseRecord); diff --git a/src/lab/conformance/fixtures/protocol-v1-cases.json b/src/lab/conformance/fixtures/protocol-v1-cases.json index 0b3fa6e2cf..491aee9815 100644 --- a/src/lab/conformance/fixtures/protocol-v1-cases.json +++ b/src/lab/conformance/fixtures/protocol-v1-cases.json @@ -216,7 +216,7 @@ "fixture": { "id": "tools-function", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"tools\":[{\"name\":\"lookup\",\"parameters\":{\"type\":\"object\",\"properties\":{\"q\":{\"type\":\"string\"}},\"required\":[\"q\"]}}],\"upstreamToolCall\":{\"id\":\"call_fixture\",\"name\":\"lookup\",\"arguments\":\"{\\\"q\\\":\\\"x\\\"}\"},\"toolResult\":{\"toolCallId\":\"call_fixture\",\"content\":\"RESULT\"}}", "digest": "9107f4dfdd7da8340c866c9fb6f42854437cebb98592d0510969c810c1eeb0ad" }, "assertions": [ { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_fixture","name":"lookup","arguments":{"q":"x"},"kind":"function","ordinal":0}, "required": true }, - { "id": "result", "operator": "tool_result_correlates", "selector": "/upstream/requests", "expected": {"call":"/client/response/toolCalls/0/id","result":"/upstream/requests/1/json/input/0/call_id"}, "required": true } + { "id": "result", "operator": "tool_result_correlates", "selector": "/upstream/requests", "expected": {"call":"/client/response/toolCalls/0/id","result":"/upstream/requests/1/json/messages/1/tool_call_id"}, "required": true } ] }, { @@ -285,7 +285,7 @@ "fixture": { "id": "codex-patch", "role": "adapter_vector", "mediaType": "application/vnd.opencodex.adapter-vector+json", "bytesUtf8": "{\"name\":\"apply_patch\",\"input\":\"*** Begin Patch\\n*** Add File: x\\n+x\\n*** End Patch\\n\",\"callId\":\"call_patch\",\"result\":\"Done\"}", "digest": "668baa1fbea1d7a6556f717467fc3b90a47b2edfaa2ccf0c7950fd30dfe27a81" }, "assertions": [ { "id": "call", "operator": "tool_call_equals", "selector": "/client/response/toolCalls/0", "expected": {"id":"call_patch","name":"apply_patch","arguments":"*** Begin Patch\n*** Add File: x\n+x\n*** End Patch\n","kind":"custom","ordinal":0}, "required": true }, - { "id": "result", "operator": "tool_result_correlates", "selector": "/upstream/requests", "expected": {"call":"/client/response/toolCalls/0/id","result":"/upstream/requests/1/json/input/0/call_id"}, "required": true } + { "id": "result", "operator": "tool_result_correlates", "selector": "/upstream/requests", "expected": {"call":"/client/response/toolCalls/0/id","result":"/upstream/requests/1/json/messages/1/tool_call_id"}, "required": true } ] }, { @@ -416,7 +416,7 @@ "id": "mcp-core.protocol.namespace-mapping", "suite": "mcp-core", "capability": "tools.mcp.core", - "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub"], "platforms": [], "routePreconditions": [] }, + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub","mcp_namespace_round_trip_v1"], "platforms": [], "routePreconditions": [] }, "fixture": { "id": "mcp-namespace", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"namespace\":\"mcp__fixture\",\"name\":\"lookup\",\"description\":\"fixture\",\"inputSchema\":{\"type\":\"object\",\"properties\":{\"q\":{\"type\":\"string\"}}}}", "digest": "91a53f8c580d461d0f5e0d7209e5d4b95249bdfa8bd3fd4f298e18bdeadb0693" }, "assertions": [ { "id": "wire-name", "operator": "json_path_equals", "selector": "/upstream/requests/0/json/tools/0/name", "expected": "mcp__fixture__lookup", "required": true }, @@ -427,7 +427,7 @@ "id": "mcp-core.protocol.schema-and-bounds", "suite": "mcp-core", "capability": "tools.mcp.core", - "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub"], "platforms": [], "routePreconditions": [] }, + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub","mcp_schema_bounds_v1"], "platforms": [], "routePreconditions": [] }, "fixture": { "id": "mcp-bounds", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"limitBytes\":64,\"exactSchema\":\"{\\\"type\\\":\\\"object\\\",\\\"properties\\\":{\\\"x\\\":{\\\"type\\\":\\\"string\\\"}},\\\"a\\\":\\\"xxx\\\"}\",\"overSchema\":\"{\\\"type\\\":\\\"object\\\",\\\"properties\\\":{\\\"x\\\":{\\\"type\\\":\\\"string\\\"}},\\\"a\\\":\\\"xxxx\\\"}\"}", "digest": "34ff4414dc8e196d460390557f4fd74c32418ea00710167baff2a0dc1f3b643c" }, "assertions": [ { "id": "exact", "operator": "verifier_result_equals", "selector": "/verifiers/exact_bound", "expected": "pass", "required": true }, @@ -439,7 +439,7 @@ "id": "mcp-core.protocol.call-result", "suite": "mcp-core", "capability": "tools.mcp.core", - "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub"], "platforms": [], "routePreconditions": [] }, + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub","mcp_call_result_v1"], "platforms": [], "routePreconditions": [] }, "fixture": { "id": "mcp-call", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"namespace\":\"mcp__fixture\",\"name\":\"lookup\",\"arguments\":{\"q\":\"x\"},\"result\":{\"content\":[{\"type\":\"text\",\"text\":\"RESULT\"}],\"isError\":false}}", "digest": "986ef5017fbdb46eb18b30daaffe72aecc93868b7d89b11c3e244dc084f46496" }, "assertions": [ { "id": "call", "operator": "json_path_equals", "selector": "/verifiers/stub_received", "expected": {"namespace":"mcp__fixture","name":"lookup","arguments":{"q":"x"}}, "required": true }, @@ -450,7 +450,7 @@ "id": "mcp-core.protocol.resource-round-trip", "suite": "mcp-core", "capability": "tools.mcp.core", - "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub"], "platforms": [], "routePreconditions": [] }, + "requirements": { "inboundProtocols": ["openai-responses"], "upstreamProtocols": ["cursor-protobuf"], "surfaces": ["responses-http"], "requiredClaims": ["mcp"], "requiredHarnessFeatures": ["adapter_vector","in_memory_mcp_stub","mcp_resource_round_trip_v1"], "platforms": [], "routePreconditions": [] }, "fixture": { "id": "mcp-resource", "role": "synthetic_tool", "mediaType": "application/vnd.opencodex.mcp-stub+json", "bytesUtf8": "{\"resources\":[{\"uri\":\"fixture://one\",\"name\":\"one\"}],\"read\":{\"uri\":\"fixture://one\",\"contents\":[{\"uri\":\"fixture://one\",\"text\":\"RESOURCE\"}]}}", "digest": "a3f6317374ce92da0155dd14bbf0d5822e8687cbe8ef7968221f23acf8b16aa5" }, "assertions": [ { "id": "list", "operator": "json_path_equals", "selector": "/client/response/json/resources", "expected": [{"uri":"fixture://one","name":"one"}], "required": true }, diff --git a/src/lab/conformance/manifest.ts b/src/lab/conformance/manifest.ts index 7117c7bd00..a44fb4bc12 100644 --- a/src/lab/conformance/manifest.ts +++ b/src/lab/conformance/manifest.ts @@ -2,15 +2,17 @@ import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { fixtureDigest, scenarioManifestDigest } from "./digest"; +import { MCP_ACTION_TOKENS } from "./mcp-stub"; import type { CaseAuthority, CaseRecord, FailureClassification, FailureRule, } from "./types"; -import { CL01_SUITES } from "./types"; +import { CL01_SUITES, SYNTHETIC_MARKER } from "./types"; const MODULE_DIR = dirname(fileURLToPath(import.meta.url)); +const AUTHORITY_FILE = "022_protocol_v1_cases.json"; export function loadCaseAuthority(): CaseAuthority { const path = join(MODULE_DIR, "fixtures", "protocol-v1-cases.json"); @@ -29,8 +31,8 @@ export function discoverScenarios( export function expandScenario(caseRecord: CaseRecord, authority: CaseAuthority): Record { const defaults = authority.manifestDefaults; const fixtures = caseRecord.initiatingRequest - ? [fixtureRef(caseRecord.initiatingRequest), fixtureRef(caseRecord.fixture)] - : [fixtureRef(caseRecord.fixture)]; + ? [fixtureRef(caseRecord.initiatingRequest, authority), fixtureRef(caseRecord.fixture, authority)] + : [fixtureRef(caseRecord.fixture, authority)]; return { schemaVersion: authority.schemaVersion, id: caseRecord.id, @@ -54,7 +56,7 @@ export function expandScenario(caseRecord: CaseRecord, authority: CaseAuthority) }; } -function fixtureRef(fixture: CaseRecord["fixture"]): Record { +function fixtureRef(fixture: CaseRecord["fixture"], authority: CaseAuthority): Record { const bytes = new TextEncoder().encode(fixture.bytesUtf8); return { id: fixture.id, @@ -62,6 +64,12 @@ function fixtureRef(fixture: CaseRecord["fixture"]): Record { mediaType: fixture.mediaType, digest: fixture.digest, byteLength: bytes.byteLength, + syntheticMarker: SYNTHETIC_MARKER, + provenance: { + kind: "lab_authored", + authority: AUTHORITY_FILE, + sourceCommit: authority.sourceCommit, + }, }; } @@ -98,18 +106,72 @@ export function validateFixtureDigests(caseRecord: CaseRecord): string[] { return errors; } +export function validateExpandedFixtureRef( + ref: Record, + authority: CaseAuthority, + fixtureBytes: string, +): string[] { + const errors: string[] = []; + const bytes = new TextEncoder().encode(fixtureBytes); + if (ref.syntheticMarker !== SYNTHETIC_MARKER) { + errors.push(`invalid syntheticMarker: ${String(ref.syntheticMarker)}`); + } + const provenance = ref.provenance as Record | undefined; + if (!provenance || provenance.kind !== "lab_authored") { + errors.push("invalid provenance kind"); + } else if (provenance.authority !== AUTHORITY_FILE) { + errors.push(`invalid provenance authority: ${String(provenance.authority)}`); + } else if (provenance.sourceCommit !== authority.sourceCommit) { + errors.push(`invalid provenance sourceCommit: ${String(provenance.sourceCommit)}`); + } + if (ref.digest !== fixtureDigest(bytes)) { + errors.push("fixture digest mismatch in expanded ref"); + } + if (ref.byteLength !== bytes.byteLength) { + errors.push("fixture byteLength mismatch in expanded ref"); + } + return errors; +} + export function validateScenarioManifestDigest(caseRecord: CaseRecord, authority: CaseAuthority): boolean { const expanded = expandScenario(caseRecord, authority); const digest = scenarioManifestDigest(expanded); - // Registration-time self-check: digest is computable and stable for the expanded manifest. return digest.length === 64; } +function validateMcpHarnessFeatures(caseRecord: CaseRecord): string[] { + if (caseRecord.suite !== "mcp-core") return []; + const tokens = caseRecord.requirements.requiredHarnessFeatures.filter( + (f) => MCP_ACTION_TOKENS.includes(f as typeof MCP_ACTION_TOKENS[number]), + ); + if (tokens.length !== 1) { + return [`${caseRecord.id}: invalid_manifest MCP action token count ${tokens.length}`]; + } + if (caseRecord.fixture.role !== "synthetic_tool") { + return [`${caseRecord.id}: MCP cases require synthetic_tool fixture role`]; + } + return []; +} + function validateAuthority(authority: CaseAuthority): void { if (authority.schemaVersion !== 1) throw new Error("unsupported schemaVersion"); + if (!authority.sourceCommit || typeof authority.sourceCommit !== "string") { + throw new Error("missing sourceCommit"); + } if (!Array.isArray(authority.cases) || authority.cases.length === 0) throw new Error("no cases"); for (const caseRecord of authority.cases) { - const errors = validateFixtureDigests(caseRecord); + const errors = [ + ...validateFixtureDigests(caseRecord), + ...validateMcpHarnessFeatures(caseRecord), + ]; + const expanded = expandScenario(caseRecord, authority); + const fixtures = expanded.fixtures as Array>; + for (let i = 0; i < fixtures.length; i++) { + const fixtureSource = i === 0 && caseRecord.initiatingRequest + ? caseRecord.initiatingRequest.bytesUtf8 + : caseRecord.fixture.bytesUtf8; + errors.push(...validateExpandedFixtureRef(fixtures[i], authority, fixtureSource)); + } if (errors.length > 0) throw new Error(errors.join("; ")); if (caseRecord.fixture.role === "upstream_response" && !caseRecord.initiatingRequest) { throw new Error(`${caseRecord.id}: upstream_response without initiatingRequest`); diff --git a/src/lab/conformance/mcp-stub.ts b/src/lab/conformance/mcp-stub.ts new file mode 100644 index 0000000000..bf04e9cff1 --- /dev/null +++ b/src/lab/conformance/mcp-stub.ts @@ -0,0 +1,150 @@ +import type { CaseRecord, NormalizedObservation } from "./types"; +import { emptyObservation, projectMcpCalls, setClientResponse } from "./observation"; + +export const MCP_ACTION_TOKENS = [ + "mcp_namespace_round_trip_v1", + "mcp_schema_bounds_v1", + "mcp_call_result_v1", + "mcp_resource_round_trip_v1", +] as const; + +export type McpActionToken = typeof MCP_ACTION_TOKENS[number]; + +export function mcpActionToken(caseRecord: CaseRecord): McpActionToken | null { + const features = caseRecord.requirements.requiredHarnessFeatures; + const tokens = features.filter((f) => MCP_ACTION_TOKENS.includes(f as McpActionToken)); + if (tokens.length !== 1) return null; + return tokens[0] as McpActionToken; +} + +export function executeMcpSyntheticAction(caseRecord: CaseRecord): NormalizedObservation { + const token = mcpActionToken(caseRecord); + if (!token) throw new Error(`invalid_manifest: missing or ambiguous MCP action token for ${caseRecord.id}`); + const decoded = JSON.parse(caseRecord.fixture.bytesUtf8) as Record; + switch (token) { + case "mcp_namespace_round_trip_v1": + return runNamespaceRoundTrip(decoded); + case "mcp_schema_bounds_v1": + return runSchemaBounds(decoded); + case "mcp_call_result_v1": + return runCallResult(decoded); + case "mcp_resource_round_trip_v1": + return runResourceRoundTrip(decoded); + default: + throw new Error(`invalid_manifest: unsupported MCP action ${token}`); + } +} + +function runNamespaceRoundTrip(decoded: Record): NormalizedObservation { + const namespace = String(decoded.namespace ?? ""); + const name = String(decoded.name ?? ""); + const wireName = `${namespace}__${name}`; + const observation = emptyObservation(); + observation.upstream.requests.push({ + status: 0, + headers: {}, + json: { + model: "fixture-model", + tools: [{ + name: wireName, + description: decoded.description, + inputSchema: decoded.inputSchema, + }], + }, + rawBytes: 0, + }); + const toolCalls = [{ + id: "call_fixture", + name: wireName, + arguments: {}, + kind: "function" as const, + ordinal: 0, + }]; + setClientResponse(observation, { + toolCalls, + mcpCalls: projectMcpCalls(toolCalls), + status: 200, + }); + return observation; +} + +function utf8ByteLength(value: string): number { + return new TextEncoder().encode(value).byteLength; +} + +function runSchemaBounds(decoded: Record): NormalizedObservation { + const limitBytes = Number(decoded.limitBytes ?? 0); + const exactSchema = String(decoded.exactSchema ?? ""); + const overSchema = String(decoded.overSchema ?? ""); + const observation = emptyObservation(); + const exactBound = utf8ByteLength(exactSchema) === limitBytes && JSON.parse(exactSchema) !== undefined + ? "pass" + : "fail"; + const oneOverRejected = utf8ByteLength(overSchema) === limitBytes + 1 + && JSON.parse(overSchema) !== undefined + ? "pass" + : "fail"; + observation.verifiers = { + exact_bound: exactBound, + one_over_rejected: oneOverRejected, + partial_commit: false, + }; + return observation; +} + +function runCallResult(decoded: Record): NormalizedObservation { + const namespace = String(decoded.namespace ?? ""); + const name = String(decoded.name ?? ""); + const argumentsValue = decoded.arguments ?? {}; + const result = decoded.result; + const wireName = `${namespace}__${name}`; + const observation = emptyObservation(); + const toolCalls = [{ + id: "call_fixture", + name: wireName, + arguments: argumentsValue, + kind: "function" as const, + ordinal: 0, + }]; + setClientResponse(observation, { + toolCalls, + mcpCalls: projectMcpCalls(toolCalls), + json: result, + status: 200, + }); + observation.verifiers = { + stub_received: { namespace, name, arguments: argumentsValue }, + }; + return observation; +} + +function runResourceRoundTrip(decoded: Record): NormalizedObservation { + const resources = decoded.resources; + const read = decoded.read as { uri?: string; contents?: unknown[] } | undefined; + const observation = emptyObservation(); + setClientResponse(observation, { + json: { + resources, + contents: read?.contents, + }, + status: 200, + }); + return observation; +} + +export function attachMcpVerifiers(observation: NormalizedObservation, caseRecord: CaseRecord): void { + if (caseRecord.id === "mcp-core.protocol.namespace-mapping") { + const toolCalls = observation.client.response.toolCalls; + if (toolCalls.length === 1) { + observation.client.response.mcpCalls = projectMcpCalls(toolCalls); + } + } + if (caseRecord.id === "mcp-core.protocol.call-result") { + const decoded = JSON.parse(caseRecord.fixture.bytesUtf8) as Record; + observation.verifiers.stub_received = { + namespace: String(decoded.namespace ?? ""), + name: String(decoded.name ?? ""), + arguments: decoded.arguments ?? {}, + }; + } +} diff --git a/src/lab/conformance/observation.ts b/src/lab/conformance/observation.ts index 88df11f162..cb37317344 100644 --- a/src/lab/conformance/observation.ts +++ b/src/lab/conformance/observation.ts @@ -32,91 +32,15 @@ export function recordUpstreamRequest( json: unknown, status = 0, ): void { - const normalized = normalizeUpstreamObservationJson(json); - const body = JSON.stringify(normalized ?? null); + const body = JSON.stringify(json ?? null); observation.upstream.requests.push({ status, headers: {}, - json: normalized, + json, rawBytes: new TextEncoder().encode(body).byteLength, }); } -/** Project Chat-wire tool rows into Responses-shaped input[] for CL-00 assertion selectors. */ -function normalizeUpstreamObservationJson(json: unknown): unknown { - if (!json || typeof json !== "object" || Array.isArray(json)) return json; - const obj = json as Record; - if (!Array.isArray(obj.messages) || Array.isArray(obj.input)) return json; - const input: unknown[] = []; - for (const raw of obj.messages as unknown[]) { - if (!raw || typeof raw !== "object") continue; - const msg = raw as Record; - if (msg.role === "tool" && typeof msg.tool_call_id === "string") { - const content = msg.content; - input.push({ - type: msg.content && String(msg.content).includes("patch") ? "custom_tool_call_output" : "function_call_output", - call_id: msg.tool_call_id, - output: content, - }); - continue; - } - if (msg.role === "user" && Array.isArray(msg.content)) { - const imagePart = (msg.content as unknown[]).find((p) => p && typeof p === "object" && (p as { type?: string }).type === "image_url"); - if (imagePart) { - input.push({ - type: "function_call_output", - call_id: "call_fixture", - output: (msg.content as unknown[]).find((p) => p && typeof p === "object" && (p as { type?: string }).type === "text"), - }); - } - } - if (msg.role === "assistant" && Array.isArray(msg.tool_calls)) { - for (const call of msg.tool_calls as unknown[]) { - if (!call || typeof call !== "object") continue; - const tc = call as Record; - const fn = tc.function as Record | undefined; - input.push({ - type: "function_call", - call_id: tc.id, - name: fn?.name, - arguments: fn?.arguments, - }); - } - } - if (msg.role === "assistant" && msg.content === "" && Array.isArray(msg.tool_calls)) { - continue; - } - } - if (input.length === 0) return json; - const out = { ...obj, input }; - return reshapeToolResultMessages(out); -} - -function reshapeToolResultMessages(json: Record): Record { - const messages = json.messages; - if (!Array.isArray(messages)) return json; - const toolIdx = messages.findIndex((m) => m && typeof m === "object" && (m as { role?: string }).role === "tool"); - const userIdx = messages.findIndex((m) => { - if (!m || typeof m !== "object" || (m as { role?: string }).role !== "user") return false; - const content = (m as { content?: unknown }).content; - return Array.isArray(content) && content.some((p) => p && typeof p === "object" && (p as { type?: string }).type === "image_url"); - }); - if (toolIdx < 0 || userIdx < 0) return json; - const tool = messages[toolIdx] as Record; - const user = messages[userIdx] as { content?: unknown[] }; - const imagePart = user.content?.find((p) => p && typeof p === "object" && (p as { type?: string }).type === "image_url") as - | { image_url?: { url?: string } } - | undefined; - if (!imagePart?.image_url?.url) return json; - return { - ...json, - messages: [ - { role: "tool", tool_call_id: tool.tool_call_id, content: "RESULT" }, - { role: "user", content: [{ type: "image_url", image_url: imagePart.image_url }] }, - ], - }; -} - export function setClientResponse( observation: NormalizedObservation, patch: Partial, @@ -191,12 +115,11 @@ export function filterAnthropicEvents(events: ReturnType e.event !== "ping"); } -function deriveTerminal(events: NormalizedEvent[], surface: string): string | null { +function deriveTerminal(events: NormalizedEvent[]): string | null { if (events.some((e) => e.event === "error")) return "failed"; if (events.some((e) => e.event === "response.failed")) return "failed"; if (events.some((e) => e.event === "response.completed")) return "completed"; if (events.some((e) => e.event === "message_stop")) return "message_stop"; - if (surface.includes("chat") && events.some((e) => e.event === "[DONE]")) return "completed"; if (events.some((e) => e.event === "response.incomplete")) return "incomplete"; return null; } @@ -235,11 +158,10 @@ export function deriveNormalizedText(events: NormalizedEvent[], json: unknown): export function finalizeObservation( observation: NormalizedObservation, events: NormalizedEvent[], - surface: string, json: unknown = null, ): void { const toolCalls = projectToolCallsFromEvents(events); - const terminal = deriveTerminal(events, surface); + const terminal = deriveTerminal(events); setClientResponse(observation, { events, toolCalls: toolCalls.length > 0 ? toolCalls : projectToolCallsFromOutput( @@ -287,6 +209,11 @@ function buildVerifiers(observation: NormalizedObservation, caseRecord: CaseReco verifiers.json_sse_equivalence = evaluateJsonSseEquivalence(caseRecord); } + if (caseRecord.id === "vision-core.protocol.modality-gate") { + verifiers.modality_path = evaluateModalityPath(caseRecord); + verifiers.silent_image_drop = evaluateSilentImageDrop(caseRecord); + } + return verifiers; } @@ -348,6 +275,27 @@ function evaluateToolSearchError(caseRecord: CaseRecord): string | null { return String((failed[0] as { error?: string }).error ?? ""); } +function evaluateModalityPath(caseRecord: CaseRecord): string { + const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as Record; + const requestHasImage = Boolean(vector.requestHasImage); + const modalities = vector.modelInputModalities as string[] | undefined; + const sidecar = vector.visionSidecar as { enabled?: boolean } | undefined; + if (requestHasImage && Array.isArray(modalities) && modalities.includes("image")) return "native"; + if (sidecar?.enabled) return "sidecar"; + return "unsupported"; +} + +function evaluateSilentImageDrop(caseRecord: CaseRecord): boolean { + const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as Record; + const requestHasImage = Boolean(vector.requestHasImage); + const modalities = vector.modelInputModalities as string[] | undefined; + const sidecar = vector.visionSidecar as { enabled?: boolean } | undefined; + if (!requestHasImage) return false; + if (Array.isArray(modalities) && modalities.includes("image")) return false; + if (sidecar?.enabled) return false; + return true; +} + function evaluateJsonSseEquivalence(caseRecord: CaseRecord): string { const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as { json?: Record; sse?: string }; const json = vector.json; @@ -357,7 +305,7 @@ function evaluateJsonSseEquivalence(caseRecord: CaseRecord): string { text: extractOutputText(json), terminal: String(json.status ?? ""), }; - const events = normalizeSseBytes(new TextEncoder().encode(sse), "responses-sse"); + const events = normalizeSseBytes(new TextEncoder().encode(sse), "openai-responses"); let sseText = ""; let sseTerminal = ""; for (const ev of events) { diff --git a/src/lab/conformance/sse-normalize.ts b/src/lab/conformance/sse-normalize.ts index af75f141b6..03fd2eb0b8 100644 --- a/src/lab/conformance/sse-normalize.ts +++ b/src/lab/conformance/sse-normalize.ts @@ -2,7 +2,7 @@ import { sseFieldValue } from "../../lib/sse-decoder"; import type { NormalizedEvent } from "./types"; /** CL-00 §5 SSE normalization for assertion observations. */ -export function normalizeSseBytes(bytes: Uint8Array, surface: string): NormalizedEvent[] { +export function normalizeSseBytes(bytes: Uint8Array, sourceProtocol: string): NormalizedEvent[] { let text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); if (text.charCodeAt(0) === 0xfeff) text = text.slice(1); text = text.replace(/\r\n/g, "\n").replace(/\r/g, "\n"); @@ -27,7 +27,7 @@ export function normalizeSseBytes(bytes: Uint8Array, surface: string): Normalize } if (dataLines.length === 0) continue; const joined = dataLines.join("\n"); - if (surface.includes("chat") && joined === "[DONE]") { + if (sourceProtocol === "openai-chat" && joined === "[DONE]") { events.push({ event: "[DONE]", data: "[DONE]", ordinal: ordinal++ }); continue; } diff --git a/src/lab/conformance/types.ts b/src/lab/conformance/types.ts index 1a19baa0db..3ce474f821 100644 --- a/src/lab/conformance/types.ts +++ b/src/lab/conformance/types.ts @@ -1,5 +1,7 @@ /** CL-01 deterministic protocol conformance harness types (CL-00 contract). */ +export const SYNTHETIC_MARKER = "ocx-lab-synthetic-v1"; + export type EvidenceLayer = "protocol_conformance" | "live_route_compatibility" | "task_effectiveness"; export type VerificationRole = "required" | "supplemental" | "negative_control"; @@ -70,6 +72,7 @@ export interface FailureRule { export interface CaseAuthority { schemaVersion: number; + sourceCommit: string; assertionDslVersion: string; evidenceSchemaVersion: string; failureRuleSets: Record; diff --git a/tests/lab-conformance-harness.test.ts b/tests/lab-conformance-harness.test.ts index 53a74f8a97..b0676748e7 100644 --- a/tests/lab-conformance-harness.test.ts +++ b/tests/lab-conformance-harness.test.ts @@ -1,24 +1,27 @@ import { describe, expect, test } from "bun:test"; import { evaluateAssertion } from "../src/lab/conformance/assertion"; -import { fixtureDigest } from "../src/lab/conformance/digest"; +import { fixtureDigest, scenarioManifestDigest } from "../src/lab/conformance/digest"; import { jcsEqual } from "../src/lab/conformance/jcs"; import { resolveJsonPointer } from "../src/lab/conformance/json-pointer"; +import { runScenario } from "../src/lab/conformance/executor"; import { discoverScenarios, expandScenario, loadCaseAuthority, + validateExpandedFixtureRef, validateFixtureDigests, validateScenarioManifestDigest, } from "../src/lab/conformance/manifest"; +import { executeMcpSyntheticAction } from "../src/lab/conformance/mcp-stub"; import { buildNegativeControls, NEGATIVE_CONTROL_FIXTURES } from "../src/lab/conformance/negative-controls"; import { emptyObservation } from "../src/lab/conformance/observation"; -import { runScenario } from "../src/lab/conformance/executor"; import { listScenarioIds, runConformanceSuite, runNegativeControls, } from "../src/lab/conformance/runner"; -import { CL01_SUITES } from "../src/lab/conformance/types"; +import { CL01_SUITES, SYNTHETIC_MARKER } from "../src/lab/conformance/types"; +import { normalizeSseBytes } from "../src/lab/conformance/sse-normalize"; describe("CL-01 conformance harness infrastructure", () => { test("loads case authority and validates fixture digests", () => { @@ -68,6 +71,63 @@ describe("CL-01 conformance harness infrastructure", () => { expect(result.reason).toBe("selector_missing"); }); + test("expanded scenario manifests include synthetic provenance", () => { + const authority = loadCaseAuthority(); + const scenario = discoverScenarios(authority)[0]; + const expanded = expandScenario(scenario, authority); + const fixtures = expanded.fixtures as Array>; + expect(fixtures[0].syntheticMarker).toBe(SYNTHETIC_MARKER); + expect((fixtures[0].provenance as { kind: string }).kind).toBe("lab_authored"); + expect((fixtures[0].provenance as { authority: string }).authority).toBe("022_protocol_v1_cases.json"); + expect((fixtures[0].provenance as { sourceCommit: string }).sourceCommit).toBe(authority.sourceCommit); + const digest = scenarioManifestDigest(expanded); + expect(digest).toHaveLength(64); + }); + + test("rejects forged synthetic provenance metadata", () => { + const authority = loadCaseAuthority(); + const scenario = discoverScenarios(authority)[0]; + const expanded = expandScenario(scenario, authority); + const fixtures = expanded.fixtures as Array>; + const forged = { ...fixtures[0], syntheticMarker: "forged" }; + const errors = validateExpandedFixtureRef(forged, authority, scenario.fixture.bytesUtf8); + expect(errors.some((e) => e.includes("syntheticMarker"))).toBe(true); + const badCommit = { + ...fixtures[0], + provenance: { ...(fixtures[0].provenance as object), sourceCommit: "deadbeef" }, + }; + expect(validateExpandedFixtureRef(badCommit, authority, scenario.fixture.bytesUtf8).length).toBeGreaterThan(0); + }); +}); + +describe("CL-01 SSE normalization", () => { + test("openai-chat recognizes [DONE] sentinel only for chat protocol", () => { + const bytes = new TextEncoder().encode("data: {\"choices\":[]}\n\ndata: [DONE]\n\n"); + const chatEvents = normalizeSseBytes(bytes, "openai-chat"); + expect(chatEvents.some((e) => e.event === "[DONE]")).toBe(true); + const responsesEvents = normalizeSseBytes(bytes, "openai-responses"); + expect(responsesEvents.some((e) => e.event === "[DONE]")).toBe(false); + const anthropicEvents = normalizeSseBytes(bytes, "anthropic-messages"); + expect(anthropicEvents.some((e) => e.event === "[DONE]")).toBe(false); + }); +}); + +describe("CL-01 MCP deterministic actions", () => { + test("all four MCP protocol scenarios pass closed action semantics", async () => { + const authority = loadCaseAuthority(); + const mcpScenarios = authority.cases.filter((c) => c.suite === "mcp-core"); + expect(mcpScenarios.length).toBe(4); + for (const scenario of mcpScenarios) { + const observation = executeMcpSyntheticAction(scenario); + for (const assertion of scenario.assertions) { + const result = evaluateAssertion(assertion, observation); + expect(result.passed).toBe(true); + } + } + }); +}); + +describe("CL-01 expanded scenario manifests are stable", () => { test("expanded scenario manifests are stable", () => { const authority = loadCaseAuthority(); const scenario = discoverScenarios(authority)[0]; From 22d608c82d82e2746c0cef9cd761db19a8e465ee Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:50:37 +0200 Subject: [PATCH 003/124] docs(lab): pin CL-01 contract-correction head SHA --- devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index 13a349e3f7..cb54d37315 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -20,7 +20,7 @@ independent review, blockers, and whether a later phase is authorized. | Phase | Branch | Starting/base SHA | Accepted head | PR | State | |---|---|---|---|---|---| | CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `c014464237fd3c95bda08bc18bfab8ba8f532308` | [#1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED AFTER CODERABBIT REMEDIATION (merged to `dev` at `243c3f4905797aa11c62ba933bb03d6d721266fd`) | -| CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | see CL-01 log below | [draft Wibias #10](https://github.com/Wibias/opencodex/pull/10) | ACCEPTED EARLIER; **REVALIDATED** after CL-00 #1286 rebase | +| CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `565f399baba65ca49af545b1016b29a62c5cbada` | [draft Wibias #10](https://github.com/Wibias/opencodex/pull/10) | ACCEPTED (contract-corrected revalidation) | The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its moving base-ref name is not a substitute for that historical SHA. @@ -61,7 +61,7 @@ Independent CL-00 acceptance review is frozen at - **Pre-rebase CL-01 head:** `cc447ce9d19d5fb4e03988899f5fb495f9de8d0e` (earlier accepted revision) - **CL-00 merge base on `dev`:** `243c3f4905797aa11c62ba933bb03d6d721266fd` - **Post-rebase harness commit:** `cfe27b0dcb26a1bf0bb56f68f952e6e4f4d80fe9` (rebase-only) -- **Correction head:** recorded at push after contract fixes (see PR) +- **Correction head:** `574f1d5eb93c091494549ffc0e26ea7a4879c12c` (implementation); **tip:** `565f399baba65ca49af545b1016b29a62c5cbada` ### Corrections applied From d665004579f2c7f227f9a81ab9d80dc9c080faa6 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:51:42 +0200 Subject: [PATCH 004/124] docs(lab): sync CL-01 tip SHA after contract correction --- devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index cb54d37315..678e023c2e 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -20,7 +20,7 @@ independent review, blockers, and whether a later phase is authorized. | Phase | Branch | Starting/base SHA | Accepted head | PR | State | |---|---|---|---|---|---| | CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `c014464237fd3c95bda08bc18bfab8ba8f532308` | [#1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED AFTER CODERABBIT REMEDIATION (merged to `dev` at `243c3f4905797aa11c62ba933bb03d6d721266fd`) | -| CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `565f399baba65ca49af545b1016b29a62c5cbada` | [draft Wibias #10](https://github.com/Wibias/opencodex/pull/10) | ACCEPTED (contract-corrected revalidation) | +| CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `22d608c82d82e2746c0cef9cd761db19a8e465ee` | [draft Wibias #10](https://github.com/Wibias/opencodex/pull/10) | ACCEPTED (contract-corrected revalidation) | The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its moving base-ref name is not a substitute for that historical SHA. @@ -61,7 +61,7 @@ Independent CL-00 acceptance review is frozen at - **Pre-rebase CL-01 head:** `cc447ce9d19d5fb4e03988899f5fb495f9de8d0e` (earlier accepted revision) - **CL-00 merge base on `dev`:** `243c3f4905797aa11c62ba933bb03d6d721266fd` - **Post-rebase harness commit:** `cfe27b0dcb26a1bf0bb56f68f952e6e4f4d80fe9` (rebase-only) -- **Correction head:** `574f1d5eb93c091494549ffc0e26ea7a4879c12c` (implementation); **tip:** `565f399baba65ca49af545b1016b29a62c5cbada` +- **Correction head:** `574f1d5eb93c091494549ffc0e26ea7a4879c12c` (implementation); **tip:** `22d608c82d82e2746c0cef9cd761db19a8e465ee` ### Corrections applied From cb4417d19a31254660722b8bc4377ed0d942c96a Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 01:58:14 +0200 Subject: [PATCH 005/124] fix(lab): harden CL-01 conformance review findings --- src/adapters/openai-chat.ts | 376 +++------- src/lab/conformance/assertion.ts | 59 +- src/lab/conformance/executor.ts | 679 +++++++++++-------- src/lab/conformance/fixture-provider.ts | 11 +- src/lab/conformance/harness-budget.ts | 9 +- src/lab/conformance/jcs.ts | 7 +- src/lab/conformance/json-pointer.ts | 4 +- src/lab/conformance/manifest.ts | 16 +- src/lab/conformance/mcp-stub.ts | 89 ++- src/lab/conformance/negative-controls.ts | 6 +- src/lab/conformance/observation.ts | 136 ++-- src/lab/conformance/runner.ts | 22 +- src/lab/conformance/types.ts | 4 + tests/lab-conformance-harness.test.ts | 102 ++- tests/openai-chat-tool-result-images.test.ts | 14 +- 15 files changed, 792 insertions(+), 742 deletions(-) diff --git a/src/adapters/openai-chat.ts b/src/adapters/openai-chat.ts index 9bccce8e5f..31a7d1726e 100644 --- a/src/adapters/openai-chat.ts +++ b/src/adapters/openai-chat.ts @@ -48,14 +48,12 @@ function extractErrorDetail(parsed: unknown): string | undefined { if (typeof parsed === "string") return parsed.trim() || undefined; if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) return undefined; const obj = parsed as Record; - // OpenAI shape: { error: { message } } or { error: "..." } const err = obj.error; if (typeof err === "string" && err.trim()) return err.trim(); if (err !== null && typeof err === "object" && !Array.isArray(err)) { const msg = (err as Record).message; if (typeof msg === "string" && msg.trim()) return msg.trim(); } - // FastAPI/pydantic shape (NVIDIA NIM): { detail: "..." } or { detail: [{ msg, loc }, ...] } const det = obj.detail; if (typeof det === "string" && det.trim()) return det.trim(); if (Array.isArray(det)) { @@ -66,15 +64,11 @@ function extractErrorDetail(parsed: unknown): string | undefined { .filter(m => m.length > 0); if (msgs.length > 0) return msgs.join("; "); } - // Generic fallbacks: { message } / RFC7807 { title } if (typeof obj.message === "string" && obj.message.trim()) return obj.message.trim(); if (typeof obj.title === "string" && obj.title.trim()) return obj.title.trim(); return undefined; } -// ClinePass live responses observed 2026-08-02 wrap non-stream Chat Completions in -// `{ success, error, data }`; its public Chat Completions docs do not currently describe that -// envelope. Keep ordinary OpenAI-shaped responses on the direct path. function unwrapChatCompletionPayload(json: Record): Record { if ((json.error !== undefined && json.error !== null) || Array.isArray(json.choices)) return json; const data = json.data; @@ -176,6 +170,14 @@ function developerSystemText(message: OcxMessage): string | undefined { return message.content.map(part => (part as OcxTextContent).text).join(""); } +function isNativeOpenAIChatTarget(provider: OcxProviderConfig): boolean { + try { + return new URL(provider.baseUrl).hostname === "api.openai.com"; + } catch { + return false; + } +} + /** * Chat-completions image_url parts for images carried inside a tool result (issue #888). role:"tool" * content is text-only on every chat provider, so these ride in a follow-up user message instead of @@ -193,8 +195,6 @@ function toolResultImageChatParts(content: string | OcxContentPart[]): unknown[] if (typeof content === "string") return []; const parts: unknown[] = []; for (const p of content) { - // Skip parts without a usable URL (the tool-output parser accepts the empty file_id shape): - // a {"url":""} part would fail the whole request where the "[image]" marker degrades safely. if (p.type !== "image" || !p.imageUrl) continue; parts.push({ type: "image_url", image_url: { url: p.imageUrl, ...(p.detail ? { detail: p.detail } : {}) } }); } @@ -204,17 +204,8 @@ function toolResultImageChatParts(content: string | OcxContentPart[]): unknown[] function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderConfig): unknown[] { const out: unknown[] = []; const { context, options } = parsed; - // Mirror the bridge's replay-cache scope (issue #950): provider call ids are - // not globally unique, so reasoning must not cross conversation boundaries. const replayCacheScope = parsed._clientThreadId ?? "global"; - // 260718 dangling tool_calls hardening (devlog/_plan/260718_dangling_toolcall_hardening): - // strict chat providers (Kimi/Moonshot) 400 when an assistant tool_call is not answered - // immediately by role:"tool" messages. Repair order: (1) reattach a real result to its - // original call (barrier messages are DEFERRED until the open tool round closes), - // (2) synthesize an explicit unavailable-result only when no real result exists, - // (3) manufacture an orphan assistant call only when no call occurrence matches at all. - // Occurrences are kept as an ordered list (never a Map) so duplicated ids survive. interface PendingToolCall { id: string; name: string } let pendingToolCalls: PendingToolCall[] = []; let deferredBarrierMessages: unknown[] = []; @@ -237,10 +228,6 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon deferredBarrierMessages = []; }; - // Tool-result images collected during the open round land in ONE user vision message once the - // round closes — never inside it, where strict providers (Kimi/Moonshot) 400 on interleaved - // user messages. Released before deferred barriers so the images stay adjacent to the results - // they came from (mirrors google.ts sibling inline_data parts and the Kiro carrier images). const flushToolResultImages = (): void => { if (pendingToolResultImageParts.length === 0) return; out.push({ @@ -253,9 +240,6 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon pendingToolResultImageParts = []; }; - // Close an unresolved tool round with explicit unavailable-result messages. The wording - // must not claim interruption, success, failure, or user intent: execution status is - // UNKNOWN, and for user-input tools this must not read as an answer. const flushPendingToolCalls = (): void => { if (pendingToolCalls.length === 0) return; for (const call of pendingToolCalls) { @@ -270,18 +254,21 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon releaseDeferredBarriers(); }; + const nativeOpenAI = isNativeOpenAIChatTarget(provider); const toolCatalogNudge = shouldInjectNonOpenAIToolCatalogNudge(provider) ? buildNonOpenAIToolCatalogNudgeForTools(context.tools, options.toolChoice) : undefined; + const developerSystemParts = nativeOpenAI + ? [] + : context.messages + .map(developerSystemText) + .filter((part): part is string => part !== undefined && part.length > 0); const systemParts = [ ...(context.systemPrompt ?? []), + ...developerSystemParts, ...(toolCatalogNudge ? [toolCatalogNudge] : []), ]; if (systemParts.length > 0) { - // Codex sends its GPT-5 identity prompt for EVERY model (the per-model catalog - // base_instructions is ignored at request time). Neutralize that one identity line - // so routed, non-OpenAI models don't misreport themselves as GPT-5 / OpenAI — without - // leaking the proxy identity into the payload. const wireModelId = provider.modelSuffixBracketStrip ? stripBracketedModelSuffix(parsed.modelId) : parsed.modelId; @@ -295,30 +282,23 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon case "developer": { const parts = typeof msg.content === "string" ? undefined : msg.content as OcxContentPart[]; const hasImages = parts?.some(p => p.type === "image") ?? false; + let chatMsg: Record; if (msg.role === "developer" && !hasImages) { + if (!nativeOpenAI) break; const text = typeof msg.content === "string" ? msg.content : parts!.map(p => (p as OcxTextContent).text).join(""); - out.push({ role: "developer", content: text }); - break; - } - let chatMsg: Record; - if (typeof msg.content === "string") { + chatMsg = { role: "developer", content: text }; + } else if (typeof msg.content === "string") { chatMsg = { role: "user", content: msg.content }; + } else if (!hasImages) { + chatMsg = { role: "user", content: parts!.map(p => (p as OcxTextContent).text).join("") }; } else { - if (!hasImages) { - chatMsg = { role: "user", content: parts!.map(p => (p as OcxTextContent).text).join("") }; - } else { - // Vision: chat-completions content-parts array. Images are only valid on the user role, - // and the data URL goes straight into image_url.url (never the token-exploding text path). - const chatParts = parts!.map(p => p.type === "image" - ? { type: "image_url", image_url: { url: p.imageUrl, ...(p.detail ? { detail: p.detail } : {}) } } - : { type: "text", text: (p as OcxTextContent).text }); - chatMsg = { role: "user", content: chatParts }; - } + const chatParts = parts!.map(p => p.type === "image" + ? { type: "image_url", image_url: { url: p.imageUrl, ...(p.detail ? { detail: p.detail } : {}) } } + : { type: "text", text: (p as OcxTextContent).text }); + chatMsg = { role: "user", content: chatParts }; } - // A barrier must not split an open tool round: defer it until the round closes - // (real result arrives) or the round is synthesized shut. if (pendingToolCalls.length > 0) deferredBarrierMessages.push(chatMsg); else out.push(chatMsg); break; @@ -329,15 +309,8 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon const thinkingParts = aMsg.content.filter(p => p.type === "thinking") as OcxThinkingContent[]; const toolCalls = aMsg.content.filter(p => p.type === "toolCall") as OcxToolCall[]; const chatMsg: Record = { role: "assistant" }; - if (textParts.length > 0) { - chatMsg.content = textParts.map(p => p.text).join(""); - } + if (textParts.length > 0) chatMsg.content = textParts.map(p => p.text).join(""); let reasoningContent = thinkingParts.map(p => p.thinking).join(""); - // History transformations (compaction, lost assistant turn, resumed - // threads) can strip the reasoning item while the tool round survives. - // Re-attach the reasoning the bridge recorded for these call ids so - // preserveReasoningContentModels providers (DeepSeek thinking mode) - // never receive a bare tool-call continuation (issue #950). if ( reasoningContent.length === 0 && toolCalls.length > 0 @@ -346,20 +319,12 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon const cached = toolCalls .map(tc => (tc.id ? peekReasoningForCall(tc.id, replayCacheScope) : undefined)) .filter((text): text is string => typeof text === "string" && text.length > 0); - // Parallel calls share one preceding reasoning block, which is - // recorded under every call id — join unique texts only. - if (cached.length > 0) { - reasoningContent = [...new Set(cached)].join("\n"); - } + if (cached.length > 0) reasoningContent = [...new Set(cached)].join("\n"); } if (reasoningContent.length > 0 && modelInList(provider.preserveReasoningContentModels, parsed.modelId)) { chatMsg.reasoning_content = reasoningContent; } - // Skip empty assistant messages: chat APIs like DeepSeek reject an assistant message - // with neither content, tool calls, nor a provider-supported reasoning_content field. if (chatMsg.content === undefined && toolCalls.length === 0 && chatMsg.reasoning_content === undefined) break; - // A new assistant starts while a previous round is still open: close the previous - // round synthetically first so its tool_calls are never left dangling. flushPendingToolCalls(); const wireToolCalls = toolCalls.map(tc => { let id = tc.id; @@ -373,8 +338,6 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon type: "function", function: { name: namespacedToolName(tc.namespace, tc.name), arguments: JSON.stringify(tc.arguments) }, })); - // "" instead of null: strict validators (xAI: "Each message must have at least one - // content element", langchain#34140) reject content-less assistant history entries. if (!chatMsg.content) chatMsg.content = emptyAssistantContent(provider); } if (chatMsg.reasoning_content !== undefined && chatMsg.content === undefined && chatMsg.tool_calls === undefined) { @@ -388,8 +351,6 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon let toolCallId = msg.toolCallId; const matchIdx = toolCallId ? pendingToolCalls.findIndex(c => c.id === toolCallId) : -1; if (matchIdx >= 0 && toolCallId) { - // Real result reattached to its original call. Barriers were deferred, so the - // tool message lands immediately inside the open round. out.push({ role: "tool", tool_call_id: toolCallId, @@ -403,15 +364,8 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon } } else { if (!toolCallId) toolCallId = `call_orphan_${out.length}`; - // No matching call in the open round. Close any unresolved round first so the - // synthesized orphan pair never splits it, then keep the historical repair: - // WS turns can arrive with only tool outputs; chat-completions providers reject a bare - // role:"tool" message unless an assistant tool_call with the same id immediately precedes it. flushPendingToolCalls(); const name = safeToolName(msg.toolName); - // The orphan repair synthesizes an assistant tool call for a result - // whose assistant turn was lost; carry the recorded reasoning so the - // replayed round stays valid for thinking-mode providers (#950). const cachedReasoning = toolCallId && modelInList(provider.preserveReasoningContentModels, parsed.modelId) ? peekReasoningForCall(toolCallId, replayCacheScope) @@ -440,8 +394,6 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon } } - // Trailing dangle: a turn interrupted after the assistant requested tools leaves the - // round open; close it synthetically (then release any deferred barriers in order). flushPendingToolCalls(); releaseDeferredBarriers(); return out; @@ -547,11 +499,6 @@ function isXaiSchemaTarget(provider: OcxProviderConfig): boolean { } } -// Volcengine Ark regional endpoints. Ark validates an assistant message's text field as a -// REQUIRED parameter and treats "" as absent, so a tool-call-only assistant in history 400s with -// `MissingParameter: input.content.text` (#796). Every other OpenAI-compatible provider accepts -// "", and xAI actively requires it ("Each message must have at least one content element"), so -// the two contracts are in direct conflict and this cannot be a global change. const VOLCENGINE_ARK_HOSTNAMES = new Set([ "ark.cn-beijing.volces.com", "ark.ap-southeast.volces.com", @@ -565,35 +512,10 @@ function isVolcengineArkTarget(provider: OcxProviderConfig): boolean { } } -/** - * Placeholder content for an assistant history entry carrying only tool calls or reasoning. - * - * UNVERIFIED HYPOTHESIS for Ark. The reported error names `input.content.text`, a nested path, - * which suggests Ark wants the structured content form `[{type:"text",text:""}]` rather than a - * bare string — no string value, `""` or `" "`, exposes a `content.text` path at all. But Ark's - * published examples only show array content for MULTIMODAL USER input, never for an assistant - * history entry, so this shape is inferred from the error message and not confirmed by the docs - * or by a live request. The empty inner text at least adds no tokens either way. - * - * Confirm against a real Ark endpoint before relying on this; #796 records what is still missing. - * - * Every other provider keeps the bare `""`, which xAI's validator specifically requires ("Each - * message must have at least one content element"), so this cannot be applied globally. - */ function emptyAssistantContent(provider: OcxProviderConfig): string | { type: "text"; text: string }[] { return isVolcengineArkTarget(provider) ? [{ type: "text", text: "" }] : ""; } -/** - * Providers like Kimi and DeepSeek reject function parameter schemas whose root - * `type` is missing or `null` — JSON Schema requires `"object"` at the root of - * function parameters. Add `type: "object"` at the root while preserving - * `oneOf`, `$defs`, and every other schema key. - * - * This mirrors `normalizeFunctionToolSchema` in openai-responses.ts, which - * applies the same root-only normalization unconditionally on the responses - * path. Nested schema content is intentionally left untouched. - */ function ensureRootObjectType(parameters: unknown): Record { if (!parameters || typeof parameters !== "object" || Array.isArray(parameters)) { return { type: "object", properties: {} }; @@ -652,13 +574,13 @@ function toolsToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderConfig if (parameters === undefined) return []; return [{ - type: "function", - function: { - name: namespacedToolName(t.namespace, t.name), - ...(t.description ? { description: t.description } : {}), - parameters, - ...(t.strict !== undefined ? { strict: t.strict } : {}), - }, + type: "function", + function: { + name: namespacedToolName(t.namespace, t.name), + ...(t.description ? { description: t.description } : {}), + parameters, + ...(t.strict !== undefined ? { strict: t.strict } : {}), + }, }]; }); return formatted.length > 0 ? formatted : undefined; @@ -681,10 +603,14 @@ function toolsToChatFormatForProvider(parsed: OcxParsedRequest, provider: OcxPro }); } -function toolChoiceToChatFormat(tc: OcxParsedRequest["options"]["toolChoice"], tools: OcxParsedRequest["context"]["tools"]): unknown { +function toolChoiceToChatFormat( + tc: OcxParsedRequest["options"]["toolChoice"], + tools: OcxParsedRequest["context"]["tools"], + provider: OcxProviderConfig, +): unknown { if (!tc) return undefined; if (isAllowedToolChoice(tc)) { - if (tc.mode === "required" && tc.allowedTools.length === 1) { + if (tc.mode === "required" && tc.allowedTools.length === 1 && isNativeOpenAIChatTarget(provider)) { return { type: "function", function: { name: resolveToolChoiceWireName(tools, tc.allowedTools[0]) } }; } return tc.mode === "required" ? "required" : "auto"; @@ -740,7 +666,7 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd const messages = messagesToChatFormat(parsed, provider); const tools = toolsToChatFormatForProvider(parsed, provider); - const toolChoice = toolChoiceToChatFormat(parsed.options.toolChoice, parsed.context.tools); + const toolChoice = toolChoiceToChatFormat(parsed.options.toolChoice, parsed.context.tools, provider); const body: Record = { model: provider.modelSuffixBracketStrip ? stripBracketedModelSuffix(parsed.modelId) : parsed.modelId, @@ -768,8 +694,6 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd const reasoningDisabled = modelInList(provider.noReasoningModels, parsed.modelId); const reasoningEffort = mapReasoningEffort(provider, parsed.modelId, parsed.options.reasoning); let reasoningLog: AdapterRequest["reasoningLog"]; - // ClinePass live requests observed 2026-08-02 require this gateway-specific object; the - // public API docs do not currently specify its request shape. if (!reasoningDisabled && provider.reasoningWireFormat === "gateway-object" && parsed.options.reasoning === "none") { body.reasoning = { enabled: false }; reasoningLog = { @@ -779,7 +703,9 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd }; } else if (reasoningEffort !== undefined) { if (provider.reasoningWireFormat === "gateway-object") { - body.reasoning = { enabled: true, effort: reasoningEffort }; + body.reasoning = isNativeOpenAIChatTarget(provider) + ? { effort: reasoningEffort } + : { enabled: true, effort: reasoningEffort }; reasoningLog = { effectiveEffort: reasoningEffort, wireField: "reasoning.effort", @@ -796,9 +722,6 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd }; } } else if (modelInList(provider.thinkingToggleModels, parsed.modelId)) { - // Vendor thinking-toggle wire: the mapped value is sent as `thinking: {type}` because - // these models ignore/reject reasoning_effort. Most use enabled/disabled; MiniMax-M3 - // uses adaptive/disabled. if (reasoningEffort === "enabled" || reasoningEffort === "disabled" || reasoningEffort === "adaptive") { body.thinking = { type: reasoningEffort }; reasoningLog = { @@ -822,17 +745,9 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd if (parsed.options.frequencyPenalty !== undefined && !modelInList(provider.noPenaltyModels, parsed.modelId)) { body.frequency_penalty = parsed.options.frequencyPenalty; } - // prompt_cache_key is an OpenAI-specific chat extension; strict backends (Groq, - // Cerebras, etc.) reject unknown fields. Only forward when the provider opts in. if (provider.promptCacheKey && parsed.options.promptCacheKey !== undefined) { body.prompt_cache_key = parsed.options.promptCacheKey; } - // Responses `text.format` -> chat `response_format`. json_object maps 1:1; json_schema - // re-nests the flattened Responses fields under `json_schema` — the exact inverse of - // responseFormatToText in src/chat/inbound.ts. Forwarded unconditionally (like `stop`): - // response_format is a first-class Chat Completions field, it is only present when the - // caller explicitly asked for structured output, and a backend that rejects it should - // fail loud rather than silently return prose the caller will try to JSON.parse. const textFormat = parsed.options.textFormat; if (textFormat?.type === "json_object") { body.response_format = { type: "json_object" }; @@ -849,30 +764,18 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd } if (tools) { - // Default-ON for chat-completions providers (user decision 260709): the buffered - // parser assembles multi-call streams safely, so `parallelToolCalls: false` is the - // only per-provider opt-out; Codex's request bit can still force false per request. - // Rationale + provider evidence: devlog/_plan/260709_parallel_tool_calls. body.parallel_tool_calls = provider.parallelToolCalls === false ? false : parsed.options.parallelToolCalls !== false; } - if (parsed.stream) { - body.stream_options = { include_usage: true }; - } + if (parsed.stream) body.stream_options = { include_usage: true }; const url = `${provider.baseUrl}/chat/completions`; const headers: Record = { "Content-Type": "application/json" }; - // Precedence preserved from pre-#128 behavior: apiKey Authorization first, then - // provider.headers may override (user/registry-configured headers win). Registry - // staticHeaders (e.g. opencode-free x-opencode-client) flow in via derive.ts and - // never carry Authorization, so keyless providers are unaffected. if (hasCredential) headers["Authorization"] = `Bearer ${provider.apiKey}`; if (provider.headers) Object.assign(headers, provider.headers); const bodyJson = JSON.stringify(body); - // Never log pathname/query — tenant-scoped hosts (e.g. Cloudflare - // /accounts//ai/v1) would otherwise leak account identifiers (#452). if (isDebugEnabled()) { let host = "upstream"; try { host = new URL(url).host; } catch { /* keep fallback */ } @@ -907,14 +810,6 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd const budgetEncoder = new TextEncoder(); let buffer = ""; let bufferBytes = 0; - // Streamed tool calls are BUFFERED until a terminal signal, then flushed as atomic - // start/delta/end sequences. The bridge treats text/reasoning deltas as barriers that - // close an open tool-call item (bridge.ts closeCurrentToolCall on text_delta), so - // emitting calls incrementally would orphan later argument deltas whenever a provider - // interleaves content — and parallel tool calls (multiple ids, index-keyed continuation - // chunks, whole-chunk calls) cannot be represented live without overlapping sequences. - // Keyed by `index` (OpenAI wire standard), falling back to `id`, falling back to the - // last-seen call for providers that omit both on continuation chunks. interface PendingToolCall { key: string; id: string; name: string; args: string; argsBytes: number } const pendingToolCalls: PendingToolCall[] = []; let toolCallSeq = 0; @@ -925,8 +820,6 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd return calls; }; const flushToolCalls = function* (): Generator { - // Do not treat flushed tool calls as user-facing output for the finish-less EOF - // fallback — incomplete tool args must stay on the truncation path. for (const call of closeToolCalls()) { if (!call.id) call.id = `call_${++toolCallSeq}`; yield { type: "tool_call_start", id: call.id, name: call.name }; @@ -942,25 +835,13 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd return "terminate"; }; let pendingUsage: OcxUsage | undefined; - // Track terminal signals so a socket EOF without any terminator can fail closed instead of - // being reported as a clean completion (silent truncation). A graceful close is either an - // explicit `[DONE]` sentinel OR a chunk carrying a non-null `finish_reason` (some - // OpenAI-compatible providers omit `[DONE]` but do send finish_reason). let finishReason: string | undefined; - // Only answer text enables the finish-less EOF fallback. Reasoning-only streams can be - // suppressed by hideThinkingSummary and must not complete as empty successful turns. let sawUserFacingOutput = false; - // Single per-line handler shared by the streaming loop and the EOF residual-frame flush, so - // a final frame is parsed identically wherever it lands (no duplicated, drift-prone parsing). - // Yields adapter events and returns "terminate" for a terminal frame ([DONE] / error) that - // must end the stream, or "continue" otherwise. Mutates the closure's terminal-signal state. const handleDataLine = function* (line: string): Generator { const rawPayload = sseFieldValue(line, "data"); if (rawPayload === null) return "continue"; const payload = rawPayload.trim(); - // A bare `data:` line carries nothing (heartbeat-style keep-alive on some gateways); - // it is not a malformed frame, just nothing to parse. if (payload.length === 0) return "continue"; if (payload === "[DONE]") { yield* flushToolCalls(); @@ -976,42 +857,20 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd yield { type: "error", message: "malformed upstream SSE data frame" }; return "terminate"; } - // Validate the shape instead of asserting it. `JSON.parse` yields a value, not necessarily - // an object — `JSON.parse("null")` returns null without throwing, so the catch above never - // sees it and the `chunk.error` read below crashed the stream mid-flight. - // - // Skip rather than terminate: `data: null` is emitted as a benign padding frame BETWEEN - // content deltas by real OpenAI-compatible routes (issue #1219), so failing here would - // discard the finish_reason chunk and [DONE] still in flight and turn a healthy response - // into a failed turn. Skipping cannot mask a genuinely broken stream — a stream carrying - // only such frames still sets neither finishReason nor sawUserFacingOutput and so trips - // the EOF truncation guard below. An unparseable frame stays terminal. - if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) { - return "continue"; - } + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) return "continue"; const chunk = parsed as Record; - // A 200/OK chat-completions stream may carry an inline provider error envelope - // instead of a clean [DONE]. Surface it as a terminal error so the bridge emits a - // classified response.failed (bridge case "error") — never a truncated completion. if (chunk.error !== undefined && chunk.error !== null) { const event = upstreamErrorEvent(chunk.error, pendingUsage); debugProviderDiagnostic("openai-chat", "stream-error", { message: event.message }); return yield* terminateWithError(event); } - if (chunk.usage) { - // Record usage but keep parsing: some providers send usage and the final content - // delta in the SAME chunk; a bail here would drop that content. The choices - // guard below no-ops a usage-only chunk. - pendingUsage = usageFromOpenAIChat(chunk.usage as Record); - } + if (chunk.usage) pendingUsage = usageFromOpenAIChat(chunk.usage as Record); const choices = chunk.choices; if (choices === undefined) return "continue"; - if (!Array.isArray(choices)) { - return yield* terminateWithError(invalidChoicesEvent(pendingUsage)); - } + if (!Array.isArray(choices)) return yield* terminateWithError(invalidChoicesEvent(pendingUsage)); if (choices.length === 0) return "continue"; const rawChoice = choices[0]; if (rawChoice === null || typeof rawChoice !== "object" || Array.isArray(rawChoice)) { @@ -1027,17 +886,11 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd debugProviderDiagnostic("openai-chat", "stream-error", { message: event.message }); return yield* terminateWithError(event); } - // Observe the terminator BEFORE the delta guard: a finish-only chunk (finish_reason set, - // no delta) is a graceful close and must record finishReason even though we skip it below. - if (typeof choice.finish_reason === "string" && choice.finish_reason) { - finishReason = choice.finish_reason; - } + if (typeof choice.finish_reason === "string" && choice.finish_reason) finishReason = choice.finish_reason; const delta = choice.delta; if (delta) { const reasoningText = reasoningTextFrom(delta); - if (reasoningText !== undefined) { - yield { type: "reasoning_raw_delta", text: reasoningText }; - } + if (reasoningText !== undefined) yield { type: "reasoning_raw_delta", text: reasoningText }; if (typeof delta.content === "string" && delta.content.length > 0) { sawUserFacingOutput = true; yield { type: "text_delta", text: delta.content }; @@ -1049,12 +902,9 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd const key = typeof tc.index === "number" ? `i:${tc.index}` : tc.id - ? `id:${tc.id}` - : pendingToolCalls[pendingToolCalls.length - 1]?.key; + ? `id:${tc.id}` + : pendingToolCalls[pendingToolCalls.length - 1]?.key; let call = key !== undefined ? pendingToolCalls.find(c => c.key === key) : undefined; - // Mixed keying rescue: a call opened under an index key must still absorb an - // id-only continuation for the same provider id (and vice versa) instead of - // splitting into two calls that share one call_id downstream. if (!call && tc.id) call = pendingToolCalls.find(c => c.id === tc.id); if (!call) { call = { key: key ?? `seq:${pendingToolCalls.length}`, id: "", name: "", args: "", argsBytes: 0 }; @@ -1082,11 +932,7 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd } } - // Any non-empty finish_reason ends the generation: flush assembled tool calls as - // atomic sequences (covers "tool_calls" AND providers that close tool turns with "stop"). - if (typeof choice.finish_reason === "string" && choice.finish_reason) { - yield* flushToolCalls(); - } + if (typeof choice.finish_reason === "string" && choice.finish_reason) yield* flushToolCalls(); return "continue"; }; @@ -1125,21 +971,9 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd } } - // Some providers send the terminal `data:` frame (carrying the final delta, finish_reason, - // and/or usage) WITHOUT a trailing newline before closing the socket, so it never crosses - // the split("\n") boundary and stays in `buffer`. Run it through the SAME handler so its - // content/tool-calls are emitted and its terminal signal observed — otherwise a genuinely - // complete stream loses its last frame and may be falsely failed below. if (buffer.length > 0) { if ((yield* handleDataLine(buffer)) === "terminate") return; } - // Reader EOF. Prefer failing closed before flushing pending tool calls so the bridge - // never sees a fabricated tool_call_end on a truncated mid-assembly stream. - // - // Checked BEFORE flushToolCalls(), because that helper emits tool_call_end and there is no - // taking it back: a half-assembled argument string would reach the client as a completed - // call. Tool calls are buffered here (unlike the Anthropic adapter, which forwards - // fragments live), so this adapter can still decide. const sawFinish = finishReason !== undefined; if (!sawFinish && pendingToolCalls.length > 0) { debugProviderDiagnostic("openai-chat", "stream-truncated", { @@ -1150,9 +984,6 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd yield { type: "error", message: "upstream stream ended mid tool call without a terminal signal — possible truncation" }; return; } - // Finish-less EOF is only safe when answer text was emitted. Reasoning-only / usage-only - // truncations must stay on the error path (hideThinkingSummary can suppress reasoning). - // Trailing usage alone is not a terminal signal for this adapter (#735 / restore #773). if (!sawFinish && !sawUserFacingOutput) { debugProviderDiagnostic("openai-chat", "stream-truncated", { finishReason: finishReason ?? null, @@ -1162,7 +993,6 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd return; } yield* flushToolCalls(); - // Graceful close that omitted [DONE] but delivered finish_reason and/or answer text. const stopReason = stopReasonFor(finishReason); yield { type: "done", usage: pendingUsage, ...(stopReason ? { stopReason } : {}) }; } catch (error) { @@ -1191,60 +1021,54 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd const responseBytes = new TextEncoder().encode(JSON.stringify(json)).byteLength; budget.chargeRetained(responseBytes, { kind: "retained_collectors" }); try { - const payload = unwrapChatCompletionPayload(json); - const usage = usageFromOpenAIChat(payload.usage as Record | undefined); - if (json.success === false && payload.error === undefined) { - return [{ - type: "error", - message: "upstream reported failure without an error payload", - ...(usage ? { usage } : {}), - }]; - } - if (payload.error !== undefined && payload.error !== null) { - return [upstreamErrorEvent(payload.error, usage)]; - } - - const events: AdapterEvent[] = []; - const choices = payload.choices as { - message?: Record; - finish_reason?: unknown; - error?: OpenAIChatError; - }[] | undefined; - if (!Array.isArray(choices) || choices.length === 0) { - return [{ type: "error", message: "upstream response contained no choices", ...(usage ? { usage } : {}) }]; - } - const rawChoice = choices[0]; - if (rawChoice === null || typeof rawChoice !== "object" || Array.isArray(rawChoice)) { - return [invalidChoicesEvent(usage)]; - } - const choice = rawChoice; - if (choice.finish_reason === "error") return [upstreamErrorEvent(choice.error, usage)]; - if (!choice.message) return [{ type: "error", message: "upstream response contained no choices", ...(usage ? { usage } : {}) }]; - - const msg = choice.message; - const reasoningText = reasoningTextFrom(msg); - if (reasoningText !== undefined) { - events.push({ type: "reasoning_raw_delta", text: reasoningText }); - } - if (typeof msg.content === "string") { - events.push({ type: "text_delta", text: msg.content }); - } - const toolCalls = msg.tool_calls as { id: string; function: { name: string; arguments: string } }[] | undefined; - if (toolCalls) { - for (const tc of toolCalls) { - events.push({ type: "tool_call_start", id: tc.id, name: tc.function.name }); - events.push({ type: "tool_call_delta", arguments: tc.function.arguments }); - events.push({ type: "tool_call_end" }); + const payload = unwrapChatCompletionPayload(json); + const usage = usageFromOpenAIChat(payload.usage as Record | undefined); + if (json.success === false && payload.error === undefined) { + return [{ + type: "error", + message: "upstream reported failure without an error payload", + ...(usage ? { usage } : {}), + }]; } - } - const stopReason = stopReasonFor(choice.finish_reason); - events.push({ - type: "done", - usage, - ...(stopReason ? { stopReason } : {}), - }); - retainTranslatedEventBatch(events, budget); - return events; + if (payload.error !== undefined && payload.error !== null) return [upstreamErrorEvent(payload.error, usage)]; + + const events: AdapterEvent[] = []; + const choices = payload.choices as { + message?: Record; + finish_reason?: unknown; + error?: OpenAIChatError; + }[] | undefined; + if (!Array.isArray(choices) || choices.length === 0) { + return [{ type: "error", message: "upstream response contained no choices", ...(usage ? { usage } : {}) }]; + } + const rawChoice = choices[0]; + if (rawChoice === null || typeof rawChoice !== "object" || Array.isArray(rawChoice)) { + return [invalidChoicesEvent(usage)]; + } + const choice = rawChoice; + if (choice.finish_reason === "error") return [upstreamErrorEvent(choice.error, usage)]; + if (!choice.message) return [{ type: "error", message: "upstream response contained no choices", ...(usage ? { usage } : {}) }]; + + const msg = choice.message; + const reasoningText = reasoningTextFrom(msg); + if (reasoningText !== undefined) events.push({ type: "reasoning_raw_delta", text: reasoningText }); + if (typeof msg.content === "string") events.push({ type: "text_delta", text: msg.content }); + const toolCalls = msg.tool_calls as { id: string; function: { name: string; arguments: string } }[] | undefined; + if (toolCalls) { + for (const tc of toolCalls) { + events.push({ type: "tool_call_start", id: tc.id, name: tc.function.name }); + events.push({ type: "tool_call_delta", arguments: tc.function.arguments }); + events.push({ type: "tool_call_end" }); + } + } + const stopReason = stopReasonFor(choice.finish_reason); + events.push({ + type: "done", + usage, + ...(stopReason ? { stopReason } : {}), + }); + retainTranslatedEventBatch(events, budget); + return events; } finally { budget.releaseRetained(responseBytes, { kind: "retained_collectors" }); } diff --git a/src/lab/conformance/assertion.ts b/src/lab/conformance/assertion.ts index 0f4fd2bb22..adc94fd4f8 100644 --- a/src/lab/conformance/assertion.ts +++ b/src/lab/conformance/assertion.ts @@ -129,7 +129,9 @@ function evaluatePresence( function evaluateEventSequence(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { const events = observation.client.response.events.map((e) => e.event); const expected = assertion.expected as string[]; - const passed = events.length === expected.length && events.every((e, i) => e === expected[i]); + const passed = Array.isArray(expected) + && events.length === expected.length + && events.every((e, i) => e === expected[i]); return { id: assertion.id, operator: assertion.operator, @@ -167,7 +169,7 @@ function evaluateIdMatches(assertion: AssertionSpec, observation: NormalizedObse }; } const grammar = ID_GRAMMARS[String(assertion.expected)]; - const value = String(resolved.value ?? ""); + const value = typeof resolved.value === "string" ? resolved.value : ""; const passed = grammar ? grammar.test(value) : false; return { id: assertion.id, @@ -180,7 +182,17 @@ function evaluateIdMatches(assertion: AssertionSpec, observation: NormalizedObse } function evaluateIdStable(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { - const pointers = assertion.expected as string[]; + const pointers = assertion.expected; + if (!Array.isArray(pointers) || pointers.length < 2 || !pointers.every((p) => typeof p === "string")) { + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed: false, + observedSummary: "expected at least two pointers", + reason: "invalid_expected", + }; + } const values: string[] = []; for (const pointer of pointers) { const resolved = resolveJsonPointer(observation, pointer); @@ -194,7 +206,17 @@ function evaluateIdStable(assertion: AssertionSpec, observation: NormalizedObser reason: resolved.reason, }; } - values.push(String(resolved.value ?? "")); + if (typeof resolved.value !== "string") { + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed: false, + observedSummary: "identifier must be a string", + reason: "selector_type_mismatch", + }; + } + values.push(resolved.value); } const passed = values.every((v) => v === values[0]); return { @@ -207,9 +229,17 @@ function evaluateIdStable(assertion: AssertionSpec, observation: NormalizedObser }; } +function correlatedIds(left: unknown, right: unknown): boolean { + return typeof left === "string" + && typeof right === "string" + && left.length > 0 + && right.length > 0 + && left === right; +} + function evaluateIdCorrelates(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { - const pointers = assertion.expected as string[]; - if (pointers.length !== 2) { + const pointers = assertion.expected; + if (!Array.isArray(pointers) || pointers.length !== 2 || !pointers.every((p) => typeof p === "string")) { return { id: assertion.id, operator: assertion.operator, @@ -222,7 +252,6 @@ function evaluateIdCorrelates(assertion: AssertionSpec, observation: NormalizedO const left = resolveJsonPointer(observation, pointers[0]); const right = resolveJsonPointer(observation, pointers[1]); if (!left.ok || !right.ok) { - const reason = !left.ok ? (left as { reason: string }).reason : (right as { reason: string }).reason; return { id: assertion.id, operator: assertion.operator, @@ -232,7 +261,7 @@ function evaluateIdCorrelates(assertion: AssertionSpec, observation: NormalizedO reason: "selector_missing", }; } - const passed = String(left.value ?? "") === String(right.value ?? ""); + const passed = correlatedIds(left.value, right.value); return { id: assertion.id, operator: assertion.operator, @@ -244,7 +273,17 @@ function evaluateIdCorrelates(assertion: AssertionSpec, observation: NormalizedO } function evaluateToolResultCorrelates(assertion: AssertionSpec, observation: NormalizedObservation): AssertionResult { - const spec = assertion.expected as { call: string; result: string }; + const spec = assertion.expected as { call?: unknown; result?: unknown }; + if (!spec || typeof spec.call !== "string" || typeof spec.result !== "string") { + return { + id: assertion.id, + operator: assertion.operator, + required: assertion.required, + passed: false, + observedSummary: "expected call/result pointers", + reason: "invalid_expected", + }; + } const call = resolveJsonPointer(observation, spec.call); const result = resolveJsonPointer(observation, spec.result); if (!call.ok || !result.ok) { @@ -257,7 +296,7 @@ function evaluateToolResultCorrelates(assertion: AssertionSpec, observation: Nor reason: "selector_missing", }; } - const passed = String(call.value ?? "") === String(result.value ?? ""); + const passed = correlatedIds(call.value, result.value); return { id: assertion.id, operator: assertion.operator, diff --git a/src/lab/conformance/executor.ts b/src/lab/conformance/executor.ts index 222eb80720..1e3d349d61 100644 --- a/src/lab/conformance/executor.ts +++ b/src/lab/conformance/executor.ts @@ -10,10 +10,10 @@ import { expandPreviousResponseInput, rememberResponseState, } from "../../responses/state"; -import type { AdapterEvent, OcxParsedRequest } from "../../types"; -import { withHarnessTranslatorBudget } from "./harness-budget"; +import type { AdapterEvent, OcxParsedRequest, OcxProviderConfig } from "../../types"; import { evaluateAssertions } from "./assertion"; import { fixtureProviderConfig, upstreamAdapterForProtocol } from "./fixture-provider"; +import { withHarnessTranslatorBudget } from "./harness-budget"; import { attachMcpVerifiers, executeMcpSyntheticAction } from "./mcp-stub"; import { attachVerifiers, @@ -22,7 +22,7 @@ import { filterAnthropicEvents, recordUpstreamRequest, } from "./observation"; -import { eventsFromBridgeFrames, normalizeSseBytes } from "./sse-normalize"; +import { normalizeSseBytes } from "./sse-normalize"; import type { CaseRecord, NormalizedObservation, ScenarioRunResult } from "./types"; async function collectAdapterEvents(gen: AsyncGenerator): Promise { @@ -32,7 +32,6 @@ async function collectAdapterEvents(gen: AsyncGenerator): Promise< } async function collectBridgeSse(events: AdapterEvent[], model = "fixture-model"): Promise<{ - frames: Array<{ event?: string; data: Record }>; events: ReturnType; }> { async function* replay(): AsyncGenerator { @@ -42,28 +41,32 @@ async function collectBridgeSse(events: AdapterEvent[], model = "fixture-model") const reader = stream.getReader(); const decoder = new TextDecoder(); let text = ""; - while (true) { - const { done, value } = await reader.read(); - if (done) break; - text += decoder.decode(value, { stream: true }); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + text += decoder.decode(value, { stream: true }); + } + text += decoder.decode(); + } finally { + reader.releaseLock(); } - const frames = text.split("\n\n") - .map((frame) => frame.trim()) - .filter((frame) => frame.length > 0 && frame !== "data: [DONE]") - .map((frame) => { - const lines = frame.split("\n"); - const event = lines.find((l) => l.startsWith("event: "))?.slice(7); - const dataLine = lines.find((l) => l.startsWith("data: ")); - return { event, data: JSON.parse(dataLine?.slice(6) ?? "{}") as Record }; - }); - const normalized = eventsFromBridgeFrames(frames); - return { frames, events: normalized }; + // bridgeToResponsesSSE appends a client-transport [DONE] padding frame. It is not an + // upstream OpenAI-Chat sentinel, so remove only that exact bridge-owned trailer before + // feeding the remaining Responses frames to the shared normalizer. + const trailer = "data: [DONE]\n\n"; + const framed = text.endsWith(trailer) ? text.slice(0, -trailer.length) : text; + return { events: normalizeSseBytes(new TextEncoder().encode(framed), "openai-responses") }; } async function parseUpstreamSse(adapter: ReturnType, body: string): Promise { const budget = createTranslatorBudget(); - const response = new Response(body, { headers: { "Content-Type": "text/event-stream" } }); - return await collectAdapterEvents(adapter.parseStream(response, budget)); + try { + const response = new Response(body, { status: 200, headers: { "Content-Type": "text/event-stream" } }); + return await collectAdapterEvents(adapter.parseStream(response, budget)); + } finally { + budget.dispose(); + } } function parsedFromContext(vector: Record): OcxParsedRequest { @@ -71,24 +74,20 @@ function parsedFromContext(vector: Record): OcxParsedRequest { const options = vector.options as Record | undefined; const messages = context?.messages as Array> | undefined; const input = messages - ? messages.map((m) => { - if (m.role === "developer") return { role: "developer", content: m.content }; - return { role: m.role, content: m.content }; - }) + ? messages.map((m) => ({ role: m.role, content: m.content })) : vector.input ?? "PING"; const body: Record = { model: vector.modelId ?? "fixture-model", input, stream: vector.stream ?? false, ...(options?.temperature !== undefined ? { temperature: options.temperature } : {}), + ...(options?.reasoning !== undefined ? { reasoning: { effort: options.reasoning } } : {}), ...(options?.textFormat ? { text: { format: options.textFormat } } : {}), ...(vector.tools ? { tools: normalizeTools(vector.tools as unknown[]) } : {}), ...(vector.tool_choice ? { tool_choice: vector.tool_choice } : {}), ...(vector.text ? { text: vector.text } : {}), }; - if (context?.systemPrompt) { - body.instructions = (context.systemPrompt as string[])[0]; - } + if (context?.systemPrompt) body.instructions = (context.systemPrompt as string[])[0]; return parseRequest(body); } @@ -101,6 +100,29 @@ function normalizeTools(tools: unknown[]): unknown[] { }); } +function createHarnessAdapter(provider: OcxProviderConfig) { + return withHarnessTranslatorBudget( + provider.adapter === "openai-responses" + ? createResponsesPassthroughAdapter(provider) + : createOpenAIChatAdapter(provider), + ); +} + +async function runBuildRequest( + observation: NormalizedObservation, + parsed: OcxParsedRequest, + provider: OcxProviderConfig, +): Promise { + const adapter = createHarnessAdapter(provider); + try { + const built = await adapter.buildRequest(parsed, { headers: new Headers() }); + recordUpstreamRequest(observation, JSON.parse(built.body)); + return observation; + } finally { + adapter.dispose(); + } +} + async function executeAdapterVector(caseRecord: CaseRecord): Promise { const observation = emptyObservation(); const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as Record; @@ -112,28 +134,28 @@ async function executeAdapterVector(caseRecord: CaseRecord): Promise; const sseEvents = normalizeSseBytes(new TextEncoder().encode(String(vector.sse ?? "")), "openai-responses"); - finalizeObservation(observation, sseEvents, json); + finalizeObservation(observation, sseEvents, json, 200); attachVerifiers(observation, caseRecord); return observation; + } case "chat-core.protocol.request-mapping": return await runBuildRequest(observation, parsedFromContext(vector), provider); - case "anthropic-core.protocol.request-mapping": + case "anthropic-core.protocol.request-mapping": { const anthropicBody = JSON.parse(caseRecord.fixture.bytesUtf8); const translated = anthropicToResponsesTranslation(anthropicBody); - const parsedAnthropic = parseRequest(translated.body); - const responsesProvider = fixtureProviderConfig("openai-responses"); - return await runBuildRequest(observation, parsedAnthropic, responsesProvider); + return await runBuildRequest(observation, parseRequest(translated.body), fixtureProviderConfig("openai-responses")); + } - case "anthropic-core.protocol.tool-round-trip": + case "anthropic-core.protocol.tool-round-trip": { const toolBody = JSON.parse(caseRecord.fixture.bytesUtf8); - const toolTranslated = anthropicToResponsesTranslation(toolBody); - const parsedTool = parseRequest(toolTranslated.body); - return await runBuildRequest(observation, parsedTool, fixtureProviderConfig("openai-responses")); + const translated = anthropicToResponsesTranslation(toolBody); + return await runBuildRequest(observation, parseRequest(translated.body), fixtureProviderConfig("openai-responses")); + } case "tools-core.protocol.function-round-trip": return await runToolRoundTrip(observation, vector, provider); @@ -142,6 +164,7 @@ async function executeAdapterVector(caseRecord: CaseRecord): Promise, -): Promise { - const adapter = withHarnessTranslatorBudget( - provider.adapter === "openai-responses" - ? createResponsesPassthroughAdapter(provider) - : createOpenAIChatAdapter(provider), - ); - const built = await adapter.buildRequest(parsed, { - headers: new Headers(), - translatorBudget: createTranslatorBudget(), - }); - const json = JSON.parse(built.body); - recordUpstreamRequest(observation, json); - return observation; -} - async function runToolRoundTrip( observation: NormalizedObservation, vector: Record, - provider: ReturnType, + provider: OcxProviderConfig, ): Promise { const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(provider)); - const tools = normalizeTools(vector.tools as unknown[]); - const upstreamToolCall = vector.upstreamToolCall as Record; - const toolResult = vector.toolResult as Record; - const parsed1 = parseRequest({ - model: "fixture-model", - input: "PING", - tools, - stream: false, - }); - const built1 = await adapter.buildRequest(parsed1, { - headers: new Headers(), - translatorBudget: createTranslatorBudget(), - }); - recordUpstreamRequest(observation, JSON.parse(built1.body)); - const sseBody = [ - `data: ${JSON.stringify({ choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: upstreamToolCall.id, function: { name: upstreamToolCall.name, arguments: upstreamToolCall.arguments } }] } }] })}\n\n`, - `data: ${JSON.stringify({ choices: [{ index: 0, finish_reason: "tool_calls" }] })}\n\n`, - "data: [DONE]\n\n", - ].join(""); - const events1 = await parseUpstreamSse(adapter, sseBody); - const bridged = await collectBridgeSse(events1); - finalizeObservation(observation, bridged.events); - const parsed2 = parseRequest({ - model: "fixture-model", - input: [ - { type: "function_call", call_id: upstreamToolCall.id, name: upstreamToolCall.name, arguments: upstreamToolCall.arguments }, - { type: "function_call_output", call_id: toolResult.toolCallId, output: toolResult.content }, - ], - stream: false, - }); - const built2 = await adapter.buildRequest(parsed2, { - headers: new Headers(), - translatorBudget: createTranslatorBudget(), - }); - recordUpstreamRequest(observation, JSON.parse(built2.body)); - return observation; + try { + const tools = normalizeTools(vector.tools as unknown[]); + const upstreamToolCall = vector.upstreamToolCall as Record; + const toolResult = vector.toolResult as Record; + const parsed1 = parseRequest({ model: "fixture-model", input: "PING", tools, stream: false }); + const built1 = await adapter.buildRequest(parsed1, { headers: new Headers() }); + recordUpstreamRequest(observation, JSON.parse(built1.body)); + + const sseBody = [ + `data: ${JSON.stringify({ choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: upstreamToolCall.id, function: { name: upstreamToolCall.name, arguments: upstreamToolCall.arguments } }] } }] })}\n\n`, + `data: ${JSON.stringify({ choices: [{ index: 0, finish_reason: "tool_calls" }] })}\n\n`, + "data: [DONE]\n\n", + ].join(""); + const events1 = await parseUpstreamSse(adapter, sseBody); + const bridged = await collectBridgeSse(events1); + finalizeObservation(observation, bridged.events, null, 200); + + const parsed2 = parseRequest({ + model: "fixture-model", + input: [ + { type: "function_call", call_id: upstreamToolCall.id, name: upstreamToolCall.name, arguments: upstreamToolCall.arguments }, + { type: "function_call_output", call_id: toolResult.toolCallId, output: toolResult.content }, + ], + stream: false, + }); + const built2 = await adapter.buildRequest(parsed2, { headers: new Headers() }); + recordUpstreamRequest(observation, JSON.parse(built2.body)); + return observation; + } finally { + adapter.dispose(); + } } async function runCustomToolRoundTrip( observation: NormalizedObservation, vector: Record, ): Promise { - const provider = fixtureProviderConfig("openai-responses"); - const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(provider)); - const tool = vector.tool as Record; - const call = vector.call as Record; - const output = vector.output as Record; - const parsed1 = parseRequest({ - model: "fixture-model", - input: "PING", - tools: [tool], - stream: false, - }); - const built1 = await adapter.buildRequest(parsed1, { - headers: new Headers(), - translatorBudget: createTranslatorBudget(), - }); - recordUpstreamRequest(observation, JSON.parse(built1.body)); - const events: AdapterEvent[] = [ - { type: "tool_call_start", id: String(call.id), name: String(call.name) }, - { type: "tool_call_delta", arguments: String(call.input) }, - { type: "tool_call_end" }, - { type: "done" }, - ]; - const bridged = await collectBridgeSse(events); - finalizeObservation(observation, bridged.events); - const parsed2 = parseRequest({ - model: "fixture-model", - input: [ - { type: "custom_tool_call", call_id: call.id, name: call.name, input: call.input }, - { type: "custom_tool_call_output", call_id: output.call_id, output: output.output }, - ], - tools: [tool], - stream: false, - }); - const built2 = await adapter.buildRequest(parsed2, { - headers: new Headers(), - translatorBudget: createTranslatorBudget(), - }); - recordUpstreamRequest(observation, JSON.parse(built2.body)); - return observation; + const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(fixtureProviderConfig("openai-responses"))); + try { + const tool = vector.tool as Record; + const call = vector.call as Record; + const output = vector.output as Record; + const parsed1 = parseRequest({ model: "fixture-model", input: "PING", tools: [tool], stream: false }); + const built1 = await adapter.buildRequest(parsed1, { headers: new Headers() }); + recordUpstreamRequest(observation, JSON.parse(built1.body)); + + const bridged = await collectBridgeSse([ + { type: "tool_call_start", id: String(call.id), name: String(call.name) }, + { type: "tool_call_delta", arguments: String(call.input) }, + { type: "tool_call_end" }, + { type: "done" }, + ]); + finalizeObservation(observation, bridged.events, null, 200); + + const parsed2 = parseRequest({ + model: "fixture-model", + input: [ + { type: "custom_tool_call", call_id: call.id, name: call.name, input: call.input }, + { type: "custom_tool_call_output", call_id: output.call_id, output: output.output }, + ], + tools: [tool], + stream: false, + }); + const built2 = await adapter.buildRequest(parsed2, { headers: new Headers() }); + recordUpstreamRequest(observation, JSON.parse(built2.body)); + return observation; + } finally { + adapter.dispose(); + } } async function runToolResultContent( observation: NormalizedObservation, vector: Record, - provider: ReturnType, + provider: OcxProviderConfig, ): Promise { const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(provider)); - const content = vector.content as Array>; - const parsed = parseRequest({ - model: "fixture-model", - input: [{ type: "function_call_output", call_id: vector.callId, output: content }], - stream: false, - }); - const built = await adapter.buildRequest(parsed, { - headers: new Headers(), - translatorBudget: createTranslatorBudget(), - }); - const upstreamJson = normalizeImageToolResultUpstream(JSON.parse(built.body) as Record); - recordUpstreamRequest(observation, upstreamJson); - return observation; + try { + const content = (vector.content ?? vector.result) as Array>; + const parsed = parseRequest({ + model: "fixture-model", + input: [{ type: "function_call_output", call_id: vector.callId, output: content }], + stream: false, + }); + const built = await adapter.buildRequest(parsed, { headers: new Headers() }); + recordUpstreamRequest(observation, normalizeImageToolResultUpstream(JSON.parse(built.body) as Record)); + return observation; + } finally { + adapter.dispose(); + } } function normalizeImageToolResultUpstream(body: Record): Record { const messages = body.messages as Array> | undefined; if (!messages) return body; const toolIdx = messages.findIndex((m) => m.role === "tool"); - const userIdx = messages.findIndex((m) => { - if (m.role !== "user" || !Array.isArray(m.content)) return false; - return (m.content as unknown[]).some((p) => p && typeof p === "object" && (p as { type?: string }).type === "image_url"); - }); + const userIdx = messages.findIndex((m) => m.role === "user" && Array.isArray(m.content) + && (m.content as unknown[]).some((p) => p && typeof p === "object" && (p as { type?: string }).type === "image_url")); if (toolIdx < 0 || userIdx < 0) return body; const tool = messages[toolIdx]; const user = messages[userIdx]; @@ -314,57 +316,51 @@ function normalizeImageToolResultUpstream(body: Record): Record async function runApplyPatchTurn( observation: NormalizedObservation, vector: Record, - provider: ReturnType, + provider: OcxProviderConfig, ): Promise { const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(provider)); - const events: AdapterEvent[] = [ - { type: "tool_call_start", id: String(vector.callId), name: "apply_patch" }, - { type: "tool_call_delta", arguments: String(vector.input) }, - { type: "tool_call_end" }, - { type: "done" }, - ]; - const bridged = await collectBridgeSse(events); - finalizeObservation(observation, bridged.events); - recordUpstreamRequest(observation, { model: "fixture-model", messages: [{ role: "user", content: "PING" }] }); - const parsed2 = parseRequest({ - model: "fixture-model", - input: [ - { type: "custom_tool_call", call_id: vector.callId, name: "apply_patch", input: vector.input }, - { type: "custom_tool_call_output", call_id: vector.callId, output: vector.result }, - ], - stream: false, - }); - const built2 = await adapter.buildRequest(parsed2, { - headers: new Headers(), - translatorBudget: createTranslatorBudget(), - }); - recordUpstreamRequest(observation, JSON.parse(built2.body)); - return observation; + try { + const bridged = await collectBridgeSse([ + { type: "tool_call_start", id: String(vector.callId), name: "apply_patch" }, + { type: "tool_call_delta", arguments: String(vector.input) }, + { type: "tool_call_end" }, + { type: "done" }, + ]); + finalizeObservation(observation, bridged.events, null, 200); + recordUpstreamRequest(observation, { model: "fixture-model", messages: [{ role: "user", content: "PING" }] }); + const parsed2 = parseRequest({ + model: "fixture-model", + input: [ + { type: "custom_tool_call", call_id: vector.callId, name: "apply_patch", input: vector.input }, + { type: "custom_tool_call_output", call_id: vector.callId, output: vector.result }, + ], + stream: false, + }); + const built2 = await adapter.buildRequest(parsed2, { headers: new Headers() }); + recordUpstreamRequest(observation, JSON.parse(built2.body)); + return observation; + } finally { + adapter.dispose(); + } } async function runCodexToolContinuation( observation: NormalizedObservation, vector: Record, ): Promise { - const provider = fixtureProviderConfig("openai-responses"); - const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(provider)); - const turn1 = vector.turn1 as { output: unknown[] }; - const turn2 = vector.turn2 as { input: unknown[] }; - const parsed = parseRequest({ - model: "fixture-model", - input: turn2.input, - stream: false, - }); - const built = await adapter.buildRequest(parsed, { - headers: new Headers(), - translatorBudget: createTranslatorBudget(), - }); - const upstreamJson = JSON.parse(built.body) as { input?: unknown[] }; - if (Array.isArray(turn1.output)) { - upstreamJson.input = [...turn1.output, ...(upstreamJson.input as unknown[] ?? [])]; + const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(fixtureProviderConfig("openai-responses"))); + try { + const turn1 = vector.turn1 as { output: unknown[] }; + const turn2 = vector.turn2 as { input: unknown[] }; + const parsed = parseRequest({ model: "fixture-model", input: turn2.input, stream: false }); + const built = await adapter.buildRequest(parsed, { headers: new Headers() }); + const upstreamJson = JSON.parse(built.body) as { input?: unknown[] }; + if (Array.isArray(turn1.output)) upstreamJson.input = [...turn1.output, ...(upstreamJson.input ?? [])]; + recordUpstreamRequest(observation, upstreamJson); + return observation; + } finally { + adapter.dispose(); } - recordUpstreamRequest(observation, upstreamJson); - return observation; } async function runPreviousResponseReplay( @@ -372,33 +368,115 @@ async function runPreviousResponseReplay( vector: Record, ): Promise { clearResponseStateForTests(); - const stored = vector.stored as Record; - const next = vector.next as Record; - rememberResponseState( - { input: stored.input, store: true }, - { id: String(stored.id), output: stored.output, status: "completed" }, - undefined, - { force: true }, - ); - const requestBody = { - model: "fixture-model", - store: true, - previous_response_id: stored.id, - input: next.input, + try { + const stored = vector.stored as Record; + const next = vector.next as Record; + rememberResponseState( + { input: stored.input, store: true }, + { id: String(stored.id), output: stored.output, status: "completed" }, + undefined, + { force: true }, + ); + const expanded = expandPreviousResponseInput({ + model: "fixture-model", + store: true, + previous_response_id: stored.id, + input: next.input, + }); + const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(fixtureProviderConfig("openai-responses"))); + try { + const built = await adapter.buildRequest({ ...parseRequest(expanded), _previousResponseInputExpanded: true }, { headers: new Headers() }); + const upstreamJson = JSON.parse(built.body) as Record; + delete upstreamJson.previous_response_id; + recordUpstreamRequest(observation, upstreamJson); + return observation; + } finally { + adapter.dispose(); + } + } finally { + clearResponseStateForTests(); + } +} + +async function runReasoningEffortMapping( + observation: NormalizedObservation, + vector: Record, +): Promise { + const provider: OcxProviderConfig = { + ...fixtureProviderConfig("openai-chat"), + reasoningEffortMap: vector.reasoningEffortMap as Record, + reasoningWireFormat: vector.reasoningWireFormat as OcxProviderConfig["reasoningWireFormat"], }; - const expanded = expandPreviousResponseInput(requestBody); - const provider = fixtureProviderConfig("openai-responses"); - const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(provider)); - const parsed = parseRequest(expanded); - const built = await adapter.buildRequest({ ...parsed, _previousResponseInputExpanded: true }, { - headers: new Headers(), - translatorBudget: createTranslatorBudget(), + const parsed = parseRequest({ + model: "fixture-model", + input: "PING", + stream: false, + reasoning: { effort: vector.requested }, }); - const upstreamJson = JSON.parse(built.body) as Record; - delete upstreamJson.previous_response_id; - recordUpstreamRequest(observation, upstreamJson); + return await runBuildRequest(observation, parsed, provider); +} + +async function runReasoningReplay( + observation: NormalizedObservation, + vector: Record, +): Promise { + const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(fixtureProviderConfig("openai-responses"))); + try { + const turn1 = vector.turn1 as { + reasoning: { id: string; text: string; signature: string }; + toolCall: { callId: string }; + }; + const turn2 = vector.turn2 as { toolResult: { callId: string; output: string } }; + const first = await adapter.buildRequest(parseRequest({ model: "fixture-model", input: "PING", stream: false }), { headers: new Headers() }); + recordUpstreamRequest(observation, JSON.parse(first.body)); + const replayInput = [ + { + type: "reasoning", + id: turn1.reasoning.id, + content: [{ type: "reasoning_text", text: turn1.reasoning.text }], + signature: turn1.reasoning.signature, + }, + { + type: "function_call_output", + call_id: turn2.toolResult.callId, + output: turn2.toolResult.output, + }, + ]; + const second = await adapter.buildRequest(parseRequest({ model: "fixture-model", input: replayInput, stream: false }), { headers: new Headers() }); + recordUpstreamRequest(observation, JSON.parse(second.body)); + return observation; + } finally { + adapter.dispose(); + } +} + +async function runReasoningPrivateIsolation( + observation: NormalizedObservation, + vector: Record, +): Promise { clearResponseStateForTests(); - return observation; + try { + const origin = vector.origin as { encrypted?: string }; + rememberResponseState( + { input: "PING", store: true }, + { + id: "resp_private_fixture", + output: [{ type: "reasoning", id: "rs_private", summary: [], encrypted_content: origin.encrypted }], + status: "completed", + }, + undefined, + { force: true }, + ); + const expanded = expandPreviousResponseInput({ + model: "fixture-model", + store: true, + previous_response_id: "resp_private_fixture", + input: "NEXT", + }); + return await runBuildRequest(observation, parseRequest(expanded), fixtureProviderConfig("openai-chat")); + } finally { + clearResponseStateForTests(); + } } async function executeClientRequest(caseRecord: CaseRecord): Promise { @@ -409,20 +487,20 @@ async function executeClientRequest(caseRecord: CaseRecord): Promise { + if (!caseRecord.initiatingRequest) return; + const body = JSON.parse(caseRecord.initiatingRequest.bytesUtf8); + const inbound = caseRecord.requirements.inboundProtocols[0] ?? "openai-responses"; + const parsed = inbound === "anthropic-messages" + ? parseRequest(anthropicToResponsesTranslation(body).body) + : parseRequest(body); + const upstream = caseRecord.requirements.upstreamProtocols[0] ?? "openai-chat"; + await runBuildRequest(observation, parsed, fixtureProviderConfig(upstreamAdapterForProtocol(upstream))); } async function executeStreamScenario(caseRecord: CaseRecord): Promise { @@ -431,88 +509,94 @@ async function executeStreamScenario(caseRecord: CaseRecord): Promise; let json: Record | null = null; + let responseStatus = 200; if (upstreamProtocol === "openai-chat") { - const provider = fixtureProviderConfig("openai-chat"); - const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(provider)); - const adapterEvents = await parseUpstreamSse(adapter, caseRecord.fixture.bytesUtf8); - const bridged = await collectBridgeSse(adapterEvents); - events = bridged.events; - if (surface.includes("anthropic")) { - const budget = createTranslatorBudget(); - const bridgedStream = bridgeToResponsesSSE((async function* () { - for (const event of adapterEvents) yield event; - })(), "fixture-model"); - const anthropicStream = responsesSseToAnthropicSse(bridgedStream, "fixture-model", { translatorBudget: budget }); - const reader = anthropicStream.getReader(); - const decoder = new TextDecoder(); - let anthropicText = ""; - while (true) { - const { done, value } = await reader.read(); - if (done) break; - anthropicText += decoder.decode(value, { stream: true }); - } - events = filterAnthropicEvents(normalizeSseBytes(new TextEncoder().encode(anthropicText), "anthropic-messages")); - } - if (caseRecord.id === "codex-core.protocol.streaming-turn" && events.length > 0) { - const data = events[0].data; - if (data && typeof data === "object") { - (data as Record).phase = "final_answer"; + const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(fixtureProviderConfig("openai-chat"))); + try { + const adapterEvents = await parseUpstreamSse(adapter, caseRecord.fixture.bytesUtf8); + events = (await collectBridgeSse(adapterEvents)).events; + if (surface.includes("anthropic")) { + const budget = createTranslatorBudget(); + try { + const bridgedStream = bridgeToResponsesSSE((async function* () { + for (const event of adapterEvents) yield event; + })(), "fixture-model"); + const anthropicStream = responsesSseToAnthropicSse(bridgedStream, "fixture-model", { translatorBudget: budget }); + const reader = anthropicStream.getReader(); + const decoder = new TextDecoder(); + let text = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + text += decoder.decode(value, { stream: true }); + } + } finally { + reader.releaseLock(); + } + events = filterAnthropicEvents(normalizeSseBytes(new TextEncoder().encode(text), "anthropic-messages")); + } finally { + budget.dispose(); + } } + } finally { + adapter.dispose(); } } else if (upstreamProtocol === "openai-responses") { if (inboundProtocol === "anthropic-messages") { const budget = createTranslatorBudget(); - const responsesSse = bridgeToResponsesSSE((async function* () { - const passthrough = createResponsesPassthroughAdapter(fixtureProviderConfig("openai-responses")); - const budgetInner = createTranslatorBudget(); - const response = new Response(caseRecord.fixture.bytesUtf8, { headers: { "Content-Type": "text/event-stream" } }); - for await (const event of passthrough.parseStream(response, budgetInner)) yield event; - })(), "fixture-model"); - const anthropicStream = responsesSseToAnthropicSse(responsesSse, "fixture-model", { translatorBudget: budget }); - const reader = anthropicStream.getReader(); - const decoder = new TextDecoder(); - let text = ""; - while (true) { - const { done, value } = await reader.read(); - if (done) break; - text += decoder.decode(value, { stream: true }); - } - events = filterAnthropicEvents(normalizeSseBytes(new TextEncoder().encode(text), "anthropic-messages")); - if (caseRecord.id === "anthropic-core.protocol.terminal-errors") { - events = events.filter((e) => e.event === "error"); + const passthroughBudget = createTranslatorBudget(); + try { + const responsesSse = bridgeToResponsesSSE((async function* () { + const passthrough = createResponsesPassthroughAdapter(fixtureProviderConfig("openai-responses")); + const response = new Response(caseRecord.fixture.bytesUtf8, { status: 200, headers: { "Content-Type": "text/event-stream" } }); + for await (const event of passthrough.parseStream(response, passthroughBudget)) yield event; + })(), "fixture-model"); + const anthropicStream = responsesSseToAnthropicSse(responsesSse, "fixture-model", { translatorBudget: budget }); + const reader = anthropicStream.getReader(); + const decoder = new TextDecoder(); + let text = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + text += decoder.decode(value, { stream: true }); + } + } finally { + reader.releaseLock(); + } + events = filterAnthropicEvents(normalizeSseBytes(new TextEncoder().encode(text), "anthropic-messages")); + } finally { + passthroughBudget.dispose(); + budget.dispose(); } } else { events = normalizeSseBytes(upstreamBytes, upstreamProtocol); } } else { - events = normalizeSseBytes(upstreamBytes, upstreamProtocol); + throw new Error(`unsupported upstream protocol: ${upstreamProtocol}`); } if (caseRecord.id === "chat-core.protocol.nonstream-envelope") { - const provider = fixtureProviderConfig("openai-chat"); - const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(provider)); - const responseJson = JSON.parse(caseRecord.fixture.bytesUtf8); - const parsedEvents = adapter.parseResponse - ? await adapter.parseResponse( - new Response(caseRecord.fixture.bytesUtf8, { headers: { "Content-Type": "application/json" } }), - createTranslatorBudget(), - ) - : []; - const bridged = await collectBridgeSse(parsedEvents); - events = bridged.events; - json = buildResponseJSON(parsedEvents, "fixture-model") as Record ?? responseJson; - finalizeObservation(observation, events, json); - return observation; + const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(fixtureProviderConfig("openai-chat"))); + try { + const response = new Response(caseRecord.fixture.bytesUtf8, { status: 200, headers: { "Content-Type": "application/json" } }); + responseStatus = response.status; + const responseJson = JSON.parse(caseRecord.fixture.bytesUtf8); + const parsedEvents = adapter.parseResponse ? await adapter.parseResponse(response) : []; + events = (await collectBridgeSse(parsedEvents)).events; + json = buildResponseJSON(parsedEvents, "fixture-model") as Record ?? responseJson; + } finally { + adapter.dispose(); + } } - finalizeObservation(observation, events, json); + finalizeObservation(observation, events, json, responseStatus); attachVerifiers(observation, caseRecord); return observation; } @@ -530,7 +614,9 @@ export async function executeScenario(caseRecord: CaseRecord): Promise assertionResults.find((r) => r.id === id)?.passed); - const expectedFailureMatched = controlPassed - && requiredFailures.length === 0; + const controlPassed = listed.every((id) => assertionResults.find((r) => r.id === id)?.passed === true); + const expectedFailureMatched = controlPassed && requiredFailures.length === 0; return { scenarioId: caseRecord.id, suite: caseRecord.suite, diff --git a/src/lab/conformance/fixture-provider.ts b/src/lab/conformance/fixture-provider.ts index 1ef49185f9..d389deb673 100644 --- a/src/lab/conformance/fixture-provider.ts +++ b/src/lab/conformance/fixture-provider.ts @@ -3,7 +3,10 @@ import type { OcxProviderConfig } from "../../types"; export function fixtureProviderConfig(adapter: string): OcxProviderConfig { return { adapter, - baseUrl: "http://127.0.0.1:1/v1", + // The Chat fixture intentionally exercises native OpenAI Chat semantics (including + // role:"developer" and named single-tool selection). Other fixture adapters remain + // loopback-only and never perform network I/O. + baseUrl: adapter === "openai-chat" ? "https://api.openai.com/v1" : "http://127.0.0.1:1/v1", apiKey: "fixture-key", allowPrivateNetwork: true, models: ["fixture-model"], @@ -18,11 +21,7 @@ export function upstreamAdapterForProtocol(protocol: string): string { return "openai-chat"; case "openai-responses": return "openai-responses"; - case "anthropic-messages": - return "anthropic"; - case "cursor-protobuf": - return "cursor"; default: - return "openai-chat"; + throw new Error(`unsupported upstream protocol: ${protocol}`); } } diff --git a/src/lab/conformance/harness-budget.ts b/src/lab/conformance/harness-budget.ts index b227af3bfc..a161f5ec2e 100644 --- a/src/lab/conformance/harness-budget.ts +++ b/src/lab/conformance/harness-budget.ts @@ -11,14 +11,16 @@ type TestAdapter = Omit ReturnType>; + dispose(): void; }; -/** Inject translator budget for harness adapter calls (mirrors tests/helpers/translator-budget). */ +/** Inject one bounded translator budget for a harness adapter scope. Call dispose() in finally. */ export function withHarnessTranslatorBudget(adapter: T): TestAdapter { const budget = createTranslatorBudget(); const buildRequest = adapter.buildRequest.bind(adapter); const parseStream = adapter.parseStream.bind(adapter); const parseResponse = adapter.parseResponse?.bind(adapter); + let disposed = false; return { ...adapter, buildRequest(parsed: Parameters[0], incoming?: Partial) { @@ -36,5 +38,10 @@ export function withHarnessTranslatorBudget(adapter: return parseResponse(response, explicitBudget ?? budget); }, } : {}), + dispose() { + if (disposed) return; + disposed = true; + budget.dispose(); + }, } as unknown as TestAdapter; } diff --git a/src/lab/conformance/jcs.ts b/src/lab/conformance/jcs.ts index 4fa996107e..6bbcb923c7 100644 --- a/src/lab/conformance/jcs.ts +++ b/src/lab/conformance/jcs.ts @@ -1,7 +1,10 @@ /** RFC 8785 JSON Canonicalization Scheme (JCS) for deterministic equality. */ export function jcsStringify(value: unknown): string { - if (value === null || typeof value === "boolean" || typeof value === "number") { + if (value === undefined) throw new TypeError("jcsStringify: undefined is not representable in JCS"); + if (value === null || typeof value === "boolean") return JSON.stringify(value); + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new TypeError("jcsStringify: non-finite numbers are not representable in JCS"); return JSON.stringify(value); } if (typeof value === "string") return JSON.stringify(value); @@ -13,7 +16,7 @@ export function jcsStringify(value: unknown): string { const keys = Object.keys(obj).sort(); return `{${keys.map((k) => `${JSON.stringify(k)}:${jcsStringify(obj[k])}`).join(",")}}`; } - return JSON.stringify(value); + throw new TypeError(`jcsStringify: unsupported value type ${typeof value}`); } export function jcsEqual(a: unknown, b: unknown): boolean { diff --git a/src/lab/conformance/json-pointer.ts b/src/lab/conformance/json-pointer.ts index efc59b7120..1676d0abf1 100644 --- a/src/lab/conformance/json-pointer.ts +++ b/src/lab/conformance/json-pointer.ts @@ -26,7 +26,9 @@ export function resolveJsonPointer(root: unknown, pointer: string): PointerResul return { ok: false, reason: "selector_missing" }; } const obj = current as Record; - if (!(token in obj)) return { ok: false, reason: "selector_missing" }; + if (!Object.prototype.hasOwnProperty.call(obj, token)) { + return { ok: false, reason: "selector_missing" }; + } current = obj[token]; } return { ok: true, value: current }; diff --git a/src/lab/conformance/manifest.ts b/src/lab/conformance/manifest.ts index a44fb4bc12..df8001239b 100644 --- a/src/lab/conformance/manifest.ts +++ b/src/lab/conformance/manifest.ts @@ -1,7 +1,7 @@ import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { fixtureDigest, scenarioManifestDigest } from "./digest"; +import { fixtureDigest } from "./digest"; import { MCP_ACTION_TOKENS } from "./mcp-stub"; import type { CaseAuthority, @@ -12,6 +12,7 @@ import type { import { CL01_SUITES, SYNTHETIC_MARKER } from "./types"; const MODULE_DIR = dirname(fileURLToPath(import.meta.url)); +// Provenance is the normative CL-00 authority name, not the runtime copy's basename. const AUTHORITY_FILE = "022_protocol_v1_cases.json"; export function loadCaseAuthority(): CaseAuthority { @@ -74,7 +75,12 @@ function fixtureRef(fixture: CaseRecord["fixture"], authority: CaseAuthority): R } function expandFailureRules(caseRecord: CaseRecord, authority: CaseAuthority): FailureRule[] { - const base = [...authority.failureRuleSets[authority.manifestDefaults.failureRuleSet]]; + const setName = authority.manifestDefaults.failureRuleSet; + const ruleSet = authority.failureRuleSets[setName]; + if (!Array.isArray(ruleSet)) { + throw new Error(`harness_failure: contract_integrity unknown failureRuleSet ${setName}`); + } + const base = [...ruleSet]; if (!caseRecord.expectedFailure) return base; const template = authority.expectedFailureRuleTemplate; const controlRule: FailureRule = { @@ -133,12 +139,6 @@ export function validateExpandedFixtureRef( return errors; } -export function validateScenarioManifestDigest(caseRecord: CaseRecord, authority: CaseAuthority): boolean { - const expanded = expandScenario(caseRecord, authority); - const digest = scenarioManifestDigest(expanded); - return digest.length === 64; -} - function validateMcpHarnessFeatures(caseRecord: CaseRecord): string[] { if (caseRecord.suite !== "mcp-core") return []; const tokens = caseRecord.requirements.requiredHarnessFeatures.filter( diff --git a/src/lab/conformance/mcp-stub.ts b/src/lab/conformance/mcp-stub.ts index bf04e9cff1..ee50b3599a 100644 --- a/src/lab/conformance/mcp-stub.ts +++ b/src/lab/conformance/mcp-stub.ts @@ -30,14 +30,19 @@ export function executeMcpSyntheticAction(caseRecord: CaseRecord): NormalizedObs return runCallResult(decoded); case "mcp_resource_round_trip_v1": return runResourceRoundTrip(decoded); - default: - throw new Error(`invalid_manifest: unsupported MCP action ${token}`); } } +function nonEmptyString(value: unknown): value is string { + return typeof value === "string" && value.length > 0; +} + function runNamespaceRoundTrip(decoded: Record): NormalizedObservation { - const namespace = String(decoded.namespace ?? ""); - const name = String(decoded.name ?? ""); + if (!nonEmptyString(decoded.namespace) || !nonEmptyString(decoded.name)) { + throw new Error("invalid_manifest: MCP namespace/name must be non-empty strings"); + } + const namespace = decoded.namespace; + const name = decoded.name; const wireName = `${namespace}__${name}`; const observation = emptyObservation(); observation.upstream.requests.push({ @@ -72,29 +77,50 @@ function utf8ByteLength(value: string): number { return new TextEncoder().encode(value).byteLength; } +function parsesJson(value: string): boolean { + try { + JSON.parse(value); + return true; + } catch { + return false; + } +} + function runSchemaBounds(decoded: Record): NormalizedObservation { - const limitBytes = Number(decoded.limitBytes ?? 0); - const exactSchema = String(decoded.exactSchema ?? ""); - const overSchema = String(decoded.overSchema ?? ""); const observation = emptyObservation(); - const exactBound = utf8ByteLength(exactSchema) === limitBytes && JSON.parse(exactSchema) !== undefined - ? "pass" - : "fail"; - const oneOverRejected = utf8ByteLength(overSchema) === limitBytes + 1 - && JSON.parse(overSchema) !== undefined - ? "pass" - : "fail"; + const limitBytes = decoded.limitBytes; + const exactSchema = decoded.exactSchema; + const overSchema = decoded.overSchema; + if (!Number.isInteger(limitBytes) || (limitBytes as number) <= 0 + || typeof exactSchema !== "string" || typeof overSchema !== "string") { + observation.verifiers = { + exact_bound: "fail", + one_over_rejected: "fail", + partial_commit: false, + }; + return observation; + } + + const limit = limitBytes as number; + const exactValid = utf8ByteLength(exactSchema) === limit && parsesJson(exactSchema); + // The inert stub models two isolated catalogue transactions. The over-bound transaction + // is rejected before commit; its validity matters so this tests the byte ceiling rather + // than malformed JSON. + const overRejected = utf8ByteLength(overSchema) === limit + 1 && parsesJson(overSchema); observation.verifiers = { - exact_bound: exactBound, - one_over_rejected: oneOverRejected, + exact_bound: exactValid ? "pass" : "fail", + one_over_rejected: overRejected ? "pass" : "fail", partial_commit: false, }; return observation; } function runCallResult(decoded: Record): NormalizedObservation { - const namespace = String(decoded.namespace ?? ""); - const name = String(decoded.name ?? ""); + if (!nonEmptyString(decoded.namespace) || !nonEmptyString(decoded.name)) { + throw new Error("invalid_manifest: MCP namespace/name must be non-empty strings"); + } + const namespace = decoded.namespace; + const name = decoded.name; const argumentsValue = decoded.arguments ?? {}; const result = decoded.result; const wireName = `${namespace}__${name}`; @@ -119,13 +145,22 @@ function runCallResult(decoded: Record): NormalizedObservation } function runResourceRoundTrip(decoded: Record): NormalizedObservation { - const resources = decoded.resources; - const read = decoded.read as { uri?: string; contents?: unknown[] } | undefined; + if (!Array.isArray(decoded.resources) || !decoded.read || typeof decoded.read !== "object") { + throw new Error("invalid_manifest: invalid MCP resource fixture"); + } + const read = decoded.read as { uri?: unknown; contents?: unknown[] }; + if (!nonEmptyString(read.uri) || !Array.isArray(read.contents)) { + throw new Error("invalid_manifest: invalid MCP resource read fixture"); + } + const matching = decoded.resources.filter((resource) => + resource && typeof resource === "object" && (resource as { uri?: unknown }).uri === read.uri + ); + if (matching.length !== 1) throw new Error("invalid_manifest: MCP resource URI must resolve exactly once"); const observation = emptyObservation(); setClientResponse(observation, { json: { - resources, - contents: read?.contents, + resources: decoded.resources, + contents: read.contents, }, status: 200, }); @@ -139,12 +174,6 @@ export function attachMcpVerifiers(observation: NormalizedObservation, caseRecor observation.client.response.mcpCalls = projectMcpCalls(toolCalls); } } - if (caseRecord.id === "mcp-core.protocol.call-result") { - const decoded = JSON.parse(caseRecord.fixture.bytesUtf8) as Record; - observation.verifiers.stub_received = { - namespace: String(decoded.namespace ?? ""), - name: String(decoded.name ?? ""), - arguments: decoded.arguments ?? {}, - }; - } + // runCallResult records the literal one-invocation receipt. Do not reconstruct it here: + // assertions must inspect the action result that actually ran. } diff --git a/src/lab/conformance/negative-controls.ts b/src/lab/conformance/negative-controls.ts index 04f35a7aed..66ab70f4e7 100644 --- a/src/lab/conformance/negative-controls.ts +++ b/src/lab/conformance/negative-controls.ts @@ -54,7 +54,7 @@ export const NEGATIVE_CONTROL_FIXTURES: Array<{ mutate: (c) => ({ ...c, id: "negative.tool-result-order", - assertions: [{ id: "result", operator: "tool_result_correlates", selector: "/upstream/requests", expected: { call: "/client/response/toolCalls/0/id", result: "/upstream/requests/1/json/input/0/call_id" }, required: true }], + assertions: [{ id: "result", operator: "tool_result_correlates", selector: "/upstream/requests", expected: { call: "/client/response/toolCalls/0/id", result: "/upstream/requests/1/json/messages/1/tool_call_id" }, required: true }], fixture: { ...c.fixture, bytesUtf8: JSON.stringify({ @@ -152,7 +152,9 @@ export function buildNegativeControls(cases: CaseRecord[]): CaseRecord[] { for (const control of NEGATIVE_CONTROL_FIXTURES) { const base = baseCaseForNegativeControl(control.id, cases); if (!base) continue; - built.push(control.mutate(structuredClone(base))); + const cloned = structuredClone(base); + delete cloned.expectedFailure; + built.push(control.mutate(cloned)); } return built; } diff --git a/src/lab/conformance/observation.ts b/src/lab/conformance/observation.ts index cb37317344..287467802f 100644 --- a/src/lab/conformance/observation.ts +++ b/src/lab/conformance/observation.ts @@ -64,24 +64,27 @@ export function projectToolCallsFromOutput(output: unknown[]): ToolCallProjectio if (!item || typeof item !== "object") continue; const rec = item as Record; if (rec.type === "function_call") { - calls.push({ - id: String(rec.call_id ?? rec.id ?? ""), - name: String(rec.name ?? ""), - arguments: parseToolArguments(rec.arguments, "function"), - kind: "function", - ordinal: ordinal++, - }); + const id = rec.call_id ?? rec.id; + const name = rec.name; + if (typeof id !== "string" || id.length === 0 || typeof name !== "string" || name.length === 0) continue; + const args = parseToolArguments(rec.arguments, "function"); + if (args === null) continue; + calls.push({ id, name, arguments: args, kind: "function", ordinal: ordinal++ }); } else if (rec.type === "custom_tool_call") { + const id = rec.call_id ?? rec.id; + const name = rec.name; + if (typeof id !== "string" || id.length === 0 || typeof name !== "string" || name.length === 0) continue; calls.push({ - id: String(rec.call_id ?? rec.id ?? ""), - name: String(rec.name ?? ""), + id, + name, arguments: parseToolArguments(rec.input, "custom"), kind: "custom", ordinal: ordinal++, }); } } - return calls; + const ids = calls.map((call) => call.id); + return new Set(ids).size === ids.length ? calls : []; } export function projectToolCallsFromEvents(events: NormalizedEvent[]): ToolCallProjection[] { @@ -124,24 +127,28 @@ function deriveTerminal(events: NormalizedEvent[]): string | null { return null; } -export function deriveNormalizedText(events: NormalizedEvent[], json: unknown): string { - if (json && typeof json === "object" && !Array.isArray(json)) { - const resp = json as Record; - if (Array.isArray(resp.output)) { - let text = ""; - for (const item of resp.output) { - if (!item || typeof item !== "object") continue; - const content = (item as { content?: unknown }).content; - if (!Array.isArray(content)) continue; - for (const part of content) { - if (part && typeof part === "object" && (part as { type?: string }).type === "output_text") { - text += String((part as { text?: string }).text ?? ""); - } - } +function extractOutputText(json: Record): string { + let text = ""; + const output = json.output; + if (!Array.isArray(output)) return text; + for (const item of output) { + if (!item || typeof item !== "object") continue; + const content = (item as { content?: unknown[] }).content; + if (!Array.isArray(content)) continue; + for (const part of content) { + if (part && typeof part === "object" && (part as { type?: string }).type === "output_text") { + text += String((part as { text?: string }).text ?? ""); } - if (text) return text; } } + return text; +} + +export function deriveNormalizedText(events: NormalizedEvent[], json: unknown): string { + if (json && typeof json === "object" && !Array.isArray(json)) { + const text = extractOutputText(json as Record); + if (text) return text; + } let text = ""; for (const ev of events) { if (ev.event === "response.output_text.delta" && ev.data && typeof ev.data === "object") { @@ -159,26 +166,30 @@ export function finalizeObservation( observation: NormalizedObservation, events: NormalizedEvent[], json: unknown = null, + status = 200, ): void { - const toolCalls = projectToolCallsFromEvents(events); - const terminal = deriveTerminal(events); - setClientResponse(observation, { - events, - toolCalls: toolCalls.length > 0 ? toolCalls : projectToolCallsFromOutput( + const eventToolCalls = projectToolCallsFromEvents(events); + const resolvedToolCalls = eventToolCalls.length > 0 + ? eventToolCalls + : projectToolCallsFromOutput( json && typeof json === "object" && !Array.isArray(json) ? ((json as { output?: unknown[] }).output ?? []) : [], - ), - mcpCalls: projectMcpCalls(toolCalls), + ); + const terminal = deriveTerminal(events); + setClientResponse(observation, { + events, + toolCalls: resolvedToolCalls, + mcpCalls: projectMcpCalls(resolvedToolCalls), terminal, normalizedText: deriveNormalizedText(events, json), json, - status: 200, + status, }); } export function attachVerifiers(observation: NormalizedObservation, caseRecord: CaseRecord): void { - observation.verifiers = buildVerifiers(observation, caseRecord); + observation.verifiers = { ...observation.verifiers, ...buildVerifiers(observation, caseRecord) }; } function buildVerifiers(observation: NormalizedObservation, caseRecord: CaseRecord): Record { @@ -218,23 +229,28 @@ function buildVerifiers(observation: NormalizedObservation, caseRecord: CaseReco } function evaluateCallResultOrder(observation: NormalizedObservation): string { - const input = observation.upstream.requests[0]?.json as { input?: unknown[] } | undefined; - if (!input?.input || !Array.isArray(input.input)) return "fail"; - let sawCall = false; - for (const item of input.input) { + const request = observation.upstream.requests[0]?.json as { input?: unknown[] } | undefined; + const input = request?.input; + if (!Array.isArray(input)) return "fail"; + let pendingCallId: string | undefined; + let resultCount = 0; + for (const item of input) { if (!item || typeof item !== "object") continue; - const type = (item as { type?: string }).type; - if (type === "function_call") { - if (sawCall) return "fail"; - sawCall = true; + const record = item as { type?: string; call_id?: unknown }; + if (record.type === "function_call") { + if (pendingCallId !== undefined) return "fail"; + if (typeof record.call_id !== "string" || record.call_id.length === 0) return "fail"; + pendingCallId = record.call_id; continue; } - if (type === "function_call_output") { - if (!sawCall) return "fail"; - return "pass"; + if (record.type === "function_call_output") { + if (pendingCallId === undefined) return "fail"; + if (typeof record.call_id !== "string" || record.call_id !== pendingCallId) return "fail"; + resultCount++; + if (resultCount > 1) return "fail"; } } - return "fail"; + return pendingCallId !== undefined && resultCount === 1 ? "pass" : "fail"; } function evaluateCompactionReplayed(caseRecord: CaseRecord): boolean { @@ -287,13 +303,10 @@ function evaluateModalityPath(caseRecord: CaseRecord): string { function evaluateSilentImageDrop(caseRecord: CaseRecord): boolean { const vector = JSON.parse(caseRecord.fixture.bytesUtf8) as Record; - const requestHasImage = Boolean(vector.requestHasImage); - const modalities = vector.modelInputModalities as string[] | undefined; - const sidecar = vector.visionSidecar as { enabled?: boolean } | undefined; - if (!requestHasImage) return false; - if (Array.isArray(modalities) && modalities.includes("image")) return false; - if (sidecar?.enabled) return false; - return true; + if (!Boolean(vector.requestHasImage)) return false; + // Protocol V1's closed modality-gate vector treats the explicit `unsupported` path as a + // typed rejection, not as an omitted image. Native/sidecar paths likewise preserve it. + return !["native", "sidecar", "unsupported"].includes(evaluateModalityPath(caseRecord)); } function evaluateJsonSseEquivalence(caseRecord: CaseRecord): string { @@ -320,20 +333,3 @@ function evaluateJsonSseEquivalence(caseRecord: CaseRecord): string { const sseProjection = { text: sseText, terminal: sseTerminal }; return JSON.stringify(jsonProjection) === JSON.stringify(sseProjection) ? "pass" : "fail"; } - -function extractOutputText(json: Record): string { - let text = ""; - const output = json.output; - if (!Array.isArray(output)) return text; - for (const item of output) { - if (!item || typeof item !== "object") continue; - const content = (item as { content?: unknown[] }).content; - if (!Array.isArray(content)) continue; - for (const part of content) { - if (part && typeof part === "object" && (part as { type?: string }).type === "output_text") { - text += String((part as { text?: string }).text ?? ""); - } - } - } - return text; -} diff --git a/src/lab/conformance/runner.ts b/src/lab/conformance/runner.ts index d50057d3ef..864f887266 100644 --- a/src/lab/conformance/runner.ts +++ b/src/lab/conformance/runner.ts @@ -5,17 +5,26 @@ import type { ScenarioRunResult } from "./types"; import { CL01_SUITES } from "./types"; export interface ConformanceRunSummary { + /** Number of scenarios executed. */ total: number; + /** Number of scenarios that met their expected outcome. */ passed: number; + /** Number of scenarios that did not meet their expected outcome. */ failed: number; results: ScenarioRunResult[]; } +export interface NegativeControlRunSummary extends ConformanceRunSummary { + /** Number of deliberately defective controls correctly rejected by the harness. */ + rejected: number; +} + export async function runConformanceSuite( suites: readonly string[] = CL01_SUITES, ): Promise { const authority = loadCaseAuthority(); const scenarios = discoverScenarios(authority, suites); + if (scenarios.length === 0) throw new Error("harness_failure: no CL-01 scenarios discovered"); const results: ScenarioRunResult[] = []; for (const scenario of scenarios) { results.push(await runScenario(scenario)); @@ -24,15 +33,22 @@ export async function runConformanceSuite( return { total: results.length, passed, failed: results.length - passed, results }; } -export async function runNegativeControls(): Promise { +export async function runNegativeControls(): Promise { const authority = loadCaseAuthority(); const scenarios = buildNegativeControls(discoverScenarios(authority)); + if (scenarios.length === 0) throw new Error("harness_failure: no negative controls discovered"); const results: ScenarioRunResult[] = []; for (const scenario of scenarios) { results.push(await runScenario(scenario)); } - const passed = results.filter((r) => !r.passed).length; - return { total: results.length, passed, failed: results.length - passed, results }; + const rejected = results.filter((r) => !r.passed).length; + return { + total: results.length, + passed: rejected, + rejected, + failed: results.length - rejected, + results, + }; } export function listScenarioIds(suites: readonly string[] = CL01_SUITES): string[] { diff --git a/src/lab/conformance/types.ts b/src/lab/conformance/types.ts index 3ce474f821..f1c61bce49 100644 --- a/src/lab/conformance/types.ts +++ b/src/lab/conformance/types.ts @@ -152,10 +152,14 @@ export interface ScenarioRunResult { diagnostics: string[]; } +/** All eight protocol-conformance suites frozen by CL-00 Protocol V1. */ export const CL01_SUITES = [ "responses-core", "chat-core", "anthropic-core", "tools-core", "codex-core", + "vision-core", + "reasoning-core", + "mcp-core", ] as const; diff --git a/tests/lab-conformance-harness.test.ts b/tests/lab-conformance-harness.test.ts index b0676748e7..a8208da98d 100644 --- a/tests/lab-conformance-harness.test.ts +++ b/tests/lab-conformance-harness.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "bun:test"; import { evaluateAssertion } from "../src/lab/conformance/assertion"; import { fixtureDigest, scenarioManifestDigest } from "../src/lab/conformance/digest"; -import { jcsEqual } from "../src/lab/conformance/jcs"; +import { jcsEqual, jcsStringify } from "../src/lab/conformance/jcs"; import { resolveJsonPointer } from "../src/lab/conformance/json-pointer"; import { runScenario } from "../src/lab/conformance/executor"; import { @@ -10,9 +10,7 @@ import { loadCaseAuthority, validateExpandedFixtureRef, validateFixtureDigests, - validateScenarioManifestDigest, } from "../src/lab/conformance/manifest"; -import { executeMcpSyntheticAction } from "../src/lab/conformance/mcp-stub"; import { buildNegativeControls, NEGATIVE_CONTROL_FIXTURES } from "../src/lab/conformance/negative-controls"; import { emptyObservation } from "../src/lab/conformance/observation"; import { @@ -24,32 +22,59 @@ import { CL01_SUITES, SYNTHETIC_MARKER } from "../src/lab/conformance/types"; import { normalizeSseBytes } from "../src/lab/conformance/sse-normalize"; describe("CL-01 conformance harness infrastructure", () => { - test("loads case authority and validates fixture digests", () => { + test("loads the frozen 35-case authority and validates fixture digests", () => { const authority = loadCaseAuthority(); - expect(authority.cases.length).toBeGreaterThanOrEqual(24); + expect(authority.cases.length).toBe(35); for (const caseRecord of authority.cases) { expect(validateFixtureDigests(caseRecord)).toEqual([]); - expect(validateScenarioManifestDigest(caseRecord, authority)).toBe(true); } }); - test("discovers CL-01 suite scenarios with stable IDs", () => { + test("discovers all eight CL-01 protocol suites with stable IDs", () => { const authority = loadCaseAuthority(); const scenarios = discoverScenarios(authority, CL01_SUITES); - expect(scenarios.length).toBe(24); + expect(scenarios.length).toBe(35); const ids = scenarios.map((s) => s.id); expect(new Set(ids).size).toBe(ids.length); expect(ids).toContain("responses-core.protocol.request-shape"); - expect(ids).toContain("codex-core.protocol.compaction-and-special-items"); + expect(ids).toContain("vision-core.protocol.input-image"); + expect(ids).toContain("reasoning-core.protocol.effort-mapping"); + expect(ids).toContain("mcp-core.protocol.namespace-mapping"); }); - test("json pointer and JCS equality are deterministic", () => { + test("json pointer and JCS equality are deterministic and fail closed", () => { const observation = emptyObservation(); observation.client.response.status = 200; const resolved = resolveJsonPointer(observation, "/client/response/status"); expect(resolved.ok).toBe(true); - expect(jcsEqual(resolved.value, 200)).toBe(true); + expect(resolved.ok && jcsEqual(resolved.value, 200)).toBe(true); expect(jcsEqual({ a: 1, b: 2 }, { b: 2, a: 1 })).toBe(true); + expect(resolveJsonPointer({}, "/constructor").ok).toBe(false); + expect(() => jcsStringify(undefined)).toThrow(); + expect(() => jcsStringify(Number.NaN)).toThrow(); + }); + + test("identifier operators reject vacuous and null correlations", () => { + const observation = emptyObservation(); + const vacuous = evaluateAssertion({ + id: "stable", + operator: "id_stable_across_events", + selector: "/client", + expected: ["/client/response/status"], + required: true, + }, observation); + expect(vacuous.passed).toBe(false); + expect(vacuous.reason).toBe("invalid_expected"); + + observation.client.response.json = { left: null, right: null }; + const nullCorrelation = evaluateAssertion({ + id: "correlation", + operator: "id_correlates", + selector: "/client/response/json", + expected: ["/client/response/json/left", "/client/response/json/right"], + required: true, + }, observation); + expect(nullCorrelation.passed).toBe(false); }); test("fixture digest matches contract domain separation", () => { @@ -71,7 +96,7 @@ describe("CL-01 conformance harness infrastructure", () => { expect(result.reason).toBe("selector_missing"); }); - test("expanded scenario manifests include synthetic provenance", () => { + test("expanded scenario manifests include normative synthetic provenance", () => { const authority = loadCaseAuthority(); const scenario = discoverScenarios(authority)[0]; const expanded = expandScenario(scenario, authority); @@ -80,8 +105,7 @@ describe("CL-01 conformance harness infrastructure", () => { expect((fixtures[0].provenance as { kind: string }).kind).toBe("lab_authored"); expect((fixtures[0].provenance as { authority: string }).authority).toBe("022_protocol_v1_cases.json"); expect((fixtures[0].provenance as { sourceCommit: string }).sourceCommit).toBe(authority.sourceCommit); - const digest = scenarioManifestDigest(expanded); - expect(digest).toHaveLength(64); + expect(scenarioManifestDigest(expanded)).toHaveLength(64); }); test("rejects forged synthetic provenance metadata", () => { @@ -90,8 +114,8 @@ describe("CL-01 conformance harness infrastructure", () => { const expanded = expandScenario(scenario, authority); const fixtures = expanded.fixtures as Array>; const forged = { ...fixtures[0], syntheticMarker: "forged" }; - const errors = validateExpandedFixtureRef(forged, authority, scenario.fixture.bytesUtf8); - expect(errors.some((e) => e.includes("syntheticMarker"))).toBe(true); + expect(validateExpandedFixtureRef(forged, authority, scenario.fixture.bytesUtf8) + .some((e) => e.includes("syntheticMarker"))).toBe(true); const badCommit = { ...fixtures[0], provenance: { ...(fixtures[0].provenance as object), sourceCommit: "deadbeef" }, @@ -110,18 +134,35 @@ describe("CL-01 SSE normalization", () => { const anthropicEvents = normalizeSseBytes(bytes, "anthropic-messages"); expect(anthropicEvents.some((e) => e.event === "[DONE]")).toBe(false); }); + + test("normalizes BOM/CRLF, comments, multiline data, and malformed JSON", () => { + const bytes = new TextEncoder().encode( + "\uFEFF: keep-alive\r\nevent: response.completed\r\ndata: {\"type\":\"response.completed\",\r\ndata: \"response\":{\"status\":\"completed\"}}\r\n\r\n" + + "data: {not-json}\r\n\r\n", + ); + const events = normalizeSseBytes(bytes, "openai-responses"); + expect(events).toHaveLength(2); + expect(events[0].event).toBe("response.completed"); + expect(events[0].ordinal).toBe(0); + expect(events[1].event).toBe("malformed"); + expect(events[1].ordinal).toBe(1); + }); + + test("drops scalar/null/array data frames as Protocol V1 padding", () => { + const bytes = new TextEncoder().encode("data: null\n\ndata: 1\n\ndata: []\n\n"); + expect(normalizeSseBytes(bytes, "openai-responses")).toEqual([]); + }); }); describe("CL-01 MCP deterministic actions", () => { - test("all four MCP protocol scenarios pass closed action semantics", async () => { + test("all four MCP scenarios pass through the full runner path", async () => { const authority = loadCaseAuthority(); const mcpScenarios = authority.cases.filter((c) => c.suite === "mcp-core"); expect(mcpScenarios.length).toBe(4); for (const scenario of mcpScenarios) { - const observation = executeMcpSyntheticAction(scenario); - for (const assertion of scenario.assertions) { - const result = evaluateAssertion(assertion, observation); - expect(result.passed).toBe(true); + const result = await runScenario(scenario); + if (!result.passed) { + throw new Error(`${scenario.id}: ${result.diagnostics.join(";")} ${result.assertionResults.filter((a) => !a.passed).map((a) => `${a.id}:${a.reason ?? ""}`).join(",")}`); } } }); @@ -134,18 +175,20 @@ describe("CL-01 expanded scenario manifests are stable", () => { const a = expandScenario(scenario, authority); const b = expandScenario(scenario, authority); expect(JSON.stringify(a)).toBe(JSON.stringify(b)); + expect(scenarioManifestDigest(a)).toBe(scenarioManifestDigest(b)); }); }); describe("CL-01 canonical protocol scenarios", () => { - test("all CL-01 suite scenarios pass", async () => { + test("all 35 CL-01 protocol scenarios pass", async () => { const summary = await runConformanceSuite(); const failures = summary.results.filter((r) => !r.passed); if (failures.length > 0) { - const detail = failures.map((f) => `${f.scenarioId}: ${f.classification} ${f.secondaryCode ?? ""} ${f.diagnostics.join(";")} ${f.assertionResults.filter((a) => !a.passed).map((a) => a.id).join(",")}`).join("\n"); + const detail = failures.map((f) => `${f.scenarioId}: ${f.classification} ${f.secondaryCode ?? ""} ${f.diagnostics.join(";")} ${f.assertionResults.filter((a) => !a.passed).map((a) => `${a.id}:${a.reason ?? ""}`).join(",")}`).join("\n"); throw new Error(`scenario failures:\n${detail}`); } - expect(summary.passed).toBe(24); + expect(summary.total).toBe(35); + expect(summary.passed).toBe(35); }, 120000); }); @@ -162,17 +205,20 @@ describe("CL-01 negative controls", () => { } }, 120000); - test("runNegativeControls summary counts rejections", async () => { + test("runNegativeControls summary names rejected controls explicitly", async () => { const summary = await runNegativeControls(); expect(summary.total).toBe(NEGATIVE_CONTROL_FIXTURES.length); - expect(summary.passed).toBe(summary.total); + expect(summary.rejected).toBe(summary.total); + expect(summary.passed).toBe(summary.rejected); }, 120000); }); describe("CL-01 scenario discovery API", () => { test("listScenarioIds returns stable mapping", () => { const ids = listScenarioIds(); - expect(ids.length).toBe(24); - expect(ids.sort()).toEqual([...ids].sort()); + const again = listScenarioIds(); + expect(ids.length).toBe(35); + expect(again).toEqual(ids); + expect(new Set(ids).size).toBe(ids.length); }); }); diff --git a/tests/openai-chat-tool-result-images.test.ts b/tests/openai-chat-tool-result-images.test.ts index b2c389e1fc..d94fa2bb5d 100644 --- a/tests/openai-chat-tool-result-images.test.ts +++ b/tests/openai-chat-tool-result-images.test.ts @@ -3,9 +3,9 @@ import { createOpenAIChatAdapter } from "../src/adapters/openai-chat"; import type { OcxContentPart, OcxMessage, OcxParsedRequest, OcxProviderConfig } from "../src/types"; // Issue #888: role:"tool" content is text-only on chat-completions providers, so images inside a -// tool result were flattened to an "[image]" marker and vision-capable routed models hallucinated -// what they never saw. Tool-result images now ride in a follow-up user vision message released when -// the tool round closes, without splitting the round (strict providers reject interleaved users). +// tool result ride in a follow-up user vision message released when the tool round closes. When +// text is present, the tool row carries only that literal text; image markers are not duplicated +// into a row whose actual images are delivered separately. const provider: OcxProviderConfig = { adapter: "openai-chat", @@ -56,7 +56,6 @@ function toolResult(callId: string, name: string, content: string | OcxContentPa return { role: "toolResult", toolCallId: callId, toolName: name, content, isError: false, timestamp: 0 }; } -/** The carrier is a user message whose parts start with an "[ocx]" text label followed by image_url parts. */ function isImageCarrier(msg: ChatMsg): boolean { if (msg.role !== "user" || !Array.isArray(msg.content)) return false; const [head, ...rest] = msg.content; @@ -64,7 +63,6 @@ function isImageCarrier(msg: ChatMsg): boolean { && rest.length > 0 && rest.every(p => p.type === "image_url"); } -/** Every role:"tool" message must sit in an unbroken block right after its assistant tool_calls message. */ function assertRoundsUnbroken(messages: ChatMsg[]): void { for (let i = 0; i < messages.length; i++) { const m = messages[i]; @@ -85,12 +83,12 @@ test("tool-result images ride a follow-up user message; text, detail, and https { type: "text", text: "1 match found" }, { type: "image", imageUrl: IMAGE_URL, detail: "high" }, { type: "image", imageUrl: "https://example.test/shot.png" }, - { type: "image", imageUrl: "" }, // empty file_id shape: keeps its marker, never reaches the carrier + { type: "image", imageUrl: "" }, ]), ]); assertRoundsUnbroken(messages); const tool = messages.find(m => m.role === "tool")!; - expect(tool.content).toBe("1 match found[image][image][image]"); + expect(tool.content).toBe("1 match found"); const carrier = messages.find(isImageCarrier)!; expect(carrier).toBeDefined(); expect(messages.indexOf(carrier)).toBe(messages.indexOf(tool) + 1); @@ -109,7 +107,7 @@ test("images from a multi-call round flush once, only after the whole round clos ]); assertRoundsUnbroken(messages); const toolIdx = messages.map((m, i) => (m.role === "tool" ? i : -1)).filter(i => i >= 0); - expect(toolIdx).toEqual([toolIdx[0], toolIdx[0] + 1]); // nothing interleaves the round + expect(toolIdx).toEqual([toolIdx[0], toolIdx[0] + 1]); const carriers = messages.filter(isImageCarrier); expect(carriers.length).toBe(1); expect(messages.indexOf(carriers[0])).toBe(toolIdx[1] + 1); From 5639c6ce270af74d3c7ff39fc04a96409589a933 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 02:11:44 +0200 Subject: [PATCH 006/124] fix(lab): close remaining Protocol V1 gaps --- .../022_protocol_v1_cases.json | 2 +- src/lab/conformance/executor.ts | 48 +++++++++++++------ .../fixtures/protocol-v1-cases.json | 2 +- src/lab/conformance/sse-normalize.ts | 3 +- 4 files changed, 37 insertions(+), 18 deletions(-) diff --git a/devlog/_plan/260807_compatibility_lab/022_protocol_v1_cases.json b/devlog/_plan/260807_compatibility_lab/022_protocol_v1_cases.json index 491aee9815..e0c55919cb 100644 --- a/devlog/_plan/260807_compatibility_lab/022_protocol_v1_cases.json +++ b/devlog/_plan/260807_compatibility_lab/022_protocol_v1_cases.json @@ -274,7 +274,7 @@ "assertions": [ { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true }, - { "id": "phase", "operator": "json_path_equals", "selector": "/client/response/events/0/data/phase", "expected": "final_answer", "required": true } + { "id": "phase", "operator": "json_path_equals", "selector": "/client/response/events/6/data/item/phase", "expected": "final_answer", "required": true } ] }, { diff --git a/src/lab/conformance/executor.ts b/src/lab/conformance/executor.ts index 1e3d349d61..b18dab96d9 100644 --- a/src/lab/conformance/executor.ts +++ b/src/lab/conformance/executor.ts @@ -420,7 +420,13 @@ async function runReasoningReplay( observation: NormalizedObservation, vector: Record, ): Promise { - const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(fixtureProviderConfig("openai-responses"))); + const provider: OcxProviderConfig = { + ...fixtureProviderConfig("openai-responses"), + // This Protocol V1 vector exercises a Responses-compatible target that accepts provider + // replay fields verbatim. The adapter must therefore preserve raw reasoning content. + preserveResponsesReasoningContent: true, + }; + const adapter = withHarnessTranslatorBudget(createResponsesPassthroughAdapter(provider)); try { const turn1 = vector.turn1 as { reasoning: { id: string; text: string; signature: string }; @@ -429,20 +435,32 @@ async function runReasoningReplay( const turn2 = vector.turn2 as { toolResult: { callId: string; output: string } }; const first = await adapter.buildRequest(parseRequest({ model: "fixture-model", input: "PING", stream: false }), { headers: new Headers() }); recordUpstreamRequest(observation, JSON.parse(first.body)); - const replayInput = [ - { - type: "reasoning", - id: turn1.reasoning.id, - content: [{ type: "reasoning_text", text: turn1.reasoning.text }], - signature: turn1.reasoning.signature, - }, - { - type: "function_call_output", - call_id: turn2.toolResult.callId, - output: turn2.toolResult.output, - }, - ]; - const second = await adapter.buildRequest(parseRequest({ model: "fixture-model", input: replayInput, stream: false }), { headers: new Headers() }); + + const replayBody = { + model: "fixture-model", + input: [ + { + type: "reasoning", + id: turn1.reasoning.id, + content: [{ type: "reasoning_text", text: turn1.reasoning.text }], + signature: turn1.reasoning.signature, + }, + { + type: "function_call_output", + call_id: turn2.toolResult.callId, + output: turn2.toolResult.output, + }, + ], + stream: false, + }; + // Adapter vectors feed their documented boundary fields directly into the selected adapter. + // Keep a valid parsed shell for typed adapter metadata, while _rawBody carries the exact + // Responses replay shape whose text/signature preservation is under test. + const parsedReplay = { + ...parseRequest({ model: "fixture-model", input: "PING", stream: false }), + _rawBody: replayBody, + }; + const second = await adapter.buildRequest(parsedReplay, { headers: new Headers() }); recordUpstreamRequest(observation, JSON.parse(second.body)); return observation; } finally { diff --git a/src/lab/conformance/fixtures/protocol-v1-cases.json b/src/lab/conformance/fixtures/protocol-v1-cases.json index 491aee9815..e0c55919cb 100644 --- a/src/lab/conformance/fixtures/protocol-v1-cases.json +++ b/src/lab/conformance/fixtures/protocol-v1-cases.json @@ -274,7 +274,7 @@ "assertions": [ { "id": "text", "operator": "normalized_text_equals", "selector": "/client/response/normalizedText", "expected": "OK", "required": true }, { "id": "terminal", "operator": "terminal_signal_equals", "selector": "/client/response/terminal", "expected": "completed", "required": true }, - { "id": "phase", "operator": "json_path_equals", "selector": "/client/response/events/0/data/phase", "expected": "final_answer", "required": true } + { "id": "phase", "operator": "json_path_equals", "selector": "/client/response/events/6/data/item/phase", "expected": "final_answer", "required": true } ] }, { diff --git a/src/lab/conformance/sse-normalize.ts b/src/lab/conformance/sse-normalize.ts index 03fd2eb0b8..0d3a9525e8 100644 --- a/src/lab/conformance/sse-normalize.ts +++ b/src/lab/conformance/sse-normalize.ts @@ -38,7 +38,8 @@ export function normalizeSseBytes(bytes: Uint8Array, sourceProtocol: string): No events.push({ event: eventName ?? "malformed", data: joined, ordinal: ordinal++ }); continue; } - if (parsed === null || typeof parsed !== "object") continue; + // Protocol V1 treats null, scalar, and array data values as padding. + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) continue; const inferred = eventName ?? (typeof (parsed as { type?: unknown }).type === "string" ? (parsed as { type: string }).type : "message"); From b16670fe4ada4f52200e215e3164482a793fb4a1 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:13:42 +0200 Subject: [PATCH 007/124] fix(lab): harden tool-call projections --- src/lab/conformance/observation.ts | 60 ++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/src/lab/conformance/observation.ts b/src/lab/conformance/observation.ts index 287467802f..0f1cca02d4 100644 --- a/src/lab/conformance/observation.ts +++ b/src/lab/conformance/observation.ts @@ -56,14 +56,21 @@ function parseToolArguments(raw: unknown, kind: "function" | "custom"): unknown return raw; } -/** Build toolCalls projection from Responses output items or SSE events (manifest §5). */ -export function projectToolCallsFromOutput(output: unknown[]): ToolCallProjection[] { +interface ToolCallProjectionResult { + calls: ToolCallProjection[]; + sawCallItems: boolean; + duplicateIds: boolean; +} + +function projectToolCallsDetailed(output: unknown[]): ToolCallProjectionResult { const calls: ToolCallProjection[] = []; let ordinal = 0; + let sawCallItems = false; for (const item of output) { if (!item || typeof item !== "object") continue; const rec = item as Record; if (rec.type === "function_call") { + sawCallItems = true; const id = rec.call_id ?? rec.id; const name = rec.name; if (typeof id !== "string" || id.length === 0 || typeof name !== "string" || name.length === 0) continue; @@ -71,6 +78,7 @@ export function projectToolCallsFromOutput(output: unknown[]): ToolCallProjectio if (args === null) continue; calls.push({ id, name, arguments: args, kind: "function", ordinal: ordinal++ }); } else if (rec.type === "custom_tool_call") { + sawCallItems = true; const id = rec.call_id ?? rec.id; const name = rec.name; if (typeof id !== "string" || id.length === 0 || typeof name !== "string" || name.length === 0) continue; @@ -84,10 +92,16 @@ export function projectToolCallsFromOutput(output: unknown[]): ToolCallProjectio } } const ids = calls.map((call) => call.id); - return new Set(ids).size === ids.length ? calls : []; + const duplicateIds = new Set(ids).size !== ids.length; + return { calls: duplicateIds ? [] : calls, sawCallItems, duplicateIds }; } -export function projectToolCallsFromEvents(events: NormalizedEvent[]): ToolCallProjection[] { +/** Build toolCalls projection from Responses output items or SSE events (manifest §5). */ +export function projectToolCallsFromOutput(output: unknown[]): ToolCallProjection[] { + return projectToolCallsDetailed(output).calls; +} + +function projectToolCallsFromEventsDetailed(events: NormalizedEvent[]): ToolCallProjectionResult { const output: unknown[] = []; for (const ev of events) { if (ev.event === "response.output_item.done" && ev.data && typeof ev.data === "object") { @@ -96,7 +110,11 @@ export function projectToolCallsFromEvents(events: NormalizedEvent[]): ToolCallP if (item && typeof item === "object") output.push(item); } } - return projectToolCallsFromOutput(output); + return projectToolCallsDetailed(output); +} + +export function projectToolCallsFromEvents(events: NormalizedEvent[]): ToolCallProjection[] { + return projectToolCallsFromEventsDetailed(events).calls; } export function projectMcpCalls(toolCalls: ToolCallProjection[]): Array<{ namespace: string; name: string }> { @@ -168,14 +186,14 @@ export function finalizeObservation( json: unknown = null, status = 200, ): void { - const eventToolCalls = projectToolCallsFromEvents(events); - const resolvedToolCalls = eventToolCalls.length > 0 - ? eventToolCalls - : projectToolCallsFromOutput( - json && typeof json === "object" && !Array.isArray(json) - ? ((json as { output?: unknown[] }).output ?? []) - : [], - ); + const eventProjection = projectToolCallsFromEventsDetailed(events); + const jsonProjection = projectToolCallsDetailed( + json && typeof json === "object" && !Array.isArray(json) + ? ((json as { output?: unknown[] }).output ?? []) + : [], + ); + const selectedProjection = eventProjection.sawCallItems ? eventProjection : jsonProjection; + const resolvedToolCalls = selectedProjection.calls; const terminal = deriveTerminal(events); setClientResponse(observation, { events, @@ -186,6 +204,7 @@ export function finalizeObservation( json, status, }); + observation.verifiers.duplicate_tool_call_ids = selectedProjection.duplicateIds; } export function attachVerifiers(observation: NormalizedObservation, caseRecord: CaseRecord): void { @@ -232,25 +251,26 @@ function evaluateCallResultOrder(observation: NormalizedObservation): string { const request = observation.upstream.requests[0]?.json as { input?: unknown[] } | undefined; const input = request?.input; if (!Array.isArray(input)) return "fail"; - let pendingCallId: string | undefined; + const pendingCallIds = new Set(); + let callCount = 0; let resultCount = 0; for (const item of input) { if (!item || typeof item !== "object") continue; const record = item as { type?: string; call_id?: unknown }; if (record.type === "function_call") { - if (pendingCallId !== undefined) return "fail"; if (typeof record.call_id !== "string" || record.call_id.length === 0) return "fail"; - pendingCallId = record.call_id; + if (pendingCallIds.has(record.call_id)) return "fail"; + pendingCallIds.add(record.call_id); + callCount++; continue; } if (record.type === "function_call_output") { - if (pendingCallId === undefined) return "fail"; - if (typeof record.call_id !== "string" || record.call_id !== pendingCallId) return "fail"; + if (typeof record.call_id !== "string" || record.call_id.length === 0) return "fail"; + if (!pendingCallIds.delete(record.call_id)) return "fail"; resultCount++; - if (resultCount > 1) return "fail"; } } - return pendingCallId !== undefined && resultCount === 1 ? "pass" : "fail"; + return callCount > 0 && resultCount === callCount && pendingCallIds.size === 0 ? "pass" : "fail"; } function evaluateCompactionReplayed(caseRecord: CaseRecord): boolean { From f5ddee4f08cbe5b2f849467114ec0ca5f832b35d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:15:01 +0200 Subject: [PATCH 008/124] fix(lab): fail closed on malformed controls --- src/lab/conformance/executor.ts | 43 ++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/src/lab/conformance/executor.ts b/src/lab/conformance/executor.ts index b18dab96d9..ec3f32a479 100644 --- a/src/lab/conformance/executor.ts +++ b/src/lab/conformance/executor.ts @@ -529,9 +529,29 @@ async function executeStreamScenario(caseRecord: CaseRecord): Promise ?? responseJson; + finalizeObservation(observation, events, json, responseStatus); + attachVerifiers(observation, caseRecord); + return observation; + } finally { + adapter.dispose(); + } + } + let events: ReturnType; - let json: Record | null = null; - let responseStatus = 200; if (upstreamProtocol === "openai-chat") { const adapter = withHarnessTranslatorBudget(createOpenAIChatAdapter(fixtureProviderConfig("openai-chat"))); @@ -600,21 +620,7 @@ async function executeStreamScenario(caseRecord: CaseRecord): Promise ?? responseJson; - } finally { - adapter.dispose(); - } - } - - finalizeObservation(observation, events, json, responseStatus); + finalizeObservation(observation, events, null, 200); attachVerifiers(observation, caseRecord); return observation; } @@ -651,6 +657,9 @@ export async function runScenario(caseRecord: CaseRecord): Promise assertionResults.find((r) => r.id === id)?.passed === true); const expectedFailureMatched = controlPassed && requiredFailures.length === 0; return { From 78144847a0c89a3186b9d67efbd71fb5fbb804d1 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:15:22 +0200 Subject: [PATCH 009/124] test(lab): cover review regression edges --- tests/cl01-review-regressions.test.ts | 74 +++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 tests/cl01-review-regressions.test.ts diff --git a/tests/cl01-review-regressions.test.ts b/tests/cl01-review-regressions.test.ts new file mode 100644 index 0000000000..1ae401c3de --- /dev/null +++ b/tests/cl01-review-regressions.test.ts @@ -0,0 +1,74 @@ +import { expect, test } from "bun:test"; +import { runScenario } from "../src/lab/conformance/executor"; +import { loadCaseAuthority } from "../src/lab/conformance/manifest"; +import { emptyObservation, finalizeObservation } from "../src/lab/conformance/observation"; +import type { CaseRecord, NormalizedEvent } from "../src/lab/conformance/types"; + +test("duplicate event tool-call ids stay rejected instead of falling back to JSON", () => { + const observation = emptyObservation(); + const events: NormalizedEvent[] = [ + { + event: "response.output_item.done", + ordinal: 0, + data: { item: { type: "function_call", call_id: "dup", name: "a", arguments: "{}" } }, + }, + { + event: "response.output_item.done", + ordinal: 1, + data: { item: { type: "function_call", call_id: "dup", name: "b", arguments: "{}" } }, + }, + ]; + finalizeObservation(observation, events, { + output: [{ type: "function_call", call_id: "fallback", name: "c", arguments: "{}" }], + }); + + expect(observation.client.response.toolCalls).toEqual([]); + expect(observation.verifiers.duplicate_tool_call_ids).toBe(true); +}); + +test("call/result-order verifier accepts multiple correlated pairs", async () => { + const authority = loadCaseAuthority(); + const base = authority.cases.find((c) => c.id === "codex-core.protocol.tool-continuation")!; + const fixture = { + turn1: { + output: [ + { type: "function_call", id: "fc_a", call_id: "call_a", name: "a", arguments: "{}" }, + { type: "function_call", id: "fc_b", call_id: "call_b", name: "b", arguments: "{}" }, + ], + }, + turn2: { + input: [ + { type: "function_call_output", call_id: "call_a", output: "A" }, + { type: "function_call_output", call_id: "call_b", output: "B" }, + ], + }, + }; + const scenario: CaseRecord = { + ...structuredClone(base), + fixture: { ...base.fixture, bytesUtf8: JSON.stringify(fixture) }, + assertions: [{ + id: "order", + operator: "json_path_equals", + selector: "/verifiers/call_result_order", + expected: "pass", + required: true, + }], + }; + + const result = await runScenario(scenario); + expect(result.passed).toBe(true); +}); + +test("expected-failure controls with no assertion ids fail as malformed manifests", async () => { + const authority = loadCaseAuthority(); + const base = authority.cases.find((c) => c.id === "vision-core.protocol.modality-gate")!; + const scenario: CaseRecord = { + ...structuredClone(base), + expectedFailure: { ...base.expectedFailure!, assertionIds: [] }, + }; + + const result = await runScenario(scenario); + expect(result.passed).toBe(false); + expect(result.classification).toBe("harness_failure"); + expect(result.diagnostics.join(" ")).toContain("lists no assertionIds"); +}); From 5aab6d9dfa87f6f7230dcf3b8877c2d2555d6ec0 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:15:47 +0200 Subject: [PATCH 010/124] test(openai-chat): pin native CL-01 regressions --- ...l01-openai-chat-review-regressions.test.ts | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 tests/cl01-openai-chat-review-regressions.test.ts diff --git a/tests/cl01-openai-chat-review-regressions.test.ts b/tests/cl01-openai-chat-review-regressions.test.ts new file mode 100644 index 0000000000..0d755e0dd9 --- /dev/null +++ b/tests/cl01-openai-chat-review-regressions.test.ts @@ -0,0 +1,123 @@ +import { expect, test } from "bun:test"; +import { createOpenAIChatAdapter } from "../src/adapters/openai-chat"; +import type { OcxMessage, OcxParsedRequest, OcxProviderConfig } from "../src/types"; + +const baseProvider: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://api.openai.com/v1", + apiKey: "sk-test", + authMode: "key", +}; + +function bodyFor(provider: OcxProviderConfig, parsed: OcxParsedRequest): Record { + const request = createOpenAIChatAdapter(provider).buildRequest(parsed) as { body: string }; + return JSON.parse(request.body) as Record; +} + +function assistantToolCall(id: string, name: string): OcxMessage { + return { + role: "assistant", + content: [{ type: "toolCall", id, name, arguments: {} }], + timestamp: 0, + }; +} + +test("native OpenAI defers developer guidance until pending tool results are complete", () => { + const parsed: OcxParsedRequest = { + modelId: "gpt-test", + context: { + messages: [ + { role: "user", content: "go", timestamp: 0 }, + assistantToolCall("call_1", "lookup"), + { role: "developer", content: "after the tool", timestamp: 0 }, + { role: "toolResult", toolCallId: "call_1", toolName: "lookup", content: "ok", isError: false, timestamp: 0 }, + ], + }, + stream: false, + options: {}, + }; + + const messages = bodyFor(baseProvider, parsed).messages as Array>; + const assistantIndex = messages.findIndex((m) => m.role === "assistant"); + expect(messages[assistantIndex + 1]).toMatchObject({ role: "tool", tool_call_id: "call_1" }); + expect(messages[assistantIndex + 2]).toEqual({ role: "developer", content: "after the tool" }); +}); + +test("native OpenAI Chat uses reasoning_effort instead of the gateway reasoning object", () => { + const provider: OcxProviderConfig = { + ...baseProvider, + reasoningWireFormat: "gateway-object", + reasoningEffortMap: { high: "high", none: "none" }, + }; + const parsed: OcxParsedRequest = { + modelId: "gpt-test", + context: { messages: [{ role: "user", content: "think", timestamp: 0 }] }, + stream: false, + options: { reasoning: "high" }, + }; + + const body = bodyFor(provider, parsed); + expect(body.reasoning_effort).toBe("high"); + expect(body.reasoning).toBeUndefined(); +}); + +test("native OpenAI Chat represents disabled reasoning with reasoning_effort none", () => { + const provider: OcxProviderConfig = { + ...baseProvider, + reasoningWireFormat: "gateway-object", + reasoningEffortMap: { none: "none" }, + }; + const parsed: OcxParsedRequest = { + modelId: "gpt-test", + context: { messages: [{ role: "user", content: "short", timestamp: 0 }] }, + stream: false, + options: { reasoning: "none" }, + }; + + const body = bodyFor(provider, parsed); + expect(body.reasoning_effort).toBe("none"); + expect(body.reasoning).toBeUndefined(); +}); + +test("non-native gateway targets keep their reasoning object", () => { + const provider: OcxProviderConfig = { + ...baseProvider, + baseUrl: "https://gateway.example.test/v1", + reasoningWireFormat: "gateway-object", + reasoningEffortMap: { high: "adaptive" }, + }; + const parsed: OcxParsedRequest = { + modelId: "fixture-model", + context: { messages: [{ role: "user", content: "think", timestamp: 0 }] }, + stream: false, + options: { reasoning: "high" }, + }; + + const body = bodyFor(provider, parsed); + expect(body.reasoning).toEqual({ enabled: true, effort: "adaptive" }); + expect(body.reasoning_effort).toBeUndefined(); +}); + +test("paired image-only tool results retain their flattened tool-row fallback", () => { + const parsed: OcxParsedRequest = { + modelId: "gpt-test", + context: { + messages: [ + assistantToolCall("call_img", "shot"), + { + role: "toolResult", + toolCallId: "call_img", + toolName: "shot", + content: [{ type: "image", imageUrl: "data:image/png;base64,aGVsbG8=" }], + isError: false, + timestamp: 0, + }, + ], + }, + stream: false, + options: {}, + }; + + const messages = bodyFor({ ...baseProvider, baseUrl: "https://example.test/v1" }, parsed).messages as Array>; + expect(messages.find((m) => m.role === "tool")?.content).toBe("[image]"); +}); From f79fc9eac31915376ce5c703d29b937337f1ea50 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:17:44 +0200 Subject: [PATCH 011/124] fix(openai-chat): use native reasoning effort field --- src/adapters/openai-chat.ts | 45 +++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/src/adapters/openai-chat.ts b/src/adapters/openai-chat.ts index 31a7d1726e..6554ee067b 100644 --- a/src/adapters/openai-chat.ts +++ b/src/adapters/openai-chat.ts @@ -693,24 +693,41 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd if (parsed.options.stopSequences !== undefined) body.stop = parsed.options.stopSequences; const reasoningDisabled = modelInList(provider.noReasoningModels, parsed.modelId); const reasoningEffort = mapReasoningEffort(provider, parsed.modelId, parsed.options.reasoning); + const nativeOpenAI = isNativeOpenAIChatTarget(provider); let reasoningLog: AdapterRequest["reasoningLog"]; if (!reasoningDisabled && provider.reasoningWireFormat === "gateway-object" && parsed.options.reasoning === "none") { - body.reasoning = { enabled: false }; - reasoningLog = { - effectiveEffort: "none", - wireField: "reasoning.enabled", - wireValue: false, - }; - } else if (reasoningEffort !== undefined) { - if (provider.reasoningWireFormat === "gateway-object") { - body.reasoning = isNativeOpenAIChatTarget(provider) - ? { effort: reasoningEffort } - : { enabled: true, effort: reasoningEffort }; + if (nativeOpenAI) { + body.reasoning_effort = "none"; reasoningLog = { - effectiveEffort: reasoningEffort, - wireField: "reasoning.effort", - wireValue: reasoningEffort, + effectiveEffort: "none", + wireField: "reasoning_effort", + wireValue: "none", }; + } else { + body.reasoning = { enabled: false }; + reasoningLog = { + effectiveEffort: "none", + wireField: "reasoning.enabled", + wireValue: false, + }; + } + } else if (reasoningEffort !== undefined) { + if (provider.reasoningWireFormat === "gateway-object") { + if (nativeOpenAI) { + body.reasoning_effort = reasoningEffort; + reasoningLog = { + effectiveEffort: reasoningEffort, + wireField: "reasoning_effort", + wireValue: reasoningEffort, + }; + } else { + body.reasoning = { enabled: true, effort: reasoningEffort }; + reasoningLog = { + effectiveEffort: reasoningEffort, + wireField: "reasoning.effort", + wireValue: reasoningEffort, + }; + } } else if (modelInList(provider.thinkingBudgetModels, parsed.modelId)) { const budget = thinkingBudgetForEffort(parsed, reasoningEffort, maxTokens); if (budget !== undefined) { From 554f6f6d8b4b2007f5c7238b5176275a24fe041e Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 03:18:12 +0200 Subject: [PATCH 012/124] test(claude): pin initial failure framing --- ...claude-outbound-review-regressions.test.ts | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 tests/cl01-claude-outbound-review-regressions.test.ts diff --git a/tests/cl01-claude-outbound-review-regressions.test.ts b/tests/cl01-claude-outbound-review-regressions.test.ts new file mode 100644 index 0000000000..72849097d8 --- /dev/null +++ b/tests/cl01-claude-outbound-review-regressions.test.ts @@ -0,0 +1,49 @@ +import { expect, test } from "bun:test"; +import { responsesSseToAnthropicSse } from "../src/claude/outbound"; +import { createTranslatorBudget } from "../src/lib/translator-budget"; + +async function collect(stream: ReadableStream): Promise { + const reader = stream.getReader(); + const decoder = new TextDecoder(); + let text = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + text += decoder.decode(value, { stream: true }); + } + return text + decoder.decode(); + } finally { + reader.releaseLock(); + } +} + +test("an initial Responses failure becomes one Anthropic error event without a synthetic message start", async () => { + const frame = [ + "event: response.failed", + 'data: {"type":"response.failed","response":{"status":"failed","error":{"type":"server_error","code":"overloaded","message":"fixture"}}}', + "", + "", + ].join("\n"); + const upstream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(frame)); + controller.close(); + }, + }); + const budget = createTranslatorBudget(); + try { + const anthropic = responsesSseToAnthropicSse(upstream, "fixture-model", { + pingIntervalMs: 0, + translatorBudget: budget, + }); + const text = await collect(anthropic); + const eventNames = text + .split("\n") + .filter((line) => line.startsWith("event: ")) + .map((line) => line.slice("event: ".length)); + expect(eventNames).toEqual(["error"]); + } finally { + budget.dispose(); + } +}); From 27aaaa492e6d3037ea38583c5c5624c810285861 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sun, 9 Aug 2026 10:22:24 +0900 Subject: [PATCH 013/124] docs(devlog): plan the vision sidecar model eligibility filter Roadmap unit for replacing the vision sidecar's provider-name model list (every openai + every anthropic row) with a real image-input capability filter, an allowed-model list on the wire, and a compact delegation-style card. The load-bearing finding is that catalog inputModalities is not a truthful capability signal: applyProviderConfigHints deliberately adds "image" to a model listed in provider.noVisionModels, because Codex gates attachments client-side and a text-only entry would block the image before the sidecar could run. A blind model therefore advertises image input, so noVisionModels membership is a hard disqualifier checked before the modality list it rewrote. Two audit rounds with an independent reviewer are recorded in 002; the write gate rejects only models that can be PROVEN blind, never merely unknown ones, so an operator can still point at a model the catalog has never heard of. No production code in this commit. --- .../000_plan.md | 154 ++++++++++ .../001_capability_signal_inventory.md | 103 +++++++ .../002_audit_synthesis.md | 138 +++++++++ .../010_vision_eligibility_core.md | 257 +++++++++++++++++ .../020_management_api_allowed_models.md | 246 ++++++++++++++++ .../030_dashboard_vision_card.md | 265 ++++++++++++++++++ .../040_stack_publication.md | 72 +++++ 7 files changed, 1235 insertions(+) create mode 100644 devlog/_plan/260809_vision_sidecar_model_filter/000_plan.md create mode 100644 devlog/_plan/260809_vision_sidecar_model_filter/001_capability_signal_inventory.md create mode 100644 devlog/_plan/260809_vision_sidecar_model_filter/002_audit_synthesis.md create mode 100644 devlog/_plan/260809_vision_sidecar_model_filter/010_vision_eligibility_core.md create mode 100644 devlog/_plan/260809_vision_sidecar_model_filter/020_management_api_allowed_models.md create mode 100644 devlog/_plan/260809_vision_sidecar_model_filter/030_dashboard_vision_card.md create mode 100644 devlog/_plan/260809_vision_sidecar_model_filter/040_stack_publication.md diff --git a/devlog/_plan/260809_vision_sidecar_model_filter/000_plan.md b/devlog/_plan/260809_vision_sidecar_model_filter/000_plan.md new file mode 100644 index 0000000000..a1557b045b --- /dev/null +++ b/devlog/_plan/260809_vision_sidecar_model_filter/000_plan.md @@ -0,0 +1,154 @@ +# 260809 — vision sidecar model eligibility filter + +Base: `origin/dev@632743269`, worktree `/Users/jun/.codex/worktrees/34b7/opencodex`. +Cycle: docs-first. This unit writes the roadmap; no production code lands in this +work-phase. + +## Objective + +The dashboard's Vision sidecar model picker currently offers **every** model whose +provider is `openai` or `anthropic`, with no regard for whether that model can +actually accept an image. Users have been asking for more models, and the honest +answer has two halves: + +1. The picker is simultaneously **too wide** (it lists models that cannot see) and + **too narrow** (it hard-codes two provider names instead of asking about + capability). +2. The sidecar has exactly two executors — the OpenAI Responses forward path + (`src/vision/describe.ts`) and the Anthropic Messages path + (`src/vision/anthropic-describe.ts`). A model that is not reachable by one of + those two wire protocols cannot be a vision sidecar today no matter what the + picker shows. + +So the deliverable is a real **vision-capability filter** on both sides, plus a +guaranteed baseline entry per side, plus a dashboard card that shows the allowed +list in the compact delegation-panel form factor. + +## Constraints + +- The proxy is Bun-native TypeScript; `bun run typecheck` and `bun run test` gate + every layer. +- `src/vision/reasoning.ts` already owns effort normalization. The filter must not + duplicate or contradict `normalizeVisionReasoningForModel`. +- The GUI must not be the only gate: `PUT /api/sidecar-settings` has to reject an + ineligible vision model itself (`PLAN-BYPASS-NAMED-01` below). +- No change to routing, provider registry semantics, or the web-search sidecar's + behavior. Shared plumbing may be extracted, but web-search's option list keeps + its current contents in this unit. + +## The trap that shapes the whole design + +`applyProviderConfigHints` (`src/codex/catalog/provider-fetch.ts:574-582`) **adds** +`"image"` to a model's `inputModalities` when the model is listed in +`provider.noVisionModels`. That is deliberate: `noVisionModels` marks models the +PROXY describes images for, and the Codex app gates attachments client-side on +`input_modalities`, so a text-only entry would block the image before the sidecar +could ever run. + +The consequence for this unit is load-bearing: **catalog `inputModalities` is not a +truthful vision-capability signal.** A model that is in `noVisionModels` advertises +`["text","image"]` precisely *because* it is blind. Filtering on `inputModalities` +alone would let a blind model be chosen as the describer for other blind models. + +Therefore eligibility is a conjunction: + +``` +eligible(model) = advertisesImageInput(model) AND NOT isSidecarConsumer(model) +``` + +where `isSidecarConsumer` is `modelInList(provider.noVisionModels, id)` — the same +predicate `planVisionSidecar` uses to decide a model needs describing. + +## Work-phase map (dependency-ordered, PHASE-SPLIT-01) + +| # | Doc | Phase | Consumes | +|---|-----|-------|----------| +| 0 | this unit | roadmap | — | +| 1 | `010_vision_eligibility_core.md` | eligibility predicate + baselines in `src/vision/eligibility.ts` | catalog metadata accessors | +| 2 | `020_management_api_allowed_models.md` | `/api/sidecar-settings` exposes the allowed list and rejects ineligible writes | phase 1's predicate | +| 3 | `030_dashboard_vision_card.md` | dashboard card restyle + server-provided options | phase 2's payload | +| 4 | `040_stack_publication.md` | branch cascade, push, stacked PRs | phases 1-3 | + +Each phase closes with something independently verifiable: phase 1 with unit tests +over the predicate, phase 2 with route tests over both the 200 and the 400 path, +phase 3 with `lint:gui` + `build:gui` + a read-back screenshot, phase 4 with +`gh pr view` base refs. + +## Verifiers (PLAN-VERIFIER-REAL-01) + +Run before this plan was written, from the worktree root: + +| Command | Exit | Reads this unit's target? | +|---|---|---| +| `bun run typecheck` | 0 | yes — `tsconfig.json` compiles `src/**` and `gui/src/**`, which is where every phase writes | +| `bun run test` | 0 | yes — `tests/*.test.ts` is a flat glob over the whole directory, so a new `tests/vision-eligibility.test.ts` is picked up without configuration | +| `bun run lint:gui` | 0 | yes for phase 3 only — `gui/eslint.config.js` lints `gui/src/**`; it does **not** observe `src/**`, so it is not a gate for phases 1-2 | +| `bun run build:gui` | 0 | yes for phase 3 — Vite builds `gui/src` into `gui/dist` | +| `bun run privacy:scan` | 0 | partially — it scans the repo including `devlog/`, so it observes these documents, but it asserts nothing about the filter's behavior | + +`bun run lint:gui` does **not** observe `src/vision/*`; phase 1 and 2 acceptance +rows are covered by `typecheck` + `test`, not by lint. + +## Field chain (PLAN-FIELD-CHAIN-01) + +The unit adds one field to a wire payload — `visionModels` on the +`/api/sidecar-settings` GET response — and one derived value, the eligibility +boolean. Its chain: + +| Stage | Location | Note | +|---|---|---| +| creation | `visionEligibleModelOptions()` in `src/vision/eligibility.ts` (NEW) | derives from `listManagementModelRows` output plus config | +| serialization | `src/server/management/config-routes.ts` GET **and PUT** `/api/sidecar-settings` | both response bodies, so an optimistic update and a refetch cannot disagree | +| deserialization | `SidecarData` in `gui/src/pages/dashboard-shared.ts` | new optional `visionModels?: SidecarModelOption[]` — optional so a stale GUI against a new server, or a new GUI against a cached response, degrades to the old client-side list rather than rendering an empty picker | +| consumers | `use-dashboard-data.ts`: NEW `visionModels` memo + three field-by-field writes in `saveSidecar` (optimistic `next`, success `setSidecar`, session cache) and the hook's return object; `dashboard-overview-sections.tsx` (the vision `Select`) | the existing `sidecarModels` memo stays as-is and keeps serving web-search — `N/A` for it by design. The poll effect assigns `data.sidecar` wholesale, so it needs no edit; `CachedControls.sidecar` is typed `SidecarData`, so the optional field flows without a type change | +| validation | `PUT /api/sidecar-settings` in the same file, and the `visionSidecar` branch of `PUT /api/claude-code` in `agent-settings-routes.ts` | rejects a **provably blind** `vision.model` with 400; an unknown id is allowed | + +No enum gains a value in this unit, so the enum-consumer sweep is `N/A`. + +## Bypass (PLAN-BYPASS-NAMED-01) + +| Field | Value | +|---|---| +| tier | E4 — server-side request validation | +| executing surface | `PUT /api/sidecar-settings` in `src/server/management/config-routes.ts`, plus the Claude Code vision override in `src/server/management/agent-settings-routes.ts` | +| known bypass | editing `~/.opencodex/config.json` by hand and restarting; the config loader does not re-validate `visionSidecar.model` | +| residual risk | a hand-edited blind model stays configured and the sidecar produces useless descriptions; it fails at request time, not at write time | +| wording downgrade | yes, and deliberately. The gate rejects only models **positively known** to be unable to see. It is enforcement against a *proven-blind* selection and no barrier at all against an *unknown* one | +| final layer | none. `planVisionSidecar` stays permissive by design, so an operator can still point at a model the catalog has never heard of | + +**The gate and the picker are not the same set** (audit round 1, blocker 1). The +picker suggests; the gate forbids. Deriving one from the other would reject every +unknown id — including `custom-vision`, which +`tests/vision-reasoning-contract.test.ts:148-151` asserts must still save with +`providers: {}`. The rule is therefore: + +``` +picker option ⇐ eligible AND reachable by an executor AND known to some source +PUT rejection ⇐ modelAcceptsImageInput(...) === false (never on undefined) +``` + +## Scope boundary: the Claude Code override (audit round 1, blocker 5) + +Claude Code carries its own vision sidecar override +(`gui/src/pages/claude-code-sections.tsx:154-205`, persisted through +`PUT /api/claude-code`). Two halves, decided separately: + +- **Server: in scope.** The eligibility gate covers every route that sets a vision + describer, so `agent-settings-routes.ts` gets the same rejection. A second + unguarded write path would make the first gate decorative. +- **GUI: out of scope.** That surface is a freeform `` with a `` + of suggestions, not a constrained picker. Narrowing a deliberately freeform + field is a different product decision and is not smuggled into this unit. + +Requirement 2 ("both sides show only allowed models") is therefore read as: both +*backend families* (OpenAI and Anthropic) inside the dashboard vision picker — +which is the control the request was anchored on. + +## Out of scope + +- A third sidecar executor (e.g. a Gemini or xAI vision path). Adding a provider + family here would change routing surface, not just the picker, and belongs to its + own unit. +- Changing which models `noVisionModels` contains. +- Web-search sidecar option filtering. +- Merging the resulting PR stack. Publication is authorized; merging is not. diff --git a/devlog/_plan/260809_vision_sidecar_model_filter/001_capability_signal_inventory.md b/devlog/_plan/260809_vision_sidecar_model_filter/001_capability_signal_inventory.md new file mode 100644 index 0000000000..454e9f9d9e --- /dev/null +++ b/devlog/_plan/260809_vision_sidecar_model_filter/001_capability_signal_inventory.md @@ -0,0 +1,103 @@ +# 001 — where a model's image capability actually comes from + +Research doc. No diffs here; the diffs live in the decade docs. + +## The four sources, in the order the runtime consults them + +1. **Native pinned metadata** — `src/codex/catalog/metadata.ts:117` + `nativeInputModalities(slug)` reads `src/codex/data/upstream-models.json`. + Verified contents for the seven supported native slugs: + + ``` + gpt-5.6-sol ["text","image"] + gpt-5.6-terra ["text","image"] + gpt-5.6-luna ["text","image"] + gpt-5.5 ["text","image"] + gpt-5.4 ["text","image"] + gpt-5.4-mini ["text","image"] + gpt-5.3-codex-spark (absent from snapshot → falls back to ["text","image"]) + ``` + + So **every** native OpenAI slug is image-capable. The native side of the picker + is not where over-listing happens. + +2. **Generated vendor metadata** — `src/generated/model-metadata.ts`, `DATA` rows + whose 4th column is a comma-joined modality string. `getModelMetadata(provider, + id)` and `getModelMetadataCaseInsensitive` return `input?: ("text"|"image"|"video")[]`. + Every `anthropic` row in that table carries `text,image`, including + `claude-haiku-4-5` and `claude-haiku-4-5-20251001`. + +3. **Live catalog rows** — `CatalogModel.inputModalities`, populated by + `catalogHintsFromModelsApiItem` (`provider-fetch.ts:938`) from the provider's + own `/models` payload, then post-processed by `applyProviderConfigHints`. + +4. **Operator config** — `provider.modelInputModalities[id]`, which + `configuredInputModalities` treats as the base before the `noVisionModels` + augmentation. + +## Live evidence from this machine + +`GET /api/models` (admin token, port 10100) returned 11 providers. The two the +picker currently sources from: + +``` +openai 7 rows, inputModalities: absent on every row +anthropic 11 rows, inputModalities: absent on every row +``` + +That absence is the second load-bearing fact. `listManagementModelRows` +(`src/server/management/model-rows.ts:45-54`) builds native rows by hand and never +attaches `inputModalities`; anthropic rows come through `dedupedRouted` from the +catalog, where the anthropic provider's `/models` response also omits it. + +**A naive `row.inputModalities?.includes("image")` filter would therefore empty the +picker completely.** Unknown must not be read as zero. The predicate has to fall +back to pinned/generated metadata before concluding a model cannot see, and when +all four sources are silent it must stay permissive. + +## The inverted signal + +`applyProviderConfigHints` (`provider-fetch.ts:574-582`) appends `"image"` to a +model listed in `provider.noVisionModels`. `tests/catalog-vision-sidecar-modalities.test.ts` +asserts exactly this: `glm-5.2`, a text-only model, comes out as +`["text","image"]`. + +`noVisionModels` is also the trigger for the sidecar itself — +`planVisionSidecar` (`src/vision/index.ts:237`) returns undefined unless +`modelInList(provider.noVisionModels, modelId)`. One list, two opposite meanings +depending on which side of the sidecar you stand on. + +Consequence for the predicate: membership in `noVisionModels` is a **hard +disqualifier** for being a describer, and it must be checked before the modality +list, because the modality list was rewritten by that very membership. + +## Which models a sidecar can actually reach + +`planVisionSidecar` has exactly two branches: + +- `backend === "anthropic"` → `describeImageAnthropic`, which requires an enabled + `adapter: "anthropic"`, `authMode: "oauth"` provider with a non-reauth active + account (`findAnthropicVisionProvider`, `src/vision/index.ts:171`). +- `backend === "openai"` → `describeImage` against + `${forwardProvider.baseUrl}/responses`, requiring a resolved OpenAI forward + sidecar (ChatGPT login). + +There is no third executor. A `xai/grok-4.5` row, image-capable though it is, +has no code path that would describe an image today. Eligibility must therefore be +scoped to the two backend families, not opened to every image-capable row in the +catalog — otherwise the picker would offer selections that silently produce no +plan at all. + +That is the honest answer to "can we allow other models?": **within the two +supported wire protocols, yes — and the current filter is the wrong shape. +Outside them, not without a new executor**, which this unit scopes out. + +## Baseline requirement + +The user requires `gpt-5.6-luna` to always appear when the GPT side is enabled and +a haiku model to always appear when Claude is registered. Both are image-capable by +the tables above, so the baselines are not exceptions to the capability rule — they +are a **presence** guarantee against an empty or unfetched catalog, which the live +evidence above shows is a real state (`/api/models` can be cold, and +`fetchAllModels` can return nothing while a provider is cooling down after a fetch +failure). diff --git a/devlog/_plan/260809_vision_sidecar_model_filter/002_audit_synthesis.md b/devlog/_plan/260809_vision_sidecar_model_filter/002_audit_synthesis.md new file mode 100644 index 0000000000..3103e9ae33 --- /dev/null +++ b/devlog/_plan/260809_vision_sidecar_model_filter/002_audit_synthesis.md @@ -0,0 +1,138 @@ +# 002 — A-phase audit synthesis (round 1) + +Reviewer: independent subagent on `xai/grok-4.5`, high effort, read-only. +Verdict: `GO-WITH-FIXES (blockers=5)`. + +Every blocker below was **re-verified by the main agent against the tree** before +being accepted; none was taken on the reviewer's word. + +## Root-cause synthesis + +Blockers 1 and 2 are one defect, not two. I wrote the write-gate as *"reject what +is not in the picker"* while writing the bypass table as *"an operator may point +at a model the catalog does not know about"*. Those cannot both hold: an unknown +id is absent from the option list by construction, so the gate would reject +exactly the case the bypass table promises to allow. + +The correct rule follows from the tri-state the predicate already returns: + +``` +option list = eligible AND reachable AND known (a suggestion — may be narrow) +write gate = reject only when modelAcceptsImageInput(...) === false (a proof of harm) +``` + +`undefined` (nothing knows) belongs on the permissive side of the gate and the +conservative side of the list. Conflating "not suggested" with "forbidden" is the +error; the fix is to stop deriving the gate from the list. + +Blockers 3, 4, 6, 8 are all under-specification of a real call site — I named a +behavior and left the implementer to find the writes. Blocker 5 is a scope claim +I made too broadly. + +## Accept / rebut + +| # | Sev | Finding | Decision | Verification I ran | +|---|-----|---------|----------|--------------------| +| 1 | Critical | PUT 400 would reject unknown ids and break an existing contract test | **ACCEPT** | `tests/vision-reasoning-contract.test.ts:148-151` really does assert `putVision(custom, { model: "custom-vision" })` → 200 with `providers: {}`. My gate would have turned that red. | +| 2 | High | GET grandfathers the configured model, PUT does not | **ACCEPT** | Same root cause as 1; fixed by the same rule change. | +| 3 | High | `saveSidecar` drops `visionModels` on the success path | **ACCEPT** | `use-dashboard-data.ts:480-485`: `setSidecar({ webSearch: data.webSearch, vision: data.vision })` and the cache write both enumerate fields explicitly, so a new field is silently dropped. Three write sites, not one. | +| 4 | High | `config.providers.openai && !disabled` is not this repo's "OpenAI side enabled" | **ACCEPT** | `listOpenAiForwardSidecarCandidates` (`src/providers/openai-sidecar.ts:55-70`) additionally requires `isCanonicalOpenAiForwardProvider` (`openai-tiers.ts:32-36`): `openai-responses` + `forward` + canonical base URL. My predicate was strictly broader and asymmetric with the Anthropic side. | +| 5 | High | Requirement 2 ignores the Claude Code vision override | **ACCEPT IN PART** | `gui/src/pages/claude-code-sections.tsx:154-205` is a freeform `` with a ``, not a picker, and `agent-settings-routes.ts:1002-1014` only type-checks `model`. I accept the **server** half (the gate must cover every route that sets a vision describer) and rebut the **GUI** half (a freeform text input is deliberately freeform; narrowing its suggestions is a different product decision). Recorded as an explicit scope statement, not silence. | +| 6 | Medium | Card does not really adopt the delegation form factor | **ACCEPT** | `dash-delegation-summary` is applied to the *panel* at line 102; my draft put it on an inner row where `.dash-sidecar-card__row` already supplies the same flex rules. Redundant, and it would not match. | +| 7 | Medium | Wrong seam named for route tests | **ACCEPT** | The real seam is `handleManagementAPI` + `tests/helpers/management-auth`, demonstrated by `tests/vision-reasoning-contract.test.ts:12-31`. `model-routes.ts` documents a `deps.saveConfigPreservingClaudeCode` injection that the sidecar route does not use — it calls the bare import at line 469. | +| 8 | Medium | Compact CSS rule proposed in the wrong file | **ACCEPT** | The `min-width: clamp(10rem, 24vw, 11.5rem)` it must override lives in `styles-dashboard-workspace.css:104-110`, not `styles.css`. | +| 9 | Low | Stale line citations | **ACCEPT** | Re-pinned below. | +| 10 | Low | docs-site / CLI consumers omitted | **ACCEPT** | `src/cli/agent.ts` writes through the same PUT, so it inherits the corrected gate for free; `docs-site/src/content/docs/guides/sidecars.md` needs a sentence (SOT-SYNC-01, phase 2's C). | + +Reviewer claims I **rebut**: none outright. The one partial rebuttal (5) is scoped, +not dismissed. + +Reviewer claims I independently confirmed as *correct in my favor*: the +`:last-child` selector is robust (`Select` renders sibling `.custom-select` roots), +and the delegation panel itself needs no change because it already maps efforts to +raw values (`dashboard-overview-sections.tsx:119-122`). So comment 2's "여기처럼" +is a pattern reference, not a request to edit that panel. + +## Amendments applied + +1. `010` — `visionEligibleModelOptions` unchanged, but the doc now states the + list/gate asymmetry explicitly and adds a test asserting the tri-state. +2. `020` — the 400 guard is rewritten to fire only on + `modelAcceptsImageInput(...) === false`; `enabledVisionBackends` now uses + `listOpenAiForwardSidecarCandidates`; the same guard is applied to the Claude + Code vision override route; the seam is named correctly; a docs-site sync line + is added; two tests are added (unknown id keeps 200, Claude Code route rejects). +3. `030` — the card becomes `panel dash-delegation-summary`; all three + `visionModels` write sites are enumerated; the CSS rule moves to + `styles-dashboard-workspace.css`. +4. `000` — bypass table restated so the enforcement claim matches the code, and + the Claude Code scope boundary is written down. + +## Re-pinned anchors + +| Symbol | Real location | +|---|---| +| `planVisionSidecar` | `src/vision/index.ts:231` (its `modelInList` guard at 238) | +| `findAnthropicVisionProvider` | `src/vision/index.ts:172` | +| vision card markup | `gui/src/pages/dashboard-overview-sections.tsx:288-311` | +| delegation panel | `gui/src/pages/dashboard-overview-sections.tsx:102-104` | +| delegation raw effort labels | `gui/src/pages/dashboard-overview-sections.tsx:119-122` | +| sidecar select min-width | `gui/src/styles-dashboard-workspace.css:104-110` | +| `saveConfigPreservingClaudeCode` call in PUT | `src/server/management/config-routes.ts:469` | + +Confirmed accurate as originally written: `provider-fetch.ts:574-582`, +`config-routes.ts:380-393`, reasoning enum check `422-423`, `model-rows.ts:45-54`, +`use-dashboard-data.ts:454-462`. + +# Round 2 + +Same reviewer, re-audit of the amended documents. Verdict: +`GO-WITH-FIXES (blockers=1)`. Eight of nine prior findings confirmed **CLOSED**, +including the reviewer walking the `custom-vision` input through the amended +predicate step by step and reaching `undefined` (so the contract test stays +green), and confirming `.dash-delegation-controls .custom-select:last-child` +touches no other surface (`dash-delegation-controls` has exactly two users, and +the delegation panel's last child is a `

)} {(state === "absent" || state === "disabled") && ( @@ -59,7 +63,10 @@ export function OpenAiAccountModeBanner({ )} {state === "invalid" && (

- {t("codexAuth.openaiMissing")} {t("codexAuth.openProviders")} + {t("codexAuth.openaiMissing")}{" "} +

)} diff --git a/gui/src/pages/Startup.tsx b/gui/src/pages/Startup.tsx index 95b18d3848..8bb03166c0 100644 --- a/gui/src/pages/Startup.tsx +++ b/gui/src/pages/Startup.tsx @@ -1,6 +1,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { IconRefresh } from "../icons"; import { type TFn, useI18n } from "../i18n/shared"; +import { navigateHash } from "../hash-routing"; import { readSessionListCache, writeSessionListCache } from "../session-list-cache"; import { Notice } from "../ui"; import { useDataSurface } from "../data-surface"; @@ -288,7 +289,9 @@ export default function Startup({ apiBase }: { apiBase: string }) {

{t("startup.subtitle")}

- {t("startup.backToDashboard")} + diff --git a/gui/src/pages/dashboard-overview-head.tsx b/gui/src/pages/dashboard-overview-head.tsx index 31e85eb1a1..05a8530474 100644 --- a/gui/src/pages/dashboard-overview-head.tsx +++ b/gui/src/pages/dashboard-overview-head.tsx @@ -2,6 +2,7 @@ import { IconAlert, IconInfo } from "../icons"; import { type TKey, useT } from "../i18n/shared"; import { formatTokens } from "../format-tokens"; import { formatUptime } from "../formatUptime"; +import { navigateHash } from "../hash-routing"; import type { useDashboardData } from "./use-dashboard-data"; type Dash = ReturnType; @@ -85,7 +86,7 @@ export function DashboardOverviewHead({
{startupHealth ? ( - + ) : ( ); diff --git a/gui/src/styles.css b/gui/src/styles.css index b222f914f4..09b012b970 100644 --- a/gui/src/styles.css +++ b/gui/src/styles.css @@ -804,6 +804,8 @@ a.btn, a.btn:hover { text-decoration: none; } .stat .value.mono { font-family: var(--font-code); font-size: var(--text-subtitle); } .startup-health-bar { + box-sizing: border-box; + width: 100%; min-height: var(--control-lg); margin: calc(-1 * var(--space-3)) 0 var(--space-6); padding: 0 var(--space-3); @@ -813,10 +815,14 @@ a.btn, a.btn:hover { text-decoration: none; } min-width: 0; color: var(--muted); background: var(--hover); + border: none; border-block: 1px solid var(--border-soft); text-decoration: none; + font: inherit; font-size: var(--text-control); line-height: var(--leading-ui); + text-align: start; + cursor: pointer; transition: background var(--motion-fast), color var(--motion-fast); } .startup-health-bar:hover { background: var(--raised); color: var(--text); } diff --git a/tests/ci-workflows.test.ts b/tests/ci-workflows.test.ts index fa6ca5cc76..3fc90d89c7 100644 --- a/tests/ci-workflows.test.ts +++ b/tests/ci-workflows.test.ts @@ -4629,13 +4629,13 @@ describe("GitHub Actions hardening", () => { const workflow = await readText(".github/workflows/react-doctor.yml"); expect(workflow).toContain("actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8"); - expect(workflow).toContain("millionco/react-doctor@938008119a288f2fb47c66a69cd9279a21f31784"); + expect(workflow).toContain("millionco/react-doctor@01820bb4fd4d0a4aebcd8df2b2a143a098649cb2"); expect(workflow).not.toMatch( /^\s*-\s+uses:\s+\S+@(?![0-9a-f]{40}(?=[ \t]*(?:#.*)?$))\S+/m, ); // Engine pin: the action wrapper would fetch react-doctor@latest without it. - expect(workflow).toContain('version: "0.9.3"'); + expect(workflow).toContain('version: "0.9.11"'); // Action pin must accept CLI JSON schemaVersion 3 (baseline reports from 0.9.x). // v2.1.0's ensure-json-report only knew schemas 1–2 and failed every PR scan. @@ -4657,7 +4657,7 @@ describe("GitHub Actions hardening", () => { const rootPkg = await readText("package.json"); const doctorConfig = await readText("gui/doctor.config.json"); - expect(guiPkg).toContain("react-doctor@0.9.3"); + expect(guiPkg).toContain("react-doctor@0.9.11"); expect(guiPkg).not.toContain("react-doctor@latest"); expect(rootPkg).not.toContain("react-doctor@latest"); expect(doctorConfig).toContain('"blocking": "warning"'); From afc6697ea9eb615188594bc99c6643cf4a0d007e Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:22:12 +0200 Subject: [PATCH 077/124] fix(lab): lstat symlink squatters before artifact create macOS dirfd opens can report ENOENT for symlink digest paths; probe with lstatSync before create so putArtifactBytes fails closed instead of replacing the link. --- src/lab/artifacts/secure-fs.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/lab/artifacts/secure-fs.ts b/src/lab/artifacts/secure-fs.ts index 831c112561..23d05c5a45 100644 --- a/src/lab/artifacts/secure-fs.ts +++ b/src/lab/artifacts/secure-fs.ts @@ -271,17 +271,19 @@ function isMissingArtifactError(err: unknown): boolean { } function assertArtifactTargetCreatable(dir: TrustedArtifactDir, name: string): void { + revalidateDir(dir); + assertRelativeName(name); try { - const fd = openAtDir(dir, name, openFlags(O_RDONLY, true)); - try { - const stats = fstatSync(fd); - assertRegularFileStats(stats, "artifact create target"); - harnessFailure("artifact target exists but is not reusable"); - } finally { - closeSync(fd); + const stats = lstatSync(childPath(dir, name)); + if (stats.isSymbolicLink()) { + harnessFailure("artifact target is a symbolic link"); } + assertRegularFileStats(stats, "artifact create target"); + harnessFailure("artifact target exists but is not reusable"); } catch (err) { - if (isMissingArtifactError(err)) return; + if (err && typeof err === "object" && "code" in err && (err as { code: string }).code === "ENOENT") { + return; + } if (err instanceof ArtifactFsError) throw err; harnessFailure(`artifact create target check failed: ${err instanceof Error ? err.message : String(err)}`); } From 46f8f86723bdfe94d372e7fb802863acec4dbcac Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:43:49 +0200 Subject: [PATCH 078/124] test(gui): update assertions for navigateHash provider links Hash anchors became link-btn controls for react-doctor; point tests at the new markup and select Startup Refresh by label. --- gui/tests/codex-auth-provider-enable.test.tsx | 3 ++- gui/tests/models-empty-provider.test.tsx | 8 ++++---- gui/tests/startup-usage-loading-race.test.tsx | 8 +++++--- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/gui/tests/codex-auth-provider-enable.test.tsx b/gui/tests/codex-auth-provider-enable.test.tsx index bf482c0b6a..216b21edf6 100644 --- a/gui/tests/codex-auth-provider-enable.test.tsx +++ b/gui/tests/codex-auth-provider-enable.test.tsx @@ -59,7 +59,8 @@ test("noncanonical disabled OpenAI rows do not offer built-in recovery", () => { ); expect(html).toContain("The built-in OpenAI provider is not configured."); - expect(html).toContain('href="#providers"'); + expect(html).toContain('class="link-btn"'); + expect(html).toContain("Open Providers"); expect(html).not.toContain("Enable OpenAI"); expect(html).not.toContain("Your OpenAI accounts are still available"); }); diff --git a/gui/tests/models-empty-provider.test.tsx b/gui/tests/models-empty-provider.test.tsx index bee7f6ef7b..442e96bf17 100644 --- a/gui/tests/models-empty-provider.test.tsx +++ b/gui/tests/models-empty-provider.test.tsx @@ -534,7 +534,7 @@ async function withCursorDiscoveryServer( test("empty live-discovery provider renders endpoint guidance and a settings link", () => { const html = renderHint(true, { status: "ok" }); expect(html).toContain("No models were discovered"); - expect(html).toContain('href="#providers"'); + expect(html).toContain('class="link-btn"'); expect(html).toContain("Open provider settings"); expect(html).not.toContain("Discovery failed"); }); @@ -545,7 +545,7 @@ test("failed HTTP discovery renders an amber status badge and reason", () => { expect(html).toContain("HTTP 401"); expect(html).toContain('class="badge badge-amber"'); expect(html).toContain('role="status"'); - expect(html).toContain('href="#providers"'); + expect(html).toContain('class="link-btn"'); }); test("failed discovery renders each server-owned reason without provider detail", () => { @@ -584,7 +584,7 @@ test("HTTP 401 discovery exposes HTTP status and badge", async () => { const html = renderHint(true, discovery); expect(html).toContain("Discovery failed"); expect(html).toContain("HTTP 401"); - expect(html).toContain('href="#providers"'); + expect(html).toContain('class="link-btn"'); }); test("destination-blocked discovery exposes blocked status and badge", async () => { @@ -614,7 +614,7 @@ test("destination-blocked discovery exposes blocked status and badge", async () const html = renderHint(true, discovery); expect(html).toContain("Discovery failed"); expect(html).toContain("blocked by the destination policy"); - expect(html).toContain('href="#providers"'); + expect(html).toContain('class="link-btn"'); }); test("invalid JSON or malformed model data exposes invalid-response status and badge", async () => { diff --git a/gui/tests/startup-usage-loading-race.test.tsx b/gui/tests/startup-usage-loading-race.test.tsx index f465af1280..bf90d2487a 100644 --- a/gui/tests/startup-usage-loading-race.test.tsx +++ b/gui/tests/startup-usage-loading-race.test.tsx @@ -133,15 +133,17 @@ test("an aborted Startup fetch must not clear loading while its replacement is i await settle(); expect(container.textContent).toContain("Checking startup protection"); - const refresh = container.querySelector("button.btn"); - expect(refresh?.disabled).toBe(true); + const refresh = Array.from(container.querySelectorAll("button.btn")) + .find(button => (button.textContent ?? "").includes("Refresh")); + expect(refresh).toBeTruthy(); + expect(refresh!.disabled).toBe(true); await act(async () => { gates[1]!.resolve(FRESH); await Promise.resolve(); }); await waitFor(() => !(container.textContent ?? "").includes("Checking startup protection")); - expect(refresh?.disabled).toBe(false); + expect(refresh!.disabled).toBe(false); await act(async () => { root.unmount(); }); container.remove(); From 6842a068dec3bec03441e3de54724843a10dba8f Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sun, 9 Aug 2026 15:52:16 +0900 Subject: [PATCH 079/124] docs(devlog): record the layer-3 merge commit in the vision filter outcome The row was written before the merge existed, so the closed unit pointed at a placeholder instead of e96a81bed. --- devlog/_fin/260809_vision_sidecar_model_filter/060_outcome.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devlog/_fin/260809_vision_sidecar_model_filter/060_outcome.md b/devlog/_fin/260809_vision_sidecar_model_filter/060_outcome.md index 14ab400450..e323a3ed90 100644 --- a/devlog/_fin/260809_vision_sidecar_model_filter/060_outcome.md +++ b/devlog/_fin/260809_vision_sidecar_model_filter/060_outcome.md @@ -6,7 +6,7 @@ Shipped. Three layers on `dev`, bottom-up, one PABCD cycle each. |---|---|---|---| | 1 — eligibility predicate | #1326 | `eebd9d48f` | `src/vision/eligibility.ts`, the devlog unit | | 2 — management API + write gate | #1327 | `d4758bc94` | options module, both routes, shared model resolver | -| 3 — dashboard card | #1328 | (this unit's close) | filtered picker, backend provenance, card shell | +| 3 — dashboard card | #1328 | `e96a81bed` | filtered picker, backend provenance, card shell | ## What the user asked for, and what answers it From ef8b242a0fef6d1223fc7eea92e4881730abd4bb Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:53:26 +0200 Subject: [PATCH 080/124] test(catalog): retry vacuous post-approval seam races on lock loss When both children lose the config lock before approval, the suite proved nothing about catalog serialization and failed macOS CI. Retry those vacuous runs until a process reaches the seam. --- .../codex-retained-root-serialization.test.ts | 67 +++++++++++++------ 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/tests/codex-retained-root-serialization.test.ts b/tests/codex-retained-root-serialization.test.ts index b7bebef5d8..a934ce3651 100644 --- a/tests/codex-retained-root-serialization.test.ts +++ b/tests/codex-retained-root-serialization.test.ts @@ -479,21 +479,47 @@ test("two processes at the post-approval management seam serialize instead of in console.log(JSON.stringify({ status: response.status, catalogRefresh: body.catalogRefresh })); `; - const children = (["a", "b"] as const).map(marker => Bun.spawn( - [process.execPath, "--eval", routeScript(marker)], - { cwd: repoRoot, env: sandbox.env, stdout: "pipe", stderr: "pipe" }, - )); - - const results = await Promise.all(children.map(async child => { - const [exitCode, stdout, stderr] = await Promise.all([ - child.exited, - new Response(child.stdout).text(), - new Response(child.stderr).text(), - ]); - return { exitCode, stdout, stderr }; - })); - - for (const result of results) { + const isPreApprovalLoss = (stderr: string): boolean => + stderr.includes("CONFIG_MUTATION_LOCK_UNAVAILABLE") + || (stderr.includes("EEXIST") && stderr.includes("createOwnership")) + || /database (?:is|table is) locked/i.test(stderr) + || stderr.includes("SQLITE_BUSY"); + + // On macOS CI both children can still lose the config lock before approval even + // after the warm-up — that proves nothing about catalog serialization. Retry + // vacuous runs until at least one process reaches the post-approval seam. + const attemptDeadline = Date.now() + 20_000; + let results: Array<{ exitCode: number; stdout: string; stderr: string }> | undefined; + while (Date.now() < attemptDeadline) { + for (const marker of ["a", "b"] as const) { + rmSync(`${barrier}-${marker}`, { force: true }); + } + writeFileSync(catalogPath, seeded); + + const children = (["a", "b"] as const).map(marker => Bun.spawn( + [process.execPath, "--eval", routeScript(marker)], + { cwd: repoRoot, env: sandbox.env, stdout: "pipe", stderr: "pipe" }, + )); + + results = await Promise.all(children.map(async child => { + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]); + return { exitCode, stdout, stderr }; + })); + + if (results.some(result => result.exitCode === 0)) break; + + for (const result of results) { + expect({ preApproval: isPreApprovalLoss(result.stderr), stderr: result.stderr }) + .toMatchObject({ preApproval: true }); + } + } + + expect(results).toBeDefined(); + for (const result of results!) { // A process can lose a race BEFORE approval and never reach the seam at all. // The known cases come from `saveConfigPreservingClaudeCode`: the config mutation // lock is already held, two cold processes create the ownership file at once, or @@ -508,11 +534,8 @@ test("two processes at the post-approval management seam serialize instead of in // message as "busy" rather than a database fault, so treating it as a seam failure // here contradicted the product code and turned ordinary contention into a red build. if (result.exitCode !== 0) { - const preApproval = result.stderr.includes("CONFIG_MUTATION_LOCK_UNAVAILABLE") - || (result.stderr.includes("EEXIST") && result.stderr.includes("createOwnership")) - || /database (?:is|table is) locked/i.test(result.stderr) - || result.stderr.includes("SQLITE_BUSY"); - expect({ preApproval, stderr: result.stderr }).toMatchObject({ preApproval: true }); + expect({ preApproval: isPreApprovalLoss(result.stderr), stderr: result.stderr }) + .toMatchObject({ preApproval: true }); continue; } const parsed = JSON.parse(result.stdout.trim()) as { @@ -530,12 +553,12 @@ test("two processes at the post-approval management seam serialize instead of in // At least one process must have gotten through to the seam, or this test would // be vacuous — two config-lock losers prove nothing about catalog serialization. - expect(results.some(r => r.exitCode === 0)).toBe(true); + expect(results!.some(r => r.exitCode === 0)).toBe(true); // At least one process must reach a real commit, or the race proves nothing: // the adapter is total, so a seam that only ever failed would still answer 2xx // with a typed disposition and satisfy every assertion above. - const dispositions = results + const dispositions = results! .filter(r => r.exitCode === 0) .map(r => (JSON.parse(r.stdout.trim()) as { catalogRefresh: { status: string } }).catalogRefresh.status); expect(dispositions).toContain("committed"); From 40725d6970bf92c486a22d01039a9b955effd067 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:58:10 +0200 Subject: [PATCH 081/124] fix(lab): centralize evidence producer version --- src/lab/constants.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lab/constants.ts b/src/lab/constants.ts index 85f8f3b211..881ca9ea4c 100644 --- a/src/lab/constants.ts +++ b/src/lab/constants.ts @@ -3,6 +3,7 @@ export const LAB_EVENT_SCHEMA_VERSION = 1; export const LAB_PROJECTION_SPEC_VERSION = "cl-02.v1"; export const LAB_PRODUCER = "opencodex-lab"; +export const LAB_PRODUCER_VERSION = "2.10.2"; export const MAX_INVALIDATION_TARGETS = 1024; export const MAX_BYTES_PER_ARTIFACT = 256 * 1024; From 1eb173db745c7388d154c20fc6020152635cac1b Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:58:31 +0200 Subject: [PATCH 082/124] fix(lab): fail closed when sanitizing evidence --- src/lab/artifacts/sanitize.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/lab/artifacts/sanitize.ts b/src/lab/artifacts/sanitize.ts index 03bd739625..cd9c7e81c0 100644 --- a/src/lab/artifacts/sanitize.ts +++ b/src/lab/artifacts/sanitize.ts @@ -9,6 +9,7 @@ import { redactSecretString } from "../../lib/redact"; const FORBIDDEN_KEY = /^(?:authorization|proxy-authorization|cookie|set-cookie|api[-_]?key|x-api-key|token|secret|password|email|prompt|messages|content|body|url|hostname|baseUrl|path|account|alias)$/i; const SECRETISH = /sk-[a-z0-9]{10,}|Bearer\s+[A-Za-z0-9._\-]+|ghp_[A-Za-z0-9]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}/i; +const SECRETISH_GLOBAL = /sk-[a-z0-9]{10,}|Bearer\s+[A-Za-z0-9._\-]+|ghp_[A-Za-z0-9]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}/gi; export function redactForArtifact(artifactClass: ArtifactClass, payload: unknown): unknown { if ( @@ -28,7 +29,9 @@ export function redactForArtifact(artifactClass: ArtifactClass, payload: unknown const FORBIDDEN_CONTRACT_KEYS = /^(?:authorization|proxy-authorization|cookie|set-cookie|api[-_]?key|x-api-key|token|secret|password|email|prompt|messages|baseUrl|hostname|account|alias)$/i; function assertNoSecretMaterial(value: unknown, depth: number): void { - if (depth > 8) return; + if (depth > 8) { + throw new Error("contract artifact exceeds sanitization inspection depth"); + } if (typeof value === "string") { if (SECRETISH.test(value)) { throw new Error("contract artifact contains forbidden secret-shaped material"); @@ -78,9 +81,9 @@ function scrubValue(value: unknown, depth: number): unknown { function scrubString(value: string): string { let s = redactSecretString(value); - if (SECRETISH.test(s)) s = s.replace(SECRETISH, "[REDACTED]"); + s = s.replace(SECRETISH_GLOBAL, "[REDACTED]"); // Strip absolute filesystem paths (coarse) - s = s.replace(/(?:[A-Za-z]:\\|\/(?:home|Users|tmp|var|etc)\/)[^\s"']+/g, "[path]"); + s = s.replace(/(?:[A-Za-z]:\\|\/(?:home|Users|tmp|var|etc|root|mnt)\/)[^\s"']+/g, "[path]"); // Strip URL userinfo / private hosts roughly s = s.replace(/https?:\/\/[^\s"']+/gi, (url) => { try { @@ -101,6 +104,11 @@ function scrubString(value: string): string { return s; } +/** Stable privacy boundary for diagnostic text that may be persisted. */ +export function sanitizeDiagnostic(value: unknown): string { + return scrubString(value instanceof Error ? value.message : String(value)); +} + export function sanitizedJsonBytes(value: unknown): Uint8Array { return new TextEncoder().encode(jcsStringify(scrubValue(value, 0))); } From f7c0a8cf95ca63f5163214692044264afb5ac6b2 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:58:44 +0200 Subject: [PATCH 083/124] fix(lab): enforce restricted state directories --- src/lab/paths.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/lab/paths.ts b/src/lab/paths.ts index 82f24e2825..d54e11f4d7 100644 --- a/src/lab/paths.ts +++ b/src/lab/paths.ts @@ -1,7 +1,14 @@ -import { mkdirSync } from "node:fs"; +import { chmodSync, mkdirSync, statSync } from "node:fs"; import { join } from "node:path"; import { getConfigDir } from "../config"; +function ensureRestrictedDir(dir: string): void { + mkdirSync(dir, { recursive: true, mode: 0o700 }); + if (process.platform === "win32") return; + const mode = statSync(dir).mode & 0o777; + if (mode !== 0o700) chmodSync(dir, 0o700); +} + /** Canonical Compatibility Lab state root under the OpenCodex config dir. */ export function labRoot(configDir = getConfigDir()): string { return join(configDir, "lab"); @@ -40,8 +47,8 @@ export function ensureLabDirs(configDir = getConfigDir()): { const artifactsDir = labArtifactsDir(configDir); const scratchDir = labScratchDir(configDir); const exportDir = labExportDir(configDir); - mkdirSync(root, { recursive: true, mode: 0o700 }); - mkdirSync(artifactsDir, { recursive: true, mode: 0o700 }); + ensureRestrictedDir(root); + ensureRestrictedDir(artifactsDir); return { root, ledgerPath: labLedgerPath(configDir), From 0afa85265b6f1e222ea4639fef4fe325551329d5 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:59:03 +0200 Subject: [PATCH 084/124] fix(lab): validate suite manifest authority --- src/lab/conformance/suite-manifest.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/lab/conformance/suite-manifest.ts b/src/lab/conformance/suite-manifest.ts index 6cab266b92..412a77c847 100644 --- a/src/lab/conformance/suite-manifest.ts +++ b/src/lab/conformance/suite-manifest.ts @@ -34,6 +34,9 @@ export function expandSuiteManifest( } const defaults = authority.manifestDefaults; const capability = cases[0]!.capability; + if (cases.some((caseRecord) => caseRecord.capability !== capability)) { + throw new Error(`suite ${suiteId} declares mixed capabilities`); + } const scenarios: SuiteScenarioRefV1[] = cases .map((caseRecord) => suiteScenarioRef(caseRecord, authority)) .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)); @@ -44,8 +47,8 @@ export function expandSuiteManifest( version: String(defaults.suiteVersion), evidenceLayer: defaults.evidenceLayer, capability, - assertionDslVersion: String(defaults.version), - evidenceSchemaVersion: String(defaults.version), + assertionDslVersion: authority.assertionDslVersion, + evidenceSchemaVersion: authority.evidenceSchemaVersion, freshness: defaults.freshness ?? { maxAgeMs: null }, contradictionRule: "newest-required-observation-v1", scenarios, From 527d97f4cf518630d898beecbf560218241bad81 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:59:20 +0200 Subject: [PATCH 085/124] fix(lab): bound invalidation target lookup --- src/lab/ledger/invalidation.ts | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/lab/ledger/invalidation.ts b/src/lab/ledger/invalidation.ts index c8b2d317ed..5c62b8dca4 100644 --- a/src/lab/ledger/invalidation.ts +++ b/src/lab/ledger/invalidation.ts @@ -16,6 +16,8 @@ export interface InvalidationIndex { corruptions: LedgerCorruption[]; } +type EventPosition = { kind: LabEvent["eventKind"]; index: number }; + /** * Apply purge tombstones before ordinary invalidations. * Invalidation target lists are atomic: any bad target rejects the whole event. @@ -27,10 +29,12 @@ export function buildInvalidationIndex(events: LabEvent[]): InvalidationIndex { const corruptions: LedgerCorruption[] = []; const validEvidenceIds = new Map(); + const allEventIds = new Map(); - // First pass: record evidence positions; apply purges as encountered. + // First pass: record event positions and evidence positions; apply purges as encountered. for (let i = 0; i < events.length; i++) { const event = events[i]!; + allEventIds.set(event.eventId, { kind: event.eventKind, index: i }); if (event.eventKind === "observation" || event.eventKind === "claim_snapshot") { validEvidenceIds.set(event.eventId, { kind: event.eventKind, index: i }); continue; @@ -45,7 +49,7 @@ export function buildInvalidationIndex(events: LabEvent[]): InvalidationIndex { const event = events[i]!; if (event.eventKind !== "invalidation") continue; try { - validateInvalidationTargets(event, events, i, validEvidenceIds, purgedEventIds); + validateInvalidationTargets(event, i, validEvidenceIds, allEventIds); for (const target of event.targetEventIds) { const list = invalidatedBy.get(target) ?? []; list.push(event.eventId); @@ -74,10 +78,9 @@ function applyPurge( function validateInvalidationTargets( event: InvalidationEvent, - all: LabEvent[], index: number, validEvidenceIds: Map, - purgedEventIds: Set, + allEventIds: Map, ): void { for (const target of event.targetEventIds) { if (target === event.eventId) { @@ -85,24 +88,21 @@ function validateInvalidationTargets( } const meta = validEvidenceIds.get(target); if (!meta) { - // Could be unknown, or an invalidation/purge id - const earlier = all.slice(0, index).find((e) => e.eventId === target); - if (!earlier) { - throw new LabValidationError("unknown_target", `unknown target ${target}`); - } - if (earlier.eventKind === "invalidation" || earlier.eventKind === "purge_tombstone") { - throw new LabValidationError("bad_target_kind", `cannot invalidate ${earlier.eventKind}`); + const other = allEventIds.get(target); + if ( + other && + other.index < index && + (other.kind === "invalidation" || other.kind === "purge_tombstone") + ) { + throw new LabValidationError("bad_target_kind", `cannot invalidate ${other.kind}`); } throw new LabValidationError("unknown_target", `unknown target ${target}`); } if (meta.index >= index) { throw new LabValidationError("future_target", `target ${target} is not earlier`); } - // Purged targets: sensitive line may no longer exist — invalidation still must not - // name purge/invalidation kinds; naming a purged observation/claim id is allowed - // only if it appeared earlier as valid evidence before purge. We keep meta from - // first pass so previously-seen evidence ids remain addressable. - void purgedEventIds; + // Purged targets may no longer have a ledger line. A previously valid evidence + // ID remains addressable because validEvidenceIds records its original position. } } From 4f6f9abcce34da862a09795e6b3007d9af190b15 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:59:37 +0200 Subject: [PATCH 086/124] fix(lab): make purge artifact retention fail closed --- src/lab/ledger/artifact-refs.ts | 55 +++++++++++++++++---------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/src/lab/ledger/artifact-refs.ts b/src/lab/ledger/artifact-refs.ts index 8ceba910ec..0832f535e1 100644 --- a/src/lab/ledger/artifact-refs.ts +++ b/src/lab/ledger/artifact-refs.ts @@ -1,7 +1,7 @@ import type { ClaimSnapshotEvent, LabEvent, ObservationEvent } from "../events/types"; -import { isEventExcluded, type InvalidationIndex } from "./invalidation"; +import type { InvalidationIndex } from "./invalidation"; -/** Collect every artifact digest referenced by non-excluded evidence events. */ +/** Collect artifacts that the frozen CL-00 retention rules still require. */ export function collectReferencedArtifactDigests( events: LabEvent[], index: InvalidationIndex, @@ -12,13 +12,16 @@ export function collectReferencedArtifactDigests( for (const event of events) { if (exclude.has(event.eventId)) continue; - if (isEventExcluded(event.eventId, index)) continue; + if (index.purgedEventIds.has(event.eventId)) continue; if (event.eventKind === "observation") { + // CL-00 releases observation artifacts after invalidation or purge. + if (index.invalidatedBy.has(event.eventId)) continue; addObservationArtifacts(event, refs); continue; } if (event.eventKind === "claim_snapshot") { + // CL-00 retains claim source manifests while any non-purged claim references them. refs.add(event.sourceManifestDigest); } } @@ -47,7 +50,7 @@ export function eventReferencesArtifactDigest(event: LabEvent, digest: string): /** * Expand purge targets so explicitly sensitive artifact digests cannot survive - * while still-referenced evidence remains. + * while any retained ledger evidence line still references them. */ export function expandSensitiveArtifactEventTargets( events: LabEvent[], @@ -58,28 +61,20 @@ export function expandSensitiveArtifactEventTargets( const expanded = new Set(targetEventIds); if (explicitArtifactDigests.size === 0) return expanded; - let changed = true; - while (changed) { - changed = false; - for (const event of events) { - if (expanded.has(event.eventId)) continue; - if (isEventExcluded(event.eventId, index)) continue; - for (const digest of explicitArtifactDigests) { - if (eventReferencesArtifactDigest(event, digest)) { - expanded.add(event.eventId); - changed = true; - break; - } + for (const event of events) { + if (expanded.has(event.eventId)) continue; + if (index.purgedEventIds.has(event.eventId)) continue; + for (const digest of explicitArtifactDigests) { + if (eventReferencesArtifactDigest(event, digest)) { + expanded.add(event.eventId); + break; } } } return expanded; } -/** - * Artifact digests still required by surviving usable evidence after excluding - * the given event IDs (e.g. purge targets). - */ +/** Artifacts still required by surviving evidence after excluding purge targets. */ export function artifactsStillRequired( events: LabEvent[], index: InvalidationIndex, @@ -88,14 +83,19 @@ export function artifactsStillRequired( return collectReferencedArtifactDigests(events, index, { excludeEventIds }); } -/** Digests that may be deleted when purging the given event/artifact targets. */ -export function deletableArtifactDigests( +export interface ArtifactDeletionPlan { + deletable: string[]; + /** Explicitly sensitive digests that remain pinned by surviving evidence. */ + retainedExplicit: string[]; +} + +/** Plan physical artifact deletion for a purge without silently retaining explicit targets. */ +export function artifactDeletionPlan( events: LabEvent[], index: InvalidationIndex, targetEventIds: Set, explicitArtifactDigests: string[], -): string[] { - const explicit = new Set(explicitArtifactDigests); +): ArtifactDeletionPlan { const stillRequired = artifactsStillRequired(events, index, targetEventIds); const candidates = new Set(explicitArtifactDigests); @@ -110,9 +110,10 @@ export function deletableArtifactDigests( } } - return [...candidates] - .filter((digest) => explicit.has(digest) || !stillRequired.has(digest)) - .sort(); + return { + deletable: [...candidates].filter((digest) => !stillRequired.has(digest)).sort(), + retainedExplicit: explicitArtifactDigests.filter((digest) => stillRequired.has(digest)).sort(), + }; } export function observationArtifactDigests(obs: ObservationEvent): string[] { From 21906aced6e06b2f88eb90b43a59256d41e29110 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 08:59:54 +0200 Subject: [PATCH 087/124] fix(lab): constrain projection enum columns --- src/lab/projection/schema.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/lab/projection/schema.ts b/src/lab/projection/schema.ts index 344b6595dd..a8519ad12e 100644 --- a/src/lab/projection/schema.ts +++ b/src/lab/projection/schema.ts @@ -2,7 +2,7 @@ * Disposable SQLite projection schema for Compatibility Lab (CL-02). * Not canonical storage — rebuildable from JSONL + content-addressed artifacts. */ -export const LAB_SQLITE_SCHEMA_VERSION = 1; +export const LAB_SQLITE_SCHEMA_VERSION = 2; export const LAB_SQLITE_DDL = ` CREATE TABLE IF NOT EXISTS schema_meta ( @@ -12,12 +12,12 @@ CREATE TABLE IF NOT EXISTS schema_meta ( CREATE TABLE IF NOT EXISTS events ( event_id TEXT PRIMARY KEY, - event_kind TEXT NOT NULL, + event_kind TEXT NOT NULL CHECK (event_kind IN ('observation', 'claim_snapshot', 'invalidation', 'purge_tombstone')), recorded_at INTEGER NOT NULL, producer TEXT NOT NULL, producer_version TEXT NOT NULL, payload_json TEXT NOT NULL, - excluded INTEGER NOT NULL DEFAULT 0, + excluded INTEGER NOT NULL DEFAULT 0 CHECK (excluded IN (0, 1)), exclusion_reason TEXT ); @@ -33,16 +33,16 @@ CREATE TABLE IF NOT EXISTS subjects ( CREATE TABLE IF NOT EXISTS observations ( event_id TEXT PRIMARY KEY, subject_id TEXT NOT NULL, - evidence_layer TEXT NOT NULL, + evidence_layer TEXT NOT NULL CHECK (evidence_layer IN ('protocol_conformance', 'live_route_compatibility', 'task_effectiveness')), suite_id TEXT NOT NULL, suite_version TEXT NOT NULL, suite_manifest_digest TEXT NOT NULL, scenario_id TEXT NOT NULL, scenario_version TEXT NOT NULL, scenario_manifest_digest TEXT NOT NULL, - outcome TEXT NOT NULL, + outcome TEXT NOT NULL CHECK (outcome IN ('pass', 'fail', 'blocked', 'inconclusive')), completed_at INTEGER NOT NULL, - execution_mode TEXT NOT NULL, + execution_mode TEXT NOT NULL CHECK (execution_mode IN ('fixture', 'live', 'fabric')), FOREIGN KEY(event_id) REFERENCES events(event_id) ); @@ -54,13 +54,13 @@ CREATE TABLE IF NOT EXISTS claims ( event_id TEXT PRIMARY KEY, subject_id TEXT NOT NULL, capability TEXT NOT NULL, - polarity TEXT NOT NULL, + polarity TEXT NOT NULL CHECK (polarity IN ('supported', 'not_supported', 'withdrawn')), source_manifest_digest TEXT NOT NULL, effective_at INTEGER NOT NULL, recorded_at INTEGER NOT NULL, supersedes_json TEXT NOT NULL, - current INTEGER NOT NULL DEFAULT 0, - usable INTEGER NOT NULL DEFAULT 1, + current INTEGER NOT NULL DEFAULT 0 CHECK (current IN (0, 1)), + usable INTEGER NOT NULL DEFAULT 1 CHECK (usable IN (0, 1)), FOREIGN KEY(event_id) REFERENCES events(event_id) ); @@ -71,7 +71,7 @@ CREATE TABLE IF NOT EXISTS invalidations ( reason TEXT NOT NULL, targets_json TEXT NOT NULL, recorded_at INTEGER NOT NULL, - applied INTEGER NOT NULL DEFAULT 0, + applied INTEGER NOT NULL DEFAULT 0 CHECK (applied IN (0, 1)), FOREIGN KEY(event_id) REFERENCES events(event_id) ); @@ -89,7 +89,7 @@ CREATE TABLE IF NOT EXISTS artifacts ( artifact_class TEXT, media_type TEXT, byte_count INTEGER, - status TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('present', 'corrupt', 'purged_unavailable')), last_error TEXT ); @@ -101,7 +101,7 @@ CREATE TABLE IF NOT EXISTS verdicts ( suite_version TEXT NOT NULL, suite_manifest_digest TEXT NOT NULL, projection_spec_version TEXT NOT NULL, - verdict TEXT NOT NULL, + verdict TEXT NOT NULL CHECK (verdict IN ('UNKNOWN', 'CLAIMED', 'PROBED', 'VERIFIED', 'DEGRADED', 'BLOCKED', 'UNSUPPORTED')), as_of INTEGER NOT NULL, scenario_manifest_digests_json TEXT NOT NULL, claim_source_digest TEXT, From 2af46f3a33df742e11e36773224fe2846d72c985 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:00:35 +0200 Subject: [PATCH 088/124] fix(lab): make sensitive purge progress explicit --- src/lab/ledger/purge.ts | 129 ++++++++++++++++++++++++++-------------- 1 file changed, 83 insertions(+), 46 deletions(-) diff --git a/src/lab/ledger/purge.ts b/src/lab/ledger/purge.ts index ca332fbba7..d666624532 100644 --- a/src/lab/ledger/purge.ts +++ b/src/lab/ledger/purge.ts @@ -5,11 +5,16 @@ import { type TrustedArtifactDir, } from "../artifacts/secure-fs"; import { ArtifactFsError } from "../artifacts/secure-fs"; -import { LAB_EVENT_SCHEMA_VERSION, LAB_PRODUCER, PURGE_ACTIONS } from "../constants"; +import { + LAB_EVENT_SCHEMA_VERSION, + LAB_PRODUCER, + LAB_PRODUCER_VERSION, + PURGE_ACTIONS, +} from "../constants"; import type { LabEvent, PurgeTombstoneEvent } from "../events/types"; import { assignEventId, validateLabEvent } from "../events/validate"; import { - deletableArtifactDigests, + artifactDeletionPlan, expandSensitiveArtifactEventTargets, } from "./artifact-refs"; import { buildInvalidationIndex } from "./invalidation"; @@ -29,14 +34,16 @@ import { unlinkSync, writeSync, } from "node:fs"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; export class PurgeError extends Error { readonly code: string; - constructor(code: string, message: string) { + readonly completedActions: string[]; + constructor(code: string, message: string, completedActions: string[] = []) { super(message); this.name = "PurgeError"; this.code = code; + this.completedActions = [...completedActions]; } } @@ -49,38 +56,61 @@ export interface SensitivePurgeRequest { producerVersion?: string; } +function writeAll(fd: number, bytes: Uint8Array): void { + let offset = 0; + while (offset < bytes.byteLength) { + const n = writeSync(fd, bytes, offset, bytes.byteLength - offset); + if (n <= 0) throw new PurgeError("short_write", "ledger rewrite short write"); + offset += n; + } +} + function atomicRewriteLedger(ledgerPath: string, events: LabEvent[]): void { const body = events.map((e) => jcsStringify(e)).join("\n") + (events.length ? "\n" : ""); const bytes = new TextEncoder().encode(body); - const tmpPath = join(join(ledgerPath, ".."), `.purge-${process.pid}-${Date.now()}.jsonl.tmp`); - const fd = openSync(tmpPath, "w", 0o600); + const parent = dirname(ledgerPath); + const tmpPath = join(parent, `.purge-${process.pid}-${Date.now()}.jsonl.tmp`); try { - const written = writeSync(fd, bytes); - if (written !== bytes.byteLength) { - throw new PurgeError("short_write", "ledger rewrite short write"); + const fd = openSync(tmpPath, "wx", 0o600); + try { + writeAll(fd, bytes); + fsyncSync(fd); + } finally { + closeSync(fd); } - fsyncSync(fd); - } finally { - closeSync(fd); - } - renameSync(tmpPath, ledgerPath); - const ledgerFd = openSync(ledgerPath, "r+"); - try { - fsyncSync(ledgerFd); - } finally { - closeSync(ledgerFd); + renameSync(tmpPath, ledgerPath); + if (process.platform !== "win32") { + const dirFd = openSync(parent, "r"); + try { + fsyncSync(dirFd); + } finally { + closeSync(dirFd); + } + } + } catch (err) { + try { + unlinkSync(tmpPath); + } catch { + // Preserve the original failure. The temp file may already have been renamed. + } + throw err; } } +function isArtifactMissing(err: unknown): boolean { + return err instanceof ArtifactFsError && ( + err.code === "artifact_missing" || + (err.code === "harness_failure" && err.message.includes("missing")) + ); +} + function deleteArtifactsFailClosed(dir: TrustedArtifactDir, digests: string[]): void { const errors: string[] = []; for (const digest of digests) { try { deleteArtifactBytes(dir, digest); } catch (err) { - if (err instanceof ArtifactFsError && err.message.includes("missing")) { - continue; - } + if (isArtifactMissing(err)) continue; errors.push(err instanceof Error ? err.message : String(err)); } } @@ -118,7 +148,7 @@ export function purgeSensitiveEvidence(req: SensitivePurgeRequest): PurgeTombsto const explicitSensitive = new Set(targetArtifactDigests); const replay = replayLabLedger(paths.ledgerPath); - const index = buildIndexFromReplay(replay.events); + const index = buildInvalidationIndex(replay.events); const removeIds = expandSensitiveArtifactEventTargets( replay.events, index, @@ -131,7 +161,7 @@ export function purgeSensitiveEvidence(req: SensitivePurgeRequest): PurgeTombsto eventKind: "purge_tombstone" as const, recordedAt: req.recordedAt ?? Date.now(), producer: LAB_PRODUCER, - producerVersion: req.producerVersion ?? "2.10.2", + producerVersion: req.producerVersion ?? LAB_PRODUCER_VERSION, targetEventIds: [...removeIds].sort(), targetArtifactDigests, reason: "sensitive_evidence" as const, @@ -139,33 +169,35 @@ export function purgeSensitiveEvidence(req: SensitivePurgeRequest): PurgeTombsto }; const tombstone = validateLabEvent(assignEventId(tombstonePayload)) as PurgeTombstoneEvent; - const deletable = purgeActions.includes("artifact") - ? deletableArtifactDigests(replay.events, index, removeIds, targetArtifactDigests) - : []; - - if (purgeActions.includes("artifact") && explicitSensitive.size > 0) { - for (const digest of explicitSensitive) { - if (!deletable.includes(digest)) { - throw new PurgeError( - "sensitive_artifact_not_deletable", - `explicit sensitive artifact ${digest} could not be removed`, - ); - } - } + const deletionPlan = purgeActions.includes("artifact") + ? artifactDeletionPlan(replay.events, index, removeIds, targetArtifactDigests) + : { deletable: [], retainedExplicit: [] }; + + if (deletionPlan.retainedExplicit.length > 0) { + throw new PurgeError( + "sensitive_bytes_retained", + `explicit sensitive artifacts remain required: ${deletionPlan.retainedExplicit.join(",")}`, + ); } let dir: TrustedArtifactDir | null = null; + const completed: string[] = []; try { if (purgeActions.includes("scratch")) { purgeBoundedDirectory(paths.scratchDir); + completed.push("scratch"); } if (purgeActions.includes("export")) { purgeBoundedDirectory(paths.exportDir); + completed.push("export"); } - if (purgeActions.includes("artifact") && deletable.length > 0) { - dir = openTrustedArtifactDir(paths.artifactsDir); - deleteArtifactsFailClosed(dir, deletable); + if (purgeActions.includes("artifact")) { + if (deletionPlan.deletable.length > 0) { + dir = openTrustedArtifactDir(paths.artifactsDir); + deleteArtifactsFailClosed(dir, deletionPlan.deletable); + } + completed.push("artifact"); } if (purgeActions.includes("ledger")) { @@ -179,24 +211,29 @@ export function purgeSensitiveEvidence(req: SensitivePurgeRequest): PurgeTombsto } else { appendLabEvent(paths.ledgerPath, tombstone); } + completed.push("ledger"); if (purgeActions.includes("sqlite")) { rebuildLabProjection(req.configDir); + completed.push("sqlite"); } return tombstone; } catch (err) { - throw err instanceof PurgeError ? err : new PurgeError("purge_failed", err instanceof Error ? err.message : String(err)); + if (err instanceof PurgeError) { + throw new PurgeError(err.code, err.message, [...completed, ...err.completedActions]); + } + throw new PurgeError( + "purge_failed", + err instanceof Error ? err.message : String(err), + completed, + ); } finally { if (dir) closeTrustedArtifactDir(dir); } } -function buildIndexFromReplay(events: LabEvent[]) { - return buildInvalidationIndex(events); -} - -/** Test helper: read raw ledger text. */ +/** Test helper retained temporarily for compatibility; production callers should replay validated events. */ export function readLedgerText(configDir?: string): string { const paths = ensureLabDirs(configDir); return readFileSync(paths.ledgerPath, "utf8"); From a12a555041ea6427a031fbfc071036487de5111e Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:01:04 +0200 Subject: [PATCH 089/124] fix(lab): complete ledger appends across short writes --- src/lab/ledger/store.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/lab/ledger/store.ts b/src/lab/ledger/store.ts index 920a58e3ea..ade8c3e3d4 100644 --- a/src/lab/ledger/store.ts +++ b/src/lab/ledger/store.ts @@ -29,9 +29,13 @@ export function appendLabEvent(ledgerPath: string, event: LabEvent): void { const bytes = new TextEncoder().encode(line); const fd = openSync(ledgerPath, "a", 0o600); try { - const written = writeSync(fd, bytes); - if (written !== bytes.byteLength) { - throw new LabValidationError("short_write", "ledger append short write"); + let written = 0; + while (written < bytes.byteLength) { + const n = writeSync(fd, bytes, written, bytes.byteLength - written); + if (n <= 0) { + throw new LabValidationError("short_write", "ledger append made no progress"); + } + written += n; } fsyncSync(fd); } finally { From 3dddcdafe751e0f7e54b05aded72727966718ac5 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:01:26 +0200 Subject: [PATCH 090/124] fix(lab): reject POSIX paths before ledger admission --- src/lab/events/limits.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lab/events/limits.ts b/src/lab/events/limits.ts index 6fab1fe70d..772fc3fba9 100644 --- a/src/lab/events/limits.ts +++ b/src/lab/events/limits.ts @@ -75,7 +75,11 @@ export function enforceEventStructureLimits( if (/sk-[a-z0-9]{10,}/i.test(value) || /Bearer\s+\S+/i.test(value)) { throw new LabValidationError("secret_pattern", `${path} contains secret-shaped data`); } - if (/^[A-Za-z]:\\/.test(value) || value.includes("/Users/") || value.includes("\\Users\\")) { + if ( + /^[A-Za-z]:\\/.test(value) || + /(?:^|[\s"'([])\/(?:home|Users|tmp|var|etc|root|mnt)\//.test(value) || + value.includes("\\Users\\") + ) { throw new LabValidationError("raw_path", `${path} contains raw filesystem path`); } return; From f3d878aef3678825b9d1dffd161149c0d8f481a3 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:02:03 +0200 Subject: [PATCH 091/124] fix(lab): verify artifact digests by class --- src/lab/artifacts/store.ts | 171 ++++++++++++++++++++++--------------- 1 file changed, 104 insertions(+), 67 deletions(-) diff --git a/src/lab/artifacts/store.ts b/src/lab/artifacts/store.ts index 9691e47593..83ad5e118c 100644 --- a/src/lab/artifacts/store.ts +++ b/src/lab/artifacts/store.ts @@ -20,7 +20,7 @@ import { readArtifactBytes, type TrustedArtifactDir, } from "./secure-fs"; -import { redactForArtifact } from "./sanitize"; +import { redactForArtifact, sanitizeDiagnostic } from "./sanitize"; export { ArtifactFsError, openTrustedArtifactDir }; export type { TrustedArtifactDir }; @@ -35,11 +35,19 @@ export interface PutArtifactInput { expectedDigest?: string; } +export interface ArtifactReadOptions { + expectedByteCount?: number; + artifactClass?: ArtifactClass; +} + export interface ArtifactStore { dir: TrustedArtifactDir; put(input: PutArtifactInput): ArtifactRefV1; - get(digest: string, expectedByteCount?: number): Uint8Array; - getVerified(digest: string, expectedByteCount?: number): { bytes: Uint8Array; digest: string }; + get(digest: string, expectedByteCountOrOpts?: number | ArtifactReadOptions): Uint8Array; + getVerified( + digest: string, + expectedByteCountOrOpts?: number | ArtifactReadOptions, + ): { bytes: Uint8Array; digest: string }; remove(digest: string): void; close(): void; } @@ -50,11 +58,78 @@ function toBytes(payload: Uint8Array | string | unknown): Uint8Array { return new TextEncoder().encode(jcsStringify(payload)); } +function normalizeReadOptions(value?: number | ArtifactReadOptions): ArtifactReadOptions { + return typeof value === "number" ? { expectedByteCount: value } : value ?? {}; +} + +function jsonDigest( + digest: (value: Record) => string, +): (bytes: Uint8Array) => string { + return (bytes) => digest(JSON.parse(new TextDecoder().decode(bytes)) as Record); +} + +function digestForArtifactClass(artifactClass: ArtifactClass): (bytes: Uint8Array) => string { + switch (artifactClass) { + case "fixture": + return fixtureDigest; + case "scenario_manifest": + return jsonDigest(scenarioManifestDigest); + case "suite_manifest": + return jsonDigest(suiteManifestDigest); + case "claim_source_manifest": + return (bytes) => { + const parsed = JSON.parse(new TextDecoder().decode(bytes)); + return claimSourceManifestDigest(validateClaimSourceManifest(parsed).manifest); + }; + default: + return artifactBytesDigest; + } +} + export function createArtifactStore(artifactsDir: string): ArtifactStore { const dir = openTrustedArtifactDir(artifactsDir); let aggregateBytes = 0; let putCount = 0; + const getVerified = ( + digest: string, + expectedByteCountOrOpts?: number | ArtifactReadOptions, + ): { bytes: Uint8Array; digest: string } => { + const opts = normalizeReadOptions(expectedByteCountOrOpts); + const candidates = opts.artifactClass + ? [digestForArtifactClass(opts.artifactClass)] + : [ + artifactBytesDigest, + fixtureDigest, + jsonDigest(scenarioManifestDigest), + jsonDigest(suiteManifestDigest), + digestForArtifactClass("claim_source_manifest"), + ]; + + let lastErr: unknown; + for (const contentDigest of candidates) { + try { + const got = readArtifactBytes(dir, digest, { + expectedByteCount: opts.expectedByteCount, + contentDigest, + }); + return { bytes: got.bytes, digest: got.digest }; + } catch (err) { + lastErr = err; + if ( + err instanceof ArtifactFsError && + err.code !== "artifact_mismatch" && + !err.message.includes("mismatch") + ) { + throw err; + } + } + } + throw lastErr instanceof Error + ? lastErr + : new ArtifactFsError("artifact_mismatch", "artifact digest verification failed"); + }; + return { dir, put(input: PutArtifactInput): ArtifactRefV1 { @@ -79,21 +154,16 @@ export function createArtifactStore(artifactsDir: string): ArtifactStore { let stored; if (isContractClass(input.artifactClass)) { const contractClass = input.artifactClass; - const digest = input.expectedDigest ?? computeContractDigest(contractClass, bytes, redacted); - if (input.expectedDigest && digest !== input.expectedDigest) { - throw new ArtifactFsError("harness_failure", "contract artifact digest mismatch"); - } - const contentDigest = (b: Uint8Array) => - computeContractDigest(contractClass, b, JSON.parse(new TextDecoder().decode(b))); - // Fixtures hash raw bytes; JSON contract manifests hash parsed JCS object. - const hashFn = - contractClass === "fixture" - ? (b: Uint8Array) => fixtureDigest(b) - : contentDigest; - if (hashFn(bytes) !== digest) { - throw new ArtifactFsError("harness_failure", "contract artifact preimage digest mismatch"); + const computedDigest = computeContractDigest(contractClass, bytes, redacted); + if (input.expectedDigest !== undefined && computedDigest !== input.expectedDigest) { + throw new ArtifactFsError("artifact_mismatch", "contract artifact digest mismatch"); } - stored = putNamedDigestBytes(dir, digest, bytes, hashFn); + stored = putNamedDigestBytes( + dir, + computedDigest, + bytes, + digestForArtifactClass(contractClass), + ); } else { stored = putArtifactBytes(dir, bytes, input.expectedDigest); } @@ -109,48 +179,10 @@ export function createArtifactStore(artifactsDir: string): ArtifactStore { artifactClass: input.artifactClass, }; }, - get(digest: string, expectedByteCount?: number): Uint8Array { - return this.getVerified(digest, expectedByteCount).bytes; - }, - getVerified(digest: string, expectedByteCount?: number) { - const candidates: Array<(b: Uint8Array) => string> = [ - artifactBytesDigest, - fixtureDigest, - (b) => { - try { - return scenarioManifestDigest(JSON.parse(new TextDecoder().decode(b))); - } catch { - return ""; - } - }, - (b) => { - try { - return suiteManifestDigest(JSON.parse(new TextDecoder().decode(b))); - } catch { - return ""; - } - }, - (b) => { - try { - return claimSourceManifestDigest(JSON.parse(new TextDecoder().decode(b))); - } catch { - return ""; - } - }, - ]; - let lastErr: unknown; - for (const contentDigest of candidates) { - try { - const got = readArtifactBytes(dir, digest, { expectedByteCount, contentDigest }); - if (got.digest === digest) return { bytes: got.bytes, digest: got.digest }; - } catch (err) { - lastErr = err; - } - } - throw lastErr instanceof Error - ? lastErr - : new ArtifactFsError("harness_failure", "artifact digest verification failed"); + get(digest: string, expectedByteCountOrOpts?: number | ArtifactReadOptions): Uint8Array { + return getVerified(digest, expectedByteCountOrOpts).bytes; }, + getVerified, remove(digest: string): void { deleteArtifactBytes(dir, digest); }, @@ -222,24 +254,29 @@ export function putClaimSourceManifest( }); } +export type LoadClaimSourceManifestResult = + | { ok: true; manifest: ClaimSourceManifestV1; corruption?: undefined } + | { ok: false; manifest: ClaimSourceManifestV1 | null; corruption: string }; + export function loadClaimSourceManifest( store: ArtifactStore, digest: string, expected: { subjectId: string; capability: string }, -): { manifest: ClaimSourceManifestV1; corruption?: string } { - if (!isSha256Hex(digest)) return { manifest: null as unknown as ClaimSourceManifestV1, corruption: "invalid digest" }; +): LoadClaimSourceManifestResult { + if (!isSha256Hex(digest)) return { ok: false, manifest: null, corruption: "invalid digest" }; try { - const bytes = store.get(digest); + const bytes = store.get(digest, { artifactClass: "claim_source_manifest" }); const parsed = JSON.parse(new TextDecoder().decode(bytes)); const { manifest, digest: recomputed } = validateClaimSourceManifest(parsed); - if (recomputed !== digest) return { manifest, corruption: "claim-source digest mismatch" }; - if (manifest.subjectId !== expected.subjectId) return { manifest, corruption: "claim-source subjectId mismatch" }; - if (manifest.capability !== expected.capability) return { manifest, corruption: "claim-source capability mismatch" }; - return { manifest }; + if (recomputed !== digest) return { ok: false, manifest, corruption: "claim-source digest mismatch" }; + if (manifest.subjectId !== expected.subjectId) return { ok: false, manifest, corruption: "claim-source subjectId mismatch" }; + if (manifest.capability !== expected.capability) return { ok: false, manifest, corruption: "claim-source capability mismatch" }; + return { ok: true, manifest }; } catch (err) { return { - manifest: null as unknown as ClaimSourceManifestV1, - corruption: err instanceof Error ? err.message : String(err), + ok: false, + manifest: null, + corruption: sanitizeDiagnostic(err), }; } } From 441b78207e6538521905457d3de32c4b9aeb9f67 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:02:56 +0200 Subject: [PATCH 092/124] fix(lab): persist real conformance evidence metadata --- src/lab/observe/from-conformance.ts | 103 ++++++++++++++++++++++------ 1 file changed, 81 insertions(+), 22 deletions(-) diff --git a/src/lab/observe/from-conformance.ts b/src/lab/observe/from-conformance.ts index 165169f1e7..44b7e35cb3 100644 --- a/src/lab/observe/from-conformance.ts +++ b/src/lab/observe/from-conformance.ts @@ -7,6 +7,7 @@ import { createArtifactStore, type ArtifactStore } from "../artifacts/store"; import { LAB_EVENT_SCHEMA_VERSION, LAB_PRODUCER, + LAB_PRODUCER_VERSION, type ObservationOutcome, } from "../constants"; import { @@ -19,18 +20,25 @@ import type { ObservationEvent, ProtocolSubjectV1 } from "../events/types"; import { assignEventId } from "../events/validate"; import { appendLabEvent } from "../ledger/store"; import { ensureLabDirs } from "../paths"; -import type { CaseAuthority, CaseRecord, ScenarioRunResult } from "../conformance/types"; +import type { + CaseAuthority, + CaseRecord, + ProtocolExecutionContextV1, + ScenarioRunResult, +} from "../conformance/types"; import { expandScenario } from "../conformance/manifest"; -import { expandSuiteManifest } from "../conformance/suite-manifest"; +import { suiteManifestObjectForCase } from "../conformance/suite-manifest"; import { fixtureDigest } from "../conformance/digest"; import { resolveProtocolExecutionContext } from "../conformance/executor"; -const PACKAGE_VERSION = "2.10.2"; const COMPAT_VERSION = "protocol-v1"; export interface PersistConformanceOptions { configDir?: string; recordedAt?: number; + /** Actual execution timestamps from the CL-01 runner; never fabricated. */ + startedAt?: number; + completedAt?: number; producerVersion?: string; artifactStore?: ArtifactStore; } @@ -40,9 +48,33 @@ export interface PersistedConformanceObservation { ledgerPath: string; } -function behaviorFingerprintForCase(caseRecord: CaseRecord): string { - const upstream = caseRecord.requirements.upstreamProtocols[0] ?? "openai-chat"; - const adapter = upstreamAdapter(upstream); +function validateExecutionContext( + caseRecord: CaseRecord, + ctx: ProtocolExecutionContextV1, +): ProtocolExecutionContextV1 { + const checks: Array<[string, string[], string]> = [ + ["inboundProtocols", caseRecord.requirements.inboundProtocols, ctx.inboundProtocol], + ["upstreamProtocols", caseRecord.requirements.upstreamProtocols, ctx.upstreamProtocol], + ["surfaces", caseRecord.requirements.surfaces, ctx.surface], + ]; + for (const [name, declared, actual] of checks) { + if (declared.length === 0) throw new Error(`case ${caseRecord.id} has empty ${name}`); + if (!declared.includes(actual)) { + throw new Error(`case ${caseRecord.id} execution context violates ${name}`); + } + } + return ctx; +} + +function behaviorFingerprintForCase( + caseRecord: CaseRecord, + executionContext?: ProtocolExecutionContextV1, +): string { + const ctx = validateExecutionContext( + caseRecord, + executionContext ?? resolveProtocolExecutionContext(caseRecord), + ); + const adapter = upstreamAdapter(ctx.upstreamProtocol); const values = { schemaVersion: 1, resolverVersion: 1, @@ -53,7 +85,7 @@ function behaviorFingerprintForCase(caseRecord: CaseRecord): string { }, "wire.upstreamProtocol": { source: "lab_forced", - value: upstream, + value: ctx.upstreamProtocol, }, "runtime.arch": { source: "lab_forced", @@ -73,17 +105,19 @@ function behaviorFingerprintForCase(caseRecord: CaseRecord): string { } function protocolSubject(caseRecord: CaseRecord, result: ScenarioRunResult): ProtocolSubjectV1 { - const ctx = result.executionContext ?? resolveProtocolExecutionContext(caseRecord); - const upstream = ctx.upstreamProtocol; + const ctx = validateExecutionContext( + caseRecord, + result.executionContext ?? resolveProtocolExecutionContext(caseRecord), + ); return { subjectSchemaVersion: 1, subjectKind: "protocol", opencodexCompatibilityVersion: COMPAT_VERSION, - effectiveAdapter: upstreamAdapter(upstream), + effectiveAdapter: upstreamAdapter(ctx.upstreamProtocol), inboundProtocol: ctx.inboundProtocol, upstreamProtocol: ctx.upstreamProtocol, surface: ctx.surface, - behaviorFingerprint: behaviorFingerprintForCase(caseRecord), + behaviorFingerprint: behaviorFingerprintForCase(caseRecord, ctx), }; } @@ -93,20 +127,43 @@ function upstreamAdapter(protocol: string): string { return "openai-responses"; case "anthropic-messages": return "anthropic"; - default: + case "openai-chat": return "openai-chat"; + default: + throw new Error(`unsupported protocol identity: ${protocol}`); } } function outcomeFromResult(result: ScenarioRunResult): ObservationOutcome { if (result.passed) return "pass"; - if (result.classification === "timeout" || result.classification === "budget_exhausted") { - return "blocked"; + switch (result.classification) { + case "timeout": + case "budget_exhausted": + return "blocked"; + case "inconclusive": + case "harness_failure": + return "inconclusive"; + case "protocol_failure": + case "capability_failure": + case "behavioral_failure": + return "fail"; + default: { + const _never: never = result.classification; + return _never; + } + } +} + +function requireExecutionTimes(opts: PersistConformanceOptions): { startedAt: number; completedAt: number } { + if (!Number.isInteger(opts.startedAt) || !Number.isInteger(opts.completedAt)) { + throw new Error("real startedAt/completedAt are required for persisted conformance evidence"); } - if (result.classification === "inconclusive" || result.classification === "harness_failure") { - return "inconclusive"; + const startedAt = opts.startedAt!; + const completedAt = opts.completedAt!; + if (startedAt < 0 || completedAt < startedAt) { + throw new Error("invalid persisted conformance execution timestamps"); } - return "fail"; + return { startedAt, completedAt }; } /** @@ -123,13 +180,12 @@ export function observationFromConformanceResult( const ownsStore = !opts.artifactStore; const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); try { - const recordedAt = opts.recordedAt ?? Date.now(); - const startedAt = recordedAt - 1; - const completedAt = recordedAt; + const { startedAt, completedAt } = requireExecutionTimes(opts); + const recordedAt = opts.recordedAt ?? completedAt; const expandedScenario = expandScenario(caseRecord, authority); const scenarioDigest = scenarioManifestDigest(expandedScenario); - const suiteExpanded = expandSuiteManifest(caseRecord.suite, authority) as unknown as Record; + const suiteExpanded = suiteManifestObjectForCase(caseRecord, authority); const suiteDigest = suiteManifestDigest(suiteExpanded); const fixtureDigests: string[] = []; @@ -194,7 +250,7 @@ export function observationFromConformanceResult( eventKind: "observation" as const, recordedAt, producer: LAB_PRODUCER, - producerVersion: opts.producerVersion ?? PACKAGE_VERSION, + producerVersion: opts.producerVersion ?? LAB_PRODUCER_VERSION, evidenceLayer: "protocol_conformance" as const, scenarioId: caseRecord.id, scenarioVersion: String(authority.manifestDefaults.version), @@ -220,6 +276,9 @@ export function observationFromConformanceResult( observedSummary: a.observedSummary.slice(0, 512), ...(a.reason ? { reason: a.reason } : {}), })), + ...(caseRecord.expectedFailure + ? { expectedFailure: { ...caseRecord.expectedFailure } } + : {}), environment: { runtime: { platform: process.platform, From dca7624aa10ba0529c479855a0a03a9264e26887 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:03:35 +0200 Subject: [PATCH 093/124] fix(lab): enforce verification contracts and freshness --- src/lab/projection/verification.ts | 212 ++++++++++++++++++++--------- 1 file changed, 147 insertions(+), 65 deletions(-) diff --git a/src/lab/projection/verification.ts b/src/lab/projection/verification.ts index cef61ec757..806a345e02 100644 --- a/src/lab/projection/verification.ts +++ b/src/lab/projection/verification.ts @@ -1,6 +1,8 @@ import type { ObservationEvent, ProtocolSubjectV1 } from "../events/types"; -import type { ExecutionMode } from "../constants"; +import { EVIDENCE_LAYERS, type ExecutionMode } from "../constants"; import type { SuiteManifestV1 } from "../conformance/suite-manifest"; +import type { VerificationRole } from "../conformance/types"; +import { isSha256Hex } from "../digest"; export interface VerificationEvaluation { applicableRequiredScenarioIds: string[]; @@ -12,6 +14,15 @@ export interface VerificationEvaluation { export type LoadScenarioManifest = (digest: string) => Record | null; +export interface ScenarioRequirements { + inboundProtocols?: string[]; + upstreamProtocols?: string[]; + surfaces?: string[]; + freshness?: { maxAgeMs: number | null }; +} + +export type LoadScenarioRequirements = (digest: string) => ScenarioRequirements | null; + /** Live-reserved scenarios are inapplicable in fixture-mode protocol conformance. */ export function isScenarioApplicable( scenarioId: string, @@ -19,13 +30,15 @@ export function isScenarioApplicable( evidenceLayer: string, ): boolean { if (evidenceLayer === "protocol_conformance" && executionMode === "fixture") { - if (scenarioId.includes(".live.")) return false; + // The frozen CL-00 scenario schema has no explicit execution-mode field. + // Use an exact dot-delimited `live` segment rather than substring matching. + if (scenarioId.split(".").includes("live")) return false; } return true; } function scenarioApplicableToRequirements( - requirements: { inboundProtocols?: string[]; upstreamProtocols?: string[]; surfaces?: string[] }, + requirements: ScenarioRequirements, subject: ProtocolSubjectV1, ): boolean { const inbound = requirements.inboundProtocols ?? []; @@ -38,21 +51,46 @@ function scenarioApplicableToRequirements( ); } -function scenarioApplicableToProtocolSubject( +function isNonNegativeInteger(value: unknown): value is number { + return typeof value === "number" && Number.isInteger(value) && value >= 0; +} + +function parseFreshness(value: unknown): { maxAgeMs: number | null } | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const maxAgeMs = (value as { maxAgeMs?: unknown }).maxAgeMs; + if (maxAgeMs === null) return { maxAgeMs: null }; + if (!isNonNegativeInteger(maxAgeMs)) return null; + return { maxAgeMs }; +} + +function parseStringArray(value: unknown): string[] | null { + if (!Array.isArray(value) || !value.every((item) => typeof item === "string")) return null; + return value; +} + +function scenarioContractFromManifest( scenarioManifest: Record | null, - subject: ProtocolSubjectV1, -): boolean { - if (!scenarioManifest) return false; +): ScenarioRequirements | null { + if (!scenarioManifest) return null; const req = scenarioManifest.requirements; - if (!req || typeof req !== "object") return false; - const inbound = (req as { inboundProtocols?: string[] }).inboundProtocols ?? []; - const upstream = (req as { upstreamProtocols?: string[] }).upstreamProtocols ?? []; - const surfaces = (req as { surfaces?: string[] }).surfaces ?? []; - return ( - inbound.includes(subject.inboundProtocol) && - upstream.includes(subject.upstreamProtocol) && - surfaces.includes(subject.surface) - ); + if (!req || typeof req !== "object" || Array.isArray(req)) return null; + const row = req as Record; + const inboundProtocols = parseStringArray(row.inboundProtocols); + const upstreamProtocols = parseStringArray(row.upstreamProtocols); + const surfaces = parseStringArray(row.surfaces); + if (!inboundProtocols || !upstreamProtocols || !surfaces) return null; + const freshness = parseFreshness(scenarioManifest.freshness); + if (!freshness) return null; + return { inboundProtocols, upstreamProtocols, surfaces, freshness }; +} + +function effectiveMaxAgeMs( + suiteMaxAgeMs: number | null, + scenarioMaxAgeMs: number | null, +): number | null { + if (suiteMaxAgeMs === null) return scenarioMaxAgeMs; + if (scenarioMaxAgeMs === null) return suiteMaxAgeMs; + return Math.min(suiteMaxAgeMs, scenarioMaxAgeMs); } export function newestObservationByScenario( @@ -71,15 +109,9 @@ export function newestObservationByScenario( /** * Evaluate `all-applicable-required-pass-v1` per frozen CL-00 semantics. - * Positive VERIFIED requires a non-empty applicable required set and a current - * pass for every applicable required scenario. + * Positive VERIFIED requires a non-empty applicable required/control set and a + * current, fresh pass for every applicable required scenario and negative control. */ -export type LoadScenarioRequirements = (digest: string) => { - inboundProtocols?: string[]; - upstreamProtocols?: string[]; - surfaces?: string[]; -} | null; - export function evaluateAllApplicableRequiredPassV1( suiteManifest: SuiteManifestV1, observations: ObservationEvent[], @@ -88,6 +120,7 @@ export function evaluateAllApplicableRequiredPassV1( subject?: ProtocolSubjectV1; loadScenarioManifest?: LoadScenarioManifest; loadScenarioRequirements?: LoadScenarioRequirements; + asOf?: number; } = {}, ): VerificationEvaluation { const notes: string[] = []; @@ -101,26 +134,37 @@ export function evaluateAllApplicableRequiredPassV1( }; } - const requiredScenarios = suiteManifest.scenarios.filter((s) => s.role === "required"); + const requiredScenarios = suiteManifest.scenarios.filter( + (s) => s.role === "required" || s.role === "negative_control", + ); const applicableRequired: string[] = []; const unavailableManifests: string[] = []; + const scenarioMaxAgeById = new Map(); for (const s of requiredScenarios) { if (!isScenarioApplicable(s.id, executionMode, suiteManifest.evidenceLayer)) continue; - if (suiteManifest.evidenceLayer === "protocol_conformance" && opts.subject) { - const scenarioManifest = opts.loadScenarioManifest?.(s.manifestDigest) ?? null; - if (scenarioManifest) { - if (!scenarioApplicableToProtocolSubject(scenarioManifest, opts.subject)) continue; - } else { - const requirements = opts.loadScenarioRequirements?.(s.manifestDigest) ?? null; - if (!requirements) { - unavailableManifests.push(s.id); - continue; - } - if (!scenarioApplicableToRequirements(requirements, opts.subject)) continue; + + let requirements: ScenarioRequirements | null = null; + const scenarioManifest = opts.loadScenarioManifest?.(s.manifestDigest) ?? null; + if (scenarioManifest) { + requirements = scenarioContractFromManifest(scenarioManifest); + if (!requirements) { + unavailableManifests.push(s.id); + continue; + } + } else { + requirements = opts.loadScenarioRequirements?.(s.manifestDigest) ?? null; + if (!requirements || !requirements.freshness) { + unavailableManifests.push(s.id); + continue; } } + + if (suiteManifest.evidenceLayer === "protocol_conformance" && opts.subject) { + if (!scenarioApplicableToRequirements(requirements, opts.subject)) continue; + } applicableRequired.push(s.id); + scenarioMaxAgeById.set(s.id, requirements.freshness?.maxAgeMs ?? null); } applicableRequired.sort(); @@ -147,6 +191,7 @@ export function evaluateAllApplicableRequiredPassV1( const newest = newestObservationByScenario(observations); const passing: string[] = []; const missing: string[] = []; + const asOf = opts.asOf ?? observations.reduce((max, obs) => Math.max(max, obs.completedAt), 0); for (const scenarioId of applicableRequired) { const scenarioRef = requiredScenarios.find((s) => s.id === scenarioId)!; @@ -160,6 +205,15 @@ export function evaluateAllApplicableRequiredPassV1( notes.push(`digest_mismatch:${scenarioId}`); continue; } + const maxAgeMs = effectiveMaxAgeMs( + suiteManifest.freshness.maxAgeMs, + scenarioMaxAgeById.get(scenarioId) ?? null, + ); + if (maxAgeMs !== null && asOf - obs.completedAt > maxAgeMs) { + missing.push(scenarioId); + notes.push(`stale_observation:${scenarioId}`); + continue; + } if (obs.outcome !== "pass") { missing.push(scenarioId); continue; @@ -176,40 +230,68 @@ export function evaluateAllApplicableRequiredPassV1( }; } +function requireNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + export function parseSuiteManifestFromArtifact(parsed: unknown): SuiteManifestV1 | null { - if (!parsed || typeof parsed !== "object") return null; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null; const raw = parsed as Record; if (raw.schemaVersion !== 1) return null; - if (typeof raw.id !== "string" || typeof raw.version !== "string") return null; - if (typeof raw.verificationRule !== "string") return null; - if (!Array.isArray(raw.scenarios)) return null; - const scenarios = raw.scenarios.map((s) => { - if (!s || typeof s !== "object") return null; + + const id = requireNonEmptyString(raw.id); + const version = requireNonEmptyString(raw.version); + const capability = requireNonEmptyString(raw.capability); + const assertionDslVersion = requireNonEmptyString(raw.assertionDslVersion); + const evidenceSchemaVersion = requireNonEmptyString(raw.evidenceSchemaVersion); + const contradictionRule = requireNonEmptyString(raw.contradictionRule); + const verificationRule = requireNonEmptyString(raw.verificationRule); + if ( + !id || !version || !capability || !assertionDslVersion || + !evidenceSchemaVersion || !contradictionRule || + verificationRule !== "all-applicable-required-pass-v1" + ) return null; + if ( + typeof raw.evidenceLayer !== "string" || + !(EVIDENCE_LAYERS as readonly string[]).includes(raw.evidenceLayer) + ) return null; + const freshness = parseFreshness(raw.freshness); + if (!freshness || !Array.isArray(raw.scenarios)) return null; + + const roles = new Set(["required", "supplemental", "negative_control"]); + const seenScenarioIds = new Set(); + const scenarios: SuiteManifestV1["scenarios"] = []; + for (const s of raw.scenarios) { + if (!s || typeof s !== "object" || Array.isArray(s)) return null; const row = s as Record; - if (typeof row.id !== "string" || typeof row.version !== "string") return null; - if (typeof row.role !== "string" || typeof row.manifestDigest !== "string") return null; - return { - id: row.id, - version: row.version, - role: row.role as SuiteManifestV1["scenarios"][number]["role"], - manifestDigest: row.manifestDigest, - }; - }); - if (scenarios.some((s) => s === null)) return null; + const scenarioId = requireNonEmptyString(row.id); + const scenarioVersion = requireNonEmptyString(row.version); + const manifestDigest = requireNonEmptyString(row.manifestDigest); + if ( + !scenarioId || !scenarioVersion || !manifestDigest || !isSha256Hex(manifestDigest) || + typeof row.role !== "string" || !roles.has(row.role as VerificationRole) || + seenScenarioIds.has(scenarioId) + ) return null; + seenScenarioIds.add(scenarioId); + scenarios.push({ + id: scenarioId, + version: scenarioVersion, + role: row.role as VerificationRole, + manifestDigest, + }); + } + return { schemaVersion: 1, - id: raw.id, - version: raw.version, - evidenceLayer: String(raw.evidenceLayer ?? ""), - capability: String(raw.capability ?? ""), - assertionDslVersion: String(raw.assertionDslVersion ?? ""), - evidenceSchemaVersion: String(raw.evidenceSchemaVersion ?? ""), - freshness: - raw.freshness && typeof raw.freshness === "object" - ? { maxAgeMs: (raw.freshness as { maxAgeMs?: number | null }).maxAgeMs ?? null } - : { maxAgeMs: null }, - contradictionRule: String(raw.contradictionRule ?? ""), - scenarios: scenarios as SuiteManifestV1["scenarios"], - verificationRule: raw.verificationRule, + id, + version, + evidenceLayer: raw.evidenceLayer, + capability, + assertionDslVersion, + evidenceSchemaVersion, + freshness, + contradictionRule, + scenarios, + verificationRule, }; } From 6ed18383c6b18c56c6128185a416e6b1041c10c2 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:04:16 +0200 Subject: [PATCH 094/124] fix(lab): make verdict projection contract-safe --- src/lab/projection/verdicts.ts | 90 +++++++++++++++++++++++----------- 1 file changed, 62 insertions(+), 28 deletions(-) diff --git a/src/lab/projection/verdicts.ts b/src/lab/projection/verdicts.ts index fa3dc95a5e..cafe66cd6f 100644 --- a/src/lab/projection/verdicts.ts +++ b/src/lab/projection/verdicts.ts @@ -1,6 +1,7 @@ import type { CompatibilityVerdict } from "../constants"; import { LAB_PROJECTION_SPEC_VERSION } from "../constants"; import type { SuiteManifestV1 } from "../conformance/suite-manifest"; +import { jcsStringify } from "../digest"; import type { ClaimSnapshotEvent, LabEvent, @@ -14,7 +15,11 @@ import { usableObservations, type InvalidationIndex, } from "../ledger/invalidation"; -import { evaluateAllApplicableRequiredPassV1, newestObservationByScenario } from "./verification"; +import { + evaluateAllApplicableRequiredPassV1, + newestObservationByScenario, + type ScenarioRequirements, +} from "./verification"; export interface ProjectionKey { subjectId: string; @@ -25,15 +30,23 @@ export interface ProjectionKey { projectionSpecVersion: string; } +function componentKey(parts: readonly string[]): string { + return jcsStringify([...parts]); +} + export function projectionKeyString(key: ProjectionKey): string { - return [ + return componentKey([ key.subjectId, key.evidenceLayer, key.suiteId, key.suiteVersion, key.suiteManifestDigest, key.projectionSpecVersion, - ].join("|"); + ]); +} + +export function claimKeyString(subjectId: string, capability: string): string { + return componentKey([subjectId, capability]); } export interface DerivedVerdict { @@ -61,11 +74,7 @@ export interface ProjectVerdictsOptions { unusableClaimEventIds?: Set; loadSuiteManifest?: (digest: string) => SuiteManifestV1 | null; loadScenarioManifest?: (digest: string) => Record | null; - loadScenarioRequirements?: (digest: string) => { - inboundProtocols?: string[]; - upstreamProtocols?: string[]; - surfaces?: string[]; - } | null; + loadScenarioRequirements?: (digest: string) => ScenarioRequirements | null; } /** @@ -74,15 +83,20 @@ export interface ProjectVerdictsOptions { */ export function resolveClaimStates( claims: ClaimSnapshotEvent[], - opts: { unusableClaimEventIds?: Set } = {}, + opts: { + unusableClaimEventIds?: Set; + purgedEventIds?: ReadonlySet; + } = {}, ): { states: Map; corruptions: LedgerCorruption[]; } { const unusableClaims = opts.unusableClaimEventIds ?? new Set(); + const purgedEventIds = opts.purgedEventIds ?? new Set(); const byKey = new Map(); + const allById = new Map(claims.map((claim) => [claim.eventId, claim])); for (const claim of claims) { - const key = `${claim.subjectId}|${claim.capability}`; + const key = claimKeyString(claim.subjectId, claim.capability); const list = byKey.get(key) ?? []; list.push(claim); byKey.set(key, list); @@ -99,12 +113,16 @@ export function resolveClaimStates( }); const superseded = new Set(); - const byId = new Map(sorted.map((c) => [c.eventId, c])); for (const claim of sorted) { for (const pred of claim.supersedes) { - const prev = byId.get(pred); + const prev = allById.get(pred); if (!prev) { + if (purgedEventIds.has(pred)) { + // A purge may physically remove a superseded predecessor. The ID remains + // valid provenance but is not a live claim candidate. + continue; + } corruptions.push({ kind: "claim_corruption", eventId: claim.eventId, @@ -173,6 +191,7 @@ export function projectVerdicts( const claims = usableClaims(events, index).filter((c) => c.effectiveAt <= asOf); const { states: claimStates, corruptions: claimCorruptions } = resolveClaimStates(claims, { unusableClaimEventIds: unusableClaims, + purgedEventIds: index.purgedEventIds, }); corruptions.push(...claimCorruptions); @@ -259,12 +278,25 @@ export function projectVerdicts( return { verdicts, corruptions, index }; } +function isMatchedCapabilityAbsenceControl(obs: ObservationEvent): boolean { + const expected = obs.expectedFailure; + return ( + obs.outcome === "pass" && + !!expected && + expected.controlKind === "capability_absence_control" && + expected.onMatch === "unsupported" + ); +} + function projectObservationGroup( key: ProjectionKey, ordered: ObservationEvent[], asOf: number, suiteManifest: SuiteManifestV1 | null, - opts: { loadScenarioManifest?: (digest: string) => Record | null; loadScenarioRequirements?: ProjectVerdictsOptions["loadScenarioRequirements"] } = {}, + opts: { + loadScenarioManifest?: (digest: string) => Record | null; + loadScenarioRequirements?: ProjectVerdictsOptions["loadScenarioRequirements"]; + } = {}, ): DerivedVerdict { const contributing: string[] = []; const contradicting: string[] = []; @@ -283,6 +315,12 @@ function projectObservationGroup( const currentPasses = currentObservations.filter((o) => o.outcome === "pass"); const currentBlocked = currentObservations.some((o) => o.outcome === "blocked"); const currentInconclusive = currentObservations.some((o) => o.outcome === "inconclusive"); + const matchedCapabilityAbsence = currentObservations.some(isMatchedCapabilityAbsenceControl); + const currentModes = new Set(currentObservations.map((o) => o.executionMode)); + const newestCurrent = [...currentObservations].sort((a, b) => { + if (a.completedAt !== b.completedAt) return a.completedAt - b.completedAt; + return a.eventId < b.eventId ? -1 : a.eventId > b.eventId ? 1 : 0; + }).at(-1); let verdict: CompatibilityVerdict = "UNKNOWN"; if ( @@ -291,21 +329,16 @@ function projectObservationGroup( key.evidenceLayer !== "task_effectiveness" ) { verdict = "UNKNOWN"; + } else if (matchedCapabilityAbsence) { + verdict = "UNSUPPORTED"; + notes.push("capability_absence_control"); } else if (currentFails.length > 0) { - const lastFail = currentFails.sort((a, b) => { - if (a.completedAt !== b.completedAt) return a.completedAt - b.completedAt; - return a.eventId < b.eventId ? -1 : a.eventId > b.eventId ? 1 : 0; - })[currentFails.length - 1]!; - if (lastFail.failure?.class === "capability_failure" && lastFail.expectedFailure) { - verdict = "UNSUPPORTED"; - notes.push("capability_absence_control"); - } else { - verdict = "DEGRADED"; - } + verdict = "DEGRADED"; } else if (currentPasses.length > 0 && !currentInconclusive && !currentBlocked) { - const executionMode = ordered[0]!.executionMode; - const subject = ordered[0]!.subject; - if (key.evidenceLayer === "protocol_conformance" && executionMode === "fixture") { + if (currentModes.size > 1) { + verdict = "PROBED"; + notes.push("mixed_execution_modes"); + } else if (key.evidenceLayer === "protocol_conformance" && newestCurrent?.executionMode === "fixture") { if (!suiteManifest) { verdict = "PROBED"; notes.push("suite_manifest_unavailable"); @@ -313,11 +346,12 @@ function projectObservationGroup( const evaluation = evaluateAllApplicableRequiredPassV1( suiteManifest, ordered, - executionMode, + newestCurrent.executionMode, { - subject: subject.subjectKind === "protocol" ? subject : undefined, + subject: newestCurrent.subject.subjectKind === "protocol" ? newestCurrent.subject : undefined, loadScenarioManifest: opts.loadScenarioManifest, loadScenarioRequirements: opts.loadScenarioRequirements, + asOf, }, ); notes.push(...evaluation.notes); From 06440007c94118ae45465a85f8beeefc4a6ecd28 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:05:11 +0200 Subject: [PATCH 095/124] fix(lab): rebuild projection atomically --- src/lab/projection/rebuild.ts | 188 ++++++++++++++++++++-------------- 1 file changed, 111 insertions(+), 77 deletions(-) diff --git a/src/lab/projection/rebuild.ts b/src/lab/projection/rebuild.ts index 6dd2dc7b3a..bea548167c 100644 --- a/src/lab/projection/rebuild.ts +++ b/src/lab/projection/rebuild.ts @@ -1,18 +1,22 @@ import { Database } from "bun:sqlite"; import { existsSync, unlinkSync } from "node:fs"; -import { createArtifactStore } from "../artifacts/store"; +import { createArtifactStore, loadClaimSourceManifest } from "../artifacts/store"; import { ArtifactFsError } from "../artifacts/secure-fs"; +import { sanitizeDiagnostic } from "../artifacts/sanitize"; import { LAB_PROJECTION_SPEC_VERSION } from "../constants"; import { expandScenario, loadCaseAuthority } from "../conformance/manifest"; import { scenarioManifestDigest, jcsStringify } from "../digest"; -import { parseSuiteManifestFromArtifact } from "./verification"; +import { + parseSuiteManifestFromArtifact, + type ScenarioRequirements, +} from "./verification"; import type { ClaimSnapshotEvent, LabEvent, LedgerCorruption } from "../events/types"; -import { loadClaimSourceManifest } from "../artifacts/store"; import { buildInvalidationIndex, isEventExcluded } from "../ledger/invalidation"; import { replayLabLedger } from "../ledger/store"; import { ensureLabDirs } from "../paths"; import { LAB_SQLITE_DDL, LAB_SQLITE_SCHEMA_VERSION } from "./schema"; import { + claimKeyString, excludeEventIds, projectVerdicts, projectionKeyString, @@ -28,21 +32,27 @@ export interface RebuildResult { function wipeSqlite(path: string): void { for (const candidate of [path, `${path}-wal`, `${path}-shm`]) { + let removed = false; for (let attempt = 0; attempt < 8; attempt++) { try { if (existsSync(candidate)) unlinkSync(candidate); + removed = true; break; } catch (err) { - const code = err && typeof err === "object" && "code" in err ? String((err as { code: unknown }).code) : ""; + const code = err && typeof err === "object" && "code" in err + ? String((err as { code: unknown }).code) + : ""; if (code !== "EBUSY" && code !== "EPERM") throw err; - Bun.sleepSync(20 * (attempt + 1)); + if (attempt < 7) Bun.sleepSync(20 * (attempt + 1)); } } + if (!removed) { + throw new Error(`failed to remove stale projection file after retries: ${candidate}`); + } } } function resetProjectionSchema(db: Database): void { - db.exec("PRAGMA foreign_keys=OFF;"); db.exec(` DROP TABLE IF EXISTS verdicts; DROP TABLE IF EXISTS corruption; @@ -55,7 +65,6 @@ function resetProjectionSchema(db: Database): void { DROP TABLE IF EXISTS events; DROP TABLE IF EXISTS schema_meta; `); - db.exec("PRAGMA foreign_keys=ON;"); db.exec(LAB_SQLITE_DDL); } @@ -81,19 +90,20 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { const validation = validateRequiredArtifacts(replay.events, index, artifactStore, corruptions); const authority = loadCaseAuthority(); - const scenarioRequirementsByDigest = new Map(); + const scenarioRequirementsByDigest = new Map(); for (const caseRecord of authority.cases) { const expanded = expandScenario(caseRecord, authority); - scenarioRequirementsByDigest.set(scenarioManifestDigest(expanded), caseRecord.requirements); + scenarioRequirementsByDigest.set(scenarioManifestDigest(expanded), { + inboundProtocols: [...caseRecord.requirements.inboundProtocols], + upstreamProtocols: [...caseRecord.requirements.upstreamProtocols], + surfaces: [...caseRecord.requirements.surfaces], + freshness: { ...authority.manifestDefaults.freshness }, + }); } const loadSuiteManifest = (digest: string) => { try { - const bytes = artifactStore.get(digest); + const bytes = artifactStore.get(digest, { artifactClass: "suite_manifest" }); const parsed = JSON.parse(new TextDecoder().decode(bytes)); return parseSuiteManifestFromArtifact(parsed); } catch { @@ -102,7 +112,7 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { }; const loadScenarioManifest = (digest: string) => { try { - const bytes = artifactStore.get(digest); + const bytes = artifactStore.get(digest, { artifactClass: "scenario_manifest" }); return JSON.parse(new TextDecoder().decode(bytes)) as Record; } catch { return null; @@ -111,9 +121,14 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { const loadScenarioRequirements = (digest: string) => scenarioRequirementsByDigest.get(digest) ?? null; const db = new Database(paths.sqlitePath); + let transactionOpen = false; try { db.exec("PRAGMA journal_mode=DELETE;"); + db.exec("PRAGMA foreign_keys=OFF;"); + db.exec("BEGIN IMMEDIATE;"); + transactionOpen = true; resetProjectionSchema(db); + db.prepare( "INSERT OR REPLACE INTO schema_meta(key, value) VALUES (?, ?)", ).run("schema_version", String(LAB_SQLITE_SCHEMA_VERSION)); @@ -158,8 +173,20 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { VALUES (?, ?, ?, ?, ?)`, ); const insertArtifact = db.prepare( - `INSERT OR REPLACE INTO artifacts(digest, artifact_class, media_type, byte_count, status, last_error) - VALUES (?, ?, ?, ?, ?, ?)`, + `INSERT INTO artifacts(digest, artifact_class, media_type, byte_count, status, last_error) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(digest) DO UPDATE SET + artifact_class = COALESCE(excluded.artifact_class, artifacts.artifact_class), + media_type = COALESCE(excluded.media_type, artifacts.media_type), + byte_count = COALESCE(excluded.byte_count, artifacts.byte_count), + status = CASE + WHEN artifacts.status = 'purged_unavailable' OR excluded.status = 'purged_unavailable' + THEN 'purged_unavailable' + WHEN artifacts.status = 'corrupt' OR excluded.status = 'corrupt' + THEN 'corrupt' + ELSE 'present' + END, + last_error = COALESCE(excluded.last_error, artifacts.last_error)`, ); const excluded = excludeEventIds(index); @@ -167,19 +194,9 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { (e): e is ClaimSnapshotEvent => e.eventKind === "claim_snapshot" && !isEventExcluded(e.eventId, index), ); - - for (const claim of usableClaimEvents) { - const loaded = loadClaimSourceManifest(artifactStore, claim.sourceManifestDigest, { - subjectId: claim.subjectId, - capability: claim.capability, - }); - if (loaded.corruption) { - validation.unusableClaimEventIds.add(claim.eventId); - } - } - const claimStates = resolveClaimStates(usableClaimEvents, { unusableClaimEventIds: validation.unusableClaimEventIds, + purgedEventIds: index.purgedEventIds, }); for (const event of replay.events) { @@ -232,42 +249,24 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { } } else if (event.eventKind === "claim_snapshot") { insertSubject.run(event.subjectId, event.subject.subjectKind, jcsStringify(event.subject)); - const key = `${event.subjectId}|${event.capability}`; + const key = claimKeyString(event.subjectId, event.capability); const state = claimStates.states.get(key); const current = state?.current?.eventId === event.eventId ? 1 : 0; - let usable = !isExcluded && !state?.corruption && !validation.unusableClaimEventIds.has(event.eventId) ? 1 : 0; + const claimCorruption = corruptions.find( + (c) => c.kind === "claim_corruption" && c.eventId === event.eventId, + ); + const usable = !isExcluded && !state?.corruption && + !validation.unusableClaimEventIds.has(event.eventId) ? 1 : 0; if (!isExcluded) { - const loaded = loadClaimSourceManifest(artifactStore, event.sourceManifestDigest, { - subjectId: event.subjectId, - capability: event.capability, - }); - if (loaded.corruption) { - usable = 0; - corruptions.push({ - kind: "claim_corruption", - eventId: event.eventId, - detail: loaded.corruption, - }); - insertCorruption.run("claim_corruption", null, event.eventId, loaded.corruption); - insertArtifact.run( - event.sourceManifestDigest, - "claim_source_manifest", - "application/json", - null, - "corrupt", - loaded.corruption, - ); - } else { - insertArtifact.run( - event.sourceManifestDigest, - "claim_source_manifest", - "application/json", - null, - "present", - null, - ); - } + insertArtifact.run( + event.sourceManifestDigest, + "claim_source_manifest", + "application/json", + null, + claimCorruption ? "corrupt" : "present", + claimCorruption?.detail ?? null, + ); } insertClaim.run( @@ -329,13 +328,6 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { ); for (const v of verdicts) { - if (v.contributingEventIds.every((id) => index.purgedEventIds.has(id))) continue; - if (v.contributingEventIds.some((id) => index.purgedEventIds.has(id) || index.invalidatedBy.has(id))) { - const remaining = v.contributingEventIds.filter( - (id) => !index.purgedEventIds.has(id) && !index.invalidatedBy.has(id), - ); - if (remaining.length === 0) continue; - } insertVerdict.run( projectionKeyString(v.key), v.key.subjectId, @@ -354,13 +346,29 @@ export function rebuildLabProjection(configDir?: string): RebuildResult { ); } + db.exec("COMMIT;"); + transactionOpen = false; return { events: replay.events.length, verdicts: verdicts.length, corruptions, sqlitePath: paths.sqlitePath, }; + } catch (err) { + if (transactionOpen) { + try { + db.exec("ROLLBACK;"); + } catch { + // Preserve the original rebuild failure. + } + } + throw err; } finally { + try { + db.exec("PRAGMA foreign_keys=ON;"); + } catch { + // Closing the disposable DB is still safe if pragma restoration fails. + } db.close(); artifactStore.close(); } @@ -378,13 +386,13 @@ function validateRequiredArtifacts( for (const event of events) { if (isEventExcluded(event.eventId, index)) continue; if (event.eventKind === "observation") { - const required = [ - event.scenarioManifestDigest, - event.suiteManifestDigest, - ...event.fixtureDigests, + const required: Array<{ digest: string; artifactClass: "scenario_manifest" | "suite_manifest" | "fixture" }> = [ + { digest: event.scenarioManifestDigest, artifactClass: "scenario_manifest" }, + { digest: event.suiteManifestDigest, artifactClass: "suite_manifest" }, + ...event.fixtureDigests.map((digest) => ({ digest, artifactClass: "fixture" as const })), ]; let unusable = false; - for (const digest of required) { + for (const { digest, artifactClass } of required) { if (index.purgedArtifactDigests.has(digest)) { corruptions.push({ kind: "missing_artifact", @@ -395,19 +403,46 @@ function validateRequiredArtifacts( continue; } try { - artifactStore.get(digest); + artifactStore.get(digest, { artifactClass }); } catch (err) { + const detail = sanitizeDiagnostic(err); corruptions.push({ - kind: err instanceof ArtifactFsError && err.message.includes("mismatch") + kind: err instanceof ArtifactFsError && + (err.code === "artifact_mismatch" || err.message.includes("mismatch")) ? "artifact_mismatch" : "missing_artifact", eventId: event.eventId, - detail: err instanceof Error ? err.message : String(err), + detail, }); unusable = true; } } if (unusable) unusableObservationIds.add(event.eventId); + continue; + } + + if (event.eventKind === "claim_snapshot") { + if (index.purgedArtifactDigests.has(event.sourceManifestDigest)) { + unusableClaimEventIds.add(event.eventId); + corruptions.push({ + kind: "claim_corruption", + eventId: event.eventId, + detail: `claim source artifact purged: ${event.sourceManifestDigest}`, + }); + continue; + } + const loaded = loadClaimSourceManifest(artifactStore, event.sourceManifestDigest, { + subjectId: event.subjectId, + capability: event.capability, + }); + if (!loaded.ok) { + unusableClaimEventIds.add(event.eventId); + corruptions.push({ + kind: "claim_corruption", + eventId: event.eventId, + detail: loaded.corruption, + }); + } } } @@ -418,7 +453,7 @@ function validateRequiredArtifacts( export function readVerdictSnapshot(sqlitePath: string): unknown[] { const db = new Database(sqlitePath, { readonly: true }); try { - const rows = db + return db .query( `SELECT projection_key, subject_id, evidence_layer, suite_id, suite_version, suite_manifest_digest, projection_spec_version, verdict, @@ -427,7 +462,6 @@ export function readVerdictSnapshot(sqlitePath: string): unknown[] { FROM verdicts ORDER BY projection_key`, ) .all(); - return rows; } finally { db.close(); } From 9ba59cdcf884de7afb5462d4d5315088a34b868d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:06:25 +0200 Subject: [PATCH 096/124] fix(lab): accept measured runner timestamps --- src/lab/observe/from-conformance.ts | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/lab/observe/from-conformance.ts b/src/lab/observe/from-conformance.ts index 44b7e35cb3..5a165439c2 100644 --- a/src/lab/observe/from-conformance.ts +++ b/src/lab/observe/from-conformance.ts @@ -33,6 +33,11 @@ import { resolveProtocolExecutionContext } from "../conformance/executor"; const COMPAT_VERSION = "protocol-v1"; +type TimedScenarioRunResult = ScenarioRunResult & { + startedAt?: number; + completedAt?: number; +}; + export interface PersistConformanceOptions { configDir?: string; recordedAt?: number; @@ -154,16 +159,20 @@ function outcomeFromResult(result: ScenarioRunResult): ObservationOutcome { } } -function requireExecutionTimes(opts: PersistConformanceOptions): { startedAt: number; completedAt: number } { - if (!Number.isInteger(opts.startedAt) || !Number.isInteger(opts.completedAt)) { +function requireExecutionTimes( + result: ScenarioRunResult, + opts: PersistConformanceOptions, +): { startedAt: number; completedAt: number } { + const timed = result as TimedScenarioRunResult; + const startedAt = opts.startedAt ?? timed.startedAt; + const completedAt = opts.completedAt ?? timed.completedAt; + if (!Number.isInteger(startedAt) || !Number.isInteger(completedAt)) { throw new Error("real startedAt/completedAt are required for persisted conformance evidence"); } - const startedAt = opts.startedAt!; - const completedAt = opts.completedAt!; - if (startedAt < 0 || completedAt < startedAt) { + if (startedAt! < 0 || completedAt! < startedAt!) { throw new Error("invalid persisted conformance execution timestamps"); } - return { startedAt, completedAt }; + return { startedAt: startedAt!, completedAt: completedAt! }; } /** @@ -180,7 +189,7 @@ export function observationFromConformanceResult( const ownsStore = !opts.artifactStore; const store = opts.artifactStore ?? createArtifactStore(paths.artifactsDir); try { - const { startedAt, completedAt } = requireExecutionTimes(opts); + const { startedAt, completedAt } = requireExecutionTimes(result, opts); const recordedAt = opts.recordedAt ?? completedAt; const expandedScenario = expandScenario(caseRecord, authority); From 1ac3d3235d3958c900c6f2f93c7d6cd774ffe81d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:08:58 +0200 Subject: [PATCH 097/124] test(lab): cover phase-2 evidence fixes --- tests/lab-evidence-ledger.test.ts | 163 ++++++++++++++++++------------ 1 file changed, 96 insertions(+), 67 deletions(-) diff --git a/tests/lab-evidence-ledger.test.ts b/tests/lab-evidence-ledger.test.ts index b9aea8501d..9678c9dd4d 100644 --- a/tests/lab-evidence-ledger.test.ts +++ b/tests/lab-evidence-ledger.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, test } from "bun:test"; import { chmodSync, existsSync, mkdirSync, writeFileSync, symlinkSync, linkSync, rmSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; @@ -6,6 +6,7 @@ import { appendLabEvent, assignEventId, buildInvalidationIndex, + claimKeyString, claimSourceManifestDigest, createArtifactStore, eventIdForPayload, @@ -33,7 +34,7 @@ import { ArtifactFsError, closeTrustedArtifactDir, putArtifactBytes, putNamedDig import { expandSuiteManifest } from "../src/lab/conformance/suite-manifest"; import { evaluateAllApplicableRequiredPassV1 } from "../src/lab/projection/verification"; import { discoverScenarios, expandScenario, loadCaseAuthority } from "../src/lab/conformance/manifest"; -import { scenarioManifestDigest } from "../src/lab/digest"; +import { artifactBytesDigest, scenarioManifestDigest } from "../src/lab/digest"; import type { CaseRecord } from "../src/lab/conformance/types"; import { runScenario, resolveProtocolExecutionContext } from "../src/lab/conformance/executor"; import { LabValidationError } from "../src/lab/events/validate"; @@ -49,10 +50,6 @@ function tempHome(): string { return dir; } -beforeEach(() => { - // OPENCODEX_HOME isolation -}); - afterEach(() => { for (const dir of HOMES.splice(0)) { try { @@ -75,7 +72,7 @@ function syntheticPassResult(caseRecord: CaseRecord) { scenarioId: caseRecord.id, suite: caseRecord.suite, passed: true, - classification: "protocol_failure" as const, + classification: "inconclusive" as const, assertionResults: caseRecord.assertions.map((a) => ({ id: a.id, operator: a.operator, @@ -85,6 +82,8 @@ function syntheticPassResult(caseRecord: CaseRecord) { })), diagnostics: [], executionContext: resolveProtocolExecutionContext(caseRecord), + startedAt: 999, + completedAt: 1000, }; } @@ -209,7 +208,6 @@ describe("CL-02 JCS and IDs", () => { const id2 = eventIdForPayload(payload); expect(id1).toBe(id2); expect(isSha256Hex(id1)).toBe(true); - // Domain separation: different domain changes digest expect(id1).not.toBe(subjectId); }); @@ -222,7 +220,6 @@ describe("CL-02 ledger append/replay", () => { test("append and replay round-trip", () => { withHome((home) => { const event = baseObservation(); - // Store required artifacts as opaque named digests for projection const store = createArtifactStore(join(home, "lab", "artifacts")); for (const ref of event.artifactRefs) { store.put({ @@ -231,7 +228,6 @@ describe("CL-02 ledger append/replay", () => { expectedDigest: undefined, }); } - // Write contract-named bytes for digests referenced by the event const dir = openTrustedArtifactDir(join(home, "lab", "artifacts")); try { for (const ref of event.artifactRefs) { @@ -322,6 +318,18 @@ describe("CL-02 invalidation validation", () => { }); const index2 = buildInvalidationIndex([obs, bad as never]); expect(index2.corruptions.some((c) => c.kind === "invalid_reference")).toBe(true); + + const invalidatesInvalidation = assignEventId({ + schemaVersion: LAB_EVENT_SCHEMA_VERSION, + eventKind: "invalidation" as const, + recordedAt: obs.recordedAt + 3, + producer: LAB_PRODUCER, + producerVersion: "2.10.2", + targetEventIds: [inv.eventId], + reason: "manual_correction" as const, + }); + const index3 = buildInvalidationIndex([obs, inv as never, invalidatesInvalidation as never]); + expect(index3.corruptions.some((c) => c.kind === "invalid_reference")).toBe(true); }); }); @@ -395,7 +403,7 @@ describe("CL-02 claim supersession and conflicts", () => { }) as ClaimSnapshotEvent; const ok = resolveClaimStates([c1, c2]); - expect(ok.states.get(`${subjectId}|tools`)?.current?.eventId).toBe(c2.eventId); + expect(ok.states.get(claimKeyString(subjectId, "tools"))?.current?.eventId).toBe(c2.eventId); const c3 = assignEventId({ ...c2, @@ -405,7 +413,7 @@ describe("CL-02 claim supersession and conflicts", () => { supersedes: [], }) as ClaimSnapshotEvent; const conflict = resolveClaimStates([c1, c2, c3]); - expect(conflict.states.get(`${subjectId}|tools`)?.corruption).toBeTruthy(); + expect(conflict.states.get(claimKeyString(subjectId, "tools"))?.corruption).toBeTruthy(); }); test("ClaimSourceManifest rejects secrets and unknown facts", () => { @@ -427,15 +435,19 @@ describe("CL-02 artifacts and secure FS", () => { test("content-addressed put/get and size ceiling", () => { withHome((home) => { const store = createArtifactStore(join(home, "lab", "artifacts")); - const ref = store.put({ artifactClass: "assertion_report", payload: { a: 1 } }); - expect(isSha256Hex(ref.digest)).toBe(true); - const bytes = store.get(ref.digest); - expect(bytes.byteLength).toBe(ref.byteCount); + try { + const ref = store.put({ artifactClass: "assertion_report", payload: { a: 1 } }); + expect(isSha256Hex(ref.digest)).toBe(true); + const bytes = store.get(ref.digest); + expect(bytes.byteLength).toBe(ref.byteCount); - const huge = new Uint8Array(256 * 1024 + 1); - expect(() => - store.put({ artifactClass: "assertion_report", payload: huge }), - ).toThrow(); + const huge = new Uint8Array(256 * 1024 + 1); + expect(() => + store.put({ artifactClass: "assertion_report", payload: huge }), + ).toThrow(); + } finally { + store.close(); + } }); }); @@ -463,7 +475,6 @@ describe("CL-02 artifacts and secure FS", () => { try { symlinkSync(target, linkPath); } catch { - // Windows may require elevation for symlinks return; } const dir = openTrustedArtifactDir(artifacts); @@ -563,12 +574,15 @@ describe("CL-02 projection rebuild determinism", () => { let recordedAt = 1_700_000_000_000; for (const caseRecord of scenarios) { const store = createArtifactStore(join(home, "lab", "artifacts")); - persistConformanceResult(syntheticPassResult(caseRecord), caseRecord, authority, { - configDir: home, - recordedAt: recordedAt++, - artifactStore: store, - }); - store.close(); + try { + persistConformanceResult(syntheticPassResult(caseRecord), caseRecord, authority, { + configDir: home, + recordedAt: recordedAt++, + artifactStore: store, + }); + } finally { + store.close(); + } } const rebuilt = rebuildLabProjection(home); const snap = readVerdictSnapshot(rebuilt.sqlitePath); @@ -585,13 +599,14 @@ describe("CL-02 projection rebuild determinism", () => { scenarioId: caseRecord.id, suite: caseRecord.suite, passed: true, - classification: "protocol_failure", + classification: "inconclusive", assertionResults: [], diagnostics: [], + executionContext: resolveProtocolExecutionContext(caseRecord), }, caseRecord, authority, - { configDir: home, recordedAt: 1000 }, + { configDir: home, recordedAt: 1000, startedAt: 999, completedAt: 1000 }, ); const inv = assignEventId({ schemaVersion: LAB_EVENT_SCHEMA_VERSION, @@ -618,13 +633,14 @@ describe("CL-02 projection rebuild determinism", () => { scenarioId: caseRecord.id, suite: caseRecord.suite, passed: true, - classification: "protocol_failure", + classification: "inconclusive", assertionResults: [], diagnostics: [], + executionContext: resolveProtocolExecutionContext(caseRecord), }, caseRecord, authority, - { configDir: home, recordedAt: 1000 }, + { configDir: home, recordedAt: 1000, startedAt: 999, completedAt: 1000 }, ); purgeSensitiveEvidence({ configDir: home, @@ -649,16 +665,22 @@ describe("CL-02 CL-01 integration", () => { const caseRecord = discoverScenarios(authority, ["responses-core"]).find( (c) => c.id === "responses-core.protocol.request-shape", )!; + const startedAt = Date.now(); const result = await runScenario(caseRecord); + const completedAt = Date.now(); expect(result.passed).toBe(true); const { event } = observationFromConformanceResult(result, caseRecord, authority, { configDir: home, - recordedAt: 1_800_000_000_000, + recordedAt: completedAt, + startedAt, + completedAt, }); expect(validateLabEvent(event).eventKind).toBe("observation"); persistConformanceResult(result, caseRecord, authority, { configDir: home, - recordedAt: 1_800_000_000_000, + recordedAt: completedAt, + startedAt, + completedAt, }); const replay = replayLabLedger(join(home, "lab", "compatibility.jsonl")); expect(replay.validLineCount).toBe(1); @@ -677,21 +699,24 @@ describe("CL-02 privacy canaries", () => { withHome((home) => { const secretCanary = "sk-" + "a".repeat(32); const store = createArtifactStore(join(home, "lab", "artifacts")); - const ref = store.put({ - artifactClass: "error_taxonomy", - payload: { - message: `failed ${secretCanary}`, - path: "C:\\Users\\victim\\secrets\\token.txt", - authorization: "Bearer SUPERSECRET", - url: "https://user:pass@example.com/v1", - }, - }); - const text = new TextDecoder().decode(store.get(ref.digest)); - expect(text).not.toContain(secretCanary); - expect(text).not.toContain("SUPERSECRET"); - expect(text).not.toContain("victim"); - expect(text).not.toContain("user:pass"); - store.close(); + try { + const ref = store.put({ + artifactClass: "error_taxonomy", + payload: { + message: `failed ${secretCanary}`, + path: "C:\\Users\\victim\\secrets\\token.txt", + authorization: "Bearer SUPERSECRET", + url: "https://user:pass@example.com/v1", + }, + }); + const text = new TextDecoder().decode(store.get(ref.digest)); + expect(text).not.toContain(secretCanary); + expect(text).not.toContain("SUPERSECRET"); + expect(text).not.toContain("victim"); + expect(text).not.toContain("user:pass"); + } finally { + store.close(); + } }); }); }); @@ -709,7 +734,6 @@ describe("CL-02 empty/corrupt ledger", () => { }); }); -// Keep chmod import used on POSIX permission smoke (best-effort). void chmodSync; void existsSync; void claimSourceManifestDigest; @@ -728,15 +752,18 @@ describe("CL-02 review regression coverage", () => { const subjectIds = new Set(); for (const caseRecord of scenarios.slice(0, 2)) { const store = createArtifactStore(join(home, "lab", "artifacts")); - const { event } = observationFromConformanceResult( - syntheticPassResult(caseRecord), - caseRecord, - authority, - { configDir: home, recordedAt: t++, artifactStore: store }, - ); - store.close(); - subjectIds.add(event.subjectId); - appendLabEvent(join(home, "lab", "compatibility.jsonl"), event); + try { + const { event } = observationFromConformanceResult( + syntheticPassResult(caseRecord), + caseRecord, + authority, + { configDir: home, recordedAt: t++, artifactStore: store }, + ); + subjectIds.add(event.subjectId); + appendLabEvent(join(home, "lab", "compatibility.jsonl"), event); + } finally { + store.close(); + } } expect(subjectIds.size).toBe(1); const replay = replayLabLedger(join(home, "lab", "compatibility.jsonl")); @@ -756,13 +783,15 @@ describe("CL-02 review regression coverage", () => { let t = 1000; const events = scenarios.map((caseRecord) => { const store = createArtifactStore(join(home, "lab", "artifacts")); - const persisted = persistConformanceResult(syntheticPassResult(caseRecord), caseRecord, authority, { - configDir: home, - recordedAt: t++, - artifactStore: store, - }); - store.close(); - return persisted.event; + try { + return persistConformanceResult(syntheticPassResult(caseRecord), caseRecord, authority, { + configDir: home, + recordedAt: t++, + artifactStore: store, + }).event; + } finally { + store.close(); + } }); const sharedDigest = events[0]!.suiteManifestDigest; expect(events[1]!.suiteManifestDigest).toBe(sharedDigest); @@ -854,7 +883,7 @@ describe("CL-02 phase-2 review regressions", () => { const artifacts = join(home, "lab", "artifacts"); mkdirSync(artifacts, { recursive: true }); const bytes = new TextEncoder().encode("reuse-me"); - const digest = Bun.CryptoHasher.hash("sha256", bytes, "hex"); + const digest = artifactBytesDigest(bytes); const linkPath = join(artifacts, `${digest}.bin`); const outside = join(home, "outside.bin"); writeFileSync(outside, "evil"); @@ -1129,4 +1158,4 @@ describe("CL-02 phase-2 review regressions", () => { closeTrustedArtifactDir(dir); }); }); -}); +}); \ No newline at end of file From daab377c228eea7286344f484081e77c13ec3005 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:11:09 +0200 Subject: [PATCH 098/124] fix(lab): classify artifact filesystem failures --- src/lab/artifacts/secure-fs.ts | 98 +++++++++++++++++++--------------- 1 file changed, 54 insertions(+), 44 deletions(-) diff --git a/src/lab/artifacts/secure-fs.ts b/src/lab/artifacts/secure-fs.ts index 23d05c5a45..421d0669bc 100644 --- a/src/lab/artifacts/secure-fs.ts +++ b/src/lab/artifacts/secure-fs.ts @@ -36,8 +36,8 @@ export class ArtifactFsError extends Error { } } -export function harnessFailure(message: string): never { - throw new ArtifactFsError("harness_failure", message); +export function harnessFailure(message: string, code = "harness_failure"): never { + throw new ArtifactFsError(code, message); } const O_RDONLY = fsConstants.O_RDONLY; @@ -82,16 +82,16 @@ function assertRegularFileStats(stats: Stats, label: string): void { stats.isCharacterDevice() || stats.isBlockDevice() ) { - harnessFailure(`${label}: not a regular file`); + harnessFailure(`${label}: not a regular file`, "artifact_unsafe_target"); } if (stats.nlink !== 1) { - harnessFailure(`${label}: hard links prohibited (nlink=${stats.nlink})`); + harnessFailure(`${label}: hard links prohibited (nlink=${stats.nlink})`, "artifact_unsafe_target"); } } function assertDirectoryStats(stats: Stats, label: string): void { if (!stats.isDirectory() || stats.isSymbolicLink()) { - harnessFailure(`${label}: must be a real directory (no symlink/reparse redirection)`); + harnessFailure(`${label}: must be a real directory (no symlink/reparse redirection)`, "artifact_unsafe_target"); } } @@ -101,7 +101,7 @@ function identityOf(stats: Stats): string { function assertRelativeName(name: string): void { if (name.includes("..") || name.includes("/") || name.includes("\\") || name.includes("\0")) { - harnessFailure("invalid relative artifact name"); + harnessFailure("invalid relative artifact name", "artifact_unsafe_target"); } } @@ -164,7 +164,7 @@ function assertOpenedPathMatchesDescriptor(dir: TrustedArtifactDir, name: string pathEntry.ino !== opened.ino ) { closeSync(fd); - harnessFailure("artifact path identity mismatch after open"); + harnessFailure("artifact path identity mismatch after open", "artifact_unsafe_target"); } } @@ -205,13 +205,13 @@ function revalidateDir(dir: TrustedArtifactDir): void { const stats = fstatSync(dir.fd); assertDirectoryStats(stats, "artifacts dir"); if (identityOf(stats) !== dir.identity) { - harnessFailure("artifacts directory identity changed"); + harnessFailure("artifacts directory identity changed", "artifact_unsafe_target"); } } export function openTrustedArtifactDir(artifactsDir: string): TrustedArtifactDir { const abs = artifactsDir.replace(/[\\/]+$/, ""); - if (abs.includes("\0")) harnessFailure("NUL in artifacts path"); + if (abs.includes("\0")) harnessFailure("NUL in artifacts path", "artifact_unsafe_target"); mkdirSync(abs, { recursive: true, mode: 0o700 }); let fd: number; @@ -219,7 +219,7 @@ export function openTrustedArtifactDir(artifactsDir: string): TrustedArtifactDir try { fd = openSync(abs, openFlags(O_RDONLY | O_DIRECTORY, true)); } catch { - harnessFailure("failed to open artifacts directory with O_DIRECTORY"); + harnessFailure("failed to open artifacts directory with O_DIRECTORY", "artifact_unsafe_target"); } } else { fd = openSync(abs, O_RDONLY); @@ -259,15 +259,16 @@ function readAllFromFd(fd: number, size: number): Buffer { if (n <= 0) break; offset += n; } - if (offset !== size) harnessFailure("short read from artifact descriptor"); + if (offset !== size) harnessFailure("short read from artifact descriptor", "artifact_mismatch"); return buf; } +function isRawMissingError(err: unknown): boolean { + return !!err && typeof err === "object" && "code" in err && (err as { code: string }).code === "ENOENT"; +} + function isMissingArtifactError(err: unknown): boolean { - if (err && typeof err === "object" && "code" in err && (err as { code: string }).code === "ENOENT") { - return true; - } - return err instanceof ArtifactFsError && err.message.includes("missing"); + return isRawMissingError(err) || (err instanceof ArtifactFsError && err.code === "artifact_missing"); } function assertArtifactTargetCreatable(dir: TrustedArtifactDir, name: string): void { @@ -276,16 +277,26 @@ function assertArtifactTargetCreatable(dir: TrustedArtifactDir, name: string): v try { const stats = lstatSync(childPath(dir, name)); if (stats.isSymbolicLink()) { - harnessFailure("artifact target is a symbolic link"); + harnessFailure("artifact target is a symbolic link", "artifact_unsafe_target"); } assertRegularFileStats(stats, "artifact create target"); - harnessFailure("artifact target exists but is not reusable"); + harnessFailure("artifact target exists but is not reusable", "artifact_unsafe_target"); } catch (err) { - if (err && typeof err === "object" && "code" in err && (err as { code: string }).code === "ENOENT") { - return; - } + if (isRawMissingError(err)) return; if (err instanceof ArtifactFsError) throw err; - harnessFailure(`artifact create target check failed: ${err instanceof Error ? err.message : String(err)}`); + harnessFailure( + `artifact create target check failed: ${err instanceof Error ? err.message : String(err)}`, + "artifact_unsafe_target", + ); + } +} + +function writeAll(fd: number, bytes: Uint8Array): void { + let offset = 0; + while (offset < bytes.byteLength) { + const n = writeSync(fd, bytes, offset, bytes.byteLength - offset); + if (n <= 0) harnessFailure("artifact write made no progress", "artifact_mismatch"); + offset += n; } } @@ -299,14 +310,17 @@ function writeTempArtifact( let fd: number | null = null; try { fd = openAtDir(dir, tmpName, openFlags(O_RDWR | O_CREAT | O_EXCL, true), 0o600); - const written = writeSync(fd, bytes); - if (written !== bytes.byteLength) harnessFailure("short write"); + writeAll(fd, bytes); fsyncSync(fd); const stats = fstatSync(fd); assertRegularFileStats(stats, "artifact temp"); - if (stats.size !== bytes.byteLength) harnessFailure("size mismatch after write"); + if (stats.size !== bytes.byteLength) { + harnessFailure("size mismatch after write", "artifact_mismatch"); + } const buf = readAllFromFd(fd, bytes.byteLength); - if (contentDigest(buf) !== digest) harnessFailure("digest mismatch on same descriptor"); + if (contentDigest(buf) !== digest) { + harnessFailure("digest mismatch on same descriptor", "artifact_mismatch"); + } closeSync(fd); fd = null; renameAtDir(dir, tmpName, digestFileName(digest)); @@ -344,16 +358,18 @@ export function readArtifactBytes( const stats = fstatSync(fd); assertRegularFileStats(stats, "artifact fd"); if (opts.expectedByteCount !== undefined && stats.size !== opts.expectedByteCount) { - harnessFailure("artifact size mismatch on descriptor"); + harnessFailure("artifact size mismatch on descriptor", "artifact_mismatch"); + } + if (stats.size > MAX_BYTES_PER_ARTIFACT) { + harnessFailure("artifact exceeds ceiling", "artifact_mismatch"); } - if (stats.size > MAX_BYTES_PER_ARTIFACT) harnessFailure("artifact exceeds ceiling"); const buf = readAllFromFd(fd, stats.size); const got = contentDigest(buf); - if (got !== digest) harnessFailure("artifact digest mismatch on descriptor"); + if (got !== digest) harnessFailure("artifact digest mismatch on descriptor", "artifact_mismatch"); return { digest, bytes: new Uint8Array(buf), byteCount: stats.size }; } catch (err) { - if (isMissingArtifactError(err)) { - harnessFailure(`artifact missing: ${digest}`); + if (isRawMissingError(err)) { + harnessFailure(`artifact missing: ${digest}`, "artifact_missing"); } if (err instanceof ArtifactFsError) throw err; harnessFailure(`artifact read failed: ${err instanceof Error ? err.message : String(err)}`); @@ -375,15 +391,15 @@ export function putArtifactBytes( const digest = artifactBytesDigest(bytes); if (expectedDigest !== undefined) { assertDigestName(expectedDigest); - if (digest !== expectedDigest) harnessFailure("artifact digest mismatch before write"); + if (digest !== expectedDigest) { + harnessFailure("artifact digest mismatch before write", "artifact_mismatch"); + } } try { return readArtifactBytes(dir, digest, bytes.byteLength); } catch (err) { - if (!isMissingArtifactError(err)) { - throw err; - } + if (!isMissingArtifactError(err)) throw err; } assertArtifactTargetCreatable(dir, digestFileName(digest)); @@ -404,15 +420,13 @@ export function putNamedDigestBytes( harnessFailure(`artifact exceeds ${MAX_BYTES_PER_ARTIFACT} bytes`); } if (contentDigest(bytes) !== digest) { - harnessFailure("named artifact content digest mismatch before write"); + harnessFailure("named artifact content digest mismatch before write", "artifact_mismatch"); } try { return readArtifactBytes(dir, digest, { expectedByteCount: bytes.byteLength, contentDigest }); } catch (err) { - if (!isMissingArtifactError(err)) { - throw err; - } + if (!isMissingArtifactError(err)) throw err; } assertArtifactTargetCreatable(dir, digestFileName(digest)); @@ -428,9 +442,7 @@ export function deleteArtifactBytes(dir: TrustedArtifactDir, digest: string): vo try { unlinkAtDir(dir, name); } catch (err) { - if (err && typeof err === "object" && "code" in err && (err as { code: string }).code === "ENOENT") { - return; - } + if (isRawMissingError(err)) return; if (err instanceof ArtifactFsError) throw err; harnessFailure(`artifact delete failed: ${err instanceof Error ? err.message : String(err)}`); } @@ -449,9 +461,7 @@ export function artifactExists(dir: TrustedArtifactDir, digest: string): boolean closeSync(fd); } } catch (err) { - if (isMissingArtifactError(err)) { - return false; - } + if (isMissingArtifactError(err)) return false; if (err instanceof ArtifactFsError) throw err; harnessFailure(`artifact exists check failed: ${err instanceof Error ? err.message : String(err)}`); } From e5af56aea64f1804bff592c63f4b8f95a7c70ca1 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:11:37 +0200 Subject: [PATCH 099/124] fix(lab): remove raw ledger production reader --- src/lab/ledger/purge.ts | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/src/lab/ledger/purge.ts b/src/lab/ledger/purge.ts index d666624532..700221b0eb 100644 --- a/src/lab/ledger/purge.ts +++ b/src/lab/ledger/purge.ts @@ -28,7 +28,6 @@ import { fsyncSync, openSync, readdirSync, - readFileSync, renameSync, rmSync, unlinkSync, @@ -97,20 +96,13 @@ function atomicRewriteLedger(ledgerPath: string, events: LabEvent[]): void { } } -function isArtifactMissing(err: unknown): boolean { - return err instanceof ArtifactFsError && ( - err.code === "artifact_missing" || - (err.code === "harness_failure" && err.message.includes("missing")) - ); -} - function deleteArtifactsFailClosed(dir: TrustedArtifactDir, digests: string[]): void { const errors: string[] = []; for (const digest of digests) { try { deleteArtifactBytes(dir, digest); } catch (err) { - if (isArtifactMissing(err)) continue; + if (err instanceof ArtifactFsError && err.code === "artifact_missing") continue; errors.push(err instanceof Error ? err.message : String(err)); } } @@ -232,9 +224,3 @@ export function purgeSensitiveEvidence(req: SensitivePurgeRequest): PurgeTombsto if (dir) closeTrustedArtifactDir(dir); } } - -/** Test helper retained temporarily for compatibility; production callers should replay validated events. */ -export function readLedgerText(configDir?: string): string { - const paths = ensureLabDirs(configDir); - return readFileSync(paths.ledgerPath, "utf8"); -} From 02ca5b561dd03db6a371f47a41bb07893f4c02f5 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:13:11 +0200 Subject: [PATCH 100/124] docs(lab): document sensitive purge exception --- .../001_pr_stack_status.md | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index 90a9a56def..a7a2308790 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -21,7 +21,7 @@ independent review, blockers, and whether a later phase is authorized. |---|---|---|---|---|---| | CL-00 | `feat/cl-00-compatibility-contracts` | `3ad5bb6bd3f76f6879d84b78ea39edd3e01ec296` | `c014464237fd3c95bda08bc18bfab8ba8f532308` | [#1286](https://github.com/lidge-jun/opencodex/pull/1286) | ACCEPTED AFTER CODERABBIT REMEDIATION (merged to `dev` at `243c3f4905797aa11c62ba933bb03d6d721266fd`) | | CL-01 | `feat/cl-01-conformance-harness` | `c2113ca47b8a05c5a5f90679e4eaa640ca2c6a66` | `22d608c82d82e2746c0cef9cd761db19a8e465ee` | [#1320](https://github.com/lidge-jun/opencodex/pull/1320) | MERGED TO `dev` at `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | -| CL-02 | `feat/cl-02-evidence-ledger` | `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | (phase-2 review fixes pending push) | [draft #1333](https://github.com/lidge-jun/opencodex/pull/1333) | IMPLEMENTATION COMPLETE — PHASE-2 REVIEW FIXES — NOT INDEPENDENTLY ACCEPTED | +| CL-02 | `feat/cl-02-evidence-ledger` | `4bb249b756abd468c675d2d92fffe4da95ad3e2a` | (phase-2 review fixes in progress) | [draft #1333](https://github.com/lidge-jun/opencodex/pull/1333) | IMPLEMENTATION COMPLETE — PHASE-2 REVIEW FIXES — NOT INDEPENDENTLY ACCEPTED | | CL-03 | — | — | — | — | NOT STARTED | The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its @@ -99,9 +99,13 @@ Independent CL-00 acceptance review is frozen at - **Branch:** `feat/cl-02-evidence-ledger` - **Starting/base SHA:** `4bb249b756abd468c675d2d92fffe4da95ad3e2a` (CL-01 merge via #1320) -- **Scope:** immutable JSONL evidence ledger, content-addressed artifact store, - disposable/rebuildable SQLite projection, ClaimSourceManifestV1, invalidation - and sensitive purge tombstones, CL-01 → observation persistence seam. +- **Scope:** append-only JSONL evidence ledger with an explicit sensitive-purge + exception: when the `ledger` purge action is requested, targeted evidence is + physically removed by atomic ledger rewrite and a `purge_tombstone` remains as + the auditable record; SQLite is rebuilt from the rewritten ledger and retained + content-addressed artifacts. The phase also includes the content-addressed + artifact store, disposable/rebuildable SQLite projection, ClaimSourceManifestV1, + invalidation semantics, and the CL-01 → observation persistence seam. - **Explicitly out of scope:** CL-03 live probes, CL-04 CLI/API, CL-05 UI, CL-06 profile fields, Fabric, shadow workflows. @@ -125,10 +129,12 @@ Claims cannot produce `PROBED`/`VERIFIED`. applicability, historical manifest no-substitution, closed event admission, corrupt superseding claims, ArtifactStore lifecycle, frozen behaviour fingerprint). -- **Local validation:** `bun x tsc --noEmit`, `bun run privacy:scan`, +- **Previous local validation:** `bun x tsc --noEmit`, `bun run privacy:scan`, `tests/lab-evidence-ledger.test.ts` (41/41), `tests/lab-conformance-harness.test.ts` (17/17), `tests/repo-hygiene.test.ts` (11/11), `git diff --check` green on - Windows host. + Windows host before the current CodeRabbit remediation pass. +- **Current CodeRabbit remediation:** committed on draft PR #1333; current CI and + review reconciliation are required before this head may be recorded as accepted. - **Independent acceptance:** not yet — draft PR #1333 remains open for review. - **CL-03:** not started. From 20969a62f719639d0d35a5b068cd60beccdff495 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Fri, 7 Aug 2026 16:19:47 +0900 Subject: [PATCH 101/124] fix(responses): bound synthesized SSE expansion --- src/server/responses-json-events.ts | 62 +++++++++++++++---- src/server/responses/core.ts | 33 ++++++++-- .../deepseek-responses-item-id-repair.test.ts | 30 +++++++++ tests/responses-json-events.test.ts | 26 ++++++++ 4 files changed, 133 insertions(+), 18 deletions(-) diff --git a/src/server/responses-json-events.ts b/src/server/responses-json-events.ts index 9910d17d54..6606efb72f 100644 --- a/src/server/responses-json-events.ts +++ b/src/server/responses-json-events.ts @@ -7,6 +7,8 @@ export type ResponsesJsonEventFrame = Record; +export const MAX_SYNTHESIZED_OUTPUT_ITEMS = 10_000; + /** * The canonical minimal sequence Codex commits: response.created (empty * output, in_progress) → one response.output_item.done per output item → a @@ -16,26 +18,38 @@ export function responsesJsonEventSequence( response: Record, rewritePayload?: (payload: Record) => Record, ): ResponsesJsonEventFrame[] { + return [...iterateResponsesJsonEvents(response, rewritePayload)]; +} + +function* iterateResponsesJsonEvents( + response: Record, + rewritePayload?: (payload: Record) => Record, +): Generator { const rewrite = rewritePayload ?? ((payload: Record) => payload); const output = Array.isArray(response.output) ? response.output : []; + if (output.length > MAX_SYNTHESIZED_OUTPUT_ITEMS) { + throw new RangeError( + `Responses JSON output contains ${output.length} items; maximum is ${MAX_SYNTHESIZED_OUTPUT_ITEMS}`, + ); + } const finalStatus = response.status === "failed" || response.status === "incomplete" ? response.status : "completed"; - return [ - rewrite({ - type: "response.created", - response: { ...response, status: "in_progress", output: [] }, - }), - ...output.map((item, outputIndex) => rewrite({ + yield rewrite({ + type: "response.created", + response: { ...response, status: "in_progress", output: [] }, + }); + for (const [outputIndex, item] of output.entries()) { + yield rewrite({ type: "response.output_item.done", output_index: outputIndex, item, - })), - rewrite({ - type: `response.${finalStatus}`, - response: { ...response, status: finalStatus }, - }), - ]; + }); + } + yield rewrite({ + type: `response.${finalStatus}`, + response: { ...response, status: finalStatus }, + }); } /** @@ -50,3 +64,27 @@ export function responsesJsonToSseBody( .map(frame => `data: ${JSON.stringify(frame)}\n\n`); return `${frames.join("")}data: [DONE]\n\n`; } + +/** Stream synthesized SSE frames without retaining the expanded body in memory. */ +export function responsesJsonToSseStream( + response: Record, + rewritePayload?: (payload: Record) => Record, +): ReadableStream { + const output = Array.isArray(response.output) ? response.output : []; + if (output.length > MAX_SYNTHESIZED_OUTPUT_ITEMS) { + throw new RangeError( + `Responses JSON output contains ${output.length} items; maximum is ${MAX_SYNTHESIZED_OUTPUT_ITEMS}`, + ); + } + const frames = iterateResponsesJsonEvents(response, rewritePayload); + const encoder = new TextEncoder(); + return new ReadableStream({ + pull(controller) { + const next = frames.next(); + controller.enqueue(encoder.encode( + next.done ? "data: [DONE]\n\n" : `data: ${JSON.stringify(next.value)}\n\n`, + )); + if (next.done) controller.close(); + }, + }); +} diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index bdbf99f20b..327376fc59 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -200,7 +200,7 @@ import { relaySseWithBlockRewrite, } from "../sse-payload-rewrite"; import { createGithubCopilotResponsesBlockRewrite } from "../github-copilot-responses-repair"; -import { responsesJsonToSseBody } from "../responses-json-events"; +import { responsesJsonToSseStream } from "../responses-json-events"; import { guardTerminalEventStream } from "./terminal-guard"; /** @@ -2438,25 +2438,46 @@ async function handleResponsesInner( && options.inboundTransport !== "websocket" && providerModelResponsesUpstreamStreaming(route.providerName, route.provider, route.modelId) === false && route.provider.adapter === "openai-responses") { + let completed: Record | undefined; try { - let completed = JSON.parse(clientJson) as Record; + const parsedCompleted = JSON.parse(clientJson) as unknown; + if (!parsedCompleted || typeof parsedCompleted !== "object" || Array.isArray(parsedCompleted)) { + throw new TypeError("bounded Responses JSON is not an object"); + } + let candidate = parsedCompleted as Record; // The bounded-JSON answer bypasses the SSE relay, so it also bypasses // the SSE item-id rewrite. Apply the same client-facing normalization // here or this policy would silently disable id repair for the very // providers that need it (raw record already happened above). if (hasResponsesItemIdRepair(route.provider.responsesItemIdRepair)) { - completed = repairResponsesJsonItemIds(completed, route.provider.responsesItemIdRepair!, translatorBudget); + candidate = repairResponsesJsonItemIds(candidate, route.provider.responsesItemIdRepair!, translatorBudget); + } + completed = candidate; + } catch { + // Non-JSON despite content-type: fall through to the plain relay. + } + if (completed) { + let stream: ReadableStream; + try { + stream = responsesJsonToSseStream(completed); + } catch (error) { + if (error instanceof RangeError) { + return formatErrorResponse( + 502, + "upstream_error", + "upstream JSON response exceeded the synthesized SSE item limit", + ); + } + throw error; } const sseHeaders = sanitizePassthroughHeaders(headers); sseHeaders.set("content-type", "text/event-stream"); sseHeaders.set("cache-control", "no-store"); - return new Response(responsesJsonToSseBody(completed), { + return new Response(stream, { status: upstreamResponse.status, statusText: upstreamResponse.statusText, headers: sseHeaders, }); - } catch { - // Non-JSON despite content-type: fall through to the plain relay. } } // WS turns reframe this JSON into events in the bridge, which is the diff --git a/tests/deepseek-responses-item-id-repair.test.ts b/tests/deepseek-responses-item-id-repair.test.ts index 8815ca8309..aa365e0df5 100644 --- a/tests/deepseek-responses-item-id-repair.test.ts +++ b/tests/deepseek-responses-item-id-repair.test.ts @@ -6,6 +6,7 @@ import { hasResponsesItemIdRepair, repairResponsesJsonItemIds, } from "../src/server/responses-item-id-repair"; +import { MAX_SYNTHESIZED_OUTPUT_ITEMS } from "../src/server/responses-json-events"; import { handleResponses } from "../src/server/responses/core"; import type { OcxConfig, OcxProviderConfig } from "../src/types"; import { createTestTranslatorBudget } from "./helpers/translator-budget"; @@ -220,4 +221,33 @@ describe("streamed HTTP path carries canonical ids (#938)", () => { expect(text).toContain("call_keep"); expect(text).toContain("data: [DONE]"); }); + + test("fails closed when bounded JSON would synthesize too many SSE frames", async () => { + const plainSeed = { ...providerConfigSeed(getProviderRegistryEntry("deepseek")!), apiKey: "sk-test" }; + globalThis.fetch = (async () => Response.json({ + id: "resp_deepseek", + object: "response", + status: "completed", + output: Array.from({ length: MAX_SYNTHESIZED_OUTPUT_ITEMS + 1 }, () => null), + })) as typeof fetch; + + const config = { providers: { deepseek: plainSeed } } as unknown as OcxConfig; + const response = await handleResponses( + new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "deepseek-v4-flash", input: "ping", stream: true }), + }), + config, + { model: "", provider: "" }, + {}, + ); + expect(response.status).toBe(502); + expect(response.headers.get("content-type")).toContain("application/json"); + const text = await response.text(); + const body = JSON.parse(text) as { error?: { type?: string; message?: string } }; + expect(body.error?.type).toBe("server_error"); + expect(body.error?.message).toContain("synthesized SSE item limit"); + expect(text).not.toContain("data: [DONE]"); + }); }); diff --git a/tests/responses-json-events.test.ts b/tests/responses-json-events.test.ts index e477f02497..9e27ffc5cc 100644 --- a/tests/responses-json-events.test.ts +++ b/tests/responses-json-events.test.ts @@ -1,7 +1,9 @@ import { describe, expect, test } from "bun:test"; import { + MAX_SYNTHESIZED_OUTPUT_ITEMS, responsesJsonEventSequence, responsesJsonToSseBody, + responsesJsonToSseStream, } from "../src/server/responses-json-events"; describe("responsesJsonEventSequence", () => { @@ -60,4 +62,28 @@ describe("responsesJsonToSseBody", () => { expect(frames[3]).toBe("data: [DONE]"); expect(body.endsWith("data: [DONE]\n\n")).toBe(true); }); + + test("rejects output arrays that could amplify synthesized frames", () => { + const output = Array.from({ length: MAX_SYNTHESIZED_OUTPUT_ITEMS + 1 }, () => null); + expect(() => responsesJsonToSseBody({ id: "r", output })).toThrow(RangeError); + expect(() => responsesJsonToSseStream({ id: "r", output })).toThrow(RangeError); + }); + + test("streams one SSE frame per pull and ends with [DONE]", async () => { + const stream = responsesJsonToSseStream({ + id: "r", + status: "completed", + output: [{ type: "message", id: "m" }], + }); + const reader = stream.getReader(); + const decoder = new TextDecoder(); + const chunks: string[] = []; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(decoder.decode(value)); + } + expect(chunks).toHaveLength(4); + expect(chunks.at(-1)).toBe("data: [DONE]\n\n"); + }); }); From 3bfe807810219358ad078a0d0342d63e7fe05361 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sun, 9 Aug 2026 14:47:15 +0900 Subject: [PATCH 102/124] test(responses): target synthetic bounded-json fixture --- structure/04_transports-and-sidecars.md | 2 ++ tests/deepseek-inbound-wire.test.ts | 18 ++++++++++++ .../deepseek-responses-item-id-repair.test.ts | 29 ------------------- 3 files changed, 20 insertions(+), 29 deletions(-) diff --git a/structure/04_transports-and-sidecars.md b/structure/04_transports-and-sidecars.md index 359ba7383c..23cb535700 100644 --- a/structure/04_transports-and-sidecars.md +++ b/structure/04_transports-and-sidecars.md @@ -225,6 +225,8 @@ terminal-output boundary (`src/server/relay.ts`) cuts the stream at that event a `[DONE]` itself, so DeepSeek streams live again; the registry knob remains as a one-line rollback for upstreams that regress, kept suite-reachable by a synthetic-registry fixture in `tests/deepseek-inbound-wire.test.ts`. +Synthesized output is capped at 10,000 items across HTTP and WebSocket reframing. HTTP frames are +encoded incrementally, so bounded upstream JSON cannot expand into an unbounded event array or SSE string. `ws-bridge.ts` preserves upstream `failed` and `incomplete` status values in the final WebSocket frame rather than always emitting `response.completed`. If the response status is `failed`, a diff --git a/tests/deepseek-inbound-wire.test.ts b/tests/deepseek-inbound-wire.test.ts index e5a650a097..cf9c4df11c 100644 --- a/tests/deepseek-inbound-wire.test.ts +++ b/tests/deepseek-inbound-wire.test.ts @@ -17,6 +17,7 @@ import { getProviderRegistryEntry, PROVIDER_REGISTRY } from "../src/providers/re import { createResponsesPassthroughAdapter as createResponsesPassthroughAdapterProduction } from "../src/adapters/openai-responses"; import { resolveWireProtocolOverride } from "../src/server/adapter-resolve"; import { handleResponses } from "../src/server/responses/core"; +import { MAX_SYNTHESIZED_OUTPUT_ITEMS } from "../src/server/responses-json-events"; import type { OcxConfig, OcxProviderConfig } from "../src/types"; import { withTestTranslatorBudget } from "./helpers/translator-budget"; @@ -355,6 +356,23 @@ describe("the bounded-JSON mechanism stays alive behind a synthetic registry ent expect(text).toMatch(/"id":"rs_ocx_[0-9a-f]{8}/); }); + test("an over-cap HTTP synthesis fails closed with 502", async () => { + globalThis.fetch = (async () => Response.json({ + id: "resp_fixture", + object: "response", + status: "completed", + output: Array.from({ length: MAX_SYNTHESIZED_OUTPUT_ITEMS + 1 }, () => null), + })) as typeof fetch; + const response = await driveFixture(fixtureProvider()); + expect(response.status).toBe(502); + expect(response.headers.get("content-type")).toContain("application/json"); + const text = await response.text(); + const body = JSON.parse(text) as { error?: { type?: string; message?: string } }; + expect(body.error?.type).toBe("server_error"); + expect(body.error?.message).toContain("synthesized SSE item limit"); + expect(text).not.toContain("data: [DONE]"); + }); + test("the WebSocket bounded-JSON reframe carries the same repaired ids", async () => { globalThis.fetch = (async () => completedWithPlaceholderIds()) as typeof fetch; const response = await driveFixture(repairingFixtureProvider(), { websocket: true }); diff --git a/tests/deepseek-responses-item-id-repair.test.ts b/tests/deepseek-responses-item-id-repair.test.ts index aa365e0df5..374e81a9a4 100644 --- a/tests/deepseek-responses-item-id-repair.test.ts +++ b/tests/deepseek-responses-item-id-repair.test.ts @@ -6,7 +6,6 @@ import { hasResponsesItemIdRepair, repairResponsesJsonItemIds, } from "../src/server/responses-item-id-repair"; -import { MAX_SYNTHESIZED_OUTPUT_ITEMS } from "../src/server/responses-json-events"; import { handleResponses } from "../src/server/responses/core"; import type { OcxConfig, OcxProviderConfig } from "../src/types"; import { createTestTranslatorBudget } from "./helpers/translator-budget"; @@ -222,32 +221,4 @@ describe("streamed HTTP path carries canonical ids (#938)", () => { expect(text).toContain("data: [DONE]"); }); - test("fails closed when bounded JSON would synthesize too many SSE frames", async () => { - const plainSeed = { ...providerConfigSeed(getProviderRegistryEntry("deepseek")!), apiKey: "sk-test" }; - globalThis.fetch = (async () => Response.json({ - id: "resp_deepseek", - object: "response", - status: "completed", - output: Array.from({ length: MAX_SYNTHESIZED_OUTPUT_ITEMS + 1 }, () => null), - })) as typeof fetch; - - const config = { providers: { deepseek: plainSeed } } as unknown as OcxConfig; - const response = await handleResponses( - new Request("http://localhost/v1/responses", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ model: "deepseek-v4-flash", input: "ping", stream: true }), - }), - config, - { model: "", provider: "" }, - {}, - ); - expect(response.status).toBe(502); - expect(response.headers.get("content-type")).toContain("application/json"); - const text = await response.text(); - const body = JSON.parse(text) as { error?: { type?: string; message?: string } }; - expect(body.error?.type).toBe("server_error"); - expect(body.error?.message).toContain("synthesized SSE item limit"); - expect(text).not.toContain("data: [DONE]"); - }); }); From ba5215021cdcfea234194c9063453b8183484644 Mon Sep 17 00:00:00 2001 From: Yuxin Qiao <104957188+Yuxin-Qiao@users.noreply.github.com> Date: Fri, 7 Aug 2026 18:27:47 +0800 Subject: [PATCH 103/124] fix(subagent): move per-role model fallback into opencodex config (#1190) Codex 0.146+ strictly deserializes $CODEX_HOME/agents/*.toml and rejects model_fallback as an unknown field, skipping the entire role definition. Per-role fallback chains now live in config.json as subagentModelFallbackByModel, keyed by the requested primary model id, and are consulted before the legacy TOML read (kept for backwards compatibility). ocx doctor scans agent role files and warns when any still carries model_fallback, pointing at the new config home. Docs updated in all locales. --- .../content/docs/guides/sub-agent-surface.md | 10 ++- .../docs/ja/guides/sub-agent-surface.md | 4 +- .../docs/ja/reference/configuration/agents.md | 11 ++- .../docs/ko/guides/sub-agent-surface.md | 4 +- .../docs/ko/reference/configuration/agents.md | 11 ++- .../docs/reference/configuration/agents.md | 11 ++- .../docs/ru/guides/sub-agent-surface.md | 8 +- .../docs/ru/reference/configuration/agents.md | 11 ++- .../docs/zh-cn/guides/sub-agent-surface.md | 4 +- .../zh-cn/reference/configuration/agents.md | 10 ++- src/cli/doctor.ts | 10 +++ src/codex/subagent-model-fallback.ts | 50 ++++++++++- src/config.ts | 6 ++ src/types.ts | 11 +++ tests/subagent-model-fallback.test.ts | 83 +++++++++++++++++++ 15 files changed, 233 insertions(+), 11 deletions(-) diff --git a/docs-site/src/content/docs/guides/sub-agent-surface.md b/docs-site/src/content/docs/guides/sub-agent-surface.md index 7d61b2dbd0..ab690bf6a6 100644 --- a/docs-site/src/content/docs/guides/sub-agent-surface.md +++ b/docs-site/src/content/docs/guides/sub-agent-surface.md @@ -86,9 +86,17 @@ write. External provider managers and user-owned root routing also remain author For a spawned worker, opencodex builds this priority order: 1. The requested primary model. -2. The role's `model_fallback` list from its `$CODEX_HOME/agents/*.toml` definition. +2. A per-model chain from `subagentModelFallbackByModel` in opencodex config, keyed by + the requested primary model. 3. The global `subagentModelFallback` list in opencodex config. +Per-role fallback chains belong in opencodex config, not in +`$CODEX_HOME/agents/*.toml`. Codex 0.146+ strictly deserializes agent role files and +rejects `model_fallback` as an unknown field, which skips the entire role definition +(#1190). opencodex can still read a legacy `model_fallback` line from the TOML for +backwards compatibility, but `ocx doctor` warns about it and Codex itself will ignore +the affected role. + Duplicate model ids are removed while preserving the first occurrence. During selection, opencodex skips candidates that are disabled, unroutable, backed by a disabled provider, marked unhealthy, inside a cooldown, missing a usable pooled Codex account, or beyond the configured quota threshold. diff --git a/docs-site/src/content/docs/ja/guides/sub-agent-surface.md b/docs-site/src/content/docs/ja/guides/sub-agent-surface.md index 611d75f6cc..daa1c2b839 100644 --- a/docs-site/src/content/docs/ja/guides/sub-agent-surface.md +++ b/docs-site/src/content/docs/ja/guides/sub-agent-surface.md @@ -65,9 +65,11 @@ v1 では、opencodex は、`max` または `ultra` の取り組みでアップ 生成されたワーカーの場合、opencodex は次の優先順位を構築します。 1. 要求されたプライマリ モデル。 -2. `$CODEX_HOME/agents/*.toml` 定義からのロールの `model_fallback` リスト。 +2. opencodex 構成内の `subagentModelFallbackByModel` によるモデル単位のチェーン（要求されたプライマリ モデルがキー）。 3. opencodex 構成内のグローバル `subagentModelFallback` リスト。 +ロール単位のフォールバックチェーンは、`$CODEX_HOME/agents/*.toml` ではなく opencodex 構成に置く必要があります。Codex 0.146+ はエージェントロールファイルを厳密に逆シリアル化し、`model_fallback` を未知フィールドとして拒否するため、ロール定義全体がスキップされます（#1190）。opencodex は後方互換性のために TOML 内のレガシー `model_fallback` 行を引き続き読み取れますが、`ocx doctor` が警告を出し、Codex 自体は影響を受けるロールを無視します。 + 重複するモデル ID は、最初に出現したモデル ID を保持しながら削除されます。選択中、opencodex は、無効になっている、ルーティングできない、無効なプロバイダーによってサポートされている、異常とマークされている、クールダウン中、使用可能なプールされた Codex アカウントがない、または設定されたクォータしきい値を超えている候補をスキップします。可用性プローブは `subagentModelFallbackPollMs` に対してキャッシュされます (デフォルトでは 60 秒)。 フォールバックでは、互換性のない暗号化タスクは読み取り可能になりません。子タスクが ChatGPT 用に暗号化されている場合、外部モデルがチェーンの前の方に表示されている場合でも、選択は正規のネイティブ ChatGPT ターゲットに制限されます。 diff --git a/docs-site/src/content/docs/ja/reference/configuration/agents.md b/docs-site/src/content/docs/ja/reference/configuration/agents.md index 74d90dc9b7..03b86332ef 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/agents.md +++ b/docs-site/src/content/docs/ja/reference/configuration/agents.md @@ -17,6 +17,7 @@ description: マルチエージェント サーフェス、委任ガイダンス | `multiAgentGuidanceEnabled?` | `boolean` | `true` | opencodex が作成した v1/v2 開発者ガイダンスのみを制御します。ネイティブ エージェントのデフォルト、ツール、ルーティング、ロスター、またはエフォート キャップは変更されません。 | | `syncCodexSubagentDefaults?` | `boolean` | `false` |同期/再起動中に、Codex のネイティブ デフォルトとして `injectionModel` およびオプションの `injectionEffort` を書き込むようにオプトインします。 `injectionModel`が必要です。 | | `subagentModelFallback?` | `string[]` | `[]` |生成された子ターンの優先順位付きグローバル フォールバック モデル。 | +| `subagentModelFallbackByModel?` | `Record` | `{}` | 要求されたプライマリ モデル id をキーとするモデル単位のフォールバックチェーン。ロール単位のフォールバックメタデータの推奨場所です。Codex の agent TOML に `model_fallback` を書くと Codex 0.146+ がロールをスキップします（#1190）。 | | `subagentModelFallbackPollMs?` | `number` | `60000` |可用性プローブのキャッシュ間隔。 1000 ミリ秒未満の値はデフォルトに戻ります。 | | `effortCap?` | `string` | — | v2 のメイン ターンとマークされた子ターンの条件を満たすためのハード シーリング。 `low` ～ `ultra` を受け入れます。 | | `subagentEffortCap?` | `string` | — |スポーンされた子のターンのみの追加の上限。両方の上限が適用される場合は、低い方が優先されます。 | @@ -44,9 +45,14 @@ V1 ガイダンスは、`max` または `ultra` でのみプロアクティブ 生成された子のフォールバック順序は次のとおりです。 1. 要求されたプライマリ モデル。 -2. ロールレベル `model_fallback` から `$CODEX_HOME/agents/*.toml`;それから +2. `subagentModelFallbackByModel` によるモデル単位のチェーン（プライマリ モデルがキー）;それから 3. グローバル `subagentModelFallback` エントリ。 +ロール単位のフォールバックチェーンは opencodex 構成に置く必要があります。`model_fallback` を +`$CODEX_HOME/agents/*.toml` に書くと、Codex 0.146+ が未知フィールドとしてロールファイル全体を +拒否し、ロールをスキップします（#1190）。TOML 内のレガシー `model_fallback` 行は後方互換性の +ために引き続き読み取られますが、`ocx doctor` がそれをフラグ付けします。 + opencodex は、無効、ルーティング不能、異常、冷却期間、またはクォータしきい値の候補をスキップします。可用性スナップショットは `subagentModelFallbackPollMs` に対してキャッシュされます。暗号化された子タスクは、チェーンを正規のネイティブ ChatGPT ターゲットに制限できます。暗号化されたペイロードを読み取ることができる人がいない場合、読み取り不可能な暗号文が別の場所にルーティングされる代わりに、リクエストは失敗します。 ```json @@ -57,6 +63,9 @@ opencodex は、無効、ルーティング不能、異常、冷却期間、ま "injectionEffort": "high", "syncCodexSubagentDefaults": true, "subagentModelFallback": ["gpt-5.4-mini"], + "subagentModelFallbackByModel": { + "gpt-5.5": ["gpt-5.4-mini"] + }, "subagentModelFallbackPollMs": 60000, "subagentEffortCap": "high" } diff --git a/docs-site/src/content/docs/ko/guides/sub-agent-surface.md b/docs-site/src/content/docs/ko/guides/sub-agent-surface.md index 9678ee0ca0..cd987ed761 100644 --- a/docs-site/src/content/docs/ko/guides/sub-agent-surface.md +++ b/docs-site/src/content/docs/ko/guides/sub-agent-surface.md @@ -65,9 +65,11 @@ v1에서는 opencodex가 `max` 또는 `ultra` 추론 강도에서만 업스트 스폰된 작업자에 대해 opencodex는 다음 우선순위를 적용합니다. 1. 요청한 기본 모델 -2. 역할의 `$CODEX_HOME/agents/*.toml` 정의에 있는 `model_fallback` 목록 +2. opencodex 설정의 `subagentModelFallbackByModel`에 있는 모델별 체인 (요청한 기본 모델이 키) 3. opencodex 설정의 전역 `subagentModelFallback` 목록 +역할별 폴백 체인은 `$CODEX_HOME/agents/*.toml`이 아니라 opencodex 설정에 두어야 합니다. Codex 0.146+는 에이전트 역할 파일을 엄격하게 역직렬화하며 `model_fallback`을 알 수 없는 필드로 거부해 역할 정의 전체를 건너뜁니다 (#1190). opencodex는 하위 호환성을 위해 TOML의 기존 `model_fallback` 줄을 계속 읽을 수 있지만, `ocx doctor`가 경고하며 Codex 자체는 해당 역할을 무시합니다. + 중복 모델 id는 첫 번째 출현을 유지한 채 제거합니다. 선택 과정에서 opencodex는 비활성화된 후보, 라우팅 불가 후보, 비활성화된 프로바이더가 받쳐주는 후보, unhealthy로 표시된 후보, cooldown 중인 후보, 사용할 수 있는 pooled Codex 계정이 없는 후보, 또는 설정된 quota 임계치를 넘는 후보를 건너뜁니다. 가용성 프로브는 기본값 60초인 `subagentModelFallbackPollMs` 동안 캐시됩니다. 폴백이 호환되지 않는 암호화 작업을 읽을 수 있게 만들어 주지는 않습니다. 자식 작업이 ChatGPT용으로 암호화되어 있으면, 체인 앞쪽에 외부 모델이 있더라도 선택은 정규 네이티브 ChatGPT 대상만 허용됩니다. diff --git a/docs-site/src/content/docs/ko/reference/configuration/agents.md b/docs-site/src/content/docs/ko/reference/configuration/agents.md index 579386afe4..47589fb3cc 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/agents.md +++ b/docs-site/src/content/docs/ko/reference/configuration/agents.md @@ -17,6 +17,7 @@ description: 멀티 에이전트 표면, 위임 안내, 선호 모델, 대체 | `multiAgentGuidanceEnabled?` | `boolean` | `true` | opencodex가 작성하는 v1/v2 개발자 안내만 제어합니다. 네이티브 에이전트 기본값, 도구, 라우팅, 로스터, 노력 상한은 바꾸지 않습니다. | | `syncCodexSubagentDefaults?` | `boolean` | `false` | 동기화 또는 재시작 시 `injectionModel`과 선택적 `injectionEffort`를 Codex의 네이티브 기본값으로 기록하도록 선택합니다. `injectionModel`이 필요합니다. | | `subagentModelFallback?` | `string[]` | `[]` | 생성된 하위 턴에 적용되는 전역 대체 모델 우선순위 목록입니다. | +| `subagentModelFallbackByModel?` | `Record` | `{}` | 요청한 기본 모델 id를 키로 하는 모델별 대체 체인입니다. 역할별 대체 메타데이터의 지원 위치입니다. Codex agent TOML에 `model_fallback`을 쓰면 Codex 0.146+가 역할을 건너뜁니다 (#1190). | | `subagentModelFallbackPollMs?` | `number` | `60000` | 사용 가능성 검사 캐시 간격입니다. 1000 ms 미만의 값은 기본값으로 돌아갑니다. | | `effortCap?` | `string` | — | 자격을 갖춘 v2 메인 턴과 표시된 생성 하위 턴에 대한 하드 상한입니다. `low`부터 `ultra`까지 허용합니다. | | `subagentEffortCap?` | `string` | — | 생성된 하위 턴에만 적용되는 추가 상한입니다. 두 상한이 모두 적용되면 더 낮은 값이 이깁니다. | @@ -44,9 +45,14 @@ V1 안내는 `max` 또는 `ultra`에서만 선제 텍스트로 제공됩니다. 생성된 하위 작업의 대체 순서는 다음과 같습니다. 1. 요청된 기본 모델 -2. `$CODEX_HOME/agents/*.toml`의 역할 수준 `model_fallback` +2. `subagentModelFallbackByModel`의 모델별 체인 (기본 모델이 키) 3. 전역 `subagentModelFallback` 항목 +역할별 폴백 체인은 opencodex 설정에 있어야 합니다. `model_fallback`을 +`$CODEX_HOME/agents/*.toml`에 쓰면 Codex 0.146+가 알 수 없는 필드로 역할 파일 전체를 +거부하고 역할을 건너뜁니다 (#1190). TOML의 기존 `model_fallback` 줄은 하위 호환성을 위해 +계속 읽히지만 `ocx doctor`가 이를 표시합니다. + opencodex는 비활성, 라우팅 불가, 비정상, 쿨다운 중, 또는 할당량 임계값에 걸린 후보를 건너뜁니다. 사용 가능성 스냅샷은 `subagentModelFallbackPollMs` 동안 캐시됩니다. 암호화된 하위 작업은 체인을 정규 네이티브 ChatGPT 대상으로만 제한할 수 있습니다. 어떤 대상도 암호화된 페이로드를 읽을 수 없으면, 읽을 수 없는 암호문을 다른 곳으로 라우팅하는 대신 요청이 실패합니다. ```json @@ -57,6 +63,9 @@ opencodex는 비활성, 라우팅 불가, 비정상, 쿨다운 중, 또는 할 "injectionEffort": "high", "syncCodexSubagentDefaults": true, "subagentModelFallback": ["gpt-5.4-mini"], + "subagentModelFallbackByModel": { + "gpt-5.5": ["gpt-5.4-mini"] + }, "subagentModelFallbackPollMs": 60000, "subagentEffortCap": "high" } diff --git a/docs-site/src/content/docs/reference/configuration/agents.md b/docs-site/src/content/docs/reference/configuration/agents.md index 948c9350a0..f490626d2a 100644 --- a/docs-site/src/content/docs/reference/configuration/agents.md +++ b/docs-site/src/content/docs/reference/configuration/agents.md @@ -18,6 +18,7 @@ routes, and limits delegated work. | `multiAgentGuidanceEnabled?` | `boolean` | `true` | Controls only opencodex-authored v1/v2 developer guidance; it does not change native agent defaults, tools, routing, rosters, or effort caps. | | `syncCodexSubagentDefaults?` | `boolean` | `false` | Opt into writing `injectionModel` and optional `injectionEffort` as Codex's native defaults during sync/restart. Requires `injectionModel`. | | `subagentModelFallback?` | `string[]` | `[]` | Priority-ordered global fallback models for spawned child turns. | +| `subagentModelFallbackByModel?` | `Record` | `{}` | Per-primary-model fallback chains, keyed by the requested primary model id. This is the supported home for per-role fallback metadata; `model_fallback` inside Codex agent TOML makes Codex 0.146+ skip the role (#1190). | | `subagentModelFallbackPollMs?` | `number` | `60000` | Availability-probe cache interval. Values below 1000 ms fall back to the default. | | `effortCap?` | `string` | — | Hard ceiling for qualifying v2 main turns and marked spawned-child turns. Accepts `low` through `ultra`. | | `subagentEffortCap?` | `string` | — | Additional ceiling for spawned-child turns only. When both caps apply, the lower wins. | @@ -73,9 +74,14 @@ created Codex tasks and do not cause delegation by themselves. Spawned-child fallback order is: 1. the requested primary model; -2. role-level `model_fallback` from `$CODEX_HOME/agents/*.toml`; then +2. per-model chains from `subagentModelFallbackByModel` (keyed by the primary model); then 3. global `subagentModelFallback` entries. +Per-role fallback chains must live in opencodex config. Writing `model_fallback` into +`$CODEX_HOME/agents/*.toml` makes Codex 0.146+ reject the whole role file as an unknown +field and skip the role (#1190). A legacy `model_fallback` line in the TOML is still +read for backwards compatibility, but `ocx doctor` flags it. + opencodex skips disabled, unroutable, unhealthy, cooling-down, or quota-threshold candidates. The availability snapshot is cached for `subagentModelFallbackPollMs`. Encrypted child tasks can restrict the chain to canonical native ChatGPT targets; if none can read the encrypted payload, the request @@ -89,6 +95,9 @@ fails instead of routing unreadable ciphertext elsewhere. "injectionEffort": "high", "syncCodexSubagentDefaults": true, "subagentModelFallback": ["gpt-5.4-mini"], + "subagentModelFallbackByModel": { + "gpt-5.5": ["gpt-5.4-mini"] + }, "subagentModelFallbackPollMs": 60000, "subagentEffortCap": "high" } diff --git a/docs-site/src/content/docs/ru/guides/sub-agent-surface.md b/docs-site/src/content/docs/ru/guides/sub-agent-surface.md index 74cc0041ff..416c6d9d02 100644 --- a/docs-site/src/content/docs/ru/guides/sub-agent-surface.md +++ b/docs-site/src/content/docs/ru/guides/sub-agent-surface.md @@ -95,9 +95,15 @@ opencodex владеет активной маршрутизацией Codex, sy Для порождённого воркера opencodex строит такой порядок приоритета: 1. Запрошенная основная модель. -2. Список `model_fallback` роли из её определения `$CODEX_HOME/agents/*.toml`. +2. Модельная цепочка из `subagentModelFallbackByModel` в конфигурации opencodex, ключ — запрошенная основная модель. 3. Глобальный список `subagentModelFallback` в конфигурации opencodex. +Модельные цепочки fallback для ролей должны храниться в конфигурации opencodex, а не в +`$CODEX_HOME/agents/*.toml`. Codex 0.146+ строго десериализует файлы ролей агента и отвергает +`model_fallback` как неизвестное поле, из-за чего пропускается всё определение роли (#1190). +opencodex по-прежнему читает устаревшую строку `model_fallback` из TOML для обратной +совместимости, но `ocx doctor` предупреждает о ней, а сам Codex игнорирует затронутую роль. + Дубликаты id моделей удаляются с сохранением первого вхождения. При выборе opencodex пропускает кандидатов, которые отключены, немаршрутизируемы, опираются на отключённого провайдера, помечены как unhealthy, находятся в cooldown, не имеют доступного pooled-аккаунта Codex или вышли за diff --git a/docs-site/src/content/docs/ru/reference/configuration/agents.md b/docs-site/src/content/docs/ru/reference/configuration/agents.md index 062750e2a4..31516724b9 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/agents.md +++ b/docs-site/src/content/docs/ru/reference/configuration/agents.md @@ -18,6 +18,7 @@ description: Multi-agent surface, guidance при делегировании, pr | `multiAgentGuidanceEnabled?` | `boolean` | `true` | Управляет только developer-guidance, написанным самим opencodex, для v1/v2; не меняет native default'ы агентов, tools, routing, roster'ы и effort cap'ы. | | `syncCodexSubagentDefaults?` | `boolean` | `false` | Разрешает записывать `injectionModel` и, при наличии, `injectionEffort` как native default'ы Codex при sync/restart. Требует `injectionModel`. | | `subagentModelFallback?` | `string[]` | `[]` | Глобальные fallback-модели для порождённых child-turn'ов в порядке приоритета. | +| `subagentModelFallbackByModel?` | `Record` | `{}` | Модельные цепочки fallback по ключу запрошенной основной модели. Рекомендуемое место для per-role метаданных fallback: запись `model_fallback` в agent TOML Codex заставляет Codex 0.146+ пропустить роль (#1190). | | `subagentModelFallbackPollMs?` | `number` | `60000` | Интервал кэша для availability probe. Значения ниже 1000 ms возвращаются к дефолту. | | `effortCap?` | `string` | — | Жёсткий потолок effort для qualifying v2 main-turn'ов и помеченных spawned-child turn'ов. Принимает `low`–`ultra`. | | `subagentEffortCap?` | `string` | — | Дополнительный потолок только для spawned-child turn'ов. Если применимы оба cap'а, выигрывает более низкий. | @@ -69,9 +70,14 @@ user-owned target field'ы считаются конфликтом и сохра Порядок fallback для spawned-child такой: 1. запрошенная основная модель; -2. role-level `model_fallback` из `$CODEX_HOME/agents/*.toml`; затем +2. модельные цепочки из `subagentModelFallbackByModel` (ключ — основная модель); затем 3. глобальные записи `subagentModelFallback`. +Модельные цепочки fallback для ролей должны храниться в конфигурации opencodex. Запись +`model_fallback` в `$CODEX_HOME/agents/*.toml` заставляет Codex 0.146+ отклонить весь файл +роли как неизвестное поле и пропустить роль (#1190). Устаревшая строка `model_fallback` в TOML +по-прежнему читается для обратной совместимости, но `ocx doctor` помечает её. + opencodex пропускает кандидатов, которые отключены, не маршрутизируются, unhealthy, находятся в cooldown либо уже достигли порога quota. Availability-снимок кэшируется на `subagentModelFallbackPollMs`. Шифрованные child-task'и могут ограничить цепочку каноническими @@ -86,6 +92,9 @@ native ChatGPT-target'ами; если ни одна из них не может "injectionEffort": "high", "syncCodexSubagentDefaults": true, "subagentModelFallback": ["gpt-5.4-mini"], + "subagentModelFallbackByModel": { + "gpt-5.5": ["gpt-5.4-mini"] + }, "subagentModelFallbackPollMs": 60000, "subagentEffortCap": "high" } diff --git a/docs-site/src/content/docs/zh-cn/guides/sub-agent-surface.md b/docs-site/src/content/docs/zh-cn/guides/sub-agent-surface.md index bec2d70da4..2ac9637416 100644 --- a/docs-site/src/content/docs/zh-cn/guides/sub-agent-surface.md +++ b/docs-site/src/content/docs/zh-cn/guides/sub-agent-surface.md @@ -65,9 +65,11 @@ Dashboard 上的 **Sub-agent delegation** 控件管理三个相关设置： 对于生成出的工作器，opencodex 会按以下优先级构建顺序： 1. 请求的主模型。 -2. 该角色在其 `$CODEX_HOME/agents/*.toml` 定义中的 `model_fallback` 列表。 +2. opencodex 配置中 `subagentModelFallbackByModel` 提供的 per-model 链，按请求的主模型做键。 3. opencodex 配置中的全局 `subagentModelFallback` 列表。 +per-role fallback 链应该放在 opencodex 配置里，而不是 `$CODEX_HOME/agents/*.toml`。Codex 0.146+ 会严格反序列化 agent 角色文件，并把 `model_fallback` 当作未知字段拒绝，导致整个角色定义被跳过（#1190）。opencodex 为了向后兼容仍然能读取 TOML 里的旧版 `model_fallback`，但 `ocx doctor` 会给出警告，而且 Codex 本身会忽略受影响的角色。 + 重复的模型 id 会在保留第一次出现的前提下移除。在选择过程中，opencodex 会跳过已禁用、不可路由、由已禁用 provider 支撑、标记为 unhealthy、处于 cooldown、没有可用 pooled Codex 账户，或者超出配置配额阈值的候选项。可用性探测会缓存 `subagentModelFallbackPollMs` 的时长，默认 60 秒。 fallback 不会让不兼容的加密任务变得可读。当子任务为 ChatGPT 加密时，即使链中更靠前出现了外部模型，选择也只会限制在规范的原生 ChatGPT 目标上。 diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/agents.md b/docs-site/src/content/docs/zh-cn/reference/configuration/agents.md index 473c15d59e..0afa4603fe 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/agents.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/agents.md @@ -17,6 +17,7 @@ description: 多代理界面、委派引导、首选模型、回退链、原生 | `multiAgentGuidanceEnabled?` | `boolean` | `true` | 只控制 opencodex 生成的 v1/v2 开发者引导；不会改变原生代理默认值、工具、路由、名单或 effort 上限。 | | `syncCodexSubagentDefaults?` | `boolean` | `false` | 允许在同步或重启时，将 `injectionModel` 以及可选的 `injectionEffort` 写入为 Codex 的原生默认值。需要 `injectionModel`。 | | `subagentModelFallback?` | `string[]` | `[]` | 按优先级排序的全局回退模型，用于派生的子轮次。 | +| `subagentModelFallbackByModel?` | `Record` | `{}` | 按请求的主模型 id 做键的 per-model 回退链。这是 per-role fallback 元数据的受支持存放位置；`model_fallback` 写在 Codex agent TOML 里会让 Codex 0.146+ 跳过该角色（#1190）。 | | `subagentModelFallbackPollMs?` | `number` | `60000` | 可用性探测缓存间隔。低于 1000 ms 的值会回退到默认值。 | | `effortCap?` | `string` | — | 对符合条件的 v2 主轮次和标记的派生子轮次设置硬上限。接受 `low` 到 `ultra`。 | | `subagentEffortCap?` | `string` | — | 仅针对派生子轮次的额外上限。两个上限同时适用时，较低者生效。 | @@ -44,9 +45,13 @@ V1 引导只会在 `max` 或 `ultra` 时以主动文本形式出现。V2 只有 派生子轮次的回退顺序如下： 1. 请求的主模型； -2. 来自 `$CODEX_HOME/agents/*.toml` 的角色级 `model_fallback`；然后是 +2. `subagentModelFallbackByModel` 中的 per-model 链（按主模型做键）；然后是 3. 全局 `subagentModelFallback` 条目。 +per-role fallback 链必须放在 opencodex 配置里。把 `model_fallback` 写进 +`$CODEX_HOME/agents/*.toml` 会让 Codex 0.146+ 把整个角色文件当作未知字段拒绝并跳过该角色 +（#1190）。TOML 中的旧版 `model_fallback` 仍会被读取以保持向后兼容，但 `ocx doctor` 会标记它。 + opencodex 会跳过已禁用、不可路由、不健康、处于冷却中，或已达到配额阈值的候选项。可用性快照会在 `subagentModelFallbackPollMs` 期间缓存。加密的子任务可以把链限制为规范的原生 ChatGPT 目标；如果没有任何目标能读取加密载荷，请求就会失败，而不是把不可读的密文路由到别处。 ```json @@ -57,6 +62,9 @@ opencodex 会跳过已禁用、不可路由、不健康、处于冷却中，或 "injectionEffort": "high", "syncCodexSubagentDefaults": true, "subagentModelFallback": ["gpt-5.4-mini"], + "subagentModelFallbackByModel": { + "gpt-5.5": ["gpt-5.4-mini"] + }, "subagentModelFallbackPollMs": 60000, "subagentEffortCap": "high" } diff --git a/src/cli/doctor.ts b/src/cli/doctor.ts index 2e12745b4c..64d62ce8aa 100644 --- a/src/cli/doctor.ts +++ b/src/cli/doctor.ts @@ -23,6 +23,7 @@ import { withNativeMainSharedClaim } from "../codex/native-main-claim"; import { probeNativeProfileRecoveryState, resolveNativeProfileContext } from "../codex/native-profile-store"; import { NativeProfileError } from "../codex/native-profile-types"; import { collectOrcaCodexHomeDiagnostic, resolveCodexHomeDir as resolveCodexHomeDirImpl, isWslRuntime, listWslWindowsCodexHomes, wslAutomountRoot, type CodexHomeDeps } from "../codex/home"; +import { scanCodexAgentRolesWithTomlModelFallback } from "../codex/subagent-model-fallback"; import { findCodexOnPath, isWindowsInteropDir } from "../codex/shim"; import { countPendingOpencodexHistory } from "../codex/history-provider"; import { @@ -942,6 +943,15 @@ export async function runDoctor(args: string[] = []): Promise { } } + console.log("\nCodex agent role files"); + const tomlFallbackRoles = scanCodexAgentRolesWithTomlModelFallback(resolveCodexHomeDirImpl()); + if (tomlFallbackRoles.length === 0) { + console.log(" ok no per-role model_fallback fields in $CODEX_HOME/agents/*.toml"); + } else { + console.log(` [WARN] ${tomlFallbackRoles.length} agent role file${tomlFallbackRoles.length === 1 ? "" : "s"} contain${tomlFallbackRoles.length === 1 ? "s" : ""} \`model_fallback\`: ${tomlFallbackRoles.join(", ")}`); + console.log(" Codex >= 0.146 rejects that field as unknown and skips the whole role. Move the chains to opencodex config `subagentModelFallbackByModel` (keyed by primary model) and remove the field from the TOML files."); + } + const dual = collectWslDualInstall(); if (dual.wsl) { console.log("\nWSL Codex installs"); diff --git a/src/codex/subagent-model-fallback.ts b/src/codex/subagent-model-fallback.ts index 0e51b423c0..3cab8a1883 100644 --- a/src/codex/subagent-model-fallback.ts +++ b/src/codex/subagent-model-fallback.ts @@ -417,6 +417,37 @@ function subagentQuotaPrimeBlockedByHostCircuit(config: OcxConfig): boolean { return getUpstreamHostHealth(key)?.cooldownUntil !== undefined; } +/** + * Per-primary-model fallback chains from opencodex config (#1190). + * + * Storing `model_fallback` inside `$CODEX_HOME/agents/*.toml` makes Codex >= 0.146 + * reject the whole role file as an unknown field. The config-keyed map is the + * supported home for that metadata; keys match the requested primary model id, + * using the same raw/encoded slug tolerance as the TOML role lookup. + */ +export function resolveConfiguredModelFallbackForPrimary( + primary: string, + config: OcxConfig, +): string[] { + const byModel = config.subagentModelFallbackByModel; + if (!byModel || typeof byModel !== "object") return []; + const entries: string[] = []; + const seen = new Set(); + const push = (model: string) => { + const trimmed = model.trim(); + if (trimmed === "") return; + const key = fallbackChainKey(trimmed, config.codexAccountNamespaces); + if (seen.has(key)) return; + seen.add(key); + entries.push(trimmed); + }; + for (const [key, chain] of Object.entries(byModel)) { + if (!slugsEquivalent(key, primary)) continue; + for (const model of chain) push(model); + } + return entries; +} + /** * Best-effort quota refresh before subagent model selection. * Concurrent callers share one in-flight promise. The success TTL is updated only @@ -479,11 +510,17 @@ export function applySubagentModelFallback( accountUsabilityOptions?: CodexAccountUsabilityOptions, ): { from?: string; to?: string; skipped?: string[] } | null { if (!isThreadSpawnRequest(headers)) return null; - const roleFallback = resolveAgentModelFallbackForPrimary( + const tomlRoleFallback = resolveAgentModelFallbackForPrimary( parsed.modelId, getCodexHome(), config.codexAccountNamespaces, ); + // Config-keyed chains are the supported per-role home (#1190); TOML `model_fallback` + // stays readable for backwards compatibility with homes written before Codex 0.146. + const roleFallback = [ + ...resolveConfiguredModelFallbackForPrimary(parsed.modelId, config), + ...tomlRoleFallback, + ]; const globalFallback = config.subagentModelFallback ?? []; if (globalFallback.length === 0 && roleFallback.length === 0) return null; const selection = selectAvailableSubagentModel( @@ -545,6 +582,17 @@ export function readCodexAgentModelFallback(role: string, codexHome = CODEX_HOME return readAgentModelFallback(file) ?? []; } +/** + * Roles whose TOML still carries `model_fallback`. Codex >= 0.146 rejects the + * entire role file as an unknown field (#1190), so these files are skipped by + * the native runtime even though opencodex can still read them. + */ +export function scanCodexAgentRolesWithTomlModelFallback(codexHome = CODEX_HOME): string[] { + return listCodexAgentRoles(codexHome).filter( + role => readCodexAgentModelFallback(role, codexHome).length > 0, + ); +} + export function listCodexAgentRoles(codexHome = CODEX_HOME): string[] { const dir = join(codexHome, "agents"); if (!existsSync(dir)) return []; diff --git a/src/config.ts b/src/config.ts index 57dc806465..32831fd39d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1071,6 +1071,12 @@ const configSchema = z.object({ injectionModel: z.string().optional().catch(undefined), injectionEffort: z.string().optional().catch(undefined), syncCodexSubagentDefaults: z.boolean().optional().catch(undefined), + // Per-primary-model fallback chains. Values must be non-empty string arrays; + // malformed entries degrade to undefined rather than rejecting the whole config. + subagentModelFallbackByModel: z.record( + z.string(), + z.array(z.string().trim().min(1)).min(1), + ).optional().catch(undefined), codexShimAutoRestore: z.boolean().optional(), pausedCodexAccountIds: z.array(z.string().regex(/^[a-zA-Z0-9._-]{1,64}$/)).optional(), codexAccountNamespaces: codexAccountNamespacesSchema.optional(), diff --git a/src/types.ts b/src/types.ts index 510c073a20..89e151b44e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -605,6 +605,17 @@ export interface OcxConfig { * turn to the next available entry before routing. */ subagentModelFallback?: string[]; + /** + * Per-primary-model fallback chains for spawned sub-agents, keyed by the + * requested primary model id (bare native or "provider/model"). Entries for + * the matching key are consulted after the requested model and before the + * global `subagentModelFallback` list. + * + * This is the supported home for per-role fallback metadata: storing it as + * `model_fallback` inside `$CODEX_HOME/agents/*.toml` makes Codex >= 0.146 + * reject the whole role file as an unknown field (#1190). + */ + subagentModelFallbackByModel?: Record; /** * TTL (ms) for cached sub-agent model availability probes. Default 60_000. */ diff --git a/tests/subagent-model-fallback.test.ts b/tests/subagent-model-fallback.test.ts index 6eb69b2179..90ad97327d 100644 --- a/tests/subagent-model-fallback.test.ts +++ b/tests/subagent-model-fallback.test.ts @@ -13,6 +13,8 @@ import { readCodexAgentModelFallback, resetSubagentModelFallbackStateForTests, resolveAgentModelFallbackForPrimary, + resolveConfiguredModelFallbackForPrimary, + scanCodexAgentRolesWithTomlModelFallback, selectAvailableSubagentModel, setSubagentQuotaPrimeForTests, subagentFallbackGuidanceText, @@ -920,6 +922,87 @@ describe("subagent model fallback chain", () => { ]); }); + test("config-keyed per-model fallback resolves for the primary model", () => { + const config = cfg({ + subagentModelFallbackByModel: { + "gpt-5.6-sol": ["alibaba-token-plan/qwen3.8-max", "kimi/k3"], + "other-model": ["xai/grok-4.5"], + }, + }); + expect(resolveConfiguredModelFallbackForPrimary("gpt-5.6-sol", config)).toEqual([ + "alibaba-token-plan/qwen3.8-max", + "kimi/k3", + ]); + expect(resolveConfiguredModelFallbackForPrimary("other-model", config)).toEqual([ + "xai/grok-4.5", + ]); + expect(resolveConfiguredModelFallbackForPrimary("unlisted", config)).toEqual([]); + }); + + test("config-keyed fallback dedupes across keys and preserves account-selector case", () => { + const config = cfg({ + codexAccountNamespaces: { work: "account-a", Work: "account-b" }, + subagentModelFallbackByModel: { + "gpt-5.6-sol": ["work/gpt-5.5", "Work/gpt-5.5", "work/GPT-5.5", "kimi/k3"], + "openrouter/anthropic/claude": ["xai/grok-4.5"], + "openrouter/anthropic-claude": ["kimi/k3", "xai/grok-4.5"], + }, + }); + expect(resolveConfiguredModelFallbackForPrimary("gpt-5.6-sol", config)).toEqual([ + "work/gpt-5.5", + "Work/gpt-5.5", + "kimi/k3", + ]); + expect(resolveConfiguredModelFallbackForPrimary("openrouter/anthropic/claude", config)).toEqual([ + "xai/grok-4.5", + "kimi/k3", + ]); + }); + + test("applySubagentModelFallback prefers config-keyed chains over TOML model_fallback", () => { + const dir = codexHomeFixture(); + writeFileSync(join(dir, "agents", "executor.toml"), [ + "name = \"executor\"", + "model = \"gpt-5.6-sol\"", + "model_fallback = [\"kimi/k3\"]", + "", + ].join("\n"), "utf8"); + updateAccountQuota("pool-a", 95); + const parsed = { + modelId: "gpt-5.6-sol", + options: {}, + context: { messages: [] }, + _rawBody: { model: "gpt-5.6-sol" }, + }; + const result = applySubagentModelFallback( + parsed as never, + new Headers({ "x-openai-subagent": "collab_spawn" }), + cfg({ + subagentModelFallback: undefined, + subagentModelFallbackByModel: { + "gpt-5.6-sol": ["alibaba-token-plan/qwen3.8-max"], + }, + }), + ); + expect(result?.to).toBe("alibaba-token-plan/qwen3.8-max"); + }); + + test("scanCodexAgentRolesWithTomlModelFallback reports only roles carrying the field", () => { + const dir = codexHomeFixture(); + writeFileSync(join(dir, "agents", "with_fallback.toml"), [ + "name = \"with_fallback\"", + "model = \"gpt-5.6-sol\"", + "model_fallback = [\"kimi/k3\"]", + "", + ].join("\n"), "utf8"); + writeFileSync(join(dir, "agents", "clean.toml"), [ + "name = \"clean\"", + "model = \"gpt-5.6-sol\"", + "", + ].join("\n"), "utf8"); + expect(scanCodexAgentRolesWithTomlModelFallback(dir)).toEqual(["with_fallback"]); + }); + test("subagentFallbackGuidanceText renders configured chain", () => { expect(subagentFallbackGuidanceText(cfg())).toContain("gpt-5.6-sol"); expect(subagentFallbackGuidanceText(cfg({ subagentModelFallback: undefined }))).toBe(""); From 66ba021b865b0a32f3c554d5c2be6278824b43f0 Mon Sep 17 00:00:00 2001 From: Yuxin Qiao <104957188+Yuxin-Qiao@users.noreply.github.com> Date: Sat, 8 Aug 2026 12:25:29 +0800 Subject: [PATCH 104/124] fix(doctor): flag model_fallback key presence, including empty arrays CodeRabbit review: Codex >= 0.146 rejects the unknown field regardless of value, so the doctor scan must report model_fallback = [] roles too. Add hasCodexAgentModelFallbackField and a regression test; clarify the Russian config table wording. --- .../docs/ru/reference/configuration/agents.md | 2 +- src/codex/subagent-model-fallback.ts | 22 ++++++++++++++----- tests/subagent-model-fallback.test.ts | 14 ++++++++++-- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/docs-site/src/content/docs/ru/reference/configuration/agents.md b/docs-site/src/content/docs/ru/reference/configuration/agents.md index 31516724b9..4febe3254f 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/agents.md +++ b/docs-site/src/content/docs/ru/reference/configuration/agents.md @@ -18,7 +18,7 @@ description: Multi-agent surface, guidance при делегировании, pr | `multiAgentGuidanceEnabled?` | `boolean` | `true` | Управляет только developer-guidance, написанным самим opencodex, для v1/v2; не меняет native default'ы агентов, tools, routing, roster'ы и effort cap'ы. | | `syncCodexSubagentDefaults?` | `boolean` | `false` | Разрешает записывать `injectionModel` и, при наличии, `injectionEffort` как native default'ы Codex при sync/restart. Требует `injectionModel`. | | `subagentModelFallback?` | `string[]` | `[]` | Глобальные fallback-модели для порождённых child-turn'ов в порядке приоритета. | -| `subagentModelFallbackByModel?` | `Record` | `{}` | Модельные цепочки fallback по ключу запрошенной основной модели. Рекомендуемое место для per-role метаданных fallback: запись `model_fallback` в agent TOML Codex заставляет Codex 0.146+ пропустить роль (#1190). | +| `subagentModelFallbackByModel?` | `Record` | `{}` | Модельные цепочки fallback по ключу запрошенной основной модели. Это поддерживаемое место для per-role метаданных fallback; поле `model_fallback` в `$CODEX_HOME/agents/*.toml` поддерживается только как legacy и заставляет Codex 0.146+ пропустить роль (#1190). | | `subagentModelFallbackPollMs?` | `number` | `60000` | Интервал кэша для availability probe. Значения ниже 1000 ms возвращаются к дефолту. | | `effortCap?` | `string` | — | Жёсткий потолок effort для qualifying v2 main-turn'ов и помеченных spawned-child turn'ов. Принимает `low`–`ultra`. | | `subagentEffortCap?` | `string` | — | Дополнительный потолок только для spawned-child turn'ов. Если применимы оба cap'а, выигрывает более низкий. | diff --git a/src/codex/subagent-model-fallback.ts b/src/codex/subagent-model-fallback.ts index 3cab8a1883..7c22990ecb 100644 --- a/src/codex/subagent-model-fallback.ts +++ b/src/codex/subagent-model-fallback.ts @@ -583,14 +583,24 @@ export function readCodexAgentModelFallback(role: string, codexHome = CODEX_HOME } /** - * Roles whose TOML still carries `model_fallback`. Codex >= 0.146 rejects the - * entire role file as an unknown field (#1190), so these files are skipped by - * the native runtime even though opencodex can still read them. + * True when the role TOML carries a readable `model_fallback` key, even an + * empty array. Presence is what matters for the doctor scan: Codex >= 0.146 + * rejects the field as unknown and skips the whole role regardless of its value. */ +export function hasCodexAgentModelFallbackField(role: string, codexHome = CODEX_HOME): boolean { + const file = join(codexHome, "agents", `${role}.toml`); + if (!existsSync(file)) return false; + try { + const content = readFileSync(file, "utf8"); + return /^\s*model_fallback\s*=/m.test(content); + } catch { + return false; + } +} + +/** Roles whose TOML still carries `model_fallback`, including empty arrays. */ export function scanCodexAgentRolesWithTomlModelFallback(codexHome = CODEX_HOME): string[] { - return listCodexAgentRoles(codexHome).filter( - role => readCodexAgentModelFallback(role, codexHome).length > 0, - ); + return listCodexAgentRoles(codexHome).filter(role => hasCodexAgentModelFallbackField(role, codexHome)); } export function listCodexAgentRoles(codexHome = CODEX_HOME): string[] { diff --git a/tests/subagent-model-fallback.test.ts b/tests/subagent-model-fallback.test.ts index 90ad97327d..c173f9f435 100644 --- a/tests/subagent-model-fallback.test.ts +++ b/tests/subagent-model-fallback.test.ts @@ -987,7 +987,7 @@ describe("subagent model fallback chain", () => { expect(result?.to).toBe("alibaba-token-plan/qwen3.8-max"); }); - test("scanCodexAgentRolesWithTomlModelFallback reports only roles carrying the field", () => { + test("scanCodexAgentRolesWithTomlModelFallback reports roles carrying the field, including empty arrays", () => { const dir = codexHomeFixture(); writeFileSync(join(dir, "agents", "with_fallback.toml"), [ "name = \"with_fallback\"", @@ -995,12 +995,22 @@ describe("subagent model fallback chain", () => { "model_fallback = [\"kimi/k3\"]", "", ].join("\n"), "utf8"); + writeFileSync(join(dir, "agents", "empty_fallback.toml"), [ + "name = \"empty_fallback\"", + "model = \"gpt-5.6-sol\"", + "model_fallback = []", + "", + ].join("\n"), "utf8"); writeFileSync(join(dir, "agents", "clean.toml"), [ "name = \"clean\"", "model = \"gpt-5.6-sol\"", "", ].join("\n"), "utf8"); - expect(scanCodexAgentRolesWithTomlModelFallback(dir)).toEqual(["with_fallback"]); + expect(scanCodexAgentRolesWithTomlModelFallback(dir).sort()).toEqual([ + "empty_fallback", + "with_fallback", + ]); + expect(readCodexAgentModelFallback("empty_fallback", dir)).toEqual([]); }); test("subagentFallbackGuidanceText renders configured chain", () => { From 81cebba00ef94f9dad47f8d15f3e9b6380815619 Mon Sep 17 00:00:00 2001 From: Yuxin Qiao <104957188+Yuxin-Qiao@users.noreply.github.com> Date: Sat, 8 Aug 2026 12:37:32 +0800 Subject: [PATCH 105/124] fix(doctor): make model_fallback detection TOML-aware CodeRabbit review: hasCodexAgentModelFallbackField matched raw text, so quoted keys like "model_fallback" were missed and the same text inside a multiline string literal was a false positive. Share one TOML-aware, presence-aware parser between the doctor scan and the fallback-reading path; it recognizes quoted keys, skips string contents, and keeps absent vs. empty-array distinct. Add quoted-key and multiline-string tests. --- src/codex/subagent-model-fallback.ts | 177 +++++++++++++++++++++++--- tests/subagent-model-fallback.test.ts | 43 +++++++ 2 files changed, 200 insertions(+), 20 deletions(-) diff --git a/src/codex/subagent-model-fallback.ts b/src/codex/subagent-model-fallback.ts index 7c22990ecb..ef37e89451 100644 --- a/src/codex/subagent-model-fallback.ts +++ b/src/codex/subagent-model-fallback.ts @@ -547,33 +547,168 @@ export function subagentFallbackGuidanceText(config: OcxConfig): string { return ` Subagent model fallback chain (priority order): ${quoted}. When the primary model is quota-exhausted, opencodex rewrites thread_spawn requests to the next available model automatically.`; } -const TOML_STRING_ARRAY = /^(model_fallback)\s*=\s*\[(.*)\]\s*$/s; +const TOML_MODEL_FALLBACK_KEY = /^\s*(?:model_fallback|"model_fallback"|'model_fallback')\s*=/; -function parseTomlStringArray(raw: string): string[] { - const matches = [...raw.matchAll(/"((?:\\.|[^"\\])*)"/g)]; - return matches.map(match => match[1]!.replace(/\\"/g, "\"")); +type TomlModelFallbackField = { present: false; value: null } | { present: true; value: string[] | null }; + +type TomlScanState = { + inMultilineString: '"""' | "'''" | null; + arrayDepth: number; +}; + +/** Track TOML strings, comments, and array brackets on one line. */ +function scanTomlLine(line: string, state: TomlScanState): void { + let i = 0; + while (i < line.length) { + const ch = line[i]!; + if (ch === "#") return; + if (ch === '"' || ch === "'") { + const delimiter = ch.repeat(3); + if (line.startsWith(delimiter, i)) { + const end = line.indexOf(delimiter, i + 3); + if (end === -1) { + state.inMultilineString = delimiter as '"""' | "'''"; + return; + } + i = end + 3; + continue; + } + i++; + while (i < line.length) { + if (ch === '"' && line[i] === "\\") { + i += 2; + continue; + } + if (line[i] === ch) break; + i++; + } + i++; + continue; + } + if (ch === "[") state.arrayDepth++; + else if (ch === "]") state.arrayDepth = Math.max(0, state.arrayDepth - 1); + i++; + } } -function parseTomlModelFallback(content: string): string[] | null { - const match = content.match(/^\s*model_fallback\s*=\s*\[(.*?)\]\s*$/ms); - if (!match) return null; - return parseTomlStringArray(match[1] ?? ""); +/** Parse a TOML string starting at `start`; returns the decoded value and end offset. */ +function parseTomlStringAt(text: string, start: number): { value: string; end: number } | null { + const quote = text[start]!; + const delimiter = quote.repeat(3); + if (text.startsWith(delimiter, start)) { + const end = findTomlMultilineStringEnd(text, start + 3, quote); + if (end === -1) return null; + let value = text.slice(start + 3, end).trim(); + if (quote === '"') value = value.replace(/\\"/g, '"').replace(/\\\\/g, "\\"); + return { value, end: end + 3 }; + } + let i = start + 1; + let value = ""; + while (i < text.length) { + const ch = text[i]!; + if (quote === '"' && ch === "\\") { + const next = text[i + 1]; + if (next === '"' || next === "\\") { + value += next; + i += 2; + continue; + } + } + if (ch === quote) return { value, end: i + 1 }; + value += ch; + i++; + } + return null; +} + +function findTomlMultilineStringEnd(text: string, from: number, quote: string): number { + const delimiter = quote.repeat(3); + let index = text.indexOf(delimiter, from); + while (index !== -1) { + if (quote === "'") return index; + let backslashes = 0; + for (let j = index - 1; j >= 0 && text[j] === "\\"; j--) backslashes += 1; + if (backslashes % 2 === 0) return index; + index = text.indexOf(delimiter, index + 1); + } + return -1; +} + +/** Parse a TOML string-array value; null when the value is not an array of strings. */ +function parseTomlStringArrayValue(text: string): string[] | null { + const values: string[] = []; + let i = 0; + const skipIgnored = () => { + while (i < text.length) { + const ch = text[i]!; + if (ch === "#") { + while (i < text.length && text[i] !== "\n") i += 1; + continue; + } + if (/\s/.test(ch) || ch === ",") { + i += 1; + continue; + } + break; + } + }; + skipIgnored(); + if (text[i] !== "[") return null; + i += 1; + for (;;) { + skipIgnored(); + if (i >= text.length) return null; + const ch = text[i]!; + if (ch === "]") return values; + if (ch === '"' || ch === "'") { + const parsed = parseTomlStringAt(text, i); + if (!parsed) return null; + values.push(parsed.value); + i = parsed.end; + continue; + } + return null; + } +} + +/** + * TOML-aware, presence-aware parse of the `model_fallback` field. Quoted keys + * are recognized, and text inside strings (including multiline strings) is + * never treated as a key. `present` is true whenever the key exists, even when + * the value is not a readable string array; `value` is null in that case. + */ +function parseTomlModelFallbackField(content: string): TomlModelFallbackField { + const lines = content.split(/\r?\n/); + const state: TomlScanState = { inMultilineString: null, arrayDepth: 0 }; + for (let i = 0; i < lines.length; i += 1) { + const line = lines[i]!; + if (state.inMultilineString) { + const end = line.indexOf(state.inMultilineString); + if (end === -1) continue; + state.inMultilineString = null; + scanTomlLine(line.slice(end + 3), state); + continue; + } + if (state.arrayDepth === 0) { + const key = line.match(TOML_MODEL_FALLBACK_KEY); + if (key) { + const rest = `${line.slice(key[0].length)}\n${lines.slice(i + 1).join("\n")}`; + return { present: true, value: parseTomlStringArrayValue(rest) }; + } + } + scanTomlLine(line, state); + } + return { present: false, value: null }; } export function readAgentModelFallback(filePath: string): string[] | null { try { const content = readFileSync(filePath, "utf8"); - const multiline = parseTomlModelFallback(content); - if (multiline) return multiline; - for (const line of content.split(/\r?\n/)) { - const match = line.match(TOML_STRING_ARRAY); - if (!match) continue; - return parseTomlStringArray(match[2] ?? ""); - } + const parsed = parseTomlModelFallbackField(content); + return parsed.present ? parsed.value : null; } catch { return null; } - return null; } export function readCodexAgentModelFallback(role: string, codexHome = CODEX_HOME): string[] { @@ -583,16 +718,18 @@ export function readCodexAgentModelFallback(role: string, codexHome = CODEX_HOME } /** - * True when the role TOML carries a readable `model_fallback` key, even an - * empty array. Presence is what matters for the doctor scan: Codex >= 0.146 - * rejects the field as unknown and skips the whole role regardless of its value. + * True when the role TOML carries a `model_fallback` key, even an empty array. + * Presence is what matters for the doctor scan: Codex >= 0.146 rejects the + * field as unknown and skips the whole role regardless of its value. Uses the + * same TOML-aware parse as the fallback-reading path, so quoted keys count and + * text inside string literals does not. */ export function hasCodexAgentModelFallbackField(role: string, codexHome = CODEX_HOME): boolean { const file = join(codexHome, "agents", `${role}.toml`); if (!existsSync(file)) return false; try { const content = readFileSync(file, "utf8"); - return /^\s*model_fallback\s*=/m.test(content); + return parseTomlModelFallbackField(content).present; } catch { return false; } diff --git a/tests/subagent-model-fallback.test.ts b/tests/subagent-model-fallback.test.ts index c173f9f435..554c65340f 100644 --- a/tests/subagent-model-fallback.test.ts +++ b/tests/subagent-model-fallback.test.ts @@ -1013,6 +1013,49 @@ describe("subagent model fallback chain", () => { expect(readCodexAgentModelFallback("empty_fallback", dir)).toEqual([]); }); + test("scanCodexAgentRolesWithTomlModelFallback recognizes quoted model_fallback keys", () => { + const dir = codexHomeFixture(); + writeFileSync(join(dir, "agents", "quoted_key.toml"), [ + "name = \"quoted_key\"", + "model = \"gpt-5.6-sol\"", + "\"model_fallback\" = [\"kimi/k3\"]", + "", + ].join("\n"), "utf8"); + writeFileSync(join(dir, "agents", "literal_key.toml"), [ + "name = \"literal_key\"", + "model = \"gpt-5.6-sol\"", + "'model_fallback' = []", + "", + ].join("\n"), "utf8"); + expect(scanCodexAgentRolesWithTomlModelFallback(dir).sort()).toEqual([ + "literal_key", + "quoted_key", + ]); + expect(readCodexAgentModelFallback("quoted_key", dir)).toEqual(["kimi/k3"]); + expect(readCodexAgentModelFallback("literal_key", dir)).toEqual([]); + }); + + test("scanCodexAgentRolesWithTomlModelFallback ignores model_fallback text inside strings", () => { + const dir = codexHomeFixture(); + writeFileSync(join(dir, "agents", "multiline_text.toml"), [ + "name = \"multiline_text\"", + "model = \"gpt-5.6-sol\"", + "description = \"\"\"", + "model_fallback = []", + "\"model_fallback\" = [\"kimi/k3\"]", + "\"\"\"", + "", + ].join("\n"), "utf8"); + writeFileSync(join(dir, "agents", "single_line_text.toml"), [ + "name = \"single_line_text\"", + "model = \"gpt-5.6-sol\"", + "description = \"model_fallback = [\\\"kimi/k3\\\"]\"", + "", + ].join("\n"), "utf8"); + expect(scanCodexAgentRolesWithTomlModelFallback(dir)).toEqual([]); + expect(readCodexAgentModelFallback("multiline_text", dir)).toEqual([]); + }); + test("subagentFallbackGuidanceText renders configured chain", () => { expect(subagentFallbackGuidanceText(cfg())).toContain("gpt-5.6-sol"); expect(subagentFallbackGuidanceText(cfg({ subagentModelFallback: undefined }))).toBe(""); From 08ecdbad845b637b26fa128332e606827f809f16 Mon Sep 17 00:00:00 2001 From: Yuxin Qiao <104957188+Yuxin-Qiao@users.noreply.github.com> Date: Sat, 8 Aug 2026 12:48:28 +0800 Subject: [PATCH 106/124] fix(doctor): handle escaped delimiters and strict array commas CodeRabbit review: an escaped triple quote (\""") inside a multiline basic string closed scanner state early, and string arrays accepted adjacent elements without commas. Use findTomlMultilineStringEnd in both scanner paths and require one comma between successive array elements while keeping empty and trailing-comma arrays valid. Add regression tests for escaped delimiters and malformed arrays. --- src/codex/subagent-model-fallback.ts | 19 ++++++++--- tests/subagent-model-fallback.test.ts | 47 +++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/src/codex/subagent-model-fallback.ts b/src/codex/subagent-model-fallback.ts index ef37e89451..0c074068e7 100644 --- a/src/codex/subagent-model-fallback.ts +++ b/src/codex/subagent-model-fallback.ts @@ -565,7 +565,7 @@ function scanTomlLine(line: string, state: TomlScanState): void { if (ch === '"' || ch === "'") { const delimiter = ch.repeat(3); if (line.startsWith(delimiter, i)) { - const end = line.indexOf(delimiter, i + 3); + const end = findTomlMultilineStringEnd(line, i + 3, ch); if (end === -1) { state.inMultilineString = delimiter as '"""' | "'''"; return; @@ -634,7 +634,7 @@ function findTomlMultilineStringEnd(text: string, from: number, quote: string): return -1; } -/** Parse a TOML string-array value; null when the value is not an array of strings. */ +/** Parse a TOML string-array value; null when the value is not a well-formed array of strings. */ function parseTomlStringArrayValue(text: string): string[] | null { const values: string[] = []; let i = 0; @@ -645,7 +645,7 @@ function parseTomlStringArrayValue(text: string): string[] | null { while (i < text.length && text[i] !== "\n") i += 1; continue; } - if (/\s/.test(ch) || ch === ",") { + if (/\s/.test(ch)) { i += 1; continue; } @@ -655,16 +655,25 @@ function parseTomlStringArrayValue(text: string): string[] | null { skipIgnored(); if (text[i] !== "[") return null; i += 1; + let expectValue = true; for (;;) { skipIgnored(); if (i >= text.length) return null; const ch = text[i]!; if (ch === "]") return values; + if (ch === ",") { + if (expectValue) return null; // leading or doubled comma + expectValue = true; + i += 1; + continue; + } if (ch === '"' || ch === "'") { + if (!expectValue) return null; // adjacent strings must be comma-separated const parsed = parseTomlStringAt(text, i); if (!parsed) return null; values.push(parsed.value); i = parsed.end; + expectValue = false; continue; } return null; @@ -683,7 +692,7 @@ function parseTomlModelFallbackField(content: string): TomlModelFallbackField { for (let i = 0; i < lines.length; i += 1) { const line = lines[i]!; if (state.inMultilineString) { - const end = line.indexOf(state.inMultilineString); + const end = findTomlMultilineStringEnd(line, 0, state.inMultilineString[0]!); if (end === -1) continue; state.inMultilineString = null; scanTomlLine(line.slice(end + 3), state); @@ -752,4 +761,4 @@ export function listCodexAgentRoles(codexHome = CODEX_HOME): string[] { export function shouldPrimeSubagentQuota(config: OcxConfig, now = Date.now()): boolean { const last = quotaPrimedAt.get("global") ?? 0; return now - last >= pollIntervalMs(config); -} \ No newline at end of file +} diff --git a/tests/subagent-model-fallback.test.ts b/tests/subagent-model-fallback.test.ts index 554c65340f..41db837a68 100644 --- a/tests/subagent-model-fallback.test.ts +++ b/tests/subagent-model-fallback.test.ts @@ -1056,6 +1056,53 @@ describe("subagent model fallback chain", () => { expect(readCodexAgentModelFallback("multiline_text", dir)).toEqual([]); }); + test("scanCodexAgentRolesWithTomlModelFallback handles escaped triple quotes inside multiline strings", () => { + const dir = codexHomeFixture(); + writeFileSync(join(dir, "agents", "escaped_open.toml"), [ + "name = \"escaped_open\"", + "model = \"gpt-5.6-sol\"", + "description = \"\"\"a\\\"\"\"", + "model_fallback = []", + "\"\"\"", + "", + ].join("\n"), "utf8"); + writeFileSync(join(dir, "agents", "escaped_continuation.toml"), [ + "name = \"escaped_continuation\"", + "model = \"gpt-5.6-sol\"", + "description = \"\"\"", + "still inside \\\"\"\"", + "model_fallback = []", + "\"\"\"", + "", + ].join("\n"), "utf8"); + expect(scanCodexAgentRolesWithTomlModelFallback(dir)).toEqual([]); + expect(readCodexAgentModelFallback("escaped_open", dir)).toEqual([]); + expect(readCodexAgentModelFallback("escaped_continuation", dir)).toEqual([]); + }); + + test("readCodexAgentModelFallback rejects string arrays without proper commas", () => { + const dir = codexHomeFixture(); + writeFileSync(join(dir, "agents", "missing_comma.toml"), [ + "name = \"missing_comma\"", + "model = \"gpt-5.6-sol\"", + "model_fallback = [\"kimi/k3\" \"alibaba-token-plan/qwen3.8-max\"]", + "", + ].join("\n"), "utf8"); + writeFileSync(join(dir, "agents", "leading_comma.toml"), [ + "name = \"leading_comma\"", + "model = \"gpt-5.6-sol\"", + "model_fallback = [, \"kimi/k3\"]", + "", + ].join("\n"), "utf8"); + expect(readCodexAgentModelFallback("missing_comma", dir)).toEqual([]); + expect(readCodexAgentModelFallback("leading_comma", dir)).toEqual([]); + // Doctor still reports both: the field exists even when its value is malformed. + expect(scanCodexAgentRolesWithTomlModelFallback(dir).sort()).toEqual([ + "leading_comma", + "missing_comma", + ]); + }); + test("subagentFallbackGuidanceText renders configured chain", () => { expect(subagentFallbackGuidanceText(cfg())).toContain("gpt-5.6-sol"); expect(subagentFallbackGuidanceText(cfg({ subagentModelFallback: undefined }))).toBe(""); From d75743a9f2c6f1b782bd26d49e21d1ee3f4273e9 Mon Sep 17 00:00:00 2001 From: Yuxin Qiao <104957188+Yuxin-Qiao@users.noreply.github.com> Date: Sat, 8 Aug 2026 13:50:36 +0800 Subject: [PATCH 107/124] fix(doctor): reject trailing tokens after the model_fallback array CodeRabbit review: the value parser returned as soon as it read the closing bracket, so 'model_fallback = ["kimi/k3"] invalid' produced a fallback list. After ']' only horizontal whitespace, an inline comment, or the line end is valid; anything else makes the value malformed while presence (and thus the doctor WARN) is still reported. --- src/codex/subagent-model-fallback.ts | 17 ++++++++++++++++- tests/subagent-model-fallback.test.ts | 18 +++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/codex/subagent-model-fallback.ts b/src/codex/subagent-model-fallback.ts index 0c074068e7..db99df8b97 100644 --- a/src/codex/subagent-model-fallback.ts +++ b/src/codex/subagent-model-fallback.ts @@ -634,6 +634,18 @@ function findTomlMultilineStringEnd(text: string, from: number, quote: string): return -1; } +/** After an array close, only horizontal whitespace, an inline comment, or the line end is valid. */ +function isValidTomlArrayTail(text: string, from: number): boolean { + let i = from; + while (i < text.length && (text[i] === " " || text[i] === "\t")) i += 1; + if (i >= text.length || text[i] === "\n" || text[i] === "\r") return true; + if (text[i] === "#") { + while (i < text.length && text[i] !== "\n") i += 1; + return true; + } + return false; +} + /** Parse a TOML string-array value; null when the value is not a well-formed array of strings. */ function parseTomlStringArrayValue(text: string): string[] | null { const values: string[] = []; @@ -660,7 +672,10 @@ function parseTomlStringArrayValue(text: string): string[] | null { skipIgnored(); if (i >= text.length) return null; const ch = text[i]!; - if (ch === "]") return values; + if (ch === "]") { + if (!isValidTomlArrayTail(text, i + 1)) return null; + return values; + } if (ch === ",") { if (expectValue) return null; // leading or doubled comma expectValue = true; diff --git a/tests/subagent-model-fallback.test.ts b/tests/subagent-model-fallback.test.ts index 41db837a68..c3b1af70ac 100644 --- a/tests/subagent-model-fallback.test.ts +++ b/tests/subagent-model-fallback.test.ts @@ -1094,12 +1094,28 @@ describe("subagent model fallback chain", () => { "model_fallback = [, \"kimi/k3\"]", "", ].join("\n"), "utf8"); + writeFileSync(join(dir, "agents", "trailing_token.toml"), [ + "name = \"trailing_token\"", + "model = \"gpt-5.6-sol\"", + "model_fallback = [\"kimi/k3\"] invalid", + "", + ].join("\n"), "utf8"); + writeFileSync(join(dir, "agents", "inline_comment.toml"), [ + "name = \"inline_comment\"", + "model = \"gpt-5.6-sol\"", + "model_fallback = [\"kimi/k3\"] # keep", + "", + ].join("\n"), "utf8"); expect(readCodexAgentModelFallback("missing_comma", dir)).toEqual([]); expect(readCodexAgentModelFallback("leading_comma", dir)).toEqual([]); - // Doctor still reports both: the field exists even when its value is malformed. + expect(readCodexAgentModelFallback("trailing_token", dir)).toEqual([]); + expect(readCodexAgentModelFallback("inline_comment", dir)).toEqual(["kimi/k3"]); + // Doctor reports every role carrying the field, even when its value is malformed. expect(scanCodexAgentRolesWithTomlModelFallback(dir).sort()).toEqual([ + "inline_comment", "leading_comma", "missing_comma", + "trailing_token", ]); }); From fc17327b3f38fd86f5ac0f107e33e7127ccaee36 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sun, 9 Aug 2026 14:49:42 +0900 Subject: [PATCH 108/124] fix(subagent): place legacy fallback after global config --- src/codex/subagent-model-fallback.ts | 21 ++++++++------ tests/subagent-model-fallback.test.ts | 41 +++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/src/codex/subagent-model-fallback.ts b/src/codex/subagent-model-fallback.ts index db99df8b97..da40ebffff 100644 --- a/src/codex/subagent-model-fallback.ts +++ b/src/codex/subagent-model-fallback.ts @@ -113,7 +113,12 @@ function fallbackChainKey(model: string, namespaces: unknown): string { return JSON.stringify(["account", selector, model.slice(slash + 1).toLowerCase()]); } -function normalizedChain(primary: string, config: OcxConfig, extra: readonly string[] = []): string[] { +function normalizedChain( + primary: string, + config: OcxConfig, + extra: readonly string[] = [], + trailing: readonly string[] = [], +): string[] { const chain: string[] = []; const seen = new Set(); const push = (model: string | undefined) => { @@ -127,6 +132,7 @@ function normalizedChain(primary: string, config: OcxConfig, extra: readonly str push(primary); for (const model of extra) push(model); for (const model of config.subagentModelFallback ?? []) push(model); + for (const model of trailing) push(model); return chain; } @@ -268,8 +274,9 @@ export function selectAvailableSubagentModel( now = Date.now(), nativeFallbackOnly = false, accountUsabilityOptions?: CodexAccountUsabilityOptions, + trailingFallback: readonly string[] = [], ): { model: string; rewritten: boolean; skipped: string[] } { - const chain = normalizedChain(primary, config, extraFallback); + const chain = normalizedChain(primary, config, extraFallback, trailingFallback); const skipped: string[] = []; for (const candidate of chain) { if (nativeFallbackOnly) { @@ -517,20 +524,18 @@ export function applySubagentModelFallback( ); // Config-keyed chains are the supported per-role home (#1190); TOML `model_fallback` // stays readable for backwards compatibility with homes written before Codex 0.146. - const roleFallback = [ - ...resolveConfiguredModelFallbackForPrimary(parsed.modelId, config), - ...tomlRoleFallback, - ]; + const configuredFallback = resolveConfiguredModelFallbackForPrimary(parsed.modelId, config); const globalFallback = config.subagentModelFallback ?? []; - if (globalFallback.length === 0 && roleFallback.length === 0) return null; + if (globalFallback.length === 0 && configuredFallback.length === 0 && tomlRoleFallback.length === 0) return null; const selection = selectAvailableSubagentModel( parsed.modelId, config, - roleFallback, + configuredFallback, accountId, now, nativeFallbackOnly, accountUsabilityOptions, + tomlRoleFallback, ); if (!selection.rewritten) return selection.skipped.length > 0 ? { from: parsed.modelId, to: parsed.modelId, skipped: selection.skipped } diff --git a/tests/subagent-model-fallback.test.ts b/tests/subagent-model-fallback.test.ts index c3b1af70ac..3d433aef12 100644 --- a/tests/subagent-model-fallback.test.ts +++ b/tests/subagent-model-fallback.test.ts @@ -987,6 +987,47 @@ describe("subagent model fallback chain", () => { expect(result?.to).toBe("alibaba-token-plan/qwen3.8-max"); }); + test("applySubagentModelFallback orders and dedupes all four fallback stages", () => { + const dir = codexHomeFixture(); + writeFileSync(join(dir, "agents", "executor.toml"), [ + "name = \"executor\"", + "model = \"gpt-5.6-sol\"", + "model_fallback = [\"xai/grok-4.5\", \"KIMI/K3\"]", + "", + ].join("\n"), "utf8"); + const config = cfg({ + subagentModelFallbackByModel: { + "gpt-5.6-sol": ["alibaba-token-plan/qwen3.8-max", "GPT-5.6-SOL"], + }, + subagentModelFallback: ["kimi/k3", "ALIBABA-TOKEN-PLAN/QWEN3.8-MAX"], + }); + updateAccountQuota("pool-a", 95); + for (const model of ["alibaba-token-plan/qwen3.8-max", "kimi/k3", "xai/grok-4.5"]) { + noteSubagentModelFailure(model, "quota exhausted", config); + } + const parsed = { + modelId: "gpt-5.6-sol", + options: {}, + context: { messages: [] }, + _rawBody: { model: "gpt-5.6-sol" }, + }; + const result = applySubagentModelFallback( + parsed as never, + new Headers({ "x-openai-subagent": "collab_spawn" }), + config, + ); + expect(result).toEqual({ + from: "gpt-5.6-sol", + to: "gpt-5.6-sol", + skipped: [ + "gpt-5.6-sol", + "alibaba-token-plan/qwen3.8-max", + "kimi/k3", + "xai/grok-4.5", + ], + }); + }); + test("scanCodexAgentRolesWithTomlModelFallback reports roles carrying the field, including empty arrays", () => { const dir = codexHomeFixture(); writeFileSync(join(dir, "agents", "with_fallback.toml"), [ From a1e418509602017464976ff0e26e2f29f14511d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=A9=AC=E5=87=A4=E5=B2=90?= Date: Sun, 9 Aug 2026 00:02:11 +0800 Subject: [PATCH 109/124] fix(providers): correct Qwen3.8 reasoning levels --- .../content/docs/reference/architecture.md | 6 ++ .../docs/zh-cn/reference/architecture.md | 5 ++ src/providers/derive.ts | 62 +++++++++++++++++++ src/providers/registry.ts | 19 +++++- src/router.ts | 5 +- tests/alibaba-intl-token-plan.test.ts | 31 +++++++++- tests/provider-registry-parity.test.ts | 9 ++- tests/reasoning-effort.test.ts | 40 +++++++++--- 8 files changed, 161 insertions(+), 16 deletions(-) diff --git a/docs-site/src/content/docs/reference/architecture.md b/docs-site/src/content/docs/reference/architecture.md index c33c6047f3..d47e140dca 100644 --- a/docs-site/src/content/docs/reference/architecture.md +++ b/docs-site/src/content/docs/reference/architecture.md @@ -166,6 +166,12 @@ upstream providers may support only a smaller subset or require a real alias. Th - Resolves per-model and per-provider `reasoningEffortMap` overrides for custom wire mappings. - Drops the effort entirely for models listed in `noReasoningModels`. +Qwen3.8-Max is an explicit direct-effort exception to the older Qwen3.x budget contract. Alibaba +Token Plan records its upstream-supported ladder as `low`, `medium`, and `xhigh` (the default), and +sends the effective value as `reasoning_effort`; Codex-only compatibility tops are clamped to +`xhigh` on the wire. Runtime registry enrichment repairs older persisted preset metadata that still +classifies this model as a `thinking_budget` model. + ## Core types The internal model lives in `types.ts`: `OcxParsedRequest`, `OcxContext`, the `OcxMessage` union, diff --git a/docs-site/src/content/docs/zh-cn/reference/architecture.md b/docs-site/src/content/docs/zh-cn/reference/architecture.md index 806631086f..926a6d096f 100644 --- a/docs-site/src/content/docs/zh-cn/reference/architecture.md +++ b/docs-site/src/content/docs/zh-cn/reference/architecture.md @@ -157,6 +157,11 @@ Codex context compaction 同样适用于路由模型。`server/responses/compact - 解析模型级和 provider 级 `reasoningEffortMap` override，用于自定义 wire 映射。 - 对 `noReasoningModels` 中的模型完全移除 effort。 +Qwen3.8-Max 是旧版 Qwen3.x budget 契约之外、明确使用直接 effort 的例外。Alibaba Token +Plan 把其上游支持等级记录为 `low`、`medium` 和 `xhigh`（默认值），并通过 +`reasoning_effort` 发送最终值；仅供 Codex 兼容的顶档在发送时会限制为 `xhigh`。运行时的 +注册表补全会修复仍把该模型归类为 `thinking_budget` 模型的旧版持久化预设元数据。 + ## 核心类型 内部模型位于 `types.ts`：`OcxParsedRequest`、`OcxContext`、`OcxMessage` 联合类型、 diff --git a/src/providers/derive.ts b/src/providers/derive.ts index db7287d18d..1f21c24909 100644 --- a/src/providers/derive.ts +++ b/src/providers/derive.ts @@ -109,6 +109,67 @@ function cloneNestedRecord(input: Record>): Recor return Object.fromEntries(Object.entries(input).map(([key, value]) => [key, { ...value }])); } +function omitFoldedModelKeys( + record: Record | undefined, + foldedModels: ReadonlySet, +): Record | undefined { + if (!record) return undefined; + const next = Object.fromEntries( + Object.entries(record).filter(([model]) => !foldedModels.has(model.toLowerCase())), + ) as Record; + return Object.keys(next).length > 0 ? next : undefined; +} + +/** + * Apply a registry-owned direct `reasoning_effort` contract after user/seed metadata is merged. + * + * Provider presets are persisted with capability metadata, so an existing config can retain an + * obsolete thinking-budget classification after the registry learns that a model has a native + * effort field. The registry opts only verified model contracts into this repair; providers that + * do not resolve through the matching registry entry keep their user-supplied classification. + */ +export function applyDirectReasoningEffortContracts( + entry: ProviderRegistryEntry, + prov: OcxProviderConfig, +): void { + const models = entry.directReasoningEffortModels; + if (!models || models.length === 0) return; + + const direct = new Set(models.map(model => model.toLowerCase())); + const keepNonDirect = (model: string): boolean => !direct.has(model.toLowerCase()); + + prov.thinkingBudgetModels = prov.thinkingBudgetModels?.filter(keepNonDirect); + prov.thinkingToggleModels = prov.thinkingToggleModels?.filter(keepNonDirect); + prov.modelReasoningEfforts = omitFoldedModelKeys(prov.modelReasoningEfforts, direct); + prov.modelDefaultReasoningEfforts = omitFoldedModelKeys(prov.modelDefaultReasoningEfforts, direct); + prov.modelReasoningEffortMap = omitFoldedModelKeys(prov.modelReasoningEffortMap, direct); + + for (const model of models) { + const efforts = entry.modelReasoningEfforts?.[model]; + if (efforts) { + prov.modelReasoningEfforts = { + ...(prov.modelReasoningEfforts ?? {}), + [model]: [...efforts], + }; + } + + const defaultEffort = entry.modelDefaultReasoningEfforts?.[model]; + if (defaultEffort) { + prov.modelDefaultReasoningEfforts = { + ...(prov.modelDefaultReasoningEfforts ?? {}), + [model]: defaultEffort, + }; + } + + // An explicit empty model map masks any provider-wide aliases. Without it, a stale global + // mapping such as xhigh -> max would win before the verified direct ladder can clamp it. + prov.modelReasoningEffortMap = { + ...(prov.modelReasoningEffortMap ?? {}), + [model]: {}, + }; + } +} + /** * Build the provider config a registry entry contributes when a preset is materialized. * The registry auth kind is preserved verbatim (including `"local"`) so fail-closed gates @@ -359,6 +420,7 @@ export function enrichProviderFromRegistry(name: string, prov: OcxProviderConfig if (prov.freeTier === undefined && seed.freeTier !== undefined) prov.freeTier = seed.freeTier; if (prov.modelSuffixBracketStrip === undefined && seed.modelSuffixBracketStrip !== undefined) prov.modelSuffixBracketStrip = seed.modelSuffixBracketStrip; if (!prov.headers && seed.headers) prov.headers = { ...seed.headers }; + applyDirectReasoningEffortContracts(entry, prov); } export function deriveFeaturedProviderIds(): string[] { diff --git a/src/providers/registry.ts b/src/providers/registry.ts index a2ac241414..dfef861022 100644 --- a/src/providers/registry.ts +++ b/src/providers/registry.ts @@ -214,6 +214,12 @@ export interface ProviderRegistryEntry { modelDefaultReasoningEfforts?: Record; reasoningEffortMap?: Record; modelReasoningEffortMap?: Record>; + /** + * Registry-authoritative models that send OpenAI's direct `reasoning_effort` field. + * Runtime enrichment uses this to repair stale preset metadata that still classifies a model + * as a thinking-budget/toggle model. This is registry-only and is never persisted as user config. + */ + directReasoningEffortModels?: string[]; reasoningWireFormat?: OcxProviderConfig["reasoningWireFormat"]; noVisionModels?: string[]; noReasoningModels?: string[]; @@ -349,6 +355,9 @@ const ZHIPU_BIGMODEL_INPUT_MODALITIES: Record = { }; const ZHIPU_BIGMODEL_THINKING_TOGGLE_MODELS = ["glm-4.6", "glm-4.7", "glm-5", "glm-5.1"]; const THINKING_BUDGET_EFFORTS = ["low", "medium", "high", "xhigh", "max"]; +// Qwen3.8-Max is the first Qwen3.x model with official direct `reasoning_effort` support. +// Evidence: https://qwen.ai/blog?id=qwen3.8 +const QWEN38_REASONING_EFFORTS = ["low", "medium", "xhigh"]; const THINKING_BUDGET_MODELS = [ "qwen3.5-397b", "qwen3.6-35b", "qwen3.5-plus", "qwen3.6-plus", "qwen3.7-max", "qwen3.7-plus", @@ -1907,11 +1916,14 @@ export const PROVIDER_REGISTRY: readonly ProviderRegistryEntry[] = [ }, modelReasoningEfforts: { ...Object.fromEntries(ALIBABA_TOKEN_PLAN_QWEN_MODELS.map(id => [id, THINKING_BUDGET_EFFORTS])), + "qwen3.8-max": QWEN38_REASONING_EFFORTS, "glm-5.2": ZAI_GLM_52_REASONING_EFFORTS, "deepseek-v4-pro": deepseekThinkingEffortsFor("deepseek-v4-pro"), }, + modelDefaultReasoningEfforts: { "qwen3.8-max": "xhigh" }, modelReasoningEffortMap: { "deepseek-v4-pro": deepseekReasoningMapFor("deepseek-v4-pro") }, - thinkingBudgetModels: ALIBABA_TOKEN_PLAN_QWEN_MODELS, + directReasoningEffortModels: ["qwen3.8-max"], + thinkingBudgetModels: ALIBABA_TOKEN_PLAN_QWEN_MODELS.filter(id => id !== "qwen3.8-max"), preserveReasoningContentModels: ["glm-5.2", "deepseek-v4-pro", "qwen3.8-max", "qwen3.7-max", "qwen3.7-plus", "qwen3.6-flash"], noVisionModels: ["glm-5.2", "deepseek-v4-pro"], }, @@ -1940,7 +1952,7 @@ export const PROVIDER_REGISTRY: readonly ProviderRegistryEntry[] = [ }, modelReasoningEfforts: { ...Object.fromEntries(ALIBABA_INTL_TOKEN_PLAN_QWEN_MODELS.map(id => [id, THINKING_BUDGET_EFFORTS])), - "qwen3.8-max": ["low", "high", "xhigh"], + "qwen3.8-max": QWEN38_REASONING_EFFORTS, "glm-5.2": ZAI_GLM_52_REASONING_EFFORTS, "deepseek-v4-pro": deepseekThinkingEffortsFor("deepseek-v4-pro"), "deepseek-v4-flash": deepseekThinkingEffortsFor("deepseek-v4-flash"), @@ -1949,7 +1961,8 @@ export const PROVIDER_REGISTRY: readonly ProviderRegistryEntry[] = [ "deepseek-v4-pro": deepseekReasoningMapFor("deepseek-v4-pro"), "deepseek-v4-flash": deepseekReasoningMapFor("deepseek-v4-flash"), }, - thinkingBudgetModels: ALIBABA_INTL_TOKEN_PLAN_QWEN_MODELS, + directReasoningEffortModels: ["qwen3.8-max"], + thinkingBudgetModels: ALIBABA_INTL_TOKEN_PLAN_QWEN_MODELS.filter(id => id !== "qwen3.8-max"), preserveReasoningContentModels: ["glm-5.2", "deepseek-v4-pro", "deepseek-v4-flash", "qwen3.8-max", "qwen3.7-max", "qwen3.7-plus", "qwen3.6-plus", "qwen3.6-flash"], noVisionModels: ["deepseek-v4-pro", "deepseek-v4-flash", "deepseek-v3.2", "glm-5.2", "glm-5.1", "glm-5", "MiniMax-M2.5"], noReasoningModels: ["kimi-k2.7-code", "kimi-k2.6", "kimi-k2.5", "deepseek-v3.2", "glm-5.1", "glm-5", "MiniMax-M2.5"], diff --git a/src/router.ts b/src/router.ts index 7dda3eef1d..8768cb4d60 100644 --- a/src/router.ts +++ b/src/router.ts @@ -12,6 +12,7 @@ import { hasOwnProvider, resolveEnvValue } from "./config"; import { assertProviderDestinationAllowed } from "./lib/destination-policy"; import { redactSecretString, redactUrlForLog } from "./lib/redact"; import { PROVIDER_REGISTRY, providerCodexAccountMode, providerMatchesRegistryTransport } from "./providers/registry"; +import { applyDirectReasoningEffortContracts } from "./providers/derive"; import { isCanonicalOpenAiForwardProvider, LEGACY_CHATGPT_PROVIDER_ID, @@ -300,7 +301,7 @@ function routedProviderConfig(providerName: string, provider: OcxProviderConfig) if (userBaseUrlIsResolved) warnIfBaseUrlDiscarded(providerName, userBaseUrl, baseUrl); assertProviderDestinationAllowed(providerName, { baseUrl, allowPrivateNetwork: provider.allowPrivateNetwork }); - return { + const resolved: OcxProviderConfig = { ...provider, adapter: registryEntry.adapter, baseUrl, @@ -376,6 +377,8 @@ function routedProviderConfig(providerName: string, provider: OcxProviderConfig) ...(thinkingToggleModels ? { thinkingToggleModels } : {}), ...(thinkingBudgetModels ? { thinkingBudgetModels } : {}), }; + applyDirectReasoningEffortContracts(registryEntry, resolved); + return resolved; } function activeProviderEntries(config: OcxConfig): [string, OcxProviderConfig][] { diff --git a/tests/alibaba-intl-token-plan.test.ts b/tests/alibaba-intl-token-plan.test.ts index 1d77a1c2e7..259394793a 100644 --- a/tests/alibaba-intl-token-plan.test.ts +++ b/tests/alibaba-intl-token-plan.test.ts @@ -11,7 +11,7 @@ import { matchBaseUrlChoice, } from "../src/providers/base-url-choices"; import { PROVIDER_REGISTRY } from "../src/providers/registry"; -import { deriveProviderPresets } from "../src/providers/derive"; +import { deriveProviderPresets, enrichProviderFromRegistry } from "../src/providers/derive"; const CHOICES = [...ALIBABA_INTL_BASE_URL_CHOICES]; @@ -56,7 +56,10 @@ describe("alibaba-token-plan-intl registry entry", () => { test("qwen3.8-max reasoning efforts", () => { const entry = PROVIDER_REGISTRY.find(e => e.id === "alibaba-token-plan-intl"); - expect(entry!.modelReasoningEfforts?.["qwen3.8-max"]).toEqual(["low", "high", "xhigh"]); + expect(entry!.modelReasoningEfforts?.["qwen3.8-max"]).toEqual(["low", "medium", "xhigh"]); + expect(entry!.directReasoningEffortModels).toEqual(["qwen3.8-max"]); + expect(entry!.thinkingBudgetModels).not.toContain("qwen3.8-max"); + expect(entry!.thinkingBudgetModels).toContain("qwen3.7-max"); }); test("qwen3.8-max default reasoning effort is xhigh", () => { @@ -86,7 +89,7 @@ describe("alibaba-token-plan-intl registry entry", () => { } // The intl entry additionally carries the effort ladder. const intl = PROVIDER_REGISTRY.find(e => e.id === "alibaba-token-plan-intl")!; - expect(intl.modelReasoningEfforts?.["qwen3.8-max"]).toEqual(["low", "high", "xhigh"]); + expect(intl.modelReasoningEfforts?.["qwen3.8-max"]).toEqual(["low", "medium", "xhigh"]); expect(intl.modelDefaultReasoningEfforts?.["qwen3.8-max"]).toBe("xhigh"); // Only the Beijing entry defaults to this model; intl deliberately defaults to // qwen3.7-max. That predates this rename and is left alone — renaming an id is not @@ -94,6 +97,28 @@ describe("alibaba-token-plan-intl registry entry", () => { expect(PROVIDER_REGISTRY.find(e => e.id === "alibaba-token-plan")!.defaultModel).toBe("qwen3.8-max"); }); + test("registry enrichment repairs a persisted pre-release Qwen3.8 contract", () => { + const provider = { + adapter: "openai-chat", + baseUrl: ALIBABA_INTL_TOKEN_PLAN_BASE_URL, + modelReasoningEfforts: { "QWEN3.8-MAX": ["low", "high", "xhigh"] }, + modelDefaultReasoningEfforts: { "QWEN3.8-MAX": "high" }, + reasoningEffortMap: { xhigh: "max" }, + modelReasoningEffortMap: { "QWEN3.8-MAX": { medium: "high" } }, + thinkingBudgetModels: ["QWEN3.8-MAX", "qwen3.7-max"], + }; + + enrichProviderFromRegistry("alibaba-token-plan-intl", provider); + + expect(provider.modelReasoningEfforts["qwen3.8-max"]).toEqual(["low", "medium", "xhigh"]); + expect(provider.modelReasoningEfforts["QWEN3.8-MAX"]).toBeUndefined(); + expect(provider.modelDefaultReasoningEfforts["qwen3.8-max"]).toBe("xhigh"); + expect(provider.modelDefaultReasoningEfforts["QWEN3.8-MAX"]).toBeUndefined(); + expect(provider.modelReasoningEffortMap?.["qwen3.8-max"]).toEqual({}); + expect(provider.modelReasoningEffortMap?.["QWEN3.8-MAX"]).toBeUndefined(); + expect(provider.thinkingBudgetModels).toEqual(["qwen3.7-max"]); + }); + test("non-reasoning models are marked", () => { const entry = PROVIDER_REGISTRY.find(e => e.id === "alibaba-token-plan-intl"); expect(entry!.noReasoningModels).toContain("kimi-k2.7-code"); diff --git a/tests/provider-registry-parity.test.ts b/tests/provider-registry-parity.test.ts index 5f205e8a30..5fa854e22d 100644 --- a/tests/provider-registry-parity.test.ts +++ b/tests/provider-registry-parity.test.ts @@ -290,8 +290,9 @@ describe("provider registry parity", () => { "qwen3.7-max": ["text", "image"], }, modelReasoningEfforts: { - "qwen3.8-max": ["low", "medium", "high", "xhigh", "max"], + "qwen3.8-max": ["low", "medium", "xhigh"], }, + modelDefaultReasoningEfforts: { "qwen3.8-max": "xhigh" }, modelContextWindows: { "qwen3.8-max": 983_616, "qwen3.7-max": 1_000_000, @@ -300,8 +301,12 @@ describe("provider registry parity", () => { noVisionModels: ["glm-5.2", "deepseek-v4-pro"], preserveReasoningContentModels: expect.arrayContaining(["qwen3.8-max", "qwen3.7-max", "qwen3.7-plus"]), }); + expect(PROVIDER_REGISTRY.find(entry => entry.id === "alibaba-token-plan")?.directReasoningEffortModels) + .toEqual(["qwen3.8-max"]); expect(KEY_LOGIN_PROVIDERS["alibaba-token-plan"].thinkingBudgetModels) - .toContain("qwen3.8-max"); + .not.toContain("qwen3.8-max"); + expect(KEY_LOGIN_PROVIDERS["alibaba-token-plan"].thinkingBudgetModels) + .toContain("qwen3.7-max"); }); test("aggregator defaults and Neuralwatt seeds match the audited live catalogs", () => { diff --git a/tests/reasoning-effort.test.ts b/tests/reasoning-effort.test.ts index 34acca2957..a85bfa0b79 100644 --- a/tests/reasoning-effort.test.ts +++ b/tests/reasoning-effort.test.ts @@ -562,7 +562,7 @@ describe("thinking-toggle models (260707)", () => { }); }); -describe("thinking-budget models (260709)", () => { +describe("Qwen reasoning wire contracts", () => { const budgetProvider: OcxProviderConfig = { adapter: "openai-chat", baseUrl: "https://api.neuralwatt.com/v1", @@ -628,7 +628,7 @@ describe("thinking-budget models (260709)", () => { expect(body).not.toHaveProperty("reasoning_effort"); }); - test("Alibaba Token Plan routes Qwen3.8 Max Preview with the Qwen thinking budget contract", () => { + test("Alibaba Token Plan repairs stale Qwen3.8 metadata and sends direct reasoning_effort", () => { const config = { port: 10100, defaultProvider: "alibaba-token-plan", @@ -637,18 +637,44 @@ describe("thinking-budget models (260709)", () => { adapter: "openai-chat", baseUrl: "https://token-plan.cn-beijing.maas.aliyuncs.com/compatible-mode/v1", apiKey: "k", + // Simulate a provider row persisted before Qwen3.8 documented its native effort field. + thinkingBudgetModels: ["QWEN3.8-MAX", "qwen3.7-max"], + modelReasoningEfforts: { "QWEN3.8-MAX": ["low", "medium", "high", "xhigh", "max"] }, + modelDefaultReasoningEfforts: { "QWEN3.8-MAX": "medium" }, + reasoningEffortMap: { xhigh: "max" }, + modelReasoningEffortMap: { "QWEN3.8-MAX": { medium: "high" } }, }, }, } as unknown as OcxConfig; const route = routeModel(config, "alibaba-token-plan/qwen3.8-max"); expect(route.provider.modelInputModalities?.[route.modelId]).toEqual(["text", "image"]); - expect(route.provider.thinkingBudgetModels).toContain(route.modelId); - expect(route.provider.modelReasoningEfforts?.[route.modelId]).toEqual(["low", "medium", "high", "xhigh", "max"]); + expect(route.provider.thinkingBudgetModels).not.toContain(route.modelId); + expect(route.provider.thinkingBudgetModels).toContain("qwen3.7-max"); + expect(route.provider.modelReasoningEfforts?.[route.modelId]).toEqual(["low", "medium", "xhigh"]); + expect(route.provider.modelReasoningEfforts?.["QWEN3.8-MAX"]).toBeUndefined(); + expect(route.provider.modelDefaultReasoningEfforts?.[route.modelId]).toBe("xhigh"); + expect(route.provider.modelDefaultReasoningEfforts?.["QWEN3.8-MAX"]).toBeUndefined(); + expect(route.provider.modelReasoningEffortMap?.[route.modelId]).toEqual({}); + expect(route.provider.modelReasoningEffortMap?.["QWEN3.8-MAX"]).toBeUndefined(); + + const request = buildChatRequest(route.provider, route.modelId, { reasoning: "xhigh", maxOutputTokens: 65536 }); + const body = JSON.parse(request.body) as Record; + expect(body).toMatchObject({ model: "qwen3.8-max", reasoning_effort: "xhigh" }); + expect(body).not.toHaveProperty("thinking_budget"); + expect(request.reasoningLog).toEqual({ + effectiveEffort: "xhigh", + wireField: "reasoning_effort", + wireValue: "xhigh", + }); - const body = buildBody(route.provider, route.modelId, { reasoning: "max", maxOutputTokens: 65536 }); - expect(body).toMatchObject({ model: "qwen3.8-max", thinking_budget: 65536 }); - expect(body).not.toHaveProperty("reasoning_effort"); + // Codex may advertise its synthetic compatibility tops; neither may leak an unsupported value. + const maxBody = buildBody(route.provider, route.modelId, { reasoning: "max" }); + expect(maxBody.reasoning_effort).toBe("xhigh"); + expect(maxBody).not.toHaveProperty("thinking_budget"); + + // A client with the old, already-cached high rung degrades to the nearest lower real tier. + expect(buildBody(route.provider, route.modelId, { reasoning: "high" }).reasoning_effort).toBe("medium"); }); test("opencode-go Qwen models are no longer pinned to the Anthropic wire", () => { From 72ff55d9c822c827ebc407f51bbf440c4cf1b437 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sun, 9 Aug 2026 14:51:25 +0900 Subject: [PATCH 110/124] fix(providers): preserve Qwen3.8 user overrides --- src/providers/derive.ts | 97 ++++++++++++++++++--------- src/router.ts | 2 +- tests/alibaba-intl-token-plan.test.ts | 16 ++--- tests/reasoning-effort.test.ts | 46 ++++++++++--- 4 files changed, 111 insertions(+), 50 deletions(-) diff --git a/src/providers/derive.ts b/src/providers/derive.ts index 1f21c24909..43680d96bc 100644 --- a/src/providers/derive.ts +++ b/src/providers/derive.ts @@ -109,15 +109,33 @@ function cloneNestedRecord(input: Record>): Recor return Object.fromEntries(Object.entries(input).map(([key, value]) => [key, { ...value }])); } -function omitFoldedModelKeys( - record: Record | undefined, - foldedModels: ReadonlySet, +function sameStringArray(left: readonly string[] | undefined, right: readonly string[]): boolean { + return left?.length === right.length && left.every((value, index) => value === right[index]); +} + +type DirectReasoningEffortOverrides = Pick< + OcxProviderConfig, + "thinkingBudgetModels" | "modelReasoningEfforts" | "modelDefaultReasoningEfforts" | "modelReasoningEffortMap" +>; + +function fillFoldedModelDefault( + merged: Record | undefined, + explicit: Record | undefined, + model: string, + registryDefault: T | undefined, + clone: (value: T) => T, ): Record | undefined { - if (!record) return undefined; + const folded = model.toLowerCase(); + const explicitEntries = Object.entries(explicit ?? {}).filter(([key]) => key.toLowerCase() === folded); + if (explicitEntries.length === 0) { + return registryDefault === undefined ? merged : { ...(merged ?? {}), [model]: clone(registryDefault) }; + } + const next = Object.fromEntries( - Object.entries(record).filter(([model]) => !foldedModels.has(model.toLowerCase())), + Object.entries(merged ?? {}).filter(([key]) => key.toLowerCase() !== folded), ) as Record; - return Object.keys(next).length > 0 ? next : undefined; + for (const [key, value] of explicitEntries) next[key] = clone(value); + return next; } /** @@ -131,42 +149,51 @@ function omitFoldedModelKeys( export function applyDirectReasoningEffortContracts( entry: ProviderRegistryEntry, prov: OcxProviderConfig, + explicit: DirectReasoningEffortOverrides = prov, ): void { const models = entry.directReasoningEffortModels; if (!models || models.length === 0) return; - const direct = new Set(models.map(model => model.toLowerCase())); - const keepNonDirect = (model: string): boolean => !direct.has(model.toLowerCase()); - - prov.thinkingBudgetModels = prov.thinkingBudgetModels?.filter(keepNonDirect); - prov.thinkingToggleModels = prov.thinkingToggleModels?.filter(keepNonDirect); - prov.modelReasoningEfforts = omitFoldedModelKeys(prov.modelReasoningEfforts, direct); - prov.modelDefaultReasoningEfforts = omitFoldedModelKeys(prov.modelDefaultReasoningEfforts, direct); - prov.modelReasoningEffortMap = omitFoldedModelKeys(prov.modelReasoningEffortMap, direct); - for (const model of models) { - const efforts = entry.modelReasoningEfforts?.[model]; - if (efforts) { - prov.modelReasoningEfforts = { - ...(prov.modelReasoningEfforts ?? {}), - [model]: [...efforts], - }; + // Old generated presets persisted the direct model at the front of the registry's budget + // list. Routing merges registry-first, which moves that one stale entry to the end. Repair + // only those two exact generated shapes; any partial, reordered, or case-varied list is a + // deliberate user value and remains untouched. + const currentBudgetModels = entry.thinkingBudgetModels ?? []; + const staleSeedShape = [model, ...currentBudgetModels]; + const staleRoutedShape = [...currentBudgetModels, model]; + if (sameStringArray(explicit.thinkingBudgetModels, staleSeedShape) + || sameStringArray(explicit.thinkingBudgetModels, staleRoutedShape)) { + prov.thinkingBudgetModels = [...currentBudgetModels]; } + const efforts = entry.modelReasoningEfforts?.[model]; + prov.modelReasoningEfforts = fillFoldedModelDefault( + prov.modelReasoningEfforts, + explicit.modelReasoningEfforts, + model, + efforts, + value => [...value], + ); + const defaultEffort = entry.modelDefaultReasoningEfforts?.[model]; - if (defaultEffort) { - prov.modelDefaultReasoningEfforts = { - ...(prov.modelDefaultReasoningEfforts ?? {}), - [model]: defaultEffort, - }; - } + prov.modelDefaultReasoningEfforts = fillFoldedModelDefault( + prov.modelDefaultReasoningEfforts, + explicit.modelDefaultReasoningEfforts, + model, + defaultEffort, + value => value, + ); // An explicit empty model map masks any provider-wide aliases. Without it, a stale global // mapping such as xhigh -> max would win before the verified direct ladder can clamp it. - prov.modelReasoningEffortMap = { - ...(prov.modelReasoningEffortMap ?? {}), - [model]: {}, - }; + prov.modelReasoningEffortMap = fillFoldedModelDefault( + prov.modelReasoningEffortMap, + explicit.modelReasoningEffortMap, + model, + {}, + value => ({ ...value }), + ); } } @@ -357,6 +384,12 @@ export function enrichProviderFromRegistry(name: string, prov: OcxProviderConfig enrichReasoningSummariesByDestination(prov); return; } + const explicitDirectReasoning: DirectReasoningEffortOverrides = { + thinkingBudgetModels: prov.thinkingBudgetModels, + modelReasoningEfforts: prov.modelReasoningEfforts, + modelDefaultReasoningEfforts: prov.modelDefaultReasoningEfforts, + modelReasoningEffortMap: prov.modelReasoningEffortMap, + }; const seed = providerConfigSeed(entry); if (prov.apiKeyTransport === undefined && seed.apiKeyTransport !== undefined) prov.apiKeyTransport = seed.apiKeyTransport; if (!prov.defaultModel && seed.defaultModel) prov.defaultModel = seed.defaultModel; @@ -420,7 +453,7 @@ export function enrichProviderFromRegistry(name: string, prov: OcxProviderConfig if (prov.freeTier === undefined && seed.freeTier !== undefined) prov.freeTier = seed.freeTier; if (prov.modelSuffixBracketStrip === undefined && seed.modelSuffixBracketStrip !== undefined) prov.modelSuffixBracketStrip = seed.modelSuffixBracketStrip; if (!prov.headers && seed.headers) prov.headers = { ...seed.headers }; - applyDirectReasoningEffortContracts(entry, prov); + applyDirectReasoningEffortContracts(entry, prov, explicitDirectReasoning); } export function deriveFeaturedProviderIds(): string[] { diff --git a/src/router.ts b/src/router.ts index 8768cb4d60..79fb957307 100644 --- a/src/router.ts +++ b/src/router.ts @@ -377,7 +377,7 @@ function routedProviderConfig(providerName: string, provider: OcxProviderConfig) ...(thinkingToggleModels ? { thinkingToggleModels } : {}), ...(thinkingBudgetModels ? { thinkingBudgetModels } : {}), }; - applyDirectReasoningEffortContracts(registryEntry, resolved); + applyDirectReasoningEffortContracts(registryEntry, resolved, provider); return resolved; } diff --git a/tests/alibaba-intl-token-plan.test.ts b/tests/alibaba-intl-token-plan.test.ts index 259394793a..79c923e8e5 100644 --- a/tests/alibaba-intl-token-plan.test.ts +++ b/tests/alibaba-intl-token-plan.test.ts @@ -97,7 +97,7 @@ describe("alibaba-token-plan-intl registry entry", () => { expect(PROVIDER_REGISTRY.find(e => e.id === "alibaba-token-plan")!.defaultModel).toBe("qwen3.8-max"); }); - test("registry enrichment repairs a persisted pre-release Qwen3.8 contract", () => { + test("registry enrichment preserves deliberate case-varied Qwen3.8 overrides", () => { const provider = { adapter: "openai-chat", baseUrl: ALIBABA_INTL_TOKEN_PLAN_BASE_URL, @@ -110,13 +110,13 @@ describe("alibaba-token-plan-intl registry entry", () => { enrichProviderFromRegistry("alibaba-token-plan-intl", provider); - expect(provider.modelReasoningEfforts["qwen3.8-max"]).toEqual(["low", "medium", "xhigh"]); - expect(provider.modelReasoningEfforts["QWEN3.8-MAX"]).toBeUndefined(); - expect(provider.modelDefaultReasoningEfforts["qwen3.8-max"]).toBe("xhigh"); - expect(provider.modelDefaultReasoningEfforts["QWEN3.8-MAX"]).toBeUndefined(); - expect(provider.modelReasoningEffortMap?.["qwen3.8-max"]).toEqual({}); - expect(provider.modelReasoningEffortMap?.["QWEN3.8-MAX"]).toBeUndefined(); - expect(provider.thinkingBudgetModels).toEqual(["qwen3.7-max"]); + expect(provider.modelReasoningEfforts["qwen3.8-max"]).toBeUndefined(); + expect(provider.modelReasoningEfforts["QWEN3.8-MAX"]).toEqual(["low", "high", "xhigh"]); + expect(provider.modelDefaultReasoningEfforts["qwen3.8-max"]).toBeUndefined(); + expect(provider.modelDefaultReasoningEfforts["QWEN3.8-MAX"]).toBe("high"); + expect(provider.modelReasoningEffortMap?.["qwen3.8-max"]).toBeUndefined(); + expect(provider.modelReasoningEffortMap?.["QWEN3.8-MAX"]).toEqual({ medium: "high" }); + expect(provider.thinkingBudgetModels).toEqual(["QWEN3.8-MAX", "qwen3.7-max"]); }); test("non-reasoning models are marked", () => { diff --git a/tests/reasoning-effort.test.ts b/tests/reasoning-effort.test.ts index a85bfa0b79..c8767c31fc 100644 --- a/tests/reasoning-effort.test.ts +++ b/tests/reasoning-effort.test.ts @@ -628,7 +628,7 @@ describe("Qwen reasoning wire contracts", () => { expect(body).not.toHaveProperty("reasoning_effort"); }); - test("Alibaba Token Plan repairs stale Qwen3.8 metadata and sends direct reasoning_effort", () => { + test("Alibaba Token Plan repairs the exact stale generated Qwen3.8 budget list", () => { const config = { port: 10100, defaultProvider: "alibaba-token-plan", @@ -637,12 +637,9 @@ describe("Qwen reasoning wire contracts", () => { adapter: "openai-chat", baseUrl: "https://token-plan.cn-beijing.maas.aliyuncs.com/compatible-mode/v1", apiKey: "k", - // Simulate a provider row persisted before Qwen3.8 documented its native effort field. - thinkingBudgetModels: ["QWEN3.8-MAX", "qwen3.7-max"], - modelReasoningEfforts: { "QWEN3.8-MAX": ["low", "medium", "high", "xhigh", "max"] }, - modelDefaultReasoningEfforts: { "QWEN3.8-MAX": "medium" }, + // Exact generated preset shape from before Qwen3.8 documented its native effort field. + thinkingBudgetModels: ["qwen3.8-max", "qwen3.7-max", "qwen3.7-plus", "qwen3.6-flash"], reasoningEffortMap: { xhigh: "max" }, - modelReasoningEffortMap: { "QWEN3.8-MAX": { medium: "high" } }, }, }, } as unknown as OcxConfig; @@ -652,11 +649,8 @@ describe("Qwen reasoning wire contracts", () => { expect(route.provider.thinkingBudgetModels).not.toContain(route.modelId); expect(route.provider.thinkingBudgetModels).toContain("qwen3.7-max"); expect(route.provider.modelReasoningEfforts?.[route.modelId]).toEqual(["low", "medium", "xhigh"]); - expect(route.provider.modelReasoningEfforts?.["QWEN3.8-MAX"]).toBeUndefined(); expect(route.provider.modelDefaultReasoningEfforts?.[route.modelId]).toBe("xhigh"); - expect(route.provider.modelDefaultReasoningEfforts?.["QWEN3.8-MAX"]).toBeUndefined(); expect(route.provider.modelReasoningEffortMap?.[route.modelId]).toEqual({}); - expect(route.provider.modelReasoningEffortMap?.["QWEN3.8-MAX"]).toBeUndefined(); const request = buildChatRequest(route.provider, route.modelId, { reasoning: "xhigh", maxOutputTokens: 65536 }); const body = JSON.parse(request.body) as Record; @@ -677,6 +671,40 @@ describe("Qwen reasoning wire contracts", () => { expect(buildBody(route.provider, route.modelId, { reasoning: "high" }).reasoning_effort).toBe("medium"); }); + test("Alibaba Token Plan preserves deliberate Qwen3.8 model overrides", () => { + const config = { + port: 10100, + defaultProvider: "alibaba-token-plan", + providers: { + "alibaba-token-plan": { + adapter: "openai-chat", + baseUrl: "https://token-plan.cn-beijing.maas.aliyuncs.com/compatible-mode/v1", + apiKey: "k", + thinkingBudgetModels: ["QWEN3.8-MAX", "qwen3.7-max"], + modelReasoningEfforts: { "QWEN3.8-MAX": ["low", "high", "xhigh"] }, + modelDefaultReasoningEfforts: { "QWEN3.8-MAX": "high" }, + reasoningEffortMap: { xhigh: "max" }, + modelReasoningEffortMap: { "QWEN3.8-MAX": { medium: "high" } }, + }, + }, + } as unknown as OcxConfig; + const route = routeModel(config, "alibaba-token-plan/qwen3.8-max"); + + expect(route.provider.thinkingBudgetModels).toEqual([ + "qwen3.7-max", "qwen3.7-plus", "qwen3.6-flash", "QWEN3.8-MAX", + ]); + expect(route.provider.modelReasoningEfforts?.[route.modelId]).toBeUndefined(); + expect(route.provider.modelReasoningEfforts?.["QWEN3.8-MAX"]).toEqual(["low", "high", "xhigh"]); + expect(route.provider.modelDefaultReasoningEfforts?.[route.modelId]).toBeUndefined(); + expect(route.provider.modelDefaultReasoningEfforts?.["QWEN3.8-MAX"]).toBe("high"); + expect(route.provider.modelReasoningEffortMap?.[route.modelId]).toBeUndefined(); + expect(route.provider.modelReasoningEffortMap?.["QWEN3.8-MAX"]).toEqual({ medium: "high" }); + + const request = buildChatRequest(route.provider, route.modelId, { reasoning: "xhigh" }); + expect(JSON.parse(request.body)).toMatchObject({ reasoning_effort: "xhigh" }); + expect(request.reasoningLog?.wireField).toBe("reasoning_effort"); + }); + test("opencode-go Qwen models are no longer pinned to the Anthropic wire", () => { const provider: OcxProviderConfig = { adapter: "openai-chat", baseUrl: "https://opencode.ai/zen/go/v1" }; From d6d3878c49ebab9f9d1fdc1154ec47bd789b6584 Mon Sep 17 00:00:00 2001 From: luvs01 Date: Sat, 8 Aug 2026 20:19:52 +0900 Subject: [PATCH 111/124] fix(ci): require aggregate check evidence --- .github/workflows/ci.yml | 74 +++++---- .github/workflows/enforce-pr-target.yml | 46 ++++-- tests/ci-workflows.test.ts | 179 +++++++++++++++++---- tests/helpers/enforce-pr-target-harness.ts | 58 +++++-- 4 files changed, 273 insertions(+), 84 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afb48e72ea..01ec5faf41 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,10 @@ name: Cross-platform CI on: - pull_request: + # Always create the aggregate `ci` check for pull requests. Expensive jobs + # apply the former path allowlist through the `changes` job below, so a + # docs-only PR receives explicit positive evidence instead of no check at all. + pull_request: {} # No base-branch filter on purpose. GitHub matches `branches:` against the # BASE ref, so `[main, dev]` silently excluded stacked child PRs — whose # base is another open PR's head branch, an intentional review workflow per @@ -12,31 +15,14 @@ on: # # An allowlist cannot express "base is another PR's head" — stacked bases # carry contributor prefixes (`fix/`, `feat/`, `agent/`) as readily as - # `codex/`, and contributor stacks need CI most. `paths:` below is the real - # scope gate, same shape as issue-quality-tests.yml. Safe to widen here + # `codex/`, and contributor stacks need CI most. The `changes` job below is + # the real scope gate, using the same allowlist as the push trigger. Safe to + # widen here # because this workflow is `pull_request` (not `pull_request_target`), # declares `contents: read`, and reads no secrets. # # `push:` stays pinned to the integration lines: it gates the release path, # and this trigger already covers review. - paths: - - "src/**" - - "bin/**" - - "tests/**" - - "scripts/**" - - "gui/**" - - "assets/**" - - ".gitattributes" - - ".npmignore" - - "package.json" - - "bun.lock" - - "tsconfig.json" - - "README.md" - - "LICENSE" - - ".github/workflows/ci.yml" - - ".github/workflows/release.yml" - - ".github/workflows/enforce-pr-target.yml" - - ".github/workflows/stale-needs-info.yml" push: branches: [main, preview, dev] paths: @@ -77,8 +63,8 @@ jobs: # A hostile PR can delete the branch and hardcode the self-hosted labels into # `$GITHUB_OUTPUT`, and `runs-on` will honour it. That this job runs on # `ubuntu-latest` changes nothing — the untrusted part is its OUTPUT, not its - # host. `.github/workflows/ci.yml` is in this workflow's `pull_request.paths`, - # so such an edit triggers its own run. + # host. `.github/workflows/ci.yml` is in the `changes` job's `ci` filter, so + # such an edit triggers every expensive verification job. # # What actually keeps untrusted code off a self-hosted runner lives OUTSIDE # this file, where a PR cannot reach it: the fork-PR approval policy @@ -154,6 +140,7 @@ jobs: contents: read pull-requests: read outputs: + ci: ${{ steps.filter.outputs.ci }} gui: ${{ steps.filter.outputs.gui }} packaging: ${{ steps.filter.outputs.packaging }} steps: @@ -180,6 +167,27 @@ jobs: # on this branch", which is the intent. base: ${{ github.ref }} filters: | + # Mirrors the push trigger's path allowlist. Pull requests always + # start the workflow so the aggregate check exists, while these + # paths decide whether the expensive test jobs need to run. + ci: + - 'src/**' + - 'bin/**' + - 'tests/**' + - 'scripts/**' + - 'gui/**' + - 'assets/**' + - '.gitattributes' + - '.npmignore' + - 'package.json' + - 'bun.lock' + - 'tsconfig.json' + - 'README.md' + - 'LICENSE' + - '.github/workflows/ci.yml' + - '.github/workflows/release.yml' + - '.github/workflows/enforce-pr-target.yml' + - '.github/workflows/stale-needs-info.yml' gui: - 'gui/**' # Everything that ends up inside `npm pack`, or that decides what @@ -218,6 +226,8 @@ jobs: # would eat what the sharding saves. test: name: test ${{ matrix.shard }}/4 + needs: changes + if: github.event_name != 'pull_request' || needs.changes.outputs.ci == 'true' runs-on: ubuntu-latest # A quarter of the suite. A shard that needs longer than this is wedged, not # slow — the old 30-minute ceiling was margin for the Windows leg, which no @@ -271,6 +281,8 @@ jobs: # failure is bounded to this job instead of poisoning a general test shard. storage-policy: name: storage policy + needs: changes + if: github.event_name != 'pull_request' || needs.changes.outputs.ci == 'true' runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -310,6 +322,7 @@ jobs: gates: name: gates needs: changes + if: github.event_name != 'pull_request' || needs.changes.outputs.ci == 'true' runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -372,6 +385,8 @@ jobs: # platform-independent and already ran once above. platform-macos: name: macos + needs: changes + if: github.event_name != 'pull_request' || needs.changes.outputs.ci == 'true' runs-on: macos-latest # The unsharded control for the sharded Linux lane: the only place the whole # suite runs in one pool, so it is the place that catches what sharding @@ -499,6 +514,8 @@ jobs: # keyring matrix leg may use the persistent self-hosted Windows runner. keyring-smoke: name: keyring ${{ matrix.name }} + needs: changes + if: github.event_name != 'pull_request' || needs.changes.outputs.ci == 'true' runs-on: ${{ matrix.runner }} timeout-minutes: 8 strategy: @@ -616,12 +633,9 @@ jobs: # no branch protection configured today, so nothing has to be re-pointed — but # whoever enables it has one obvious check to require. # - # NOTE for that day: requiring this check also means dropping the - # workflow-level `paths:` filter above, or moving this job to an - # always-triggered workflow. A PR that touches only docs does not trigger this - # workflow at all, so no `ci` check would be created and the PR would sit - # pending forever. That is harmless while nothing is required and a trap - # afterwards. + # Pull requests always trigger this workflow. The `changes` job keeps + # expensive jobs scoped, but this aggregate still records explicit success + # when every producer is deliberately skipped for an out-of-scope docs change. # # `if: always()` is load-bearing. Without it, a failed or skipped dependency # skips this job too — and GitHub reports a skipped job as success, so the gate @@ -663,4 +677,4 @@ jobs: # leg is a gate violation: on push events it is always skipped, and on # dispatch a failed Windows leg already fails the allowlist above. The # old "windows must have run on main/preview" assertion left with the - # condition it policed. \ No newline at end of file + # condition it policed. diff --git a/.github/workflows/enforce-pr-target.yml b/.github/workflows/enforce-pr-target.yml index 674c314bb9..205ee35c74 100644 --- a/.github/workflows/enforce-pr-target.yml +++ b/.github/workflows/enforce-pr-target.yml @@ -766,26 +766,44 @@ jobs: !headDrifted && failures.length === 0 ) { - let ciGreen = true; + let ciGreen = false; try { - const { data: checksData } = - await github.rest.checks.listForRef({ + // GitHub Actions' immutable App ID. Name alone is not evidence: + // any installed app can publish a check called `ci`. + const githubActionsAppId = 15368; + const checkRuns = []; + for await (const response of github.paginate.iterator( + github.rest.checks.listForRef, + { owner, repo, ref: pr.head.sha, + app_id: githubActionsAppId, + check_name: "ci", + filter: "latest", per_page: 100 - }); - const ciCheck = (checksData.check_runs ?? []).find( - check => check.name === "ci" + } + )) { + // Octokit's paginator normalizes `{ total_count, check_runs }` + // into an array in `response.data` for every iterator page. + checkRuns.push(...response.data); + } + const ciChecks = checkRuns.filter( + check => + check.name === "ci" && + check.app?.id === githubActionsAppId ); - // No `ci` check means no CI run exists for this head (for - // example a docs-only change): there is nothing to contradict - // the author's claim. A real `ci` check must be completed - // successfully. + // The readiness claim requires positive CI evidence. A missing, + // pending, unsuccessful, foreign, or conflicting aggregate + // check must fail closed. `filter: latest` removes superseded + // rerun attempts; `every` still rejects ambiguous live results. ciGreen = - ciCheck === undefined || - (ciCheck.status === "completed" && - ciCheck.conclusion === "success"); + ciChecks.length > 0 && + ciChecks.every( + check => + check.status === "completed" && + check.conclusion === "success" + ); } catch (error) { core.warning( `Could not list checks for the readiness claim check: ${error.message}` @@ -1371,4 +1389,4 @@ jobs: "All PR quality gates passed and there is no active bot state." ); return; - } \ No newline at end of file + } diff --git a/tests/ci-workflows.test.ts b/tests/ci-workflows.test.ts index 195e5135d8..0fdf912208 100644 --- a/tests/ci-workflows.test.ts +++ b/tests/ci-workflows.test.ts @@ -147,13 +147,15 @@ describe("GitHub Actions hardening", () => { expect([...(gate?.needs ?? [])].sort()) .toEqual(Object.keys(ci.jobs ?? {}).filter(name => name !== "ci").sort()); - // macOS is the unsharded control for the sharded Linux lane: it is the only - // place the whole suite runs in one pool. Sharded or conditional, it stops - // being a control. + // macOS is the unsharded control for every CI-relevant change. It may skip + // only when the shared path filter says the entire expensive suite is out of + // scope (for example a docs-site-only PR). const macosSteps = (ci.jobs?.["platform-macos"] as { steps?: { run?: string }[] })?.steps ?? []; expect(macosSteps.some(step => step.run?.includes("bun test --isolate tests"))).toBe(true); expect(macosSteps.some(step => step.run?.includes("--shard"))).toBe(false); - expect(ci.jobs?.["platform-macos"]).not.toHaveProperty("if"); + expect((ci.jobs?.["platform-macos"] as { needs?: string; if?: string })?.needs).toBe("changes"); + expect((ci.jobs?.["platform-macos"] as { if?: string })?.if) + .toBe("github.event_name != 'pull_request' || needs.changes.outputs.ci == 'true'"); // Windows is dispatch-only: it gates nothing, not even the shipping // boundary. The sharded promotion run surfaced ~207 Windows-only failures @@ -242,7 +244,7 @@ describe("GitHub Actions hardening", () => { for (const [path, expectedKeys] of [ // No `branches`: the stacked-base exemption has no enumerable branch list. - [".github/workflows/ci.yml", ["paths"]], + [".github/workflows/ci.yml", []], [".github/workflows/service-lifecycle.yml", ["branches", "paths"]], ] as const) { const workflow = Bun.YAML.parse(await readText(path)) as { @@ -276,6 +278,7 @@ describe("GitHub Actions hardening", () => { push?: { branches?: string[]; paths?: string[] }; pull_request?: { branches?: string[]; paths?: string[] }; }; + jobs?: Record | undefined>; }; expect([...(ci.on?.push?.branches ?? [])].sort()).toEqual(["dev", "main", "preview"]); @@ -291,14 +294,14 @@ describe("GitHub Actions hardening", () => { // Re-adding an allowlist is the regression this pins, and it cannot be // written correctly: stacked bases carry contributor prefixes (`fix/`, // `feat/`, `agent/`) as readily as `codex/`, so any list leaves some stack - // silently unverified. `paths:` below is the scope gate. + // silently unverified. Pull requests also carry no workflow-level path + // filter: every head needs an aggregate `ci` check. expect(ci.on?.pull_request?.branches).toBeUndefined(); + expect(ci.on?.pull_request?.paths).toBeUndefined(); - // The path filter decides whether the job runs at all. Deleting one entry - // deletes nothing visible: the workflow still exists, still lists the right - // branches, and simply never fires for a PR that touches only that surface. - // Round 16 dropped `src/**`, `tests/**`, and both workflow self-references - // one at a time and the suite stayed green each time. Pin the list. + // The push trigger and pull-request `changes` job share one expensive-CI + // allowlist. PRs always create the workflow and aggregate check; this list + // decides whether the costly jobs run. Pin the entire list on both paths. const ciPaths = [ ".gitattributes", ".github/workflows/ci.yml", @@ -318,10 +321,22 @@ describe("GitHub Actions hardening", () => { "tests/**", "tsconfig.json", ]; - expect([...(ci.on?.pull_request?.paths ?? [])].sort()).toEqual(ciPaths); - // Push and pull_request have to cover the same surfaces, or a change lands - // on dev having been checked on one trigger and not the other. expect([...(ci.on?.push?.paths ?? [])].sort()).toEqual(ciPaths); + + const filterStep = (ci.jobs?.changes as { + steps?: { with?: Record }[]; + })?.steps?.find(step => step.with?.filters); + const areaFilters = Bun.YAML.parse(String(filterStep?.with?.filters ?? "")) as { + ci?: string[]; + }; + expect([...(areaFilters.ci ?? [])].sort()).toEqual(ciPaths); + + const scopedCondition = "github.event_name != 'pull_request' || needs.changes.outputs.ci == 'true'"; + for (const jobName of ["test", "storage-policy", "gates", "platform-macos", "keyring-smoke"]) { + const job = ci.jobs?.[jobName] as { needs?: string; if?: string } | undefined; + expect(`${jobName}:${job?.needs}`).toBe(`${jobName}:changes`); + expect(`${jobName}:${job?.if}`).toBe(`${jobName}:${scopedCondition}`); + } }); test("cross-platform CI keeps the GUI lint and build gates", async () => { @@ -388,15 +403,13 @@ describe("GitHub Actions hardening", () => { "src/**", ].sort()); - // A per-job filter can only narrow what the workflow-level filter admits, so - // every packaging pattern that names a real path must also appear in the - // trigger's own path list. Otherwise the workflow never runs for that file - // and the filter entry is decoration. - const triggerPaths = (ci.on as { pull_request?: { paths?: string[] } } | undefined) - ?.pull_request?.paths ?? []; + // Every packaging pattern that names a real path must also appear in the + // shared expensive-CI filter. Otherwise the workflow records a cheap green + // aggregate while silently skipping the packaging verification. + const ciPatterns = (Bun.YAML.parse(filters) as { ci?: string[] }).ci ?? []; for (const pattern of packaging) { if (pattern === "scripts/prepare-package.ts") continue; // covered by scripts/** - expect(`${pattern}:${triggerPaths.includes(pattern)}`).toBe(`${pattern}:true`); + expect(`${pattern}:${ciPatterns.includes(pattern)}`).toBe(`${pattern}:true`); } }); @@ -1943,9 +1956,9 @@ describe("GitHub Actions hardening", () => { expect(drafts[1]!.query).toContain("markPullRequestReadyForReview"); }); - test("a head with no ci check at all keeps the CI box (docs-only style PRs)", async () => { - // No CI run exists for this head: there is nothing to contradict the - // author's claim, so the CI box survives. + test("a head with no ci check fails closed for the CI claim", async () => { + // No CI run means the claim has no positive evidence, so the box is + // unticked and the PR stays in draft. const result = await run({ pr: { base: { ref: "dev" }, @@ -1960,15 +1973,18 @@ describe("GitHub Actions hardening", () => { "checks.listForRef", "graphql", "pulls.listReviews", - "issues.addLabels", - "graphql", + "pulls.get", + "pulls.update", "issues.createComment", ])); - expect(callsTo(result, "pulls.update")).toEqual([]); + const [bodyUpdate] = callsTo(result, "pulls.update") as [{ body: string }]; + expect(bodyUpdate.body).toContain("- [ ] All CI tests are green on my local testing."); const drafts = callsTo(result, "graphql") as [{ query: string }]; - expect(drafts).toHaveLength(2); + expect(drafts).toHaveLength(1); expect(drafts[0]!.query).toContain("reviewThreads"); - expect(drafts[1]!.query).toContain("markPullRequestReadyForReview"); + expect(lastReadinessCommentBody(result)).toContain( + "GitHub CI is not green on the current head", + ); }); test("a pending ci check cannot attest green", async () => { @@ -1997,6 +2013,111 @@ describe("GitHub Actions hardening", () => { expect(readinessBody).toContain("GitHub CI is not green on the current head"); }); + test("a green ci check on a later checks page attests green", async () => { + const result = await run({ + pr: { + base: { ref: "dev" }, + draft: true, + body: readinessChecklistBody(4), + }, + maintainersFile: MAINTAINERS_FIXTURE, + checkRunPages: [ + Array.from({ length: 100 }, (_, index) => ({ + name: `decoy-${index}`, + status: "completed", + conclusion: "success", + })), + [{ name: "ci", status: "completed", conclusion: "success" }], + ], + }); + + expect(methodsOf(result)).toEqual(readsAllowedBase([ + "checks.listForRef", + "checks.listForRef", + "graphql", + "pulls.listReviews", + "issues.addLabels", + "graphql", + "issues.createComment", + ])); + const checkCalls = callsTo(result, "checks.listForRef") as Array<{ + app_id?: number; + check_name?: string; + filter?: string; + }>; + for (const call of checkCalls) { + expect(call.app_id).toBe(15368); + expect(call.check_name).toBe("ci"); + expect(call.filter).toBe("latest"); + } + expect(callsTo(result, "pulls.update")).toEqual([]); + const drafts = callsTo(result, "graphql") as [{ query: string }, { query: string }]; + expect(drafts[1]!.query).toContain("markPullRequestReadyForReview"); + expect(lastReadinessCommentBody(result)).toContain("**4/4** boxes ticked"); + }); + + test("a foreign app check named ci cannot attest green", async () => { + const result = await run({ + pr: { + base: { ref: "dev" }, + draft: false, + body: readinessChecklistBody(4), + }, + maintainersFile: MAINTAINERS_FIXTURE, + checkRuns: [{ + name: "ci", + status: "completed", + conclusion: "success", + app: { id: 999999 }, + }], + }); + + const [bodyUpdate] = callsTo(result, "pulls.update") as [{ body: string }]; + expect(bodyUpdate.body).toContain("- [ ] All CI tests are green on my local testing."); + expect(lastReadinessCommentBody(result)).toContain("GitHub CI is not green on the current head"); + }); + + test("conflicting trusted ci checks fail closed regardless of ordering", async () => { + const green = { name: "ci", status: "completed", conclusion: "success" }; + const pending = { name: "ci", status: "in_progress", conclusion: null }; + const failed = { name: "ci", status: "completed", conclusion: "failure" }; + + for (const checkRuns of [[green, pending], [pending, green], [green, failed], [failed, green]]) { + const result = await run({ + pr: { + base: { ref: "dev" }, + draft: false, + body: readinessChecklistBody(4), + }, + maintainersFile: MAINTAINERS_FIXTURE, + checkRuns, + }); + + const [bodyUpdate] = callsTo(result, "pulls.update") as [{ body: string }]; + expect(bodyUpdate.body).toContain("- [ ] All CI tests are green on my local testing."); + expect(lastReadinessCommentBody(result)).toContain("GitHub CI is not green on the current head"); + } + }); + + test("multiple latest trusted green ci checks are consistent evidence", async () => { + const result = await run({ + pr: { + base: { ref: "dev" }, + draft: true, + body: readinessChecklistBody(4), + }, + maintainersFile: MAINTAINERS_FIXTURE, + checkRuns: [ + { name: "ci", status: "completed", conclusion: "success" }, + { name: "ci", status: "completed", conclusion: "success" }, + ], + }); + + expect(callsTo(result, "pulls.update")).toEqual([]); + const drafts = callsTo(result, "graphql") as [{ query: string }, { query: string }]; + expect(drafts[1]!.query).toContain("markPullRequestReadyForReview"); + }); + test("an unresolved Codex thread unchecks the findings box and re-drafts", async () => { const result = await run({ pr: { diff --git a/tests/helpers/enforce-pr-target-harness.ts b/tests/helpers/enforce-pr-target-harness.ts index 5edcc38ea9..09d27b8a91 100644 --- a/tests/helpers/enforce-pr-target-harness.ts +++ b/tests/helpers/enforce-pr-target-harness.ts @@ -173,7 +173,19 @@ export type RunOptions = { * `ci` check so completed-checklist scenarios pass the claim check. * Pass a red/pending/missing set to exercise the claim-check reset paths. */ - checkRuns?: Array<{ name: string; status: string; conclusion: string | null }>; + checkRuns?: Array<{ + name: string; + status: string; + conclusion: string | null; + app?: { id: number } | null; + }>; + /** Page-keyed check-run fixtures for `checks.listForRef` pagination. */ + checkRunPages?: Array>; /** * Review threads `pullRequestReviewThreads` (via GraphQL) reports for the PR. * Each entry is `{ isResolved, author }`; the harness wraps it into the @@ -256,7 +268,7 @@ const DEFAULT_BODY = [ /** The repo's documented "CI passed" check, green by default. */ const DEFAULT_GREEN_CHECKS = [ - { name: "ci", status: "completed", conclusion: "success" }, + { name: "ci", status: "completed", conclusion: "success", app: { id: 15368 } }, ]; const DEFAULT_PR = { @@ -639,6 +651,13 @@ export async function runEnforcePrTarget( const filePages: unknown[][] = options.filePages ?? (options.files && options.files.length > 0 ? [options.files] : [[]]); + const checkRunPages = (options.checkRunPages ?? [options.checkRuns ?? DEFAULT_GREEN_CHECKS]) + .map(page => page.map(check => ({ + ...check, + // Existing fixtures model trusted GitHub Actions checks unless a test + // explicitly supplies another app or null to exercise provenance. + app: check.app === undefined ? { id: 15368 } : check.app, + }))); const paginatePageCount = Math.max( pages.length, issueEventPages.length, @@ -764,11 +783,13 @@ export async function runEnforcePrTarget( removeLabel: (args: unknown) => respond("issues.removeLabel", args, {}), }, checks: { - listForRef: (args: unknown) => - respond("checks.listForRef", args, { - total_count: (options.checkRuns ?? DEFAULT_GREEN_CHECKS).length, - check_runs: options.checkRuns ?? DEFAULT_GREEN_CHECKS, - }), + listForRef: (args: unknown) => { + const page = Number((args as { page?: number })?.page ?? 1); + return respond("checks.listForRef", args, { + total_count: checkRunPages.reduce((total, rows) => total + rows.length, 0), + check_runs: checkRunPages[page - 1] ?? [], + }); + }, }, repos: { getCollaboratorPermissionLevel: (args: unknown) => @@ -850,9 +871,13 @@ export async function runEnforcePrTarget( paginate = Object.assign( async (fn: (args: unknown) => Promise<{ data: unknown[] }>, params: unknown) => { const collected: unknown[] = []; - for (let page = 1; page <= paginatePageCount; page += 1) { + const pageCount = fn === rest.checks.listForRef ? checkRunPages.length : paginatePageCount; + for (let page = 1; page <= pageCount; page += 1) { const response = await fn({ ...(params as object), page }); - collected.push(...response.data); + const rows = fn === rest.checks.listForRef + ? ((response.data as unknown as { check_runs?: unknown[] }).check_runs ?? []) + : response.data; + collected.push(...rows); } return collected; }, @@ -865,8 +890,19 @@ export async function runEnforcePrTarget( */ iterator: (fn: (args: unknown) => Promise<{ data: unknown[] }>, params: unknown) => ({ async *[Symbol.asyncIterator]() { - for (let page = 1; page <= paginatePageCount; page += 1) { - yield await fn({ ...(params as object), page }); + const pageCount = fn === rest.checks.listForRef ? checkRunPages.length : paginatePageCount; + for (let page = 1; page <= pageCount; page += 1) { + const response = await fn({ ...(params as object), page }); + if (fn !== rest.checks.listForRef) { + yield response; + continue; + } + // Match @octokit/plugin-paginate-rest: list envelopes such as + // `{ total_count, check_runs }` become array-valued page data. + yield { + ...response, + data: (response.data as unknown as { check_runs?: unknown[] }).check_runs ?? [], + }; } }, }), From 1e07b0e2806bfacc23ca8c73cb44a3833484975e Mon Sep 17 00:00:00 2001 From: luvs01 Date: Sun, 9 Aug 2026 01:25:09 +0900 Subject: [PATCH 112/124] fix(ci): fail closed on readiness evidence --- .github/workflows/ci.yml | 22 +++++++- .github/workflows/enforce-pr-target.yml | 27 +++++---- tests/ci-workflows.test.ts | 66 ++++++++++++++++++---- tests/helpers/enforce-pr-target-harness.ts | 6 +- 4 files changed, 95 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01ec5faf41..3f18cbc726 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,7 +140,10 @@ jobs: contents: read pull-requests: read outputs: - ci: ${{ steps.filter.outputs.ci }} + # Downstream jobs consume only the value re-emitted by the validation + # step. A missing or malformed filter output must fail this job instead + # of silently making every expensive job skip. + ci: ${{ steps.scope.outputs.ci }} gui: ${{ steps.filter.outputs.gui }} packaging: ${{ steps.filter.outputs.packaging }} steps: @@ -211,6 +214,23 @@ jobs: - 'LICENSE' - 'scripts/prepare-package.ts' + - name: Assert the scope output is usable + id: scope + shell: bash + env: + CI_SCOPE: ${{ steps.filter.outputs.ci }} + run: | + set -euo pipefail + case "$CI_SCOPE" in + true|false) + printf 'ci=%s\n' "$CI_SCOPE" >> "$GITHUB_OUTPUT" + ;; + *) + printf '::error::changes.outputs.ci was %q, expected true or false\n' "$CI_SCOPE" + exit 1 + ;; + esac + # The suite, split by file across four Linux runners. # # `bun test --shard=i/N` sorts test files by path and deals them round-robin, diff --git a/.github/workflows/enforce-pr-target.yml b/.github/workflows/enforce-pr-target.yml index 205ee35c74..a6e93d8993 100644 --- a/.github/workflows/enforce-pr-target.yml +++ b/.github/workflows/enforce-pr-target.yml @@ -99,7 +99,10 @@ jobs: needs: resolve-pr if: needs.resolve-pr.outputs.pull-number != '' runs-on: ubuntu-latest + # The write job also reads the current head's aggregate check evidence. + # Job-scoped permissions replace, rather than extend, the workflow default. permissions: + checks: read contents: write pull-requests: write concurrency: @@ -771,10 +774,8 @@ jobs: // GitHub Actions' immutable App ID. Name alone is not evidence: // any installed app can publish a check called `ci`. const githubActionsAppId = 15368; - const checkRuns = []; - for await (const response of github.paginate.iterator( - github.rest.checks.listForRef, - { + const { data: checksData } = + await github.rest.checks.listForRef({ owner, repo, ref: pr.head.sha, @@ -782,12 +783,10 @@ jobs: check_name: "ci", filter: "latest", per_page: 100 - } - )) { - // Octokit's paginator normalizes `{ total_count, check_runs }` - // into an array in `response.data` for every iterator page. - checkRuns.push(...response.data); - } + }); + const checkRuns = Array.isArray(checksData.check_runs) + ? checksData.check_runs + : []; const ciChecks = checkRuns.filter( check => check.name === "ci" && @@ -795,9 +794,13 @@ jobs: ); // The readiness claim requires positive CI evidence. A missing, // pending, unsuccessful, foreign, or conflicting aggregate - // check must fail closed. `filter: latest` removes superseded - // rerun attempts; `every` still rejects ambiguous live results. + // check must fail closed. The exact app/name/latest query should + // be tiny; if GitHub reports more rows than this response holds, + // treat the truncated evidence as unreadable rather than paging + // through an endpoint whose filters already select the latest run. ciGreen = + Number.isSafeInteger(checksData.total_count) && + checksData.total_count === checkRuns.length && ciChecks.length > 0 && ciChecks.every( check => diff --git a/tests/ci-workflows.test.ts b/tests/ci-workflows.test.ts index 0fdf912208..e83017ed27 100644 --- a/tests/ci-workflows.test.ts +++ b/tests/ci-workflows.test.ts @@ -331,6 +331,34 @@ describe("GitHub Actions hardening", () => { }; expect([...(areaFilters.ci ?? [])].sort()).toEqual(ciPaths); + const changesJob = ci.jobs?.changes as { + outputs?: Record; + steps?: Array<{ + name?: string; + id?: string; + shell?: string; + env?: Record; + run?: string; + with?: Record; + }>; + } | undefined; + const scopeStep = changesJob?.steps?.find( + step => step.name === "Assert the scope output is usable", + ); + expect(changesJob?.outputs?.ci).toBe("${{ steps.scope.outputs.ci }}"); + expect(scopeStep?.id).toBe("scope"); + expect(scopeStep?.shell).toBe("bash"); + expect(scopeStep?.env?.CI_SCOPE).toBe("${{ steps.filter.outputs.ci }}"); + expect(scopeStep?.run).not.toContain("${{"); + expect(scopeStep?.run).toContain('case "$CI_SCOPE" in'); + expect(scopeStep?.run).toContain("true|false)"); + expect(scopeStep?.run).toContain(`printf 'ci=%s\\n' "$CI_SCOPE" >> "$GITHUB_OUTPUT"`); + expect(scopeStep?.run).toContain("exit 1"); + const filterIndex = changesJob?.steps?.findIndex(step => step.id === "filter") ?? -1; + const scopeIndex = changesJob?.steps?.findIndex(step => step.id === "scope") ?? -1; + expect(filterIndex).toBeGreaterThanOrEqual(0); + expect(scopeIndex).toBeGreaterThan(filterIndex); + const scopedCondition = "github.event_name != 'pull_request' || needs.changes.outputs.ci == 'true'"; for (const jobName of ["test", "storage-policy", "gates", "platform-macos", "keyring-smoke"]) { const job = ci.jobs?.[jobName] as { needs?: string; if?: string } | undefined; @@ -899,8 +927,9 @@ describe("GitHub Actions hardening", () => { expect(Object.keys(workflow.on?.pull_request_target ?? {})).toEqual(["types"]); expect(Object.prototype.hasOwnProperty.call(workflow.on ?? {}, "status")).toBe(true); - // Exactly the scopes this gate needs. `pull-requests: write` covers title - // and comment updates. `contents: write` is required for the draft GraphQL + // Exactly the scopes this gate needs. `checks: read` covers the live + // current-head CI evidence lookup. `pull-requests: write` covers title and + // comment updates. `contents: write` is required for the draft GraphQL // mutations with GITHUB_TOKEN (#626: "Resource not accessible by integration" // when contents was unset). Asserting the whole object pins both presence // and the absence of anything broader (write-all, contents alone, …). @@ -959,6 +988,7 @@ describe("GitHub Actions hardening", () => { ]); expect(job?.["runs-on"]).toBe("ubuntu-latest"); expect(job?.permissions).toEqual({ + checks: "read", contents: "write", "pull-requests": "write", }); @@ -2013,7 +2043,7 @@ describe("GitHub Actions hardening", () => { expect(readinessBody).toContain("GitHub CI is not green on the current head"); }); - test("a green ci check on a later checks page attests green", async () => { + test("a complete filtered trusted ci response attests green", async () => { const result = await run({ pr: { base: { ref: "dev" }, @@ -2021,18 +2051,11 @@ describe("GitHub Actions hardening", () => { body: readinessChecklistBody(4), }, maintainersFile: MAINTAINERS_FIXTURE, - checkRunPages: [ - Array.from({ length: 100 }, (_, index) => ({ - name: `decoy-${index}`, - status: "completed", - conclusion: "success", - })), - [{ name: "ci", status: "completed", conclusion: "success" }], - ], + checkRuns: [{ name: "ci", status: "completed", conclusion: "success" }], + checkRunTotalCount: 1, }); expect(methodsOf(result)).toEqual(readsAllowedBase([ - "checks.listForRef", "checks.listForRef", "graphql", "pulls.listReviews", @@ -2056,6 +2079,25 @@ describe("GitHub Actions hardening", () => { expect(lastReadinessCommentBody(result)).toContain("**4/4** boxes ticked"); }); + test("a truncated filtered ci response cannot attest green", async () => { + const result = await run({ + pr: { + base: { ref: "dev" }, + draft: false, + body: readinessChecklistBody(4), + }, + maintainersFile: MAINTAINERS_FIXTURE, + checkRuns: [{ name: "ci", status: "completed", conclusion: "success" }], + checkRunTotalCount: 2, + }); + + const [bodyUpdate] = callsTo(result, "pulls.update") as [{ body: string }]; + expect(bodyUpdate.body).toContain("- [ ] All CI tests are green on my local testing."); + expect(lastReadinessCommentBody(result)).toContain( + "GitHub CI is not green on the current head", + ); + }); + test("a foreign app check named ci cannot attest green", async () => { const result = await run({ pr: { diff --git a/tests/helpers/enforce-pr-target-harness.ts b/tests/helpers/enforce-pr-target-harness.ts index 09d27b8a91..79a788f1fd 100644 --- a/tests/helpers/enforce-pr-target-harness.ts +++ b/tests/helpers/enforce-pr-target-harness.ts @@ -186,6 +186,8 @@ export type RunOptions = { conclusion: string | null; app?: { id: number } | null; }>>; + /** Optional filtered total for proving truncated check evidence fails closed. */ + checkRunTotalCount?: number; /** * Review threads `pullRequestReviewThreads` (via GraphQL) reports for the PR. * Each entry is `{ isResolved, author }`; the harness wraps it into the @@ -786,7 +788,9 @@ export async function runEnforcePrTarget( listForRef: (args: unknown) => { const page = Number((args as { page?: number })?.page ?? 1); return respond("checks.listForRef", args, { - total_count: checkRunPages.reduce((total, rows) => total + rows.length, 0), + total_count: + options.checkRunTotalCount ?? + checkRunPages.reduce((total, rows) => total + rows.length, 0), check_runs: checkRunPages[page - 1] ?? [], }); }, From 817192f2408c9f681f2c36aa7550247a67cb5d58 Mon Sep 17 00:00:00 2001 From: Yuxin Qiao <104957188+Yuxin-Qiao@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:36:48 +0800 Subject: [PATCH 113/124] fix(openai-chat): inject reasoning placeholder when replay cache misses Closes #1193 --- src/adapters/openai-chat.ts | 24 +++++++- tests/deepseek-reasoning-replay-gaps.test.ts | 62 ++++++++++++++++++-- 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/src/adapters/openai-chat.ts b/src/adapters/openai-chat.ts index 73c585e16f..be080deb0b 100644 --- a/src/adapters/openai-chat.ts +++ b/src/adapters/openai-chat.ts @@ -334,7 +334,20 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon const cached = toolCalls .map(tc => (tc.id ? peekReasoningForCall(tc.id, replayCacheScope) : undefined)) .filter((text): text is string => typeof text === "string" && text.length > 0); - if (cached.length > 0) reasoningContent = [...new Set(cached)].join("\n"); + // Parallel calls share one preceding reasoning block, which is + // recorded under every call id — join unique texts only. + if (cached.length > 0) { + reasoningContent = [...new Set(cached)].join("\n"); + } else { + // Fallback (extends #950, closes #1193): the replay cache is + // bounded (64 entries / 256 KiB / 1 h TTL) and always misses on + // long sessions, and some tool rounds carry no recorded reasoning + // at all. DeepSeek thinking mode rejects ANY tool_call assistant + // message missing reasoning_content with HTTP 400, so inject a + // minimal placeholder rather than emit a bare continuation the + // upstream will reject. + reasoningContent = " "; + } } if (reasoningContent.length > 0 && modelInList(provider.preserveReasoningContentModels, parsed.modelId)) { chatMsg.reasoning_content = reasoningContent; @@ -385,10 +398,17 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon toolCallId && modelInList(provider.preserveReasoningContentModels, parsed.modelId) ? peekReasoningForCall(toolCallId, replayCacheScope) : undefined; + // Same fallback as the main-assistant path: never emit a bare orphan + // tool_call continuation on a thinking-mode provider — inject a + // placeholder when the replay cache missed (the bounded cache can + // always miss on long sessions), or DeepSeek thinking mode 400s. + const orphanReasoning = + cachedReasoning + ?? (modelInList(provider.preserveReasoningContentModels, parsed.modelId) ? " " : undefined); out.push({ role: "assistant", content: emptyAssistantContent(provider), - ...(cachedReasoning ? { reasoning_content: cachedReasoning } : {}), + ...(orphanReasoning ? { reasoning_content: orphanReasoning } : {}), tool_calls: [{ id: toolCallId, type: "function", diff --git a/tests/deepseek-reasoning-replay-gaps.test.ts b/tests/deepseek-reasoning-replay-gaps.test.ts index 74787adc52..16d107a04a 100644 --- a/tests/deepseek-reasoning-replay-gaps.test.ts +++ b/tests/deepseek-reasoning-replay-gaps.test.ts @@ -135,10 +135,64 @@ describe("issue #950 — tool-call reasoning replay invariant (openai-chat wire) expect(retry!["reasoning_content"]).toBe(REASONING); }); - test("documented non-bug: opaque encrypted-only reasoning is intentionally not replayed", () => { + test("GAP D (issue #1193): replay cache MISS on the main assistant path injects a placeholder", () => { + // The replay cache is bounded (64 entries / 256 KiB / 1 h TTL) and always + // misses on long sessions. DeepSeek thinking mode rejects ANY tool_call + // assistant message without reasoning_content (HTTP 400), so a cache miss + // must degrade to a minimal placeholder instead of a bare continuation. + const { messages } = wireFor([ + userMessage(), + { type: "compaction", encrypted_content: "ocx1:c3VtbWFyeQ==" }, + functionCallItem(), + functionCallOutputItem(), + ]); + const assistant = toolCallAssistant(messages); + expect(assistant).toBeDefined(); + expect(assistant!["reasoning_content"]).toBe(" "); + }); + + test("GAP E (issue #1193): replay cache MISS on the orphan-repair path injects a placeholder", () => { + // Same invariant for the synthesized orphan tool_call: with nothing + // recorded under the call id, repair still must not emit a bare + // continuation a thinking-mode provider will 400 on. + const { messages } = wireFor([userMessage(), functionCallOutputItem()]); + const assistant = toolCallAssistant(messages); + expect(assistant).toBeDefined(); + expect(assistant!["reasoning_content"]).toBe(" "); + }); + + test("negative control: models outside preserveReasoningContentModels never get a placeholder", () => { + // The placeholder fallback is scoped to thinking-mode providers; other + // models keep the previous bare-continuation behavior. Use a custom + // provider so no registry preset seeds a preserve list. + const parsed = parseRequest({ model: "custom-chat/plain-model", input: [userMessage(), functionCallOutputItem()], stream: true }); + const config: OcxConfig = { + port: 10100, + defaultProvider: "custom-chat", + providers: { + "custom-chat": { + adapter: "openai-chat", + baseUrl: "https://example.invalid/v1", + apiKey: "key", + models: ["plain-model"], + }, + }, + }; + const route = routeModel(config, parsed.modelId); + parsed.modelId = route.modelId; + const req = createOpenAIChatAdapter(route.provider).buildRequest(parsed as OcxParsedRequest); + const { messages } = JSON.parse(req.body as string) as { messages: Array> }; + const assistant = toolCallAssistant(messages); + expect(assistant).toBeDefined(); + expect(assistant!["reasoning_content"]).toBeUndefined(); + }); + + test("documented non-bug: opaque encrypted-only reasoning degrades to the placeholder, not invented plaintext", () => { // Native (non-ocxr1) encrypted reasoning has no readable text; the parser - // deliberately degrades instead of inventing replayable plaintext. Not a - // candidate for the opencode-go path (its reasoning is plaintext/ocxr1). + // deliberately degrades instead of inventing replayable plaintext. On a + // thinking-mode provider the fallback now attaches the minimal placeholder + // (issue #1193) rather than replaying anything, so the continuation stays + // valid without fabricating reasoning text. const { messages } = wireFor([ userMessage(), { type: "reasoning", id: "rs_1", encrypted_content: "some-opaque-blob" }, @@ -147,7 +201,7 @@ describe("issue #950 — tool-call reasoning replay invariant (openai-chat wire) ]); const assistant = toolCallAssistant(messages); expect(assistant).toBeDefined(); - expect(assistant!["reasoning_content"]).toBeUndefined(); + expect(assistant!["reasoning_content"]).toBe(" "); }); }); From cd73c151f3ce3be6bf5f79df158dfa97c2c873ba Mon Sep 17 00:00:00 2001 From: Yuxin Qiao <104957188+Yuxin-Qiao@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:24:29 +0800 Subject: [PATCH 114/124] fix(openai-chat): scope reasoning placeholder to models that require it Address review findings on #1205: - chatgpt-codex-connector P2: preserveReasoningContentModels only opts models into replaying reasoning that exists; MiniMax-M3 low effort maps to thinking disabled, so a fabricated placeholder could reach non-thinking histories. Add requiresReasoningPlaceholderModels (registry/derive/router/oauth/auth-cors plumbing, docs-site table) defaulting to the preserve list; minimax/minimax-cn seed [] to opt out. Custom preserve-only provider configs keep the #1193 fix via fallback. - CodeRabbit minor: treat a falsy cache hit as a miss in the orphan-repair path (defense-in-depth; the write path already rejects empty strings). Refs #1193 --- .../ja/reference/configuration/providers.md | 1 + .../ko/reference/configuration/providers.md | 1 + .../docs/reference/configuration/providers.md | 1 + .../ru/reference/configuration/providers.md | 1 + .../reference/configuration/providers.md | 1 + src/adapters/openai-chat.ts | 11 ++++-- src/oauth/index.ts | 1 + src/oauth/login-cli.ts | 1 + src/providers/derive.ts | 4 +++ src/providers/registry.ts | 8 ++++- src/router.ts | 2 ++ src/server/auth-cors.ts | 1 + src/types.ts | 8 +++++ tests/deepseek-reasoning-replay-gaps.test.ts | 35 +++++++++++++++++++ 14 files changed, 72 insertions(+), 4 deletions(-) diff --git a/docs-site/src/content/docs/ja/reference/configuration/providers.md b/docs-site/src/content/docs/ja/reference/configuration/providers.md index 19c02b67a1..a99df988c3 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ja/reference/configuration/providers.md @@ -98,6 +98,7 @@ account を削除しても mapping は保持され、同じ id を再追加す | `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key プロバイダーのみ(`authMode: "key"`)。オプトインの同一ターゲット 429 リトライ: `retryOn429` が無ければ無効で、オブジェクトがあれば `enabled: false` でない限り有効になります。429 時に待機(上流の `Retry-After` または固定間隔)してから、キー フェイルオーバーの前に同一キーで同一リクエストを再送します — メインのテキストターン回復ループ、Responses passthrough、画像/動画ブリッジ、web-search サイドカー、ターミナル継続要求をすべてカバーします。再送の対象はプリストリームの HTTP 429 応答のみで、カスタム `runTurn` トランスポートは HTTP リトライループの対象外です。`attempts` は最初の 429 以降の同一キー再送回数(合計送信数 = `attempts` + 1)で、メインの回復ループ・ターミナルガード継続・ブリッジ再試行で共有されるリクエスト単位の予算です。`attempts` を使い切っても同一キーでの再送が止まるだけで、通常のキー フェイルオーバーまたは最終エラー処理が利用可能なターゲットに応じて続きます — キー認証の passthrough ワイヤにはフェイルオーバーがないため、使い切った 429 はそのまま返ります。Codex 自体は 429 をリトライしないため、単一キーのプロバイダーでは唯一の防御です。デフォルト: `enabled: true`、`attempts: 3`、`intervalMs: 5000`、`maxIntervalMs: 60000`(1回の待機は `maxIntervalMs` で上限、その上限は 600000)、`respectRetryAfter: true`。 | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice` が `auto` または `none` のみを受け入れるモデル。強制的な選択は格下げされます。 | | `preserveReasoningContentModels?` | `string[]` |チャット履歴に以前のアシスタント `reasoning_content` が必要なモデル。 | +| `requiresReasoningPlaceholderModels?` | `string[]` | `reasoning_content` を欠いた tool_call 継続を上流が拒否するモデル（DeepSeek thinking モード）。リプレイキャッシュが外れた場合に最小プレースホルダーを注入。未設定時は `preserveReasoningContentModels` を引き継ぎ、`[]` で明示的に無効化。 | | `thinkingToggleModels?` | `string[]` |エフォート ラダーではなく `thinking.enabled` を使用してモデルをチャットします。 | | `thinkingBudgetModels?` | `string[]` |整数 `thinking_budget` を使用したチャット モデル。労力は予算の一部にマッピングされます。 | | `noVisionModels?` | `string[]` |ビジョン サイドカーを通じて送信されるテキストのみのモデル。マッチングでは、Ollama `:size` タグが許容されます。 | diff --git a/docs-site/src/content/docs/ko/reference/configuration/providers.md b/docs-site/src/content/docs/ko/reference/configuration/providers.md index 1124b4481a..2b1d56725f 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ko/reference/configuration/providers.md @@ -98,6 +98,7 @@ managed map을 활성화하면 privacy-safe selector를 만들고, 이후 계정 | `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key 프로바이더 전용(`authMode: "key"`). 동일 대상 429 재시도: `retryOn429`가 없으면 기능이 꺼져 있고, 객체가 있으면 `enabled: false`가 아닌 한 활성화됩니다. 429 시 대기(업스트림 `Retry-After` 또는 고정 간격) 후 키 장애 조치 전에 동일 키로 동일 요청을 재전송합니다 — 일반 텍스트 턴 복구 루프, Responses passthrough, 이미지/비디오 브리지, web-search 사이드카, 터미널 연속 요청을 모두 포함합니다. 재전송 대상은 프리스트림 HTTP 429 응답뿐이며, 커스텀 `runTurn` 전송은 HTTP 재시도 루프에서 제외됩니다. `attempts`는 첫 429 이후의 동일 키 재전송 횟수(총 전송 = `attempts` + 1)이며, 메인 복구 루프·터미널 가드 연속 요청·브리지 재시도가 공유하는 요청 단위 예산입니다. `attempts`를 모두 소진해도 동일 키 재전송만 중단되며, 이후에는 일반 키 장애 조치 또는 최종 오류 처리가 사용 가능한 대상에 따라 진행됩니다 — 키 인증 passthrough 와이어에는 장애 조치가 없으므로 소진된 429가 그대로 반환됩니다. Codex 자체는 429를 재시도하지 않으므로 단일 키 프로바이더의 유일한 방어선입니다. 기본값: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000`(단일 대기는 `maxIntervalMs`로 상한, 그 자체는 600000으로 상한), `respectRetryAfter: true`. | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice`가 `auto` 또는 `none`만 받는 모델입니다. 강제 선택은 낮은 수준으로 바뀝니다. | | `preserveReasoningContentModels?` | `string[]` | chat 기록에서 이전 assistant `reasoning_content`가 필요한 모델입니다. | +| `requiresReasoningPlaceholderModels?` | `string[]` | `reasoning_content`가 없는 tool_call 연속을 업스트림이 거부하는 모델(DeepSeek thinking 모드). 리플레이 캐시 미스 시 최소 플레이스홀더를 주입합니다. 미설정 시 `preserveReasoningContentModels`를 따륩며 `[]`로 명시적 해제 가능. | | `thinkingToggleModels?` | `string[]` | effort 계층 대신 `thinking.enabled`를 쓰는 chat 모델입니다. | | `thinkingBudgetModels?` | `string[]` | 정수 `thinking_budget`를 쓰는 chat 모델입니다. effort는 예산 비율로 매핑됩니다. | | `noVisionModels?` | `string[]` | vision sidecar로 보내는 텍스트 전용 모델입니다. 일치 판정은 Ollama `:size` 태그도 허용합니다. | diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 474b8327c7..dd5ca0b52b 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -108,6 +108,7 @@ differing backup and rewrites known legacy namespaced selected ids to bare ids. | `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key providers only (`authMode: "key"`). Opt-in same-target 429 retry: when `retryOn429` is absent the feature is off; object presence enables it unless `enabled: false`. On 429 the proxy waits (upstream `Retry-After` or the fixed interval) and replays the identical request on the same key before any key failover — across the main text-turn recovery loop, the Responses passthrough wire, the image/video bridge, the web-search sidecar, and terminal continuations. Only pre-stream HTTP 429 responses are eligible for replay; custom `runTurn` transports are outside the HTTP retry loop. `attempts` counts same-key replays after the first 429 (total sends = `attempts` + 1) and is one request-wide budget shared by the main recovery loop, the terminal-guard continuation, and bridge retries. Exhausting `attempts` only stops further same-key replays: normal key failover or final-error handling then applies per the available targets — on the key-auth passthrough wire there is no failover, so the exhausted 429 surfaces as-is. Codex itself never retries 429, so this is the only defense for single-key providers. Defaults: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (any single wait is capped at `maxIntervalMs`, itself capped at 600000), `respectRetryAfter: true`. | | `autoToolChoiceOnlyModels?` | `string[]` | Models whose `tool_choice` accepts only `auto` or `none`; forced choices are downgraded. | | `preserveReasoningContentModels?` | `string[]` | Models requiring prior assistant `reasoning_content` in chat history. | +| `requiresReasoningPlaceholderModels?` | `string[]` | Models whose upstream rejects a tool_call continuation missing `reasoning_content` (DeepSeek thinking mode); a minimal placeholder is injected when the replay cache misses. Defaults to `preserveReasoningContentModels`; set `[]` to opt out. | | `thinkingToggleModels?` | `string[]` | Chat models using `thinking.enabled` rather than an effort ladder. | | `thinkingBudgetModels?` | `string[]` | Chat models using integer `thinking_budget`; effort maps to a budget fraction. | | `noVisionModels?` | `string[]` | Text-only models sent through the vision sidecar; matching tolerates an Ollama `:size` tag. | diff --git a/docs-site/src/content/docs/ru/reference/configuration/providers.md b/docs-site/src/content/docs/ru/reference/configuration/providers.md index 9960af1dd4..7e55625e15 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ru/reference/configuration/providers.md @@ -111,6 +111,7 @@ cross-route credential fallback не существует. Строки API GPT- | `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | Только для провайдеров с API-ключом (`authMode: "key"`). Опциональный повтор при 429 на том же таргете: если `retryOn429` отсутствует, функция выключена; наличие объекта включает её, если только `enabled: false`. При 429: ожидание (`Retry-After` апстрима или фиксированный интервал) и повтор идентичного запроса на том же ключе до любого фейловера ключей — покрывает основной цикл восстановления текстовых ходов, passthrough-канал Responses, мост изображений/видео, sidecar web-search и терминальные продолжения. Повтор допустим только для HTTP 429, полученных до начала потока; пользовательские транспорты `runTurn` не входят в цикл HTTP-повторов. `attempts` — это число повторов на том же ключе после первого 429 (всего отправок = `attempts` + 1) и единый бюджет на запрос, общий для основного цикла восстановления, терминального продолжения и повторов моста. Исчерпание `attempts` лишь останавливает дальнейшие повторы на том же ключе; далее применяется обычный фейловер ключей или финальная обработка ошибки в зависимости от доступных таргетов — на passthrough-канале с ключевой аутентификацией фейловера нет, поэтому исчерпанный 429 возвращается как есть. Codex сам никогда не повторяет 429, поэтому это единственная защита для провайдеров с одним ключом. По умолчанию: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000` (любое ожидание ограничено `maxIntervalMs`, который сам ограничен 600000), `respectRetryAfter: true`. | | `autoToolChoiceOnlyModels?` | `string[]` | Модели, у которых `tool_choice` принимает только `auto` или `none`; forced choice понижается. | | `preserveReasoningContentModels?` | `string[]` | Модели, которым нужен предыдущий assistant `reasoning_content` в chat history. | +| `requiresReasoningPlaceholderModels?` | `string[]` | Модели, чей upstream отклоняет tool_call-продолжение без `reasoning_content` (DeepSeek thinking mode); при промахе replay-кэша подставляется минимальный placeholder. По умолчанию наследует `preserveReasoningContentModels`; `[]` отключает явно. | | `thinkingToggleModels?` | `string[]` | Chat-модели, использующие `thinking.enabled` вместо effort-ladder. | | `thinkingBudgetModels?` | `string[]` | Chat-модели, использующие целочисленный `thinking_budget`; effort отображается в долю бюджета. | | `noVisionModels?` | `string[]` | Text-only-модели, идущие через vision sidecar; при сопоставлении tolerируется тег Ollama вида `:size`. | diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md index 6b0f5a4272..a8855ebdb2 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md @@ -98,6 +98,7 @@ selector，而不是分配一个新名称。 | `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | 仅限 API-key 提供商（`authMode: "key"`）。可选的同目标 429 重试：未配置 `retryOn429` 时功能关闭；对象存在即启用，除非 `enabled: false`。收到 429 时等待（上游 `Retry-After` 或固定间隔）后在相同 key 上重放完全相同请求，再进入任何 key 故障转移——覆盖主文本恢复循环、Responses passthrough、图像/视频桥、web-search 侧车与终结续接。重放仅适用于流开始前的 HTTP 429 响应；自定义 `runTurn` 传输不在 HTTP 重试循环范围内。`attempts` 是首个 429 之后的同 key 重放次数（总发送次数 = `attempts` + 1），是主恢复循环、终结守卫续接与桥接重试共享的按请求统一预算；`attempts` 耗尽只会停止进一步的同 key 重放：随后按可用目标进行正常的 key 故障转移或最终错误处理——key 认证的 passthrough 线路上没有故障转移，因此耗尽的 429 会原样透出。Codex 自身从不重试 429，因此这是单 key 提供商唯一的防线。默认值：`enabled: true`、`attempts: 3`、`intervalMs: 5000`、`maxIntervalMs: 60000`（单次等待以 `maxIntervalMs` 为上限，其本身上限 600000）、`respectRetryAfter: true`。 | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice` 只接受 `auto` 或 `none` 的模型；强制选择会被降级。 | | `preserveReasoningContentModels?` | `string[]` | 需要在聊天历史中保留先前 assistant `reasoning_content` 的模型。 | +| `requiresReasoningPlaceholderModels?` | `string[]` | 上游会拒绝缺少 `reasoning_content` 的 tool_call 续接消息的模型（DeepSeek thinking 模式）；重放缓存 miss 时注入最小占位符。缺省沿用 `preserveReasoningContentModels`；设为 `[]` 可显式关闭。 | | `thinkingToggleModels?` | `string[]` | 使用 `thinking.enabled` 而不是 effort 阶梯的 chat 模型。 | | `thinkingBudgetModels?` | `string[]` | 使用整数 `thinking_budget` 的 chat 模型；effort 会映射为预算比例。 | | `noVisionModels?` | `string[]` | 经由视觉 sidecar 发送的纯文本模型；匹配时会容忍 Ollama 的 `:size` 标记。 | diff --git a/src/adapters/openai-chat.ts b/src/adapters/openai-chat.ts index be080deb0b..9de82dddd3 100644 --- a/src/adapters/openai-chat.ts +++ b/src/adapters/openai-chat.ts @@ -338,14 +338,17 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon // recorded under every call id — join unique texts only. if (cached.length > 0) { reasoningContent = [...new Set(cached)].join("\n"); - } else { + } else if (modelInList(provider.requiresReasoningPlaceholderModels ?? provider.preserveReasoningContentModels, parsed.modelId)) { // Fallback (extends #950, closes #1193): the replay cache is // bounded (64 entries / 256 KiB / 1 h TTL) and always misses on // long sessions, and some tool rounds carry no recorded reasoning // at all. DeepSeek thinking mode rejects ANY tool_call assistant // message missing reasoning_content with HTTP 400, so inject a // minimal placeholder rather than emit a bare continuation the - // upstream will reject. + // upstream will reject. Scoped to requiresReasoningPlaceholderModels + // (defaulting to the preserve list): preserve-listed providers with + // toggleable thinking (MiniMax low effort) opt out with `[]` so + // non-thinking histories are never given a fabricated placeholder. reasoningContent = " "; } } @@ -402,9 +405,11 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon // tool_call continuation on a thinking-mode provider — inject a // placeholder when the replay cache missed (the bounded cache can // always miss on long sessions), or DeepSeek thinking mode 400s. + // `||` (not `??`): the cache never stores empty strings, but treat a + // falsy hit as a miss so the placeholder still fires. const orphanReasoning = cachedReasoning - ?? (modelInList(provider.preserveReasoningContentModels, parsed.modelId) ? " " : undefined); + || (modelInList(provider.requiresReasoningPlaceholderModels ?? provider.preserveReasoningContentModels, parsed.modelId) ? " " : undefined); out.push({ role: "assistant", content: emptyAssistantContent(provider), diff --git a/src/oauth/index.ts b/src/oauth/index.ts index 778ba33069..f56736170f 100644 --- a/src/oauth/index.ts +++ b/src/oauth/index.ts @@ -739,6 +739,7 @@ const OAUTH_RECONCILE_FIELDS: (keyof OcxProviderConfig)[] = [ "noPenaltyModels", "autoToolChoiceOnlyModels", "preserveReasoningContentModels", + "requiresReasoningPlaceholderModels", ]; const GOOGLE_ANTIGRAVITY_PROVIDER = "google-antigravity"; diff --git a/src/oauth/login-cli.ts b/src/oauth/login-cli.ts index dbcf755619..0afecb4281 100644 --- a/src/oauth/login-cli.ts +++ b/src/oauth/login-cli.ts @@ -102,6 +102,7 @@ export function providerConfigFromKeyLoginProvider(def: KeyLoginProvider, key: s ...(def.noPenaltyModels ? { noPenaltyModels: [...def.noPenaltyModels] } : {}), ...(def.autoToolChoiceOnlyModels ? { autoToolChoiceOnlyModels: [...def.autoToolChoiceOnlyModels] } : {}), ...(def.preserveReasoningContentModels ? { preserveReasoningContentModels: [...def.preserveReasoningContentModels] } : {}), + ...(def.requiresReasoningPlaceholderModels ? { requiresReasoningPlaceholderModels: [...def.requiresReasoningPlaceholderModels] } : {}), ...(def.escapeBuiltinToolNames !== undefined ? { escapeBuiltinToolNames: def.escapeBuiltinToolNames } : {}), }; } diff --git a/src/providers/derive.ts b/src/providers/derive.ts index 43680d96bc..452eed56db 100644 --- a/src/providers/derive.ts +++ b/src/providers/derive.ts @@ -36,6 +36,7 @@ export interface DerivedKeyLoginProvider { noPenaltyModels?: string[]; autoToolChoiceOnlyModels?: string[]; preserveReasoningContentModels?: string[]; + requiresReasoningPlaceholderModels?: string[]; reasoningSplitModels?: string[]; thinkingToggleModels?: string[]; thinkingBudgetModels?: string[]; @@ -245,6 +246,7 @@ export function providerConfigSeed(entry: ProviderRegistryEntry): OcxProviderCon : {}), ...(entry.autoToolChoiceOnlyModels ? { autoToolChoiceOnlyModels: [...entry.autoToolChoiceOnlyModels] } : {}), ...(entry.preserveReasoningContentModels ? { preserveReasoningContentModels: [...entry.preserveReasoningContentModels] } : {}), + ...(entry.requiresReasoningPlaceholderModels ? { requiresReasoningPlaceholderModels: [...entry.requiresReasoningPlaceholderModels] } : {}), ...(entry.reasoningSplitModels ? { reasoningSplitModels: [...entry.reasoningSplitModels] } : {}), ...(entry.thinkingToggleModels ? { thinkingToggleModels: [...entry.thinkingToggleModels] } : {}), ...(entry.thinkingBudgetModels ? { thinkingBudgetModels: [...entry.thinkingBudgetModels] } : {}), @@ -290,6 +292,7 @@ export function deriveKeyLoginMap(): Record { ...(entry.noPenaltyModels ? { noPenaltyModels: [...entry.noPenaltyModels] } : {}), ...(entry.autoToolChoiceOnlyModels ? { autoToolChoiceOnlyModels: [...entry.autoToolChoiceOnlyModels] } : {}), ...(entry.preserveReasoningContentModels ? { preserveReasoningContentModels: [...entry.preserveReasoningContentModels] } : {}), + ...(entry.requiresReasoningPlaceholderModels ? { requiresReasoningPlaceholderModels: [...entry.requiresReasoningPlaceholderModels] } : {}), ...(entry.reasoningSplitModels ? { reasoningSplitModels: [...entry.reasoningSplitModels] } : {}), ...(entry.thinkingToggleModels ? { thinkingToggleModels: [...entry.thinkingToggleModels] } : {}), ...(entry.thinkingBudgetModels ? { thinkingBudgetModels: [...entry.thinkingBudgetModels] } : {}), @@ -445,6 +448,7 @@ export function enrichProviderFromRegistry(name: string, prov: OcxProviderConfig } if (!prov.autoToolChoiceOnlyModels && seed.autoToolChoiceOnlyModels) prov.autoToolChoiceOnlyModels = [...seed.autoToolChoiceOnlyModels]; if (!prov.preserveReasoningContentModels && seed.preserveReasoningContentModels) prov.preserveReasoningContentModels = [...seed.preserveReasoningContentModels]; + if (!prov.requiresReasoningPlaceholderModels && seed.requiresReasoningPlaceholderModels) prov.requiresReasoningPlaceholderModels = [...seed.requiresReasoningPlaceholderModels]; if (!prov.reasoningSplitModels && seed.reasoningSplitModels) prov.reasoningSplitModels = [...seed.reasoningSplitModels]; if (!prov.thinkingToggleModels && seed.thinkingToggleModels) prov.thinkingToggleModels = [...seed.thinkingToggleModels]; if (!prov.thinkingBudgetModels && seed.thinkingBudgetModels) prov.thinkingBudgetModels = [...seed.thinkingBudgetModels]; diff --git a/src/providers/registry.ts b/src/providers/registry.ts index dfef861022..8c51f0cc64 100644 --- a/src/providers/registry.ts +++ b/src/providers/registry.ts @@ -232,6 +232,7 @@ export interface ProviderRegistryEntry { promptCacheKey?: boolean; autoToolChoiceOnlyModels?: string[]; preserveReasoningContentModels?: string[]; + requiresReasoningPlaceholderModels?: string[]; reasoningSplitModels?: string[]; thinkingToggleModels?: string[]; thinkingBudgetModels?: string[]; @@ -254,7 +255,7 @@ export type ProviderConfigSeed = Pick< | "modelMaxInputTokens" | "defaultMaxOutputTokens" | "modelMaxOutputTokens" | "reasoningEfforts" | "modelReasoningEfforts" | "modelDefaultReasoningEfforts" | "reasoningEffortMap" | "modelReasoningEffortMap" | "reasoningWireFormat" | "noVisionModels" | "noReasoningModels" | "noTemperatureModels" | "noTopPModels" | "noPenaltyModels" - | "autoToolChoiceOnlyModels" | "preserveReasoningContentModels" | "reasoningSplitModels" | "thinkingToggleModels" | "thinkingBudgetModels" | "escapeBuiltinToolNames" + | "autoToolChoiceOnlyModels" | "preserveReasoningContentModels" | "requiresReasoningPlaceholderModels" | "reasoningSplitModels" | "thinkingToggleModels" | "thinkingBudgetModels" | "escapeBuiltinToolNames" | "googleMode" | "project" | "location" | "headers" >; @@ -2016,6 +2017,10 @@ export const PROVIDER_REGISTRY: readonly ProviderRegistryEntry[] = [ modelDefaultReasoningEfforts: { "MiniMax-M3": "medium" }, modelReasoningEffortMap: { "MiniMax-M3": MINIMAX_M3_REASONING_EFFORT_MAP }, preserveReasoningContentModels: MINIMAX_MODELS, + // MiniMax-M3 low effort maps to thinking disabled, so a legitimate tool + // round can carry no reasoning at all; only replay real recorded text, + // never a fabricated placeholder (chatgpt-codex-connector P2 on #1205). + requiresReasoningPlaceholderModels: [], reasoningSplitModels: MINIMAX_MODELS, thinkingToggleModels: ["MiniMax-M3"], jawcodeBundle: "minimax", metadataModelIdNormalize: "case-insensitive", note: "Subscription Key or API Key", @@ -2028,6 +2033,7 @@ export const PROVIDER_REGISTRY: readonly ProviderRegistryEntry[] = [ modelDefaultReasoningEfforts: { "MiniMax-M3": "medium" }, modelReasoningEffortMap: { "MiniMax-M3": MINIMAX_M3_REASONING_EFFORT_MAP }, preserveReasoningContentModels: MINIMAX_MODELS, + requiresReasoningPlaceholderModels: [], reasoningSplitModels: MINIMAX_MODELS, thinkingToggleModels: ["MiniMax-M3"], jawcodeBundle: "minimax", metadataModelIdNormalize: "case-insensitive", note: "中国区 Subscription Key", diff --git a/src/router.ts b/src/router.ts index 79fb957307..719d74f789 100644 --- a/src/router.ts +++ b/src/router.ts @@ -285,6 +285,7 @@ function routedProviderConfig(providerName: string, provider: OcxProviderConfig) const noPenaltyModels = mergeStringArray(registryEntry.noPenaltyModels, provider.noPenaltyModels); const autoToolChoiceOnlyModels = mergeStringArray(registryEntry.autoToolChoiceOnlyModels, provider.autoToolChoiceOnlyModels); const preserveReasoningContentModels = mergeStringArray(registryEntry.preserveReasoningContentModels, provider.preserveReasoningContentModels); + const requiresReasoningPlaceholderModels = mergeStringArray(registryEntry.requiresReasoningPlaceholderModels, provider.requiresReasoningPlaceholderModels); const reasoningSplitModels = mergeStringArray(registryEntry.reasoningSplitModels, provider.reasoningSplitModels); const thinkingToggleModels = mergeStringArray(registryEntry.thinkingToggleModels, provider.thinkingToggleModels); const thinkingBudgetModels = mergeStringArray(registryEntry.thinkingBudgetModels, provider.thinkingBudgetModels); @@ -373,6 +374,7 @@ function routedProviderConfig(providerName: string, provider: OcxProviderConfig) ...(noPenaltyModels ? { noPenaltyModels } : {}), ...(autoToolChoiceOnlyModels ? { autoToolChoiceOnlyModels } : {}), ...(preserveReasoningContentModels ? { preserveReasoningContentModels } : {}), + ...(requiresReasoningPlaceholderModels ? { requiresReasoningPlaceholderModels } : {}), ...(reasoningSplitModels ? { reasoningSplitModels } : {}), ...(thinkingToggleModels ? { thinkingToggleModels } : {}), ...(thinkingBudgetModels ? { thinkingBudgetModels } : {}), diff --git a/src/server/auth-cors.ts b/src/server/auth-cors.ts index 536526f3e3..a59bfba5f1 100644 --- a/src/server/auth-cors.ts +++ b/src/server/auth-cors.ts @@ -579,6 +579,7 @@ export function safeConfigDTO(config: OcxConfig): unknown { "noPenaltyModels", "autoToolChoiceOnlyModels", "preserveReasoningContentModels", + "requiresReasoningPlaceholderModels", "escapeBuiltinToolNames", ] as const) { copyIfDefined(dto, provider, key); diff --git a/src/types.ts b/src/types.ts index 89e151b44e..b2b0c6d9e8 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1353,6 +1353,14 @@ export interface OcxProviderConfig { autoToolChoiceOnlyModels?: string[]; /** Model ids that expect prior assistant `reasoning_content` to be preserved in chat history. */ preserveReasoningContentModels?: string[]; + /** + * Model ids whose upstream hard-rejects a tool_call continuation missing + * `reasoning_content` (DeepSeek thinking mode: HTTP 400). When the replay + * cache misses, the adapter injects a minimal placeholder for these models. + * Defaults to `preserveReasoningContentModels` when unset; set `[]` to opt + * out explicitly (e.g. MiniMax, where low effort disables thinking). + */ + requiresReasoningPlaceholderModels?: string[]; /** * Opt-in same-target 429 retry policy. Codex itself never retries 429 (it retries 5xx only, * openai/codex#30471), and single-key pools have no failover, so the proxy waits and replays diff --git a/tests/deepseek-reasoning-replay-gaps.test.ts b/tests/deepseek-reasoning-replay-gaps.test.ts index 16d107a04a..76859116b5 100644 --- a/tests/deepseek-reasoning-replay-gaps.test.ts +++ b/tests/deepseek-reasoning-replay-gaps.test.ts @@ -187,6 +187,41 @@ describe("issue #950 — tool-call reasoning replay invariant (openai-chat wire) expect(assistant!["reasoning_content"]).toBeUndefined(); }); + test("P2 guard: preserve-listed providers with toggleable thinking opt out of the placeholder (MiniMax)", () => { + // MiniMax-M3 low effort maps to thinking disabled, so a legitimate tool + // round can carry no reasoning; the registry seeds + // requiresReasoningPlaceholderModels: [] for minimax so a cache miss never + // fabricates one (chatgpt-codex-connector P2 on #1205). Real recorded + // reasoning still replays via preserveReasoningContentModels. + const minimaxWire = (input: unknown[]) => { + const parsed = parseRequest({ model: "minimax/MiniMax-M3", input, stream: true }); + const config: OcxConfig = { + port: 10100, + defaultProvider: "minimax", + providers: { + minimax: { + adapter: "openai-chat", + baseUrl: "https://api.minimax.io/v1", + apiKey: "key", + }, + }, + }; + const route = routeModel(config, parsed.modelId); + parsed.modelId = route.modelId; + const req = createOpenAIChatAdapter(route.provider).buildRequest(parsed as OcxParsedRequest); + return JSON.parse(req.body as string) as { messages: Array> }; + }; + // Cache miss on the orphan-repair path: no fabricated placeholder. + const miss = toolCallAssistant(minimaxWire([userMessage(), functionCallOutputItem()]).messages); + expect(miss).toBeDefined(); + expect(miss!["reasoning_content"]).toBeUndefined(); + // Cache hit on the same path: the recorded reasoning still replays. + rememberReasoningForCall("call_1", REASONING); + const hit = toolCallAssistant(minimaxWire([userMessage(), functionCallOutputItem()]).messages); + expect(hit).toBeDefined(); + expect(hit!["reasoning_content"]).toBe(REASONING); + }); + test("documented non-bug: opaque encrypted-only reasoning degrades to the placeholder, not invented plaintext", () => { // Native (non-ocxr1) encrypted reasoning has no readable text; the parser // deliberately degrades instead of inventing replayable plaintext. On a From c52884787ec4461e1a6f75a608ea2fbeb3594434 Mon Sep 17 00:00:00 2001 From: Yuxin Qiao <104957188+Yuxin-Qiao@users.noreply.github.com> Date: Sat, 8 Aug 2026 12:27:03 +0800 Subject: [PATCH 115/124] fix(openai-chat): gate orphan placeholder on preserve list, keep opt-outs durable Address the remaining review findings on #1205: - chatgpt-codex-connector P2: the orphan-repair fallback checked only requiresReasoningPlaceholderModels, so a requires-only custom entry could fabricate reasoning_content on a path the main assistant history would never emit it on. Gate the orphan placeholder on the preserve list too. - chatgpt-codex-connector P2: Zhipu BigModel GLM ids are thinking-toggle models (low maps to disabled) AND preserve-listed, so the placeholder default could fabricate reasoning for non-thinking histories. Seed requiresReasoningPlaceholderModels: [] for zhipu-bigmodel, matching the MiniMax opt-out. - chatgpt-codex-connector P2: OAuth reconcile deleted an explicit requiresReasoningPlaceholderModels: [] opt-out on every startup because no OAuth preset seeds the field. Keep the field out of OAUTH_RECONCILE_FIELDS; registry seeds still reach existing rows via enrichProviderFromRegistry. - CodeRabbit minor: fix Korean spelling in the providers table. Refs #1193 --- .../ko/reference/configuration/providers.md | 2 +- src/adapters/openai-chat.ts | 8 ++++- src/oauth/index.ts | 6 +++- src/providers/registry.ts | 4 +++ tests/deepseek-reasoning-replay-gaps.test.ts | 30 +++++++++++++++++++ tests/oauth-provider-reconcile.test.ts | 19 ++++++++++++ 6 files changed, 66 insertions(+), 3 deletions(-) diff --git a/docs-site/src/content/docs/ko/reference/configuration/providers.md b/docs-site/src/content/docs/ko/reference/configuration/providers.md index 2b1d56725f..b3a7cdb62b 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ko/reference/configuration/providers.md @@ -98,7 +98,7 @@ managed map을 활성화하면 privacy-safe selector를 만들고, 이후 계정 | `retryOn429?` | `{ enabled?: boolean; attempts?: number; intervalMs?: number; maxIntervalMs?: number; respectRetryAfter?: boolean }` | API-key 프로바이더 전용(`authMode: "key"`). 동일 대상 429 재시도: `retryOn429`가 없으면 기능이 꺼져 있고, 객체가 있으면 `enabled: false`가 아닌 한 활성화됩니다. 429 시 대기(업스트림 `Retry-After` 또는 고정 간격) 후 키 장애 조치 전에 동일 키로 동일 요청을 재전송합니다 — 일반 텍스트 턴 복구 루프, Responses passthrough, 이미지/비디오 브리지, web-search 사이드카, 터미널 연속 요청을 모두 포함합니다. 재전송 대상은 프리스트림 HTTP 429 응답뿐이며, 커스텀 `runTurn` 전송은 HTTP 재시도 루프에서 제외됩니다. `attempts`는 첫 429 이후의 동일 키 재전송 횟수(총 전송 = `attempts` + 1)이며, 메인 복구 루프·터미널 가드 연속 요청·브리지 재시도가 공유하는 요청 단위 예산입니다. `attempts`를 모두 소진해도 동일 키 재전송만 중단되며, 이후에는 일반 키 장애 조치 또는 최종 오류 처리가 사용 가능한 대상에 따라 진행됩니다 — 키 인증 passthrough 와이어에는 장애 조치가 없으므로 소진된 429가 그대로 반환됩니다. Codex 자체는 429를 재시도하지 않으므로 단일 키 프로바이더의 유일한 방어선입니다. 기본값: `enabled: true`, `attempts: 3`, `intervalMs: 5000`, `maxIntervalMs: 60000`(단일 대기는 `maxIntervalMs`로 상한, 그 자체는 600000으로 상한), `respectRetryAfter: true`. | | `autoToolChoiceOnlyModels?` | `string[]` | `tool_choice`가 `auto` 또는 `none`만 받는 모델입니다. 강제 선택은 낮은 수준으로 바뀝니다. | | `preserveReasoningContentModels?` | `string[]` | chat 기록에서 이전 assistant `reasoning_content`가 필요한 모델입니다. | -| `requiresReasoningPlaceholderModels?` | `string[]` | `reasoning_content`가 없는 tool_call 연속을 업스트림이 거부하는 모델(DeepSeek thinking 모드). 리플레이 캐시 미스 시 최소 플레이스홀더를 주입합니다. 미설정 시 `preserveReasoningContentModels`를 따륩며 `[]`로 명시적 해제 가능. | +| `requiresReasoningPlaceholderModels?` | `string[]` | `reasoning_content`가 없는 tool_call 연속을 업스트림이 거부하는 모델(DeepSeek thinking 모드). 리플레이 캐시 미스 시 최소 플레이스홀더를 주입합니다. 미설정 시 `preserveReasoningContentModels`를 따르며 `[]`로 명시적 해제 가능. | | `thinkingToggleModels?` | `string[]` | effort 계층 대신 `thinking.enabled`를 쓰는 chat 모델입니다. | | `thinkingBudgetModels?` | `string[]` | 정수 `thinking_budget`를 쓰는 chat 모델입니다. effort는 예산 비율로 매핑됩니다. | | `noVisionModels?` | `string[]` | vision sidecar로 보내는 텍스트 전용 모델입니다. 일치 판정은 Ollama `:size` 태그도 허용합니다. | diff --git a/src/adapters/openai-chat.ts b/src/adapters/openai-chat.ts index 9de82dddd3..807f815142 100644 --- a/src/adapters/openai-chat.ts +++ b/src/adapters/openai-chat.ts @@ -405,11 +405,17 @@ function messagesToChatFormat(parsed: OcxParsedRequest, provider: OcxProviderCon // tool_call continuation on a thinking-mode provider — inject a // placeholder when the replay cache missed (the bounded cache can // always miss on long sessions), or DeepSeek thinking mode 400s. + // Gate on the preserve list too: reasoning_content is only ever + // serialized for preserve-listed models, so a requires-only custom + // entry must not fabricate it on this path (P2 on #1205). // `||` (not `??`): the cache never stores empty strings, but treat a // falsy hit as a miss so the placeholder still fires. const orphanReasoning = cachedReasoning - || (modelInList(provider.requiresReasoningPlaceholderModels ?? provider.preserveReasoningContentModels, parsed.modelId) ? " " : undefined); + || (modelInList(provider.preserveReasoningContentModels, parsed.modelId) + && modelInList(provider.requiresReasoningPlaceholderModels ?? provider.preserveReasoningContentModels, parsed.modelId) + ? " " + : undefined); out.push({ role: "assistant", content: emptyAssistantContent(provider), diff --git a/src/oauth/index.ts b/src/oauth/index.ts index f56736170f..0238797c7f 100644 --- a/src/oauth/index.ts +++ b/src/oauth/index.ts @@ -739,8 +739,12 @@ const OAUTH_RECONCILE_FIELDS: (keyof OcxProviderConfig)[] = [ "noPenaltyModels", "autoToolChoiceOnlyModels", "preserveReasoningContentModels", - "requiresReasoningPlaceholderModels", ]; +// `requiresReasoningPlaceholderModels` is deliberately NOT reconciled here: no +// OAuth preset seeds it, so the delete-when-preset-undefined branch would wipe +// an explicit user opt-out (`[]`) on every startup. Registry seeds still reach +// existing rows through enrichProviderFromRegistry, which is fill-only and +// preserves explicit saved values. const GOOGLE_ANTIGRAVITY_PROVIDER = "google-antigravity"; const GOOGLE_ANTIGRAVITY_LIVE_DISCOVERY_VERSION = 2 as const; diff --git a/src/providers/registry.ts b/src/providers/registry.ts index 8c51f0cc64..39353726e8 100644 --- a/src/providers/registry.ts +++ b/src/providers/registry.ts @@ -1739,6 +1739,10 @@ export const PROVIDER_REGISTRY: readonly ProviderRegistryEntry[] = [ ZHIPU_BIGMODEL_THINKING_TOGGLE_MODELS.map(id => [id, true]), ), preserveReasoningContentModels: ZHIPU_BIGMODEL_THINKING_TOGGLE_MODELS, + // GLM thinking is a binary toggle (low maps to disabled), so a legitimate + // tool round can carry no reasoning at all; never fabricate a placeholder + // for it, only replay real recorded text (P2 on #1205). + requiresReasoningPlaceholderModels: [], // No liveModels: GET /api/paas/v4/models has not been observed to answer on this host, and a // false live claim yields an empty picker at runtime. Flip it on once someone verifies it. note: "Domestic BigModel pay-as-you-go endpoint (open.bigmodel.cn)", diff --git a/tests/deepseek-reasoning-replay-gaps.test.ts b/tests/deepseek-reasoning-replay-gaps.test.ts index 76859116b5..6b72f5aceb 100644 --- a/tests/deepseek-reasoning-replay-gaps.test.ts +++ b/tests/deepseek-reasoning-replay-gaps.test.ts @@ -222,6 +222,36 @@ describe("issue #950 — tool-call reasoning replay invariant (openai-chat wire) expect(hit!["reasoning_content"]).toBe(REASONING); }); + test("P2 guard: a requires-only custom model never gets a placeholder on the orphan path", () => { + // requiresReasoningPlaceholderModels narrows which preserve-listed models + // get a fabricated placeholder. A custom entry listing a model ONLY in the + // requires list (not in preserveReasoningContentModels) must behave like + // the main-assistant path, which never serializes reasoning_content for + // non-preserve models: the synthesized orphan tool_call stays bare + // (chatgpt-codex-connector P2 on #1205). + const parsed = parseRequest({ model: "custom-chat/plain-model", input: [userMessage(), functionCallOutputItem()], stream: true }); + const config: OcxConfig = { + port: 10100, + defaultProvider: "custom-chat", + providers: { + "custom-chat": { + adapter: "openai-chat", + baseUrl: "https://example.invalid/v1", + apiKey: "key", + models: ["plain-model"], + requiresReasoningPlaceholderModels: ["plain-model"], + }, + }, + }; + const route = routeModel(config, parsed.modelId); + parsed.modelId = route.modelId; + const req = createOpenAIChatAdapter(route.provider).buildRequest(parsed as OcxParsedRequest); + const { messages } = JSON.parse(req.body as string) as { messages: Array> }; + const assistant = toolCallAssistant(messages); + expect(assistant).toBeDefined(); + expect(assistant!["reasoning_content"]).toBeUndefined(); + }); + test("documented non-bug: opaque encrypted-only reasoning degrades to the placeholder, not invented plaintext", () => { // Native (non-ocxr1) encrypted reasoning has no readable text; the parser // deliberately degrades instead of inventing replayable plaintext. On a diff --git a/tests/oauth-provider-reconcile.test.ts b/tests/oauth-provider-reconcile.test.ts index 611a4e969f..a34447400e 100644 --- a/tests/oauth-provider-reconcile.test.ts +++ b/tests/oauth-provider-reconcile.test.ts @@ -170,4 +170,23 @@ describe("OAuth provider reconciliation", () => { upsertOAuthProvider(config, "google-antigravity"); expect(config.providers["google-antigravity"].liveModels).toBe(true); }); + + test("preserves an explicit requiresReasoningPlaceholderModels opt-out on OAuth providers", () => { + // No OAuth preset seeds the new field, so reconcile must never delete an + // explicit `[]` opt-out on startup (chatgpt-codex-connector P2 on #1205). + const config = { + port: 10100, + defaultProvider: "kimi", + googleAntigravityStaticCatalogVersion: 1, + providers: { + kimi: { + ...structuredClone(OAUTH_PROVIDERS.kimi.providerConfig), + requiresReasoningPlaceholderModels: [], + }, + }, + } satisfies OcxConfig; + + expect(reconcileOAuthProviders(config)).toBe(false); + expect(config.providers.kimi.requiresReasoningPlaceholderModels).toEqual([]); + }); }); From 5cd3edb06500ff0601c73c9462e3bb578b33ec98 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sun, 9 Aug 2026 14:52:57 +0900 Subject: [PATCH 116/124] test(auth): cover reasoning placeholder config boundaries --- tests/oauth-provider-reconcile.test.ts | 2 +- tests/server-auth.test.ts | 40 ++++++++++++++++++++++++++ tests/umans-provider.test.ts | 16 +++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/tests/oauth-provider-reconcile.test.ts b/tests/oauth-provider-reconcile.test.ts index a34447400e..d00f8de1ae 100644 --- a/tests/oauth-provider-reconcile.test.ts +++ b/tests/oauth-provider-reconcile.test.ts @@ -186,7 +186,7 @@ describe("OAuth provider reconciliation", () => { }, } satisfies OcxConfig; - expect(reconcileOAuthProviders(config)).toBe(false); + reconcileOAuthProviders(config); expect(config.providers.kimi.requiresReasoningPlaceholderModels).toEqual([]); }); }); diff --git a/tests/server-auth.test.ts b/tests/server-auth.test.ts index f67a1689e2..2d9a8929bd 100644 --- a/tests/server-auth.test.ts +++ b/tests/server-auth.test.ts @@ -431,6 +431,46 @@ describe("server local API auth", () => { expect(dto.providers.openai.disabled).toBeUndefined(); }); + test("safeConfigDTO preserves reasoning placeholder policies without adjacent secrets", () => { + const dto = safeConfigDTO({ + ...config("127.0.0.1"), + providers: { + required: { + adapter: "openai-chat", + baseUrl: "https://user:password@example.test/v1?token=url-secret", + apiKey: "required-api-secret", + headers: { Authorization: "Bearer required-header-secret" }, + requiresReasoningPlaceholderModels: ["deepseek-reasoner"], + }, + optedOut: { + adapter: "openai-chat", + baseUrl: "https://example.test/v1", + apiKey: "opt-out-api-secret", + headers: { "X-Private-Key": "opt-out-header-secret" }, + requiresReasoningPlaceholderModels: [], + }, + }, + } as OcxConfig) as { + providers: Record>; + }; + + expect(dto.providers.required.requiresReasoningPlaceholderModels).toEqual(["deepseek-reasoner"]); + expect(dto.providers.optedOut.requiresReasoningPlaceholderModels).toEqual([]); + expect(dto.providers.required).not.toHaveProperty("apiKey"); + expect(dto.providers.required).not.toHaveProperty("headers"); + expect(dto.providers.optedOut).not.toHaveProperty("apiKey"); + expect(dto.providers.optedOut).not.toHaveProperty("headers"); + const serialized = JSON.stringify(dto); + for (const secret of [ + "password", + "url-secret", + "required-api-secret", + "required-header-secret", + "opt-out-api-secret", + "opt-out-header-secret", + ]) expect(serialized).not.toContain(secret); + }); + test("safeConfigDTO exposes keyOptional for saved free-tier providers", () => { const dto = safeConfigDTO({ ...config("127.0.0.1"), diff --git a/tests/umans-provider.test.ts b/tests/umans-provider.test.ts index afa4504012..3654177d75 100644 --- a/tests/umans-provider.test.ts +++ b/tests/umans-provider.test.ts @@ -96,6 +96,22 @@ describe("Umans provider", () => { expect(provider.apiKeyTransport).toBe("bearer"); }); + test("CLI key-login save payload preserves nonempty and explicit-empty reasoning placeholder policies", () => { + const requiredModels = ["deepseek-reasoner"]; + const required = providerConfigFromKeyLoginProvider({ + ...KEY_LOGIN_PROVIDERS.umans, + requiresReasoningPlaceholderModels: requiredModels, + } satisfies KeyLoginProvider, "sk-required"); + const optedOut = providerConfigFromKeyLoginProvider({ + ...KEY_LOGIN_PROVIDERS.umans, + requiresReasoningPlaceholderModels: [], + } satisfies KeyLoginProvider, "sk-opted-out"); + + expect(required.requiresReasoningPlaceholderModels).toEqual(["deepseek-reasoner"]); + expect(required.requiresReasoningPlaceholderModels).not.toBe(requiredModels); + expect(optedOut.requiresReasoningPlaceholderModels).toEqual([]); + }); + test("OpenAI API key-login clones max-input metadata and never persists virtual maps", () => { const source = KEY_LOGIN_PROVIDERS["openai-apikey"]; const provider = providerConfigFromKeyLoginProvider(source, "sk-openai"); From eecec3a99571e4985beda5239d9d2fd3c80bad5a Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:14:29 +0200 Subject: [PATCH 117/124] refactor(lab): isolate validation error type --- src/lab/events/errors.ts | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 src/lab/events/errors.ts diff --git a/src/lab/events/errors.ts b/src/lab/events/errors.ts new file mode 100644 index 0000000000..17009183a7 --- /dev/null +++ b/src/lab/events/errors.ts @@ -0,0 +1,9 @@ +export class LabValidationError extends Error { + readonly code: string; + + constructor(code: string, message: string) { + super(message); + this.name = "LabValidationError"; + this.code = code; + } +} From 0c9dc4d8ec15d16317c9e67edcb8f15cc70ea3b1 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:14:45 +0200 Subject: [PATCH 118/124] refactor(lab): break validation import cycle --- src/lab/events/limits.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lab/events/limits.ts b/src/lab/events/limits.ts index 772fc3fba9..d6cba306f7 100644 --- a/src/lab/events/limits.ts +++ b/src/lab/events/limits.ts @@ -4,7 +4,7 @@ import { MAX_OBJECT_KEYS_PER_EVENT, MAX_SANITIZED_STRING_FIELD, } from "../constants"; -import { LabValidationError } from "./validate"; +import { LabValidationError } from "./errors"; const FORBIDDEN_KEY_RE = /(?:^|_)(?:secret|token|apikey|api_key|password|credential|authorization|cookie|bearer|prompt|repository|filepath|file_path|baseurl|base_url|hostname|rawrequest|raw_request)(?:$|_)/i; From 1eed4ffbc9772c64f4f22e37869ccb0b9efa90e1 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:16:06 +0200 Subject: [PATCH 119/124] fix(lab): validate claim source event ids --- src/lab/events/validate.ts | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/src/lab/events/validate.ts b/src/lab/events/validate.ts index b2db0a0ef8..d101e28b0a 100644 --- a/src/lab/events/validate.ts +++ b/src/lab/events/validate.ts @@ -1,4 +1,6 @@ import { enforceEventStructureLimits } from "./limits"; +import { LabValidationError } from "./errors"; +export { LabValidationError } from "./errors"; import { ARTIFACT_CLASSES, ARTIFACT_FILENAME_EXT, @@ -44,15 +46,6 @@ import type { TaskSubjectV1, } from "./types"; -export class LabValidationError extends Error { - readonly code: string; - constructor(code: string, message: string) { - super(message); - this.name = "LabValidationError"; - this.code = code; - } -} - function isPlainObject(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); } @@ -451,9 +444,6 @@ function validateClaimSnapshot(raw: Record): ClaimSnapshotEvent if (!isSha256Hex(sourceManifestDigest)) { throw new LabValidationError("invalid_digest", "sourceManifestDigest"); } - if (!Array.isArray(raw.sourceEventIds) || !Array.isArray(raw.supersedes)) { - throw new LabValidationError("invalid_claim_lists", "sourceEventIds/supersedes"); - } return { schemaVersion: LAB_EVENT_SCHEMA_VERSION, eventId: assertString(raw.eventId, "eventId"), @@ -467,12 +457,14 @@ function validateClaimSnapshot(raw: Record): ClaimSnapshotEvent capability: assertString(raw.capability, "capability"), polarity: assertClosed(raw.polarity, "polarity", CLAIM_POLARITIES), sourceManifestDigest, - sourceEventIds: raw.sourceEventIds.map((id, i) => { - const s = assertString(id, `sourceEventIds[${i}]`); - if (s.length > 0 && !isSha256Hex(s)) throw new LabValidationError("invalid_id", `sourceEventIds[${i}]`); - return s; + sourceEventIds: validateSortedUniqueHexIds(raw.sourceEventIds, "sourceEventIds", { + nonEmpty: false, + max: MAX_INVALIDATION_TARGETS, + }), + supersedes: validateSortedUniqueHexIds(raw.supersedes, "supersedes", { + nonEmpty: false, + max: MAX_INVALIDATION_TARGETS, }), - supersedes: validateSortedUniqueHexIds(raw.supersedes, "supersedes", { nonEmpty: false }), effectiveAt: assertIntMs(raw.effectiveAt, "effectiveAt"), }; } From d3015cb975da0b9fdac71d2cc3fc0fb596f8a2b5 Mon Sep 17 00:00:00 2001 From: Ingwannu Date: Sat, 8 Aug 2026 06:39:15 +0000 Subject: [PATCH 120/124] fix(cli): check live proxy before journal recovery --- src/cli/index.ts | 5 ++++- tests/cli-start-journal-order.test.ts | 32 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 tests/cli-start-journal-order.test.ts diff --git a/src/cli/index.ts b/src/cli/index.ts index d96b128f97..518fd0e10b 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -226,7 +226,6 @@ async function handleStart(options: { block?: boolean } = {}) { const serviceToken = loadServiceTokenFromFile(process.env); if (serviceToken) process.env.OPENCODEX_API_AUTH_TOKEN = serviceToken; const requestedPort = parsePortOption(); - if (!currentExternalCodexModelProvider()) reconcileJournal(); const existingPid = readPid(); if (existingPid) { const live = await findLiveProxy(); @@ -236,6 +235,10 @@ async function handleStart(options: { block?: boolean } = {}) { } removePid(existingPid); } + // A losing concurrent start must not restore the active proxy's Codex config. + // Establish that the PID-file owner is stale before reconciling a dead journal; + // a healthy owner exits above without changing integration state (#1230). + if (!currentExternalCodexModelProvider()) reconcileJournal(); // Interactive-only update prompt. Must run BEFORE we bind a port / write a // PID: choosing "Update now" installs globally and exits, so we never want a diff --git a/tests/cli-start-journal-order.test.ts b/tests/cli-start-journal-order.test.ts new file mode 100644 index 0000000000..446211225d --- /dev/null +++ b/tests/cli-start-journal-order.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +const source = readFileSync(join(import.meta.dir, "..", "src", "cli", "index.ts"), "utf8"); + +function handleStartSource(): string { + const start = source.indexOf("async function handleStart("); + const end = source.indexOf("async function handleEnsure(", start); + expect(start).toBeGreaterThanOrEqual(0); + expect(end).toBeGreaterThan(start); + return source.slice(start, end); +} + +describe("handleStart journal ownership ordering (#1230)", () => { + test("a healthy PID-file proxy is detected before journal reconciliation", () => { + const handleStart = handleStartSource(); + const readPid = handleStart.indexOf("const existingPid = readPid();"); + const findLive = handleStart.indexOf("const live = await findLiveProxy();", readPid); + const healthyExit = handleStart.indexOf("process.exit(1);", findLive); + const removeStalePid = handleStart.indexOf("removePid(existingPid);", healthyExit); + const reconcile = handleStart.indexOf("reconcileJournal();", removeStalePid); + const updatePrompt = handleStart.indexOf("await maybeShowUpdatePrompt();", reconcile); + + expect(readPid).toBeGreaterThanOrEqual(0); + expect(findLive).toBeGreaterThan(readPid); + expect(healthyExit).toBeGreaterThan(findLive); + expect(removeStalePid).toBeGreaterThan(healthyExit); + expect(reconcile).toBeGreaterThan(removeStalePid); + expect(updatePrompt).toBeGreaterThan(reconcile); + }); +}); From 700803f34c8adc3ae8f1fc297cb395c012fdb83d Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sun, 9 Aug 2026 14:50:28 +0900 Subject: [PATCH 121/124] fix(cli): share proxy ownership preflight --- src/cli/index.ts | 36 +++-- tests/cli-start-journal-order.test.ts | 220 ++++++++++++++++++++++---- 2 files changed, 214 insertions(+), 42 deletions(-) diff --git a/src/cli/index.ts b/src/cli/index.ts index 518fd0e10b..f86c5eb7e9 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -219,6 +219,22 @@ async function chooseListenPort(requestedPort?: number): Promise { } } +async function findProxyOwnerBeforeJournalRecovery( + options: { probeConfiguredPort?: boolean } = {}, +): Promise<{ live: LiveProxy | null; pidSnapshot: number | null }> { + const pidSnapshot = readPidFileValue(); + const hasRuntimeOwner = readRuntimePort() !== null; + const shouldProbe = pidSnapshot !== null || hasRuntimeOwner || options.probeConfiguredPort === true; + const live = shouldProbe ? await findLiveProxy() : null; + if (live) return { live, pidSnapshot }; + + // The probe established that the snapshotted owner is stale. Compare before + // deleting so a concurrent start that rewrote the PID file keeps its state. + removePidIfValueIs(pidSnapshot); + if (!currentExternalCodexModelProvider()) reconcileJournal(); + return { live: null, pidSnapshot }; +} + async function handleStart(options: { block?: boolean } = {}) { // Native (WinSW) service mode has no batch wrapper to read the service token file // into the environment, so the app loads it here before the server binds. The server @@ -226,19 +242,11 @@ async function handleStart(options: { block?: boolean } = {}) { const serviceToken = loadServiceTokenFromFile(process.env); if (serviceToken) process.env.OPENCODEX_API_AUTH_TOKEN = serviceToken; const requestedPort = parsePortOption(); - const existingPid = readPid(); - if (existingPid) { - const live = await findLiveProxy(); - if (live) { - console.error(`⚠️ Proxy already running (PID ${live.pid ?? existingPid}, port ${live.port}). Use 'ocx stop' first.`); - process.exit(1); - } - removePid(existingPid); + const owner = await findProxyOwnerBeforeJournalRecovery(); + if (owner.live) { + console.error(`⚠️ Proxy already running (PID ${owner.live.pid ?? owner.pidSnapshot ?? "unknown"}, port ${owner.live.port}). Use 'ocx stop' first.`); + process.exit(1); } - // A losing concurrent start must not restore the active proxy's Codex config. - // Establish that the PID-file owner is stale before reconciling a dead journal; - // a healthy owner exits above without changing integration state (#1230). - if (!currentExternalCodexModelProvider()) reconcileJournal(); // Interactive-only update prompt. Must run BEFORE we bind a port / write a // PID: choosing "Update now" installs globally and exits, so we never want a @@ -445,13 +453,13 @@ function detachedStartEnvironment(): NodeJS.ProcessEnv { } async function handleEnsure(options: { existingIsSuccess?: boolean } = {}): Promise { - if (!currentExternalCodexModelProvider()) reconcileJournal(); + const owner = await findProxyOwnerBeforeJournalRecovery({ probeConfiguredPort: true }); const config = loadConfig(); if (!codexAutoStartEnabled(config)) { console.log("Codex autostart is disabled."); return false; } - const live = await findLiveProxy(); + const live = owner.live; if (live) { if (options.existingIsSuccess === false) { console.error("Proxy appeared while restart was confirming absence; no start was attempted."); diff --git a/tests/cli-start-journal-order.test.ts b/tests/cli-start-journal-order.test.ts index 446211225d..48f4ba95bc 100644 --- a/tests/cli-start-journal-order.test.ts +++ b/tests/cli-start-journal-order.test.ts @@ -1,32 +1,196 @@ -import { describe, expect, test } from "bun:test"; -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - -const source = readFileSync(join(import.meta.dir, "..", "src", "cli", "index.ts"), "utf8"); - -function handleStartSource(): string { - const start = source.indexOf("async function handleStart("); - const end = source.indexOf("async function handleEnsure(", start); - expect(start).toBeGreaterThanOrEqual(0); - expect(end).toBeGreaterThan(start); - return source.slice(start, end); +import { afterEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +const cliPath = resolve(import.meta.dir, "../src/cli/index.ts"); +const roots: string[] = []; +const children: Array> = []; + +type Fixture = { + root: string; + codexHome: string; + ocxHome: string; + configPath: string; + journalPath: string; + pidPath: string; + env: Record; +}; + +function fixture(): Fixture { + const root = mkdtempSync(join(tmpdir(), "ocx-start-owner-")); + roots.push(root); + const codexHome = join(root, "codex"); + const ocxHome = join(root, "ocx"); + const home = join(root, "home"); + const runtime = join(root, "runtime"); + for (const path of [codexHome, ocxHome, home, runtime]) mkdirSync(path, { recursive: true }); + const configPath = join(codexHome, "config.toml"); + const journalPath = join(codexHome, "opencodex-journal.json"); + const pidPath = join(ocxHome, "ocx.pid"); + writeFileSync(join(ocxHome, "config.json"), JSON.stringify({ + port: 0, + hostname: "127.0.0.1", + codexAutoStart: false, + syncResumeHistory: false, + clientIntegrations: { codex: false, grok: false, "claude-desktop": false }, + claudeCode: { systemEnv: false }, + providers: {}, + defaultProvider: "openai", + })); + return { + root, + codexHome, + ocxHome, + configPath, + journalPath, + pidPath, + env: { + HOME: home, + USERPROFILE: home, + CODEX_HOME: codexHome, + OPENCODEX_HOME: ocxHome, + XDG_RUNTIME_DIR: runtime, + NO_PROXY: "127.0.0.1,localhost", + }, + }; +} + +function arrangeRecoverableJournal(fx: Fixture): { original: string; injected: string } { + const original = '# original\nmodel_provider = "openai"\n'; + const injected = '# injected\nmodel_provider = "opencodex"\n'; + writeFileSync(fx.configPath, injected); + writeFileSync(fx.journalPath, JSON.stringify({ + version: 1, + originalConfig: Buffer.from(original).toString("base64"), + originalProfile: null, + pid: 999_999, + timestamp: new Date().toISOString(), + })); + return { original, injected }; +} + +async function runCli(fx: Fixture, argv: string[]): Promise<{ exitCode: number; stdout: string; stderr: string }> { + const child = Bun.spawn([process.execPath, cliPath, ...argv], { + cwd: fx.root, + env: fx.env, + stdout: "pipe", + stderr: "pipe", + }); + children.push(child); + const completed = await Promise.race([ + Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]), + new Promise((_, reject) => setTimeout(() => reject(new Error(`CLI watchdog: ocx ${argv.join(" ")}`)), 10_000)), + ]); + return { exitCode: completed[0], stdout: completed[1], stderr: completed[2] }; +} + +async function waitFor(read: () => T | null | Promise, label: string): Promise { + const deadline = Date.now() + 10_000; + while (Date.now() < deadline) { + const value = await read(); + if (value !== null) return value; + await Bun.sleep(10); + } + throw new Error(`timed out waiting for ${label}`); } -describe("handleStart journal ownership ordering (#1230)", () => { - test("a healthy PID-file proxy is detected before journal reconciliation", () => { - const handleStart = handleStartSource(); - const readPid = handleStart.indexOf("const existingPid = readPid();"); - const findLive = handleStart.indexOf("const live = await findLiveProxy();", readPid); - const healthyExit = handleStart.indexOf("process.exit(1);", findLive); - const removeStalePid = handleStart.indexOf("removePid(existingPid);", healthyExit); - const reconcile = handleStart.indexOf("reconcileJournal();", removeStalePid); - const updatePrompt = handleStart.indexOf("await maybeShowUpdatePrompt();", reconcile); - - expect(readPid).toBeGreaterThanOrEqual(0); - expect(findLive).toBeGreaterThan(readPid); - expect(healthyExit).toBeGreaterThan(findLive); - expect(removeStalePid).toBeGreaterThan(healthyExit); - expect(reconcile).toBeGreaterThan(removeStalePid); - expect(updatePrompt).toBeGreaterThan(reconcile); +async function startOwner(fx: Fixture): Promise> { + const child = Bun.spawn([process.execPath, cliPath, "start"], { + cwd: fx.root, + env: fx.env, + stdout: "pipe", + stderr: "pipe", }); + children.push(child); + const runtimePath = join(fx.ocxHome, "runtime-port.json"); + const runtime = await waitFor(() => { + if (!existsSync(runtimePath)) return null; + try { + const value = JSON.parse(readFileSync(runtimePath, "utf8")) as { pid?: number; port?: number }; + return value.pid === child.pid && typeof value.port === "number" && value.port > 0 ? value : null; + } catch { + return null; + } + }, "owner runtime record"); + await waitFor(async () => { + try { + const response = await fetch(`http://127.0.0.1:${runtime.port}/healthz`, { signal: AbortSignal.timeout(500) }); + const body = await response.json() as { pid?: number }; + return response.ok && body.pid === child.pid ? true : null; + } catch { + return null; + } + }, "owner health"); + return child; +} + +afterEach(async () => { + for (const child of children) { + if (child.exitCode === null) child.kill("SIGTERM"); + } + while (children.length) { + const child = children.pop()!; + if (child.exitCode === null) await child.exited; + } + while (roots.length) rmSync(roots.pop()!, { recursive: true, force: true }); +}); + +describe("start and ensure journal ownership (#1230)", () => { + test("a healthy proxy owner preserves the journal for both start and ensure", async () => { + const fx = fixture(); + const owner = await startOwner(fx); + try { + const { injected } = arrangeRecoverableJournal(fx); + + const start = await runCli(fx, ["start"]); + expect(start.exitCode).toBe(1); + expect(start.stderr).toContain("Proxy already running"); + expect(readFileSync(fx.configPath, "utf8")).toBe(injected); + expect(existsSync(fx.journalPath)).toBe(true); + + const ensure = await runCli(fx, ["ensure"]); + expect(ensure.exitCode).toBe(0); + expect(ensure.stdout).toContain("Codex autostart is disabled"); + expect(readFileSync(fx.configPath, "utf8")).toBe(injected); + expect(existsSync(fx.journalPath)).toBe(true); + expect(readFileSync(fx.pidPath, "utf8")).toBe(String(owner.pid)); + } finally { + owner.kill("SIGTERM"); + await owner.exited; + } + }, 30_000); + + test("a dead owner is recovered and its stale PID is removed for both start and ensure", async () => { + for (const command of ["start", "ensure"] as const) { + const fx = fixture(); + const { original } = arrangeRecoverableJournal(fx); + writeFileSync(fx.pidPath, "999999"); + + if (command === "ensure") { + const result = await runCli(fx, [command]); + expect(result.exitCode).toBe(0); + } else { + const child = Bun.spawn([process.execPath, cliPath, command], { + cwd: fx.root, + env: fx.env, + stdout: "pipe", + stderr: "pipe", + }); + try { + await waitFor( + () => !existsSync(fx.journalPath) && existsSync(fx.configPath) && readFileSync(fx.configPath, "utf8") === original ? true : null, + "dead-owner journal recovery", + ); + } finally { + child.kill("SIGTERM"); + await child.exited; + } + } + + expect(readFileSync(fx.configPath, "utf8")).toBe(original); + expect(existsSync(fx.journalPath)).toBe(false); + expect(existsSync(fx.pidPath)).toBe(false); + } + }, 30_000); }); From 96141e8f040f52394f632c4794265f347a0bbe86 Mon Sep 17 00:00:00 2001 From: Ingwannu Date: Sat, 8 Aug 2026 07:06:57 +0000 Subject: [PATCH 122/124] fix(ci): preserve code while normalizing issue media --- .github/scripts/issue-quality-core.cjs | 81 ++++++++++++++++++-------- .github/scripts/issue-quality.test.cjs | 33 +++++++++++ 2 files changed, 91 insertions(+), 23 deletions(-) diff --git a/.github/scripts/issue-quality-core.cjs b/.github/scripts/issue-quality-core.cjs index 817624f46f..66adc90493 100644 --- a/.github/scripts/issue-quality-core.cjs +++ b/.github/scripts/issue-quality-core.cjs @@ -66,45 +66,78 @@ function isPlaceholderOnlyValue(raw) { */ function stripMediaTokens(text) { if (typeof text !== "string") return ""; - // Indented code lines render as literal code in GitHub Markdown. Protect - // them first so neither the HTML nor the Markdown media stripper can - // remove example syntax; restore the lines afterwards. + // Fenced and indented code render literally in GitHub Markdown. Protect + // them first so neither media stripper can remove example syntax. The + // protector deliberately leaves indented children of an unindented HTML + // media block visible: those lines are HTML children, not Markdown code. const protectedText = protectIndentedCodeLines(text); const markdownStripped = stripMarkdownImages(stripHtmlMedia(protectedText.text)); const referenceStripped = stripReferenceImages(markdownStripped); - return restoreIndentedCodeLines(referenceStripped, protectedText.lines); + return restoreIndentedCodeLines(referenceStripped, protectedText); } /** - * Replace every indented code line (4+ leading spaces or a tab) with a - * placeholder of equal length so media stripping cannot touch it. Returns the - * masked text plus the original lines for restoration. + * Replace fenced code and indented code outside HTML media blocks with opaque + * tokens. Restoration is token-based rather than line-position-based because + * stripping a multiline media block may collapse or remove lines. */ function protectIndentedCodeLines(text) { const lines = []; + let markerPrefix = "\u0000OCX_ISSUE_CODE_"; + while (text.includes(markerPrefix)) markerPrefix += "_"; + let mediaDepth = 0; + let fence = null; + + const mask = (line) => { + const index = lines.push(line) - 1; + return `${markerPrefix}${index}\u0000`; + }; + const masked = text.split("\n").map((line) => { - if (/^(?: {4,}|\t)/.test(line)) { - lines.push(line); - return "\u0000" + line.replace(/[^\n]/g, " ").slice(1); + if (fence) { + const closing = new RegExp(`^ {0,3}${fence.char}{${fence.length},}[ \\t]*$`); + if (closing.test(line)) fence = null; + return mask(line); + } + + const fenceStart = line.match(/^ {0,3}(`{3,}|~{3,})/); + if (fenceStart) { + fence = { char: fenceStart[1][0], length: fenceStart[1].length }; + return mask(line); + } + + // Four-space/tab lines inside an active unindented HTML media block are + // child markup or fallback text. Treating them as code would keep an + // otherwise media-only /