From a8f004f014a8b3084d9037391a58c2bcde160bd3 Mon Sep 17 00:00:00 2001 From: Abhishek Sharma Date: Sun, 9 Aug 2026 01:28:14 -0700 Subject: [PATCH] fix(routing): define hard cost-cap behavior when cost evidence is unknown `limits.maxEstimatedCostUsd` is documented as a hard per-request ceiling, but it never fires on the live routing path. `evaluatePolicyProfile` only excludes a candidate when the estimate is a finite number (evaluator.ts), while `routeModel` assembles cost evidence without usage (router.ts), so `estimatedUsd` is always `undefined` live and any candidate silently passes a cap the operator configured as hard. The existing coverage passed only because it supplied `usage` directly, exercising a path production does not take. Fail-closed unconditionally is not safe either: with usage unwired, it would reject every live candidate whenever a cap is set, and it would change the documented dry-run contract. This adds an explicit, opt-in policy instead: limits.onUnknownCost: "allow" | "exclude" (default "allow") - "allow" preserves today's behavior and the documented contract exactly. - "exclude" makes the ceiling genuinely hard: a candidate whose cost cannot be proven under the cap is ineligible. Unknown-cost exclusions emit a distinct `cost-limit-unknown` code so a trace distinguishes "known above the cap" from "cost is unknown", which is the operator-facing distinction #1181 asks for. Kept separate from `unknownEvidence.cost`, which governs how an unknown-cost candidate is *scored* rather than whether the *ceiling* applies. The two mechanisms now emit distinct codes and are covered by a test asserting they stay distinguishable. Fixes #1181 Co-Authored-By: Claude Opus 5 (1M context) --- src/routing/evaluator.ts | 19 ++- src/routing/profile.ts | 11 +- src/types.ts | 14 ++ tests/cost-cap-unknown-evidence.test.ts | 216 ++++++++++++++++++++++++ 4 files changed, 255 insertions(+), 5 deletions(-) create mode 100644 tests/cost-cap-unknown-evidence.test.ts diff --git a/src/routing/evaluator.ts b/src/routing/evaluator.ts index c1b3796d9..19465a1ea 100644 --- a/src/routing/evaluator.ts +++ b/src/routing/evaluator.ts @@ -304,14 +304,25 @@ export function evaluatePolicyProfile( const excludedByUnknown = unknown && profile.unknownEvidence.capability === "exclude"; const costLimit = profile.limits.maxEstimatedCostUsd; const estimatedCost = evidence.cost?.estimatedUsd; + const costEstimateKnown = typeof estimatedCost === "number" && Number.isFinite(estimatedCost); const overCostLimit = costLimit !== undefined - && typeof estimatedCost === "number" - && Number.isFinite(estimatedCost) - && estimatedCost > costLimit; + && costEstimateKnown + && estimatedCost! > costLimit; if (overCostLimit) { exclusions.push({ code: "cost-limit", detail: "maxEstimatedCostUsd" }); } - let eligible = !unsatisfied && !excludedByUnknown && !overCostLimit; + // A cap can only be *proven* satisfied when the estimate is known. The live + // routing path often has no usage evidence yet, so the default stays + // "allow" to preserve the documented dry-run contract; operators who need a + // genuine hard ceiling opt into "exclude". The distinct exclusion code lets + // a trace distinguish "known above the cap" from "cost is unknown". + const unknownCostBlocked = costLimit !== undefined + && !costEstimateKnown + && profile.limits.onUnknownCost === "exclude"; + if (unknownCostBlocked) { + exclusions.push({ code: "cost-limit-unknown", detail: "maxEstimatedCostUsd" }); + } + let eligible = !unsatisfied && !excludedByUnknown && !overCostLimit && !unknownCostBlocked; // Health scoring (RI-06): live hard cooldown is authoritative and // excludes; unknown health follows the profile's unknownEvidence policy; diff --git a/src/routing/profile.ts b/src/routing/profile.ts index b35705954..80ca01ef9 100644 --- a/src/routing/profile.ts +++ b/src/routing/profile.ts @@ -9,6 +9,7 @@ import type { OcxConfig, OcxRoutingProfileConfig, OcxRoutingUnknownEvidenceMode, + OcxRoutingUnknownCostCapMode, } from "../types"; import { codexAccountNamespaceEntries } from "../codex/account-namespaces"; import { listComboIds, resolveComboId } from "../combos"; @@ -60,7 +61,7 @@ export interface NormalizedRoutingProfile { candidates: Array<{ provider: string; model: string }>; require: NormalizedRoutingProfileRequirements; optimize: { latency: number; health: number; cost: number; quota: number }; - limits: { maxEstimatedCostUsd?: number }; + limits: { maxEstimatedCostUsd?: number; onUnknownCost?: OcxRoutingUnknownCostCapMode }; unknownEvidence: Record<"capability" | "health" | "quota" | "cost", OcxRoutingUnknownEvidenceMode>; revision: string; } @@ -317,6 +318,11 @@ export function routingProfileIssues( || limits.maxEstimatedCostUsd < 0)) { issues.push({ path: ["limits", "maxEstimatedCostUsd"], message: "maxEstimatedCostUsd must be a non-negative number" }); } + if (limits.onUnknownCost !== undefined + && limits.onUnknownCost !== "allow" + && limits.onUnknownCost !== "exclude") { + issues.push({ path: ["limits", "onUnknownCost"], message: 'onUnknownCost must be "allow" or "exclude"' }); + } } } @@ -402,6 +408,9 @@ export function normalizeRoutingProfile(id: string, raw: OcxRoutingProfileConfig ...(raw.limits?.maxEstimatedCostUsd !== undefined ? { maxEstimatedCostUsd: raw.limits.maxEstimatedCostUsd } : {}), + ...(raw.limits?.onUnknownCost !== undefined + ? { onUnknownCost: raw.limits.onUnknownCost } + : {}), }, unknownEvidence: normalizedUnknownEvidence(raw), }; diff --git a/src/types.ts b/src/types.ts index b2b0c6d9e..32a178fcd 100644 --- a/src/types.ts +++ b/src/types.ts @@ -955,9 +955,23 @@ export interface OcxRoutingProfileOptimize { quota?: number; } +/** + * Policy for the hard cost ceiling when a candidate has no finite cost + * estimate. `"allow"` (default) preserves the documented dry-run contract: + * the cap only excludes evidence known to exceed it. `"exclude"` makes the + * ceiling fail-closed, so a candidate that cannot be proven under the cap is + * ineligible. + */ +export type OcxRoutingUnknownCostCapMode = "allow" | "exclude"; + export interface OcxRoutingProfileLimits { /** Hard per-request estimated-cost ceiling in USD. */ maxEstimatedCostUsd?: number; + /** + * How `maxEstimatedCostUsd` behaves when the estimate is unknown. + * Defaults to `"allow"`; opt in to `"exclude"` for a true hard ceiling. + */ + onUnknownCost?: OcxRoutingUnknownCostCapMode; } export interface OcxRoutingProfileUnknownEvidence { diff --git a/tests/cost-cap-unknown-evidence.test.ts b/tests/cost-cap-unknown-evidence.test.ts new file mode 100644 index 000000000..e2e0e8c40 --- /dev/null +++ b/tests/cost-cap-unknown-evidence.test.ts @@ -0,0 +1,216 @@ +/** + * Reproduction for issue #1181 — "Routing: define hard cost-cap behavior when + * runtime cost evidence is unknown". + * + * The hard ceiling `limits.maxEstimatedCostUsd` is documented as a hard + * per-request cap. In the live routing path it never fires, because + * `router.ts` assembles cost evidence WITHOUT usage: + * + * costEvidenceForCandidate({ provider, model, limitUsd }) // no `usage` + * + * `costEvidenceForCandidate` then returns `{ limitUsd, incomplete: true }` + * with no `estimatedUsd`, and the evaluator's cap check + * (evaluator.ts:307-310) requires `typeof estimatedCost === "number"`, so an + * unknown estimate silently passes a cap the operator configured as hard. + * + * The existing test in cost-scoring.test.ts only exercises the cap with + * `usage: USAGE` supplied — i.e. on a code path production never takes. + * + * These tests pin both sides of `limits.onUnknownCost`: + * - the default `"allow"`, which preserves the documented dry-run contract + * and lets an unprovable candidate through, and + * - the opt-in `"exclude"`, which makes the ceiling genuinely hard and + * reports the distinct `cost-limit-unknown` exclusion. + * + * The first case was originally written as a failing reproduction before the + * evaluator change landed; it is retained to keep the fail-open default + * asserted rather than assumed. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { costEvidenceForCandidate } from "../src/routing/cost"; +import { evaluatePolicyProfile } from "../src/routing/evaluator"; +import type { OcxConfig } from "../src/types"; + +let testDir = ""; +let previousHome: string | undefined; + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + testDir = mkdtempSync(join(tmpdir(), "ocx-cost-cap-")); + process.env.OPENCODEX_HOME = testDir; +}); + +afterEach(() => { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (testDir) rmSync(testDir, { recursive: true, force: true }); +}); + +/** Mirrors the live routing path: a cap is configured, usage is NOT available. */ +function configWithCap(capUsd: number, overrides: Record = {}): OcxConfig { + return { + port: 10100, + defaultProvider: "anthropic", + providers: { + anthropic: { + adapter: "anthropic", + baseUrl: "https://api.anthropic.com/v1", + apiKey: "kan", + models: ["claude-opus-5"], + }, + }, + routingProfiles: { + cost: { + candidates: [{ provider: "anthropic", model: "claude-opus-5" }], + optimize: { cost: 0.8 }, + limits: { maxEstimatedCostUsd: capUsd }, + unknownEvidence: { capability: "allow", health: "allow", quota: "allow", cost: "allow" }, + ...overrides, + }, + }, + } as OcxConfig; +} + +describe("issue #1181 — hard cost cap under unknown evidence", () => { + test("default allow: live-path evidence carries no estimate, so the cap does not fire", async () => { + // Exactly how router.ts:515-519 builds it — no `usage` argument. + const evidence = costEvidenceForCandidate({ + provider: "anthropic", + model: "claude-opus-5", + limitUsd: 0.000001, // an absurdly low cap; nothing should realistically pass + }); + + // The evidence is explicitly unknown, and correctly so. + expect(evidence.estimatedUsd).toBeUndefined(); + expect(evidence.incomplete).toBe(true); + expect(evidence.limitUsd).toBe(0.000001); + + const result = evaluatePolicyProfile(configWithCap(0.000001), "cost", {}, [ + { + provider: "anthropic", + model: "claude-opus-5", + capability: { contextWindow: 200000 }, + cost: evidence, + }, + ]); + + // Current behaviour: the candidate is eligible and selected despite a cap + // of $0.000001. No `cost-limit` exclusion is recorded, and nothing in the + // trace distinguishes "known below cap" from "cost unknown". + expect(result.candidates[0]!.eligible).toBe(true); + expect( + result.candidates[0]!.exclusions.some(e => e.code === "cost-limit"), + ).toBe(false); + expect(result.selectedIndex).toBe(0); + }); + + test("opt-in exclude: fail-closed cap excludes unknown-cost candidates", async () => { + const evidence = costEvidenceForCandidate({ + provider: "anthropic", + model: "claude-opus-5", + limitUsd: 0.000001, + }); + + // Proposed opt-in policy: limits.onUnknownCost = "exclude". + const result = evaluatePolicyProfile( + configWithCap(0.000001, { limits: { maxEstimatedCostUsd: 0.000001, onUnknownCost: "exclude" } }), + "cost", + {}, + [ + { + provider: "anthropic", + model: "claude-opus-5", + capability: { contextWindow: 200000 }, + cost: evidence, + }, + ], + ); + + expect(result.candidates[0]!.eligible).toBe(false); + // Distinct code so operators can tell "over a known cap" from "cost unknown". + expect( + result.candidates[0]!.exclusions.some(e => e.code === "cost-limit-unknown"), + ).toBe(true); + expect(result.selectedIndex).toBeNull(); + }); + + test("cap policy and unknownEvidence.cost are distinct mechanisms", async () => { + // unknownEvidence.cost governs SCORING of an unknown-cost candidate; + // limits.onUnknownCost governs whether the hard CEILING applies to it. + // They must produce distinct exclusion codes so a trace stays diagnosable. + const evidence = costEvidenceForCandidate({ + provider: "anthropic", + model: "claude-opus-5", + limitUsd: 0.000001, + }); + + const scoringExcluded = evaluatePolicyProfile( + configWithCap(0.000001, { + unknownEvidence: { capability: "allow", health: "allow", quota: "allow", cost: "exclude" }, + }), + "cost", + {}, + [{ provider: "anthropic", model: "claude-opus-5", capability: { contextWindow: 200000 }, cost: evidence }], + ); + + const codes = scoringExcluded.candidates[0]!.exclusions.map(e => e.code); + expect(codes).toContain("unknown-price"); + expect(codes).not.toContain("cost-limit-unknown"); + expect(scoringExcluded.candidates[0]!.eligible).toBe(false); + }); + + test("no cap configured — onUnknownCost is inert", async () => { + const evidence = costEvidenceForCandidate({ provider: "anthropic", model: "claude-opus-5" }); + const noCap = { + port: 10100, + defaultProvider: "anthropic", + providers: { + anthropic: { adapter: "anthropic", baseUrl: "https://api.anthropic.com/v1", apiKey: "kan", models: ["claude-opus-5"] }, + }, + routingProfiles: { + cost: { + candidates: [{ provider: "anthropic", model: "claude-opus-5" }], + optimize: { cost: 0.8 }, + limits: { onUnknownCost: "exclude" }, // no maxEstimatedCostUsd + unknownEvidence: { capability: "allow", health: "allow", quota: "allow", cost: "allow" }, + }, + }, + } as unknown as OcxConfig; + + const result = evaluatePolicyProfile(noCap, "cost", {}, [ + { provider: "anthropic", model: "claude-opus-5", capability: { contextWindow: 200000 }, cost: evidence }, + ]); + + // Without a ceiling there is nothing to fail closed against. + expect(result.candidates[0]!.eligible).toBe(true); + expect( + result.candidates[0]!.exclusions.some(e => e.code === "cost-limit-unknown"), + ).toBe(false); + }); + + test("default stays allow — the documented contract is unchanged", async () => { + const evidence = costEvidenceForCandidate({ + provider: "anthropic", + model: "claude-opus-5", + limitUsd: 0.000001, + }); + + // No onUnknownCost configured → must behave exactly as today (fail-open), + // so existing deployments do not lose every live route on upgrade. + const result = evaluatePolicyProfile(configWithCap(0.000001), "cost", {}, [ + { + provider: "anthropic", + model: "claude-opus-5", + capability: { contextWindow: 200000 }, + cost: evidence, + }, + ]); + + expect(result.candidates[0]!.eligible).toBe(true); + expect(result.selectedIndex).toBe(0); + }); +});