From 512701dfedef52c98b565ae86adebea4cda5836a Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 05:21:55 +0900 Subject: [PATCH 01/20] fix(codex): isolate provider host transport health --- .../ja/reference/configuration/providers.md | 2 +- .../ko/reference/configuration/providers.md | 2 +- .../docs/reference/configuration/providers.md | 2 +- .../ru/reference/configuration/providers.md | 2 +- .../reference/configuration/providers.md | 2 +- src/codex/upstream-host-health.ts | 190 +++++++ src/lib/upstream-retry.ts | 45 +- src/server/responses/compact.ts | 107 +++- src/server/responses/core.ts | 98 +++- tests/codex-host-health-runtime.test.ts | 538 ++++++++++++++++++ tests/codex-upstream-host-health.test.ts | 120 ++++ tests/issue-452-empty-503.test.ts | 143 ++++- tests/responses-compaction-routing.test.ts | 254 +++++++++ tests/upstream-transient-retry.test.ts | 43 +- 14 files changed, 1502 insertions(+), 46 deletions(-) create mode 100644 src/codex/upstream-host-health.ts create mode 100644 tests/codex-host-health-runtime.test.ts create mode 100644 tests/codex-upstream-host-health.test.ts diff --git a/docs-site/src/content/docs/ja/reference/configuration/providers.md b/docs-site/src/content/docs/ja/reference/configuration/providers.md index 7c3ab0288a..8c1172e4fd 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ja/reference/configuration/providers.md @@ -21,7 +21,7 @@ description: プロバイダー エントリ、認証、エンドポイント、 | `autoSwitchThreshold?` | `number` | `80` | 使用量ベースのプロアクティブ切り替えしきい値。`quota` は紐付け済み/未紐付けタスクの次のリクエストを再評価でき、`fill-first` は未紐付け割り当ての使い切り基準としてのみ使用し、通常の `round-robin` 選択は使用しません。既知の 5 時間、週次、30 日 quota window の最大スコアを使います。`0` は使用量ベースの切り替えだけを無効にし、未紐付け割り当てや障害回復は無効にしません。 | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 新規/未紐付け Codex リクエストの割り当て戦略。live な `(parent thread id, quota scope)` affinity がなければ未紐付けで、プロキシ再起動や affinity リセット後は既存の表示タスクも未紐付けになり得ます。`quota` はアクティブアカウントがなければ既知 usage 最小の適格アカウントを選び、適格なアクティブアカウントが `autoSwitchThreshold` 未満なら維持します。しきい値到達後は、未紐付けリクエストまたは紐付け済みタスクの次のリクエストを usage の低い適格アカウントへ移せます。`round-robin` は未紐付けリクエストを均等分散し、`fill-first` は cooldown、使用不可、または drain threshold までアクティブアカウントへ割り当てます。 | | `accountPoolStickyLimit?` | `number` | `1` | 1 回の round-robin 選択で次へ進む前に保持する新規/未紐付けタスク割り当て数。カウンターは上流の成功後ではなくタスクの紐付け時に増えます。範囲 1–100。`accountPoolStrategy` が `round-robin` のときのみ。 | -| `upstreamFailoverThreshold?` | `number` | `3` |今後の新しいセッションがフェイルオーバーする前に一時的なエラーが連続して発生する。 `0` を無効に設定します。 | +| `upstreamFailoverThreshold?` | `number` | `3` | 将来の新規セッションでアカウントの failover を行えるようになるまでに必要な、アカウント単位の一時的な HTTP または意味上の失敗結果の連続回数です。`0` はアカウント failover だけを無効にします。HTTP 応答を受け取らない rejection、接続/ヘッダーの `TimeoutError`、および保守的に判定される read-then-close は、アカウントの health/affinity を変更せず、`(provider, canonical HTTP(S) origin)` をキーとするプロセスローカルな host health だけを更新します。5 分以内に論理リクエスト単位の終端 host failure が 3 回発生すると host circuit が 30 秒間開き、その後は正確に 1 件の half-open 論理リクエストだけが許可され、同時リクエストは引き続き拒否されます。実際の HTTP 応答を受け取ると以前の host state は消去されます。`503` の後に rejection が起きた場合、`503` はアカウント evidence として保持され、後の failure は host failure として記録されます。プールされた通常の Responses と native compact リクエストの Codex bearer redirect は手動処理され、追従せず、`Location` を公開せず、サイズを制限したアカウント単位の `502` に変換されます。資格情報が見える read-then-close failure では peer がリクエストを消費済みの可能性があるため別の資格情報では意図的に再送せず、正常な代替アカウントが一時的にブロックされることがあります。`200` 後の body/stream 処理は変更されず、この設定の範囲外です。 | | `modelCacheTtlMs?` | `number` | `300000` |プロバイダーごとの `/models` キャッシュの鮮度ウィンドウ。 | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic プロンプト キャッシュ ポリシー: 無効、5 分間の一時的、または 1 時間の延長。 | | `tokenGuardian?` | `OcxTokenGuardianConfig` |オフ |オプションのプロアクティブな OAuth 更新および Codex アカウントのウォームアップ ポリシー。 | diff --git a/docs-site/src/content/docs/ko/reference/configuration/providers.md b/docs-site/src/content/docs/ko/reference/configuration/providers.md index a389a79fca..d9cad66806 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ko/reference/configuration/providers.md @@ -21,7 +21,7 @@ description: 공급자 항목, 인증, 엔드포인트, 모델 카탈로그, 할 | `autoSwitchThreshold?` | `number` | `80` | 사용량 기반 선제 전환 임계값입니다. `quota`는 바인딩된 작업과 바인딩 없는 작업의 다음 요청을 모두 재평가할 수 있고, `fill-first`는 바인딩 없는 작업 배정의 소진 기준으로만 사용하며, 기본 `round-robin` 선택은 이 값을 사용하지 않습니다. 알려진 5시간, 주간, 30일 quota window 중 가장 높은 점수를 씁니다. `0`은 사용량 기반 전환만 끄며 바인딩 없는 작업 배정이나 실패 복구는 끄지 않습니다. | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 새 작업/바인딩 없는 Codex 요청의 계정 배정 전략입니다. `(parent thread id, quota scope)`의 live affinity가 없으면 바인딩 없는 요청이며, 프록시 재시작이나 affinity 초기화 뒤에는 기존에 보이던 작업도 바인딩이 없어질 수 있습니다. `quota`는 활성 계정이 없을 때 알려진 usage가 가장 낮은 적격 계정을 선택하고, 적격 활성 계정이 `autoSwitchThreshold` 미만이면 유지합니다. 임계값 도달 뒤에는 바인딩 없는 요청이나 바인딩된 작업의 다음 요청을 usage가 더 낮은 적격 계정으로 옮길 수 있습니다. `round-robin`은 바인딩 없는 요청을 균등 분배하고, `fill-first`는 cooldown, 사용 불가 또는 drain threshold까지 활성 계정에 배정합니다. | | `accountPoolStickyLimit?` | `number` | `1` | 한 round-robin 선택이 다음으로 넘어가기 전에 유지하는 새 작업/바인딩 없는 작업 배정 수입니다. 카운터는 업스트림 성공 뒤가 아니라 작업을 바인딩할 때 증가합니다. 범위 1–100이며 `accountPoolStrategy`가 `round-robin`일 때만 적용됩니다. | -| `upstreamFailoverThreshold?` | `number` | `3` | 연속된 일시적 실패가 이 횟수에 도달하면 이후 새 세션은 failover됩니다. `0`으로 두면 비활성화됩니다. | +| `upstreamFailoverThreshold?` | `number` | `3` | 이후 새 세션이 계정 failover될 수 있기 전에 필요한 연속 계정 범위의 일시적 HTTP 또는 의미상 실패 결과 수입니다. `0`은 계정 failover만 비활성화합니다. HTTP 응답이 없는 거부, 연결/헤더 `TimeoutError`, 보수적으로 분류되는 read-then-close 실패는 계정 상태나 affinity를 바꾸지 않고 `(provider, canonical HTTP(S) origin)` 키의 프로세스 로컬 host health만 갱신합니다. 5분 안에 논리 요청 단위의 최종 host failure가 3회 발생하면 host circuit이 30초 동안 열리고, 이후 정확히 하나의 half-open 논리 요청만 허용되며 동시 요청은 계속 차단됩니다. 실제 HTTP 응답은 이전 host state를 지웁니다. `503` 후 거부가 발생하면 `503`은 계정 근거로 보존되고 뒤의 실패는 host failure로 기록됩니다. 풀의 일반 Responses 및 native compact 요청에 대한 Codex bearer redirect는 수동으로 처리되어 따라가지 않고 `Location`을 노출하지 않으며, 크기가 제한된 계정 범위 `502`로 변환됩니다. 자격 증명이 보이는 read-then-close 실패는 peer가 요청을 이미 소비했을 수 있어 다른 자격 증명으로 의도적으로 재전송하지 않으므로, 정상인 대체 계정을 일시적으로 차단할 수 있습니다. `200` 이후 본문/스트림 처리는 변경되지 않으며 이 설정의 범위 밖입니다. | | `modelCacheTtlMs?` | `number` | `300000` | 공급자별 `/models` 캐시의 최신성 창입니다. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic 프롬프트 캐시 정책입니다. 비활성, 5분짜리 임시, 1시간짜리 확장 중 하나입니다. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | 꺼짐 | 선택적 선제 OAuth 갱신과 Codex 계정 워밍업 정책입니다. | diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index a9603702bf..2bc4a18df8 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -22,7 +22,7 @@ authenticated. | `autoSwitchThreshold?` | `number` | `80` | Usage threshold for proactive switching. `quota` can re-evaluate both bound and unbound tasks on their next request; `fill-first` uses it only as the drain point for unbound assignment; normal `round-robin` selection does not use it. The score uses the hottest known 5h, weekly, or 30d quota window. `0` disables usage-based proactive switching only, not unbound assignment or failure recovery. | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | Assignment strategy for new/unbound Codex requests. A request is unbound when it has no live (parent thread id, quota scope) affinity; a visible existing task can become unbound after proxy restart or affinity reset. `quota` picks the lowest-usage eligible account when no active account exists, keeps an eligible active account below `autoSwitchThreshold`, and after the threshold may move an unbound request or proactively rebind a bound task to a lower-usage eligible account. `round-robin` distributes unbound requests evenly; `fill-first` keeps assigning unbound requests to the active account until cooldown, unavailability, or the configured drain threshold. | | `accountPoolStickyLimit?` | `number` | `1` | New/unbound task assignments retained on one round-robin selection before advancing; the counter advances when a task is bound, not after an upstream success. Range 1–100. | -| `upstreamFailoverThreshold?` | `number` | `3` | Consecutive transient failures before future new sessions fail over. Set `0` to disable. | +| `upstreamFailoverThreshold?` | `number` | `3` | Consecutive account-scoped transient HTTP or semantic failure outcomes required before future new sessions may fail over; `0` disables only account failover. Rejections with no HTTP response, connect/header `TimeoutError`s, and conservatively classified read-then-close failures instead update process-local host health keyed by `(provider, canonical HTTP(S) origin)` without changing account health or affinity. Three terminal logical host failures within five minutes open the host circuit for 30 seconds; then exactly one half-open logical request is admitted while concurrent requests remain blocked. Any actual HTTP response clears prior host state. A `503` followed by a rejection retains the `503` as account evidence and records the later host failure. Codex bearer redirects for pooled regular Responses and native compact requests use manual redirect handling: they are not followed, `Location` is not exposed, and they become a bounded account-scoped `502`. A conservative credential-visible read-then-close failure can temporarily block an otherwise healthy alternate because a request the peer may have consumed is intentionally not replayed under another credential. Post-`200` body/stream handling is unchanged and outside this setting. | | `modelCacheTtlMs?` | `number` | `300000` | Freshness window for the per-provider `/models` cache. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic prompt-cache policy: disabled, 5-minute ephemeral, or 1-hour extended. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | off | Optional proactive OAuth refresh and Codex-account warmup policy. | diff --git a/docs-site/src/content/docs/ru/reference/configuration/providers.md b/docs-site/src/content/docs/ru/reference/configuration/providers.md index d2bb183b17..24f7fd4ac5 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ru/reference/configuration/providers.md @@ -22,7 +22,7 @@ description: Записи провайдеров, аутентификация, | `autoSwitchThreshold?` | `number` | `80` | Порог проактивного переключения по использованию. `quota` может повторно оценить следующий запрос как привязанной, так и непривязанной задачи; `fill-first` использует его только как точку исчерпания для непривязанных назначений; обычный `round-robin` его не использует. Оценка берёт самое горячее из окон 5 часов, недели и 30 дней. `0` отключает только переключение по использованию, но не назначение непривязанных задач и не восстановление после сбоев. | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | Стратегия назначения для новых/непривязанных запросов Codex. Запрос непривязан, если у него нет live affinity `(parent thread id, quota scope)`; видимая существующая задача может стать непривязанной после перезапуска прокси или сброса affinity. `quota` выбирает подходящий аккаунт с наименьшим известным usage, когда активного аккаунта нет, сохраняет подходящий активный аккаунт ниже `autoSwitchThreshold`, а после порога может перевести непривязанный запрос или следующий запрос привязанной задачи на подходящий аккаунт с меньшим usage. `round-robin` равномерно распределяет непривязанные запросы; `fill-first` назначает их активному аккаунту до cooldown, недоступности или порога исчерпания. | | `accountPoolStickyLimit?` | `number` | `1` | Число назначений новых/непривязанных задач на одном выборе round-robin перед переходом дальше. Счётчик растёт при привязке задачи, а не после успеха upstream. Диапазон 1–100; только при `accountPoolStrategy` = `round-robin`. | -| `upstreamFailoverThreshold?` | `number` | `3` | Сколько подряд transient failure допустить, прежде чем новые сессии начнут делать failover. `0` отключает эту логику. | +| `upstreamFailoverThreshold?` | `number` | `3` | Количество последовательных временных HTTP- или семантических результатов сбоя, относящихся к аккаунту, после которого новые сессии могут выполнить failover; `0` отключает только failover аккаунта. Отклонения без HTTP-ответа, `TimeoutError` при подключении/ожидании заголовков и консервативно классифицированные сбои read-then-close обновляют только локальное для процесса состояние хоста с ключом `(provider, canonical HTTP(S) origin)`, не меняя состояние аккаунта и affinity. Три конечных сбоя хоста на уровне логического запроса за пять минут открывают circuit хоста на 30 секунд; затем допускается ровно один half-open логический запрос, а параллельные запросы остаются заблокированными. Любой фактический HTTP-ответ очищает предыдущее состояние хоста. При `503` с последующим отклонением `503` сохраняется как свидетельство для аккаунта, а последующее событие записывается как сбой хоста. Перенаправления bearer-запросов Codex для обычных Responses из пула и native compact обрабатываются вручную: переход не выполняется, `Location` не раскрывается, а результат преобразуется в ограниченный по размеру ответ `502`, учитываемый для аккаунта. При read-then-close с видимыми учетными данными исправный альтернативный аккаунт может временно блокироваться, поскольку запрос, уже потенциально принятый peer, намеренно не повторяется с другими учетными данными. Обработка тела/потока после `200` не изменяется и находится вне области этой настройки. | | `modelCacheTtlMs?` | `number` | `300000` | Окно свежести для кэша `/models` на уровне провайдера. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Политика prompt-cache Anthropic: отключено, 5-минутный ephemeral или 1-часовой extended. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | off | Необязательная политика proactive OAuth refresh и warmup'а аккаунтов Codex. | diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md index b400ffbd43..d4e1e57744 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md @@ -21,7 +21,7 @@ description: 提供者条目、身份验证、端点、模型目录、配额、 | `autoSwitchThreshold?` | `number` | `80` | 基于用量的主动切换阈值。`quota` 可在下一次请求中重新评估已绑定和未绑定任务；`fill-first` 仅把它用作未绑定分配的耗尽点；正常 `round-robin` 不使用它。分数取已知 5 小时、周或 30 天 quota window 的最高值。`0` 只关闭基于用量的主动切换，不关闭未绑定任务分配或故障恢复。 | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 新建/未绑定 Codex 请求的分配策略。没有 live `(parent thread id, quota scope)` affinity 的请求属于未绑定；代理重启或 affinity 重置后，已有可见任务也可能未绑定。`quota` 在没有活跃账号时选择已知 usage 最低的合格账号；活跃账号合格且低于 `autoSwitchThreshold` 时继续使用；达到阈值后，可把未绑定请求或已绑定任务的下一次请求切换到 usage 更低的合格账号。`round-robin` 均匀分配未绑定请求；`fill-first` 在 cooldown、不可用或耗尽阈值前持续分配给活跃账号。 | | `accountPoolStickyLimit?` | `number` | `1` | 一次 round-robin 选择在推进前保留的新建/未绑定任务分配数。计数在任务绑定时增加，而不是在上游成功后增加。范围 1–100；仅当 `accountPoolStrategy` 为 `round-robin` 时生效。 | -| `upstreamFailoverThreshold?` | `number` | `3` | 连续发生多少次瞬态故障后，后续新会话会切换到备用上游。设为 `0` 可禁用。 | +| `upstreamFailoverThreshold?` | `number` | `3` | 新会话允许执行账户故障转移前所需的连续账户级瞬态 HTTP 或语义失败结果数；`0` 仅禁用账户故障转移。未收到 HTTP 响应的拒绝、连接/响应头 `TimeoutError`，以及保守判定的 read-then-close 故障，只更新以 `(provider, canonical HTTP(S) origin)` 为键的进程内主机健康状态，不改变账户健康状态或 affinity。五分钟内出现三次逻辑请求级最终主机故障时，主机熔断器打开 30 秒；之后只允许一个 half-open 逻辑请求，并继续阻止并发请求。任何实际 HTTP 响应都会清除先前主机状态。若先收到 `503` 后发生拒绝，`503` 保留为账户证据，后续事件记录为主机故障。池化普通 Responses 和 native compact 请求的 Codex bearer 重定向采用手动处理：不跟随、不暴露 `Location`，并转换为大小受限的账户级 `502`。对于凭据可见的 read-then-close 故障，peer 可能已经消费请求，因此不会使用其他凭据重放，这可能会暂时阻止原本健康的备用账户。`200` 之后的响应体/流处理保持不变，不属于此设置的范围。 | | `modelCacheTtlMs?` | `number` | `300000` | 每个提供者 `/models` 缓存的新鲜度窗口。 | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic 提示缓存策略：禁用、5 分钟临时缓存，或 1 小时扩展缓存。 | | `tokenGuardian?` | `OcxTokenGuardianConfig` | 关闭 | 可选的主动 OAuth 刷新与 Codex 账户预热策略。 | diff --git a/src/codex/upstream-host-health.ts b/src/codex/upstream-host-health.ts new file mode 100644 index 0000000000..83c09f7d34 --- /dev/null +++ b/src/codex/upstream-host-health.ts @@ -0,0 +1,190 @@ +export type CodexUpstreamHostKey = string & { readonly __codexUpstreamHostKey: unique symbol }; + +export interface CodexUpstreamHostHealthSnapshot { + consecutiveFailures: number; + lastFailureAt: number; + cooldownUntil?: number; +} + +export type CodexUpstreamHostProbeLease = Readonly<{ + key: CodexUpstreamHostKey; + leaseId: symbol; +}>; + +export type CodexUpstreamHostAdmission = + | { kind: "admitted"; probeLease: CodexUpstreamHostProbeLease | null } + | { kind: "blocked"; retryAfterSeconds: number }; + +type CodexUpstreamHostHealth = CodexUpstreamHostHealthSnapshot & { + lastTouchedAt: number; + halfOpenLeaseId?: symbol; +}; + +export const CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD = 3; +export const CODEX_UPSTREAM_HOST_FAILURE_WINDOW_MS = 5 * 60_000; +export const CODEX_UPSTREAM_HOST_COOLDOWN_MS = 30_000; +export const CODEX_UPSTREAM_HOST_MAX_ENTRIES = 128; + +const upstreamHostHealth = new Map(); + +function normalizedAuthority(url: URL): string | null { + if (url.protocol !== "http:" && url.protocol !== "https:") return null; + let hostname = url.hostname.trim().toLowerCase().replace(/\.+$/, ""); + if (hostname.startsWith("[") && hostname.endsWith("]")) hostname = hostname.slice(1, -1); + if (!hostname) return null; + const host = hostname.includes(":") ? `[${hostname}]` : hostname; + const port = url.port || (url.protocol === "https:" ? "443" : "80"); + return `${url.protocol}//${host}:${port}`; +} + +export function canonicalCodexUpstreamHostKey( + providerName: string, + url: string, +): CodexUpstreamHostKey | null { + const provider = providerName.trim().toLowerCase(); + if (!provider) return null; + try { + const authority = normalizedAuthority(new URL(url)); + return authority ? `${provider}\u0000${authority}` as CodexUpstreamHostKey : null; + } catch { + return null; + } +} + +function snapshot(health: CodexUpstreamHostHealth): CodexUpstreamHostHealthSnapshot { + return { + consecutiveFailures: health.consecutiveFailures, + lastFailureAt: health.lastFailureAt, + ...(health.cooldownUntil !== undefined ? { cooldownUntil: health.cooldownUntil } : {}), + }; +} + +function removeExpiredEntries(now: number): void { + for (const [key, health] of upstreamHostHealth) { + // A tripped circuit survives its cooldown so the next logical request must + // pass through the atomic half-open admission below. The bounded map still + // evicts abandoned entries when capacity is needed. + if (health.cooldownUntil === undefined + && now - health.lastFailureAt > CODEX_UPSTREAM_HOST_FAILURE_WINDOW_MS) { + upstreamHostHealth.delete(key); + } + } +} + +function makeRoom(now: number): void { + removeExpiredEntries(now); + while (upstreamHostHealth.size >= CODEX_UPSTREAM_HOST_MAX_ENTRIES) { + let oldestKey: CodexUpstreamHostKey | undefined; + let oldestAt = Number.POSITIVE_INFINITY; + for (const [key, health] of upstreamHostHealth) { + if (health.lastTouchedAt < oldestAt) { + oldestKey = key; + oldestAt = health.lastTouchedAt; + } + } + if (!oldestKey) break; + upstreamHostHealth.delete(oldestKey); + } +} + +export function getCodexUpstreamHostHealth( + key: CodexUpstreamHostKey, + now = Date.now(), +): CodexUpstreamHostHealthSnapshot | null { + const health = upstreamHostHealth.get(key); + if (!health) return null; + if (health.cooldownUntil === undefined + && now - health.lastFailureAt > CODEX_UPSTREAM_HOST_FAILURE_WINDOW_MS) { + upstreamHostHealth.delete(key); + return null; + } + return snapshot(health); +} + +export function getCodexUpstreamHostCooldownUntil( + key: CodexUpstreamHostKey, + now = Date.now(), +): number | null { + const health = upstreamHostHealth.get(key); + if (!health?.cooldownUntil) return null; + if (health.cooldownUntil <= now) return null; + return health.cooldownUntil; +} + +export function acquireCodexUpstreamHostAdmission( + key: CodexUpstreamHostKey, + now = Date.now(), +): CodexUpstreamHostAdmission { + const health = upstreamHostHealth.get(key); + if (!health) return { kind: "admitted", probeLease: null }; + if (health.cooldownUntil === undefined) { + if (now - health.lastFailureAt > CODEX_UPSTREAM_HOST_FAILURE_WINDOW_MS) { + upstreamHostHealth.delete(key); + } + return { kind: "admitted", probeLease: null }; + } + if (health.cooldownUntil > now) { + return { + kind: "blocked", + retryAfterSeconds: Math.max(1, Math.ceil((health.cooldownUntil - now) / 1_000)), + }; + } + if (health.halfOpenLeaseId !== undefined) { + return { kind: "blocked", retryAfterSeconds: 1 }; + } + + const leaseId = Symbol("codex-upstream-host-probe"); + health.halfOpenLeaseId = leaseId; + health.lastTouchedAt = now; + return { kind: "admitted", probeLease: { key, leaseId } }; +} + +/** Release a half-open probe without recording host or account evidence. */ +export function releaseCodexUpstreamHostProbeLease( + lease: CodexUpstreamHostProbeLease | null | undefined, +): boolean { + if (!lease) return false; + const health = upstreamHostHealth.get(lease.key); + if (!health || health.halfOpenLeaseId !== lease.leaseId) return false; + delete health.halfOpenLeaseId; + return true; +} + +export function recordCodexUpstreamHostFailure( + key: CodexUpstreamHostKey, + now = Date.now(), +): CodexUpstreamHostHealthSnapshot { + const current = upstreamHostHealth.get(key); + const reopensCircuit = current?.cooldownUntil !== undefined; + const stale = !current || (!reopensCircuit + && now - current.lastFailureAt > CODEX_UPSTREAM_HOST_FAILURE_WINDOW_MS); + const consecutiveFailures = reopensCircuit + ? Math.max(CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD, current.consecutiveFailures + 1) + : stale ? 1 : current.consecutiveFailures + 1; + const cooldownUntil = reopensCircuit || consecutiveFailures >= CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD + ? now + CODEX_UPSTREAM_HOST_COOLDOWN_MS + : undefined; + if (!current) makeRoom(now); + const next: CodexUpstreamHostHealth = { + consecutiveFailures, + lastFailureAt: now, + lastTouchedAt: now, + ...(cooldownUntil !== undefined ? { cooldownUntil } : {}), + }; + upstreamHostHealth.set(key, next); + return snapshot(next); +} + +/** Any HTTP response proves that the configured provider host was reachable. */ +export function recordCodexUpstreamHostResponse(key: CodexUpstreamHostKey): void { + upstreamHostHealth.delete(key); +} + +export function isCodexUpstreamRedirectStatus(status: number): boolean { + return status === 300 || status === 301 || status === 302 || status === 303 + || status === 307 || status === 308; +} + +export function clearCodexUpstreamHostHealth(): void { + upstreamHostHealth.clear(); +} diff --git a/src/lib/upstream-retry.ts b/src/lib/upstream-retry.ts index 82023549c9..50ca5361e1 100644 --- a/src/lib/upstream-retry.ts +++ b/src/lib/upstream-retry.ts @@ -234,11 +234,24 @@ export async function fetchWithAttemptDeadline( } } +export type UpstreamSendRecovery = "connection-reset" | "transient-5xx"; + +/** + * Ordered, privacy-safe evidence for one physical upstream send. Callers own the + * surrounding provider/account/host context; this leaf records no URL, headers, + * credentials, or error text. + */ +export type UpstreamAttemptObservation = + | { kind: "response"; status: number; recovery?: UpstreamSendRecovery } + | { kind: "rejection"; recovery?: UpstreamSendRecovery }; + export interface ResetRetryOptions { abortSignal?: AbortSignal; /** Short host/path label for the retry warn log (no secrets/query strings). */ label?: string; attempts?: number; + /** Ordered physical-send observer. It must not throw. */ + onAttempt?: (observation: UpstreamAttemptObservation) => void; } export interface TransientRetryOptions extends ResetRetryOptions { @@ -246,9 +259,29 @@ export interface TransientRetryOptions extends ResetRetryOptions { slowAttemptMs?: number; } -export type UpstreamSendRecovery = "connection-reset" | "transient-5xx"; type ReplayableFetch = (recovery?: UpstreamSendRecovery) => Promise; +export function lastUpstreamAttemptResponseStatus( + observations: readonly UpstreamAttemptObservation[], +): number | undefined { + for (let index = observations.length - 1; index >= 0; index--) { + const observation = observations[index]; + if (observation?.kind === "response") return observation.status; + } + return undefined; +} + +function notifyUpstreamAttempt( + observer: ResetRetryOptions["onAttempt"], + observation: UpstreamAttemptObservation, +): void { + try { + observer?.(observation); + } catch { + // Observation is diagnostic bookkeeping and must never alter transport behavior. + } +} + /** * Opt out of Bun's keep-alive pool after a connection-reset retry. * @@ -284,9 +317,17 @@ export async function fetchWithResetRetry( let lastError: unknown; for (let attempt = 0; attempt < attempts; attempt++) { if (opts.abortSignal?.aborted) throw abortError(opts.abortSignal); + const recovery = attempt === 0 ? firstRecovery : "connection-reset"; try { - return await doFetch(attempt === 0 ? firstRecovery : "connection-reset"); + const response = await doFetch(recovery); + notifyUpstreamAttempt(opts.onAttempt, { + kind: "response", + status: response.status, + ...(recovery ? { recovery } : {}), + }); + return response; } catch (err) { + notifyUpstreamAttempt(opts.onAttempt, { kind: "rejection", ...(recovery ? { recovery } : {}) }); if (opts.abortSignal?.aborted || !isConnectionResetError(err) || attempt === attempts - 1) throw err; lastError = err; console.warn( diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index fc963ad623..11cee901d6 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -66,8 +66,19 @@ import { fetchWithResetRetry, fetchWithTransientRetry, applyUpstreamRecoveryInit, + lastUpstreamAttemptResponseStatus, + type UpstreamAttemptObservation, type UpstreamSendRecovery, } from "../../lib/upstream-retry"; +import { + acquireCodexUpstreamHostAdmission, + canonicalCodexUpstreamHostKey, + isCodexUpstreamRedirectStatus, + recordCodexUpstreamHostFailure, + recordCodexUpstreamHostResponse, + releaseCodexUpstreamHostProbeLease, + type CodexUpstreamHostProbeLease, +} from "../../codex/upstream-host-health"; import { ForwardAdmissionCredentialError, validateForwardAdmissionCredential } from "../auth-cors"; import { listOpenAiForwardSidecarCandidates, resolveFirstUsableOpenAiSidecar, type ResolvedOpenAiForwardSidecar } from "../../providers/openai-sidecar"; import { isCanonicalOpenAiForwardProvider, supportsNativeResponsesCompactEndpoint } from "../../providers/openai-tiers"; @@ -351,6 +362,10 @@ export async function handleResponsesCompact( const compactUrl = `${base}/responses/compact`; const compactThreadId = req.headers.get("x-codex-parent-thread-id"); const connectMs = config.connectTimeoutMs ?? 200_000; + const compactHostKey = usesCodexForwardPoolAuth(authCtx, route.provider) + ? canonicalCodexUpstreamHostKey(route.providerName, compactUrl) + : null; + let compactHostProbeLease: CodexUpstreamHostProbeLease | null = null; // Takes its context explicitly: the alternate-account flow below records a rejection // against A while promoting B, then records B's own outcome. A closure over a single // `authCtx` cannot express either. @@ -381,10 +396,11 @@ export async function handleResponsesCompact( // wrapping reset retry — because those retries happen before any alternate is even // considered. The alternate is one bounded send: a second ladder would multiply the // work an already-rejecting pool is doing. - const sendCompactAttempt = ( + const sendCompactAttempt = async ( sendProvider: OcxProviderConfig, sendHeaders: Headers, recovery: "normal" | "single", + attempts: UpstreamAttemptObservation[], ): Promise => { const doFetch = (upstreamRecovery?: UpstreamSendRecovery) => fetchWithHeaderTimeout( compactUrl, @@ -392,33 +408,82 @@ export async function handleResponsesCompact( method: "POST", headers: sendHeaders, body: JSON.stringify({ ...compactBody, model: route.modelId }), + ...(compactHostKey ? { redirect: "manual" as const } : {}), }, upstreamRecovery), req.signal, connectMs, false, providerFetch(sendProvider), ); - return recovery === "single" - ? doFetch() - : fetchWithTransientRetry(doFetch, { abortSignal: req.signal, label: safeHostLabel(compactUrl) }); + if (recovery === "normal") { + return fetchWithTransientRetry(doFetch, { + abortSignal: req.signal, + label: safeHostLabel(compactUrl), + onAttempt: observation => attempts.push(observation), + }); + } + try { + const response = await doFetch(); + attempts.push({ kind: "response", status: response.status }); + return response; + } catch (error) { + attempts.push({ kind: "rejection" }); + throw error; + } + }; + + const compactTransportFailureResponse = ( + ctx: CodexAuthContext, + err: unknown, + attempts: readonly UpstreamAttemptObservation[], + ): Response => { + if (req.signal.aborted) { + releaseCodexUpstreamHostProbeLease(compactHostProbeLease); + recordCompactPoolOutcome(ctx, 499); + return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); + } + const observedStatus = lastUpstreamAttemptResponseStatus(attempts); + if (observedStatus !== undefined) { + recordCompactPoolOutcome(ctx, observedStatus); + if (compactHostKey) recordCodexUpstreamHostResponse(compactHostKey); + } else if (compactHostKey) { + releaseCodexAuthContextProbeLease(ctx); + } + if (compactHostKey) recordCodexUpstreamHostFailure(compactHostKey); + return formatErrorResponse(502, "upstream_error", "Failed to connect to compact upstream"); }; // The account each outcome belongs to. Reassigned only when the alternate send below // actually happens, so every recorder call names the context that produced it. let outcomeCtx = authCtx; let upstream: Response; + if (req.signal.aborted) { + recordCompactPoolOutcome(authCtx, 499); + return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); + } + const compactHostAdmission = compactHostKey + ? acquireCodexUpstreamHostAdmission(compactHostKey) + : null; + if (compactHostAdmission?.kind === "blocked") { + releaseCodexAuthContextProbeLease(authCtx); + return formatErrorResponse(502, "upstream_error", "Provider host is temporarily unavailable", { + retryAfter: String(compactHostAdmission.retryAfterSeconds), + }); + } + compactHostProbeLease = compactHostAdmission?.probeLease ?? null; + const primaryAttempts: UpstreamAttemptObservation[] = []; try { // Same connect timeout + keep-alive reset + transient-5xx recovery as /v1/responses — // compact hits the same ChatGPT host and must soft-avoid / clear affinity (#186). - upstream = await sendCompactAttempt(compactProvider, headers, "normal"); + upstream = await sendCompactAttempt(compactProvider, headers, "normal", primaryAttempts); + if (compactHostKey) recordCodexUpstreamHostResponse(compactHostKey); } catch (err) { - if (req.signal.aborted) { - recordCompactPoolOutcome(outcomeCtx, 499); - return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); - } - const outcome = err instanceof Error && err.name === "TimeoutError" ? "timeout" : "connect_error"; - recordCompactPoolOutcome(outcomeCtx, outcome); - return formatErrorResponse(502, "upstream_error", "Failed to connect to compact upstream"); + return compactTransportFailureResponse(outcomeCtx, err, primaryAttempts); + } + if (compactHostKey && isCodexUpstreamRedirectStatus(upstream.status)) { + recordCompactPoolOutcome(outcomeCtx, 502); + await upstream.body?.cancel().catch(() => undefined); + return formatErrorResponse(502, "upstream_error", "Provider returned an unsupported redirect"); } // Bounded same-request alternate: the regular /v1/responses path already does this @@ -454,6 +519,7 @@ export async function handleResponsesCompact( // quota is not ours to spend on a request nobody is waiting for. if (alternate && req.signal.aborted) { releaseCodexAuthContextProbeLease(alternate.authCtx); + releaseCodexUpstreamHostProbeLease(compactHostProbeLease); recordCompactPoolOutcome(outcomeCtx, 499); return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); } @@ -477,16 +543,17 @@ export async function handleResponsesCompact( }); await upstream.body?.cancel().catch(() => undefined); outcomeCtx = alternate.authCtx; + const alternateAttempts: UpstreamAttemptObservation[] = []; try { - upstream = await sendCompactAttempt(alternate.provider, alternate.headers, "single"); + upstream = await sendCompactAttempt(alternate.provider, alternate.headers, "single", alternateAttempts); + if (compactHostKey) recordCodexUpstreamHostResponse(compactHostKey); } catch (err) { - if (req.signal.aborted) { - recordCompactPoolOutcome(outcomeCtx, 499); - return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); - } - const outcome = err instanceof Error && err.name === "TimeoutError" ? "timeout" : "connect_error"; - recordCompactPoolOutcome(outcomeCtx, outcome); - return formatErrorResponse(502, "upstream_error", "Failed to connect to compact upstream"); + return compactTransportFailureResponse(outcomeCtx, err, alternateAttempts); + } + if (compactHostKey && isCodexUpstreamRedirectStatus(upstream.status)) { + recordCompactPoolOutcome(outcomeCtx, 502); + await upstream.body?.cancel().catch(() => undefined); + return formatErrorResponse(502, "upstream_error", "Provider returned an unsupported redirect"); } } } diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 619a2e7f08..9c2662fd63 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -91,8 +91,20 @@ import { applyUpstreamRecoveryInit, fetchWithResetRetry, fetchWithTransientRetry, + lastUpstreamAttemptResponseStatus, prepareSameTarget429Wait, + type UpstreamAttemptObservation, } from "../../lib/upstream-retry"; +import { + acquireCodexUpstreamHostAdmission, + canonicalCodexUpstreamHostKey, + isCodexUpstreamRedirectStatus, + recordCodexUpstreamHostFailure, + recordCodexUpstreamHostResponse, + releaseCodexUpstreamHostProbeLease, + type CodexUpstreamHostKey, + type CodexUpstreamHostProbeLease, +} from "../../codex/upstream-host-health"; import { ForwardAdmissionCredentialError, validateForwardAdmissionCredential } from "../auth-cors"; import { createTranslatorBudget, isTranslatorBudgetExceededError, type TranslatorBudget } from "../../lib/translator-budget"; import { listOpenAiForwardSidecarCandidates, resolveFirstUsableOpenAiSidecar, type ResolvedOpenAiForwardSidecar } from "../../providers/openai-sidecar"; @@ -310,6 +322,8 @@ type CodexPoolAccountRetryResult = kind: "transport"; error: unknown; authCtx: Extract; + attempts: UpstreamAttemptObservation[]; + hostKey: CodexUpstreamHostKey | null; }; function codexQuotaOutcomeMeta(response: Response): { @@ -423,6 +437,8 @@ async function retryCodexPoolOnAlternateAccount( config, ); + const hostKey = canonicalCodexUpstreamHostKey(route.providerName, request.url); + const attempts: UpstreamAttemptObservation[] = []; noteAttemptSend(logCtx.activeAttempt, passthroughEstimate); try { const upstreamResponse = await fetchWithHeaderTimeout( @@ -431,12 +447,15 @@ async function retryCodexPoolOnAlternateAccount( method: request.method, headers: request.headers, body: request.body, + redirect: "manual", }, upstream.signal, connectMs, stream, providerFetch(route.provider), ); + attempts.push({ kind: "response", status: upstreamResponse.status }); + if (hostKey) recordCodexUpstreamHostResponse(hostKey); return { kind: "retried", authCtx: retryAuthCtx, @@ -445,8 +464,8 @@ async function retryCodexPoolOnAlternateAccount( selectedForwardHeaders: retryHeaders, }; } catch (error) { - // Attribute the transport failure to the alternate account (already selected). - return { kind: "transport", error, authCtx: retryAuthCtx }; + attempts.push({ kind: "rejection" }); + return { kind: "transport", error, authCtx: retryAuthCtx, attempts, hostKey }; } finally { request.releaseBodyObservation?.(); } @@ -1731,26 +1750,60 @@ async function handleResponsesInner( linkAbortSignal(upstream, options.abortSignal); const connectMs = config.connectTimeoutMs ?? 200_000; let upstreamResponse: Response; - const transportFailureResponse = (err: unknown): Response => { + const tracksCodexPoolHost = usesCodexForwardPoolAuth(authCtx, route.provider); + const hostKey = tracksCodexPoolHost + ? canonicalCodexUpstreamHostKey(route.providerName, request.url) + : null; + let hostProbeLease: CodexUpstreamHostProbeLease | null = null; + const attemptHistory: UpstreamAttemptObservation[] = []; + const recordPoolTransportOutcome = (outcome: CodexUpstreamOutcome): void => { + if (!usesCodexForwardPoolAuth(authCtx, route.provider)) return; + recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, { + threadId: req.headers.get("x-codex-parent-thread-id"), + fixedAccount: authCtx.fixedAccount, + modelId: route.modelId, + probeLeaseId: codexProbeLeaseId(authCtx), + probeQuotaScope: codexProbeQuotaScope(authCtx), + writerGeneration: authCtx.writerGeneration, + }); + }; + const transportFailureResponse = ( + err: unknown, + observations: readonly UpstreamAttemptObservation[] = attemptHistory, + failedHostKey: CodexUpstreamHostKey | null = hostKey, + ): Response => { upstream.abort(); - if (options.abortSignal?.aborted) return clientCancelledResponse(); - const outcome = err instanceof Error && err.name === "TimeoutError" ? "timeout" : "connect_error"; - if (usesCodexForwardPoolAuth(authCtx, route.provider)) { - recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, { - threadId: req.headers.get("x-codex-parent-thread-id"), - fixedAccount: authCtx.fixedAccount, - modelId: route.modelId, - probeLeaseId: codexProbeLeaseId(authCtx), - probeQuotaScope: codexProbeQuotaScope(authCtx), - writerGeneration: authCtx.writerGeneration, - }); + if (options.abortSignal?.aborted) { + releaseCodexAuthContextProbeLease(authCtx); + releaseCodexUpstreamHostProbeLease(hostProbeLease); + return clientCancelledResponse(); } - const msg = outcome === "timeout" + const observedStatus = lastUpstreamAttemptResponseStatus(observations); + if (observedStatus !== undefined) { + recordPoolTransportOutcome(observedStatus); + if (failedHostKey) recordCodexUpstreamHostResponse(failedHostKey); + } else if (failedHostKey) { + releaseCodexAuthContextProbeLease(authCtx); + } + if (failedHostKey) recordCodexUpstreamHostFailure(failedHostKey); + const msg = err instanceof Error && err.name === "TimeoutError" ? `Provider connect timeout after ${connectMs}ms` : describeUpstreamConnectFailure(err, connectMs); return formatErrorResponse(502, "upstream_error", msg); }; try { + if (options.abortSignal?.aborted) { + releaseCodexAuthContextProbeLease(authCtx); + return clientCancelledResponse(); + } + const hostAdmission = hostKey ? acquireCodexUpstreamHostAdmission(hostKey) : null; + if (hostAdmission?.kind === "blocked") { + releaseCodexAuthContextProbeLease(authCtx); + return formatErrorResponse(502, "upstream_error", "Provider host is temporarily unavailable", { + retryAfter: String(hostAdmission.retryAfterSeconds), + }); + } + hostProbeLease = hostAdmission?.probeLease ?? null; // Transient-5xx pre-stream retry (devlog/_plan/260716_claudecode_hardening/010): // the ChatGPT backend emits transient 502/520s that an immediate retry absorbs. // Body is a replayable string; nothing has streamed to the client yet. @@ -1761,15 +1814,21 @@ async function handleResponsesInner( method: request.method, headers: request.headers, body: request.body, + ...(hostKey ? { redirect: "manual" as const } : {}), }, recovery), upstream.signal, connectMs, parsed.stream, providerFetch(route.provider)); }, - { abortSignal: upstream.signal, label: safeHostLabel(request.url) }, + { + abortSignal: upstream.signal, + label: safeHostLabel(request.url), + onAttempt: observation => attemptHistory.push(observation), + }, ); } catch (err) { return transportFailureResponse(err); } finally { request.releaseBodyObservation?.(); } + if (hostKey) recordCodexUpstreamHostResponse(hostKey); // Same-target 429 wait-and-retry (opt-in `retryOn429`) for key-auth providers on the // passthrough wire. This branch returns before the recovery loop below, so Responses-shaped @@ -1855,7 +1914,7 @@ async function handleResponsesInner( }); if (retry.kind === "transport") { authCtx = retry.authCtx; - return transportFailureResponse(retry.error); + return transportFailureResponse(retry.error, retry.attempts, retry.hostKey); } if (retry.kind === "retried") { authCtx = retry.authCtx; @@ -1867,6 +1926,11 @@ async function handleResponsesInner( } } } + if (hostKey && isCodexUpstreamRedirectStatus(upstreamResponse.status)) { + recordPoolTransportOutcome(502); + await upstreamResponse.body?.cancel().catch(() => undefined); + return formatErrorResponse(502, "upstream_error", "Provider returned an unsupported redirect"); + } const headers = sanitizePassthroughHeaders(upstreamResponse.headers); const resolvedModel = headers.get("openai-model")?.trim(); if (resolvedModel) logCtx.resolvedModel = resolvedModel; diff --git a/tests/codex-host-health-runtime.test.ts b/tests/codex-host-health-runtime.test.ts new file mode 100644 index 0000000000..60952833ea --- /dev/null +++ b/tests/codex-host-health-runtime.test.ts @@ -0,0 +1,538 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { createServer, type Server as NetServer, type Socket } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { saveCodexAccountCredential } from "../src/codex/account-store"; +import { clearAccountNeedsReauth, clearAccountQuota, updateAccountQuota } from "../src/codex/auth-api"; +import { + clearCodexUpstreamHealth, + clearThreadAccountMap, + getCodexUpstreamHealth, + resolveCodexAccountForThread, +} from "../src/codex/routing"; +import { + CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD, + canonicalCodexUpstreamHostKey, + clearCodexUpstreamHostHealth, + getCodexUpstreamHostHealth, +} from "../src/codex/upstream-host-health"; +import { loadConfig, saveConfig } from "../src/config"; +import { setDraining } from "../src/server/lifecycle"; +import { startServer } from "../src/server"; +import type { OcxConfig } from "../src/types"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "./helpers/isolated-codex-home"; + +const bunFetch = globalThis.fetch; +const canonicalPrefix = "/backend-api/codex"; +const canonicalHostKey = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex/responses", +)!; +const proxyEnvKeys = [ + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "NO_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + "no_proxy", + "BUN_CONFIG_NO_PROXY", +] as const; + +type StoppableServer = { + port: number; + url: URL; + stop(force?: boolean): void | Promise; +}; + +type CanonicalCall = { + url: URL; + init?: RequestInit; + headers: Headers; + accountId: string | null; +}; + +type RuntimeHarness = { + config: OcxConfig; + server: ReturnType; + send(path: "/v1/responses" | "/v1/responses/compact", threadId?: string): Promise; +}; + +const trackedServers: StoppableServer[] = []; +const trackedNetServers: NetServer[] = []; +const trackedSockets = new Set(); +const temporaryHomes: string[] = []; +let isolatedCodexHome: IsolatedCodexHome | null = null; +let previousOpencodexHome: string | undefined; +let previousApiToken: string | undefined; +let proxyEnvSnapshot: Map | null = null; + +function trackServer(server: T): T { + trackedServers.push(server); + return server; +} + +function serve(fetchHandler: (request: Request) => Response | Promise): ReturnType { + return trackServer(Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: fetchHandler })); +} + +function restoreEnvValue(key: string, value: string | undefined): void { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; +} + +function isolateProxyEnvironment(): void { + proxyEnvSnapshot = new Map(proxyEnvKeys.map(key => [key, process.env[key]])); + for (const key of proxyEnvKeys) delete process.env[key]; + // Runtime fixtures must never inherit a workstation proxy for .invalid or loopback. + process.env.NO_PROXY = "*"; + process.env.no_proxy = "*"; + process.env.BUN_CONFIG_NO_PROXY = "*"; +} + +function installCanonicalRouter(route: (call: CanonicalCall) => Promise): void { + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const value = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + const url = new URL(value); + if (url.hostname.toLowerCase() === "chatgpt.com" && url.pathname.startsWith(canonicalPrefix)) { + const headers = new Headers(init?.headers ?? (input instanceof Request ? input.headers : undefined)); + return route({ + url, + init, + headers, + accountId: headers.get("chatgpt-account-id"), + }); + } + return bunFetch(input, init); + }) as typeof fetch; +} + +function localTarget(base: string | URL, call: CanonicalCall): URL { + return new URL(`${call.url.pathname}${call.url.search}`, base); +} + +async function actualFetch(base: string | URL, call: CanonicalCall): Promise { + return bunFetch(localTarget(base, call), call.init); +} + +async function closedEphemeralPort(): Promise { + for (let attempt = 0; attempt < 5; attempt++) { + const probe = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => new Response("probe") }); + const port = probe.port; + await probe.stop(true); + try { + const unexpectedlyOpen = await bunFetch(`http://127.0.0.1:${port}/closed-port-check`); + await unexpectedlyOpen.body?.cancel().catch(() => undefined); + } catch { + return port; + } + } + throw new Error("could not reserve and verify a refused loopback port"); +} + +async function startCredentialDependentReadThenCloseServer(): Promise<{ + url: string; + requests: string[]; + successfulBRequests: string[]; +}> { + const requests: string[] = []; + const successfulBRequests: string[] = []; + const server = createServer(socket => { + trackedSockets.add(socket); + let bytes = Buffer.alloc(0); + socket.on("data", chunk => { + bytes = Buffer.concat([bytes, chunk]); + const headerEnd = bytes.indexOf("\r\n\r\n"); + if (headerEnd < 0) return; + const headerText = bytes.subarray(0, headerEnd).toString("latin1"); + const lengthMatch = /\r\ncontent-length:\s*(\d+)/i.exec(`\r\n${headerText}`); + if (!lengthMatch) return; + const contentLength = Number(lengthMatch[1]); + if (bytes.length - headerEnd - 4 < contentLength) return; + const wire = bytes.subarray(0, headerEnd + 4 + contentLength).toString("utf8"); + requests.push(wire); + socket.removeAllListeners("data"); + if (/\r\nauthorization:\s*Bearer pool-b-token\r\n/i.test(`\r\n${headerText}\r\n`)) { + successfulBRequests.push(wire); + const responseBody = JSON.stringify({ id: "healthy-b", status: "completed", output: [] }); + socket.end( + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\n" + + `content-length: ${Buffer.byteLength(responseBody)}\r\nconnection: close\r\n\r\n${responseBody}`, + ); + return; + } + socket.destroy(); + }); + socket.on("close", () => trackedSockets.delete(socket)); + }); + trackedNetServers.push(server); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + server.off("error", reject); + resolve(); + }); + }); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("raw runtime server did not bind"); + return { url: `http://127.0.0.1:${address.port}`, requests, successfulBRequests }; +} + +async function startHarness(options: { twoAccounts?: boolean; connectTimeoutMs?: number } = {}): Promise { + const home = mkdtempSync(join(tmpdir(), "ocx-host-runtime-")); + temporaryHomes.push(home); + process.env.OPENCODEX_HOME = home; + delete process.env.OPENCODEX_API_AUTH_TOKEN; + clearCodexUpstreamHealth(); + clearCodexUpstreamHostHealth(); + clearThreadAccountMap(); + clearAccountQuota(); + clearAccountNeedsReauth("pool-a"); + clearAccountNeedsReauth("pool-b"); + + const config = { + port: 0, + defaultProvider: "openai", + openaiProviderTierVersion: 2, + accountPoolStrategy: "fill-first", + upstreamFailoverThreshold: 3, + ...(options.connectTimeoutMs ? { connectTimeoutMs: options.connectTimeoutMs } : {}), + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + }, + codexAccounts: [ + { id: "main", email: "main@example.test", isMain: true }, + { id: "pool-a", email: "pool-a@example.test", isMain: false, chatgptAccountId: "acct-pool-a" }, + ...(options.twoAccounts + ? [{ id: "pool-b", email: "pool-b@example.test", isMain: false, chatgptAccountId: "acct-pool-b" }] + : []), + ], + activeCodexAccountId: "pool-a", + } as OcxConfig; + saveConfig(config); + saveCodexAccountCredential("pool-a", { + accessToken: "pool-a-token", + refreshToken: "pool-a-refresh", + expiresAt: Date.now() + 10 * 60_000, + chatgptAccountId: "acct-pool-a", + }); + updateAccountQuota("pool-a", 10); + if (options.twoAccounts) { + saveCodexAccountCredential("pool-b", { + accessToken: "pool-b-token", + refreshToken: "pool-b-refresh", + expiresAt: Date.now() + 10 * 60_000, + chatgptAccountId: "acct-pool-b", + }); + updateAccountQuota("pool-b", 20); + } + const server = trackServer(startServer(0)); + return { + config, + server, + send: (path, threadId) => bunFetch(new URL(path, server.url), { + method: "POST", + headers: { + "content-type": "application/json", + authorization: "Bearer inbound-runtime-token", + ...(threadId ? { "x-codex-parent-thread-id": threadId } : {}), + }, + body: JSON.stringify({ + model: "gpt-5.6-sol", + input: path.endsWith("/compact") ? [] : "runtime transport probe", + stream: false, + }), + }), + }; +} + +function pinAWhileActiveB(harness: RuntimeHarness, threadId: string): void { + expect(resolveCodexAccountForThread(threadId, harness.config)).toBe("pool-a"); + harness.config.activeCodexAccountId = "pool-b"; + saveConfig(harness.config); + // Reset only account health/runtime-active state. The A affinity must remain bound. + clearCodexUpstreamHealth(); + expect(loadConfig().activeCodexAccountId).toBe("pool-b"); +} + +function expectHostOnlyState(harness: RuntimeHarness, threadId: string): void { + expect(getCodexUpstreamHealth("pool-a")).toBeNull(); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(loadConfig().activeCodexAccountId).toBe("pool-b"); + expect(resolveCodexAccountForThread(threadId, harness.config)).toBe("pool-a"); +} + +function runtimeErrorLabel(error: unknown): string { + if (!(error instanceof Error)) return typeof error; + const code = "code" in error ? String((error as Error & { code?: unknown }).code ?? "") : ""; + return `${error.name}:${code}:${error.message}`; +} + +beforeEach(() => { + previousOpencodexHome = process.env.OPENCODEX_HOME; + previousApiToken = process.env.OPENCODEX_API_AUTH_TOKEN; + isolateProxyEnvironment(); + isolatedCodexHome = installIsolatedCodexHome("ocx-host-runtime-codex-"); + setDraining(false); +}); + +afterEach(async () => { + globalThis.fetch = bunFetch; + for (const socket of trackedSockets) socket.destroy(); + trackedSockets.clear(); + for (const server of trackedServers.splice(0).reverse()) { + try { await server.stop(true); } catch { /* best-effort fixture cleanup */ } + } + for (const server of trackedNetServers.splice(0).reverse()) { + if (!server.listening) continue; + await new Promise(resolve => server.close(() => resolve())); + } + setDraining(false); + clearCodexUpstreamHealth(); + clearCodexUpstreamHostHealth(); + clearThreadAccountMap(); + clearAccountQuota(); + clearAccountNeedsReauth("pool-a"); + clearAccountNeedsReauth("pool-b"); + for (const home of temporaryHomes.splice(0)) rmSync(home, { recursive: true, force: true }); + isolatedCodexHome?.restore(); + isolatedCodexHome = null; + restoreEnvValue("OPENCODEX_HOME", previousOpencodexHome); + restoreEnvValue("OPENCODEX_API_AUTH_TOKEN", previousApiToken); + if (proxyEnvSnapshot) { + for (const [key, value] of proxyEnvSnapshot) restoreEnvValue(key, value); + } + proxyEnvSnapshot = null; +}); + +describe("Codex host-health actual Bun runtime (#914/#922)", () => { + test("repeated same .invalid failures open one canonical host circuit without account rotation", async () => { + const harness = await startHarness({ twoAccounts: true }); + const threadId = "runtime-invalid-thread"; + pinAWhileActiveB(harness, threadId); + const runtimeErrors: unknown[] = []; + const accounts: Array = []; + let physicalSends = 0; + const invalidOrigin = "http://same-host-health-target.invalid"; + installCanonicalRouter(async call => { + physicalSends += 1; + accounts.push(call.accountId); + try { + return await actualFetch(invalidOrigin, call); + } catch (error) { + runtimeErrors.push(error); + throw error; + } + }); + + for (let attempt = 0; attempt < CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; attempt++) { + expect((await harness.send("/v1/responses", threadId)).status).toBe(502); + } + const sendsAtOpen = physicalSends; + const blocked = await harness.send("/v1/responses", threadId); + + expect(blocked.status).toBe(502); + expect(Number(blocked.headers.get("retry-after"))).toBeGreaterThanOrEqual(1); + expect(physicalSends).toBe(sendsAtOpen); + expect(physicalSends).toBe(CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD); + expect(accounts).toEqual(Array(CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD).fill("acct-pool-a")); + // Bun 1.3.14 on Windows has emitted more than one label for this same target. + // Activation correctness intentionally depends only on real rejection count. + expect(runtimeErrors.map(runtimeErrorLabel).every(label => label.length > 0)).toBe(true); + expect(getCodexUpstreamHostHealth(canonicalHostKey)?.cooldownUntil).toEqual(expect.any(Number)); + const invalidKey = canonicalCodexUpstreamHostKey("openai", invalidOrigin)!; + expect(invalidKey).not.toBe(canonicalHostKey); + expect(getCodexUpstreamHostHealth(invalidKey)).toBeNull(); + expectHostOnlyState(harness, threadId); + }, { timeout: 30_000 }); + + test("native compact records a real refused closed port as host-only", async () => { + const harness = await startHarness({ twoAccounts: true }); + const threadId = "runtime-compact-refused"; + pinAWhileActiveB(harness, threadId); + const port = await closedEphemeralPort(); + const target = `http://127.0.0.1:${port}`; + let physicalSends = 0; + installCanonicalRouter(async call => { + physicalSends += 1; + return actualFetch(target, call); + }); + + const response = await harness.send("/v1/responses/compact", threadId); + expect(response.status).toBe(502); + expect(physicalSends).toBe(1); + expect(getCodexUpstreamHostHealth(canonicalHostKey)?.consecutiveFailures).toBe(1); + expect(getCodexUpstreamHostHealth(canonicalCodexUpstreamHostKey("openai", target)!)).toBeNull(); + expectHostOnlyState(harness, threadId); + }, { timeout: 10_000 }); + + test("an actual delayed-header timeout is host-only", async () => { + const harness = await startHarness({ twoAccounts: true, connectTimeoutMs: 500 }); + const threadId = "runtime-header-timeout"; + pinAWhileActiveB(harness, threadId); + const seen: Array<{ authorization: string | null; accountId: string | null; body: string }> = []; + const delayed = serve(async request => { + seen.push({ + authorization: request.headers.get("authorization"), + accountId: request.headers.get("chatgpt-account-id"), + body: await request.text(), + }); + await Bun.sleep(2_000); + return Response.json({ id: "too-late", status: "completed", output: [] }); + }); + installCanonicalRouter(call => actualFetch(delayed.url, call)); + + const response = await harness.send("/v1/responses", threadId); + expect(response.status).toBe(502); + expect((await response.text()).toLowerCase()).toContain("timeout"); + expect(seen).toHaveLength(1); + expect(seen[0]).toMatchObject({ authorization: "Bearer pool-a-token", accountId: "acct-pool-a" }); + expect(seen[0]!.body).toContain("gpt-5.6-sol"); + expect(getCodexUpstreamHostHealth(canonicalHostKey)?.consecutiveFailures).toBe(1); + expectHostOnlyState(harness, threadId); + }, { timeout: 10_000 }); + + test("credential-visible 307 is manual, bounded, and never exposes Location", async () => { + const harness = await startHarness({ twoAccounts: true }); + const deadPort = await closedEphemeralPort(); + const seen: Array<{ authorization: string | null; accountId: string | null; body: string }> = []; + const redirect = serve(async request => { + seen.push({ + authorization: request.headers.get("authorization"), + accountId: request.headers.get("chatgpt-account-id"), + body: await request.text(), + }); + return new Response(null, { + status: 307, + headers: { location: `http://127.0.0.1:${deadPort}/credential-leak-target` }, + }); + }); + const redirectModes: Array = []; + installCanonicalRouter(call => { + redirectModes.push(call.init?.redirect); + return actualFetch(redirect.url, call); + }); + + const response = await harness.send("/v1/responses"); + const downstream = await response.text(); + expect(response.status).toBe(502); + expect(response.headers.get("location")).toBeNull(); + expect(downstream).not.toContain("credential-leak-target"); + expect(redirectModes).toEqual(["manual"]); + expect(seen).toHaveLength(1); + expect(seen[0]).toMatchObject({ authorization: "Bearer pool-a-token", accountId: "acct-pool-a" }); + expect(seen[0]!.body).toContain("gpt-5.6-sol"); + expect(getCodexUpstreamHostHealth(canonicalHostKey)).toBeNull(); + expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(502); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + }, { timeout: 10_000 }); + + test("a credential-consuming A failure is not replayed onto a healthy B", async () => { + const harness = await startHarness({ twoAccounts: true }); + const threadId = "runtime-read-close"; + pinAWhileActiveB(harness, threadId); + const raw = await startCredentialDependentReadThenCloseServer(); + let physicalSends = 0; + const attemptedAccounts: Array = []; + installCanonicalRouter(async call => { + physicalSends += 1; + attemptedAccounts.push(call.accountId); + return actualFetch(raw.url, call); + }); + + const response = await harness.send("/v1/responses", threadId); + expect(response.status).toBe(502); + expect(physicalSends).toBeGreaterThanOrEqual(1); + expect(raw.requests.length).toBeGreaterThanOrEqual(1); + for (const wire of raw.requests) { + expect(wire.toLowerCase()).toContain("authorization: bearer pool-a-token"); + expect(wire.toLowerCase()).toContain("chatgpt-account-id: acct-pool-a"); + expect(wire).toContain("gpt-5.6-sol"); + } + // The raw peer would return 200 for B, but A may already have consumed a + // side-effecting request. Replaying it under another credential would risk + // duplication, so this conservative residual intentionally remains host-only. + expect(attemptedAccounts.every(accountId => accountId === "acct-pool-a")).toBe(true); + expect(raw.successfulBRequests).toHaveLength(0); + expect(getCodexUpstreamHostHealth(canonicalHostKey)?.consecutiveFailures).toBe(1); + expectHostOnlyState(harness, threadId); + }, { timeout: 15_000 }); + + test("a real 503 followed by a real refused retry preserves both attribution layers", async () => { + const harness = await startHarness({ twoAccounts: true }); + const refusedPort = await closedEphemeralPort(); + let upstream503Hits = 0; + const upstream503 = serve(async request => { + upstream503Hits += 1; + await request.text(); + return Response.json({ error: { message: "busy" } }, { + status: 503, + headers: { "retry-after": "0" }, + }); + }); + let physicalSends = 0; + installCanonicalRouter(call => { + physicalSends += 1; + return physicalSends === 1 + ? actualFetch(upstream503.url, call) + : actualFetch(`http://127.0.0.1:${refusedPort}`, call); + }); + + const response = await harness.send("/v1/responses"); + expect(response.status).toBe(502); + expect(physicalSends).toBe(2); + expect(upstream503Hits).toBe(1); + expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(503); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(getCodexUpstreamHostHealth(canonicalHostKey)?.consecutiveFailures).toBe(1); + }, { timeout: 10_000 }); + + for (const path of ["/v1/responses", "/v1/responses/compact"] as const) { + test(`${path} keeps A=429 and one real B rejection separately attributed`, async () => { + const harness = await startHarness({ twoAccounts: true }); + const refusedPort = await closedEphemeralPort(); + const aSeen: Array<{ authorization: string | null; accountId: string | null; body: string }> = []; + const aQuota = serve(async request => { + aSeen.push({ + authorization: request.headers.get("authorization"), + accountId: request.headers.get("chatgpt-account-id"), + body: await request.text(), + }); + return Response.json({ error: { message: "A quota" } }, { + status: 429, + headers: { "retry-after": "60" }, + }); + }); + const counts = new Map(); + const bHeaders: Headers[] = []; + installCanonicalRouter(call => { + const accountId = call.accountId ?? "missing"; + counts.set(accountId, (counts.get(accountId) ?? 0) + 1); + if (accountId === "acct-pool-a") return actualFetch(aQuota.url, call); + bHeaders.push(call.headers); + return actualFetch(`http://127.0.0.1:${refusedPort}`, call); + }); + + const response = await harness.send(path); + expect(response.status).toBe(502); + expect(counts.get("acct-pool-a")).toBe(1); + expect(counts.get("acct-pool-b")).toBe(1); + expect([...counts.values()].reduce((sum, count) => sum + count, 0)).toBe(2); + expect(aSeen).toHaveLength(1); + expect(aSeen[0]).toMatchObject({ authorization: "Bearer pool-a-token", accountId: "acct-pool-a" }); + expect(aSeen[0]!.body).toContain("gpt-5.6-sol"); + expect(bHeaders).toHaveLength(1); + expect(bHeaders[0]!.get("authorization")).toBe("Bearer pool-b-token"); + expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(429); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(getCodexUpstreamHostHealth(canonicalHostKey)?.consecutiveFailures).toBe(1); + }, { timeout: 10_000 }); + } +}); diff --git a/tests/codex-upstream-host-health.test.ts b/tests/codex-upstream-host-health.test.ts new file mode 100644 index 0000000000..7755a1cdd7 --- /dev/null +++ b/tests/codex-upstream-host-health.test.ts @@ -0,0 +1,120 @@ +import { beforeEach, describe, expect, test } from "bun:test"; +import { + CODEX_UPSTREAM_HOST_COOLDOWN_MS, + CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD, + CODEX_UPSTREAM_HOST_FAILURE_WINDOW_MS, + CODEX_UPSTREAM_HOST_MAX_ENTRIES, + acquireCodexUpstreamHostAdmission, + canonicalCodexUpstreamHostKey, + clearCodexUpstreamHostHealth, + getCodexUpstreamHostCooldownUntil, + getCodexUpstreamHostHealth, + recordCodexUpstreamHostFailure, + recordCodexUpstreamHostResponse, + releaseCodexUpstreamHostProbeLease, +} from "../src/codex/upstream-host-health"; + +beforeEach(() => clearCodexUpstreamHostHealth()); + +describe("Codex upstream host health (#914)", () => { + test("keys normalized provider plus canonical HTTP origin only", () => { + const canonical = canonicalCodexUpstreamHostKey( + " OpenAI ", + "HTTPS://CHATGPT.COM:443/backend-api/codex/responses?account=secret#fragment", + ); + expect(canonical).toBe(canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com/other")); + expect(canonical).not.toBe(canonicalCodexUpstreamHostKey("other", "https://chatgpt.com/other")); + expect(canonical).not.toBe(canonicalCodexUpstreamHostKey("openai", "http://chatgpt.com/other")); + expect(canonicalCodexUpstreamHostKey("openai", "ftp://chatgpt.com/file")).toBeNull(); + }); + + test("opens a fixed host cooldown only at its own threshold and clears on HTTP response", () => { + const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com/backend-api/codex")!; + const now = 1_900_000_000_000; + for (let attempt = 1; attempt < CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; attempt++) { + const health = recordCodexUpstreamHostFailure(key, now + attempt); + expect(health.consecutiveFailures).toBe(attempt); + expect(health.cooldownUntil).toBeUndefined(); + } + const trippedAt = now + CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; + const tripped = recordCodexUpstreamHostFailure(key, trippedAt); + expect(tripped.cooldownUntil).toBe(trippedAt + CODEX_UPSTREAM_HOST_COOLDOWN_MS); + expect(getCodexUpstreamHostCooldownUntil(key, trippedAt)).toBe(tripped.cooldownUntil!); + recordCodexUpstreamHostResponse(key); + expect(getCodexUpstreamHostHealth(key, trippedAt)).toBeNull(); + }); + + test("an expired window starts a fresh streak", () => { + const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com")!; + recordCodexUpstreamHostFailure(key, 1_000); + const next = recordCodexUpstreamHostFailure(key, 1_000 + CODEX_UPSTREAM_HOST_FAILURE_WINDOW_MS + 1); + expect(next.consecutiveFailures).toBe(1); + }); + + test("admits exactly one half-open logical request after cooldown", () => { + const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com")!; + const now = 1_900_000_000_000; + for (let attempt = 0; attempt < CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; attempt++) { + recordCodexUpstreamHostFailure(key, now + attempt); + } + const cooldownUntil = now + CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD - 1 + + CODEX_UPSTREAM_HOST_COOLDOWN_MS; + expect(acquireCodexUpstreamHostAdmission(key, cooldownUntil - 1)).toMatchObject({ + kind: "blocked", + }); + const probe = acquireCodexUpstreamHostAdmission(key, cooldownUntil); + expect(probe.kind).toBe("admitted"); + if (probe.kind !== "admitted") throw new Error("expected half-open admission"); + expect(probe.probeLease).not.toBeNull(); + expect(acquireCodexUpstreamHostAdmission(key, cooldownUntil)).toEqual({ + kind: "blocked", + retryAfterSeconds: 1, + }); + + recordCodexUpstreamHostResponse(key); + expect(acquireCodexUpstreamHostAdmission(key, cooldownUntil)).toEqual({ + kind: "admitted", + probeLease: null, + }); + }); + + test("a half-open terminal rejection immediately reopens the cooldown", () => { + const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com")!; + const trippedAt = 2_000; + for (let attempt = 0; attempt < CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; attempt++) { + recordCodexUpstreamHostFailure(key, trippedAt); + } + const probeAt = trippedAt + CODEX_UPSTREAM_HOST_COOLDOWN_MS; + expect(acquireCodexUpstreamHostAdmission(key, probeAt).kind).toBe("admitted"); + const reopened = recordCodexUpstreamHostFailure(key, probeAt); + expect(reopened.cooldownUntil).toBe(probeAt + CODEX_UPSTREAM_HOST_COOLDOWN_MS); + expect(acquireCodexUpstreamHostAdmission(key, probeAt)).toMatchObject({ kind: "blocked" }); + }); + + test("caller abort releases a half-open probe without adding evidence", () => { + const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com")!; + const trippedAt = 3_000; + for (let attempt = 0; attempt < CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; attempt++) { + recordCodexUpstreamHostFailure(key, trippedAt); + } + const probeAt = trippedAt + CODEX_UPSTREAM_HOST_COOLDOWN_MS; + const first = acquireCodexUpstreamHostAdmission(key, probeAt); + if (first.kind !== "admitted") throw new Error("expected half-open admission"); + const before = getCodexUpstreamHostHealth(key, probeAt); + expect(releaseCodexUpstreamHostProbeLease(first.probeLease)).toBe(true); + expect(getCodexUpstreamHostHealth(key, probeAt)).toEqual(before); + const replacement = acquireCodexUpstreamHostAdmission(key, probeAt); + expect(replacement.kind).toBe("admitted"); + if (replacement.kind !== "admitted") throw new Error("expected replacement probe"); + expect(replacement.probeLease).not.toBeNull(); + }); + + test("bounds the process-local map and evicts the oldest entry", () => { + const first = canonicalCodexUpstreamHostKey("provider-0", "https://host-0.example")!; + for (let index = 0; index <= CODEX_UPSTREAM_HOST_MAX_ENTRIES; index++) { + const key = canonicalCodexUpstreamHostKey(`provider-${index}`, `https://host-${index}.example`)!; + recordCodexUpstreamHostFailure(key, 10_000 + index); + } + expect(getCodexUpstreamHostHealth(first, 10_000 + CODEX_UPSTREAM_HOST_MAX_ENTRIES)).toBeNull(); + }); +}); diff --git a/tests/issue-452-empty-503.test.ts b/tests/issue-452-empty-503.test.ts index 90188c60b5..b971a2fff2 100644 --- a/tests/issue-452-empty-503.test.ts +++ b/tests/issue-452-empty-503.test.ts @@ -3,7 +3,12 @@ import { existsSync, mkdirSync, rmSync } from "node:fs"; import { join } from "node:path"; import { saveCodexAccountCredential } from "../src/codex/account-store"; import { clearAccountNeedsReauth, clearAccountQuota, updateAccountQuota } from "../src/codex/auth-api"; -import { clearCodexUpstreamHealth, clearThreadAccountMap } from "../src/codex/routing"; +import { clearCodexUpstreamHealth, clearThreadAccountMap, getCodexUpstreamHealth } from "../src/codex/routing"; +import { + canonicalCodexUpstreamHostKey, + clearCodexUpstreamHostHealth, + getCodexUpstreamHostHealth, +} from "../src/codex/upstream-host-health"; import { saveConfig } from "../src/config"; import { createOpenAIChatAdapter } from "../src/adapters/openai-chat"; import { getDebugLogEntries, resetDebugLogBufferForTests } from "../src/lib/debug-log-buffer"; @@ -55,6 +60,7 @@ afterEach(() => { isolatedCodexHome?.restore(); isolatedCodexHome = null; clearCodexUpstreamHealth(); + clearCodexUpstreamHostHealth(); clearThreadAccountMap(); clearAccountNeedsReauth("pool-a"); clearAccountQuota(); @@ -118,12 +124,14 @@ describe("formatPassthroughUpstreamError (#452)", () => { async function withPoolPassthrough( reply: (request: Request) => Response | Promise, run: (serverUrl: string) => Promise, + options: { twoAccounts?: boolean } = {}, ): Promise { if (existsSync(TEST_DIR)) rmSync(TEST_DIR, { recursive: true, force: true }); mkdirSync(TEST_DIR, { recursive: true }); process.env.OPENCODEX_HOME = TEST_DIR; delete process.env.OPENCODEX_API_AUTH_TOKEN; clearCodexUpstreamHealth(); + clearCodexUpstreamHostHealth(); clearThreadAccountMap(); clearAccountQuota(); clearAccountNeedsReauth("pool-a"); @@ -151,6 +159,9 @@ async function withPoolPassthrough( codexAccounts: [ { id: "main", email: "main@example.test", isMain: true }, { id: "pool-a", email: "pool-a@example.test", isMain: false, chatgptAccountId: "acct-pool-a" }, + ...(options.twoAccounts + ? [{ id: "pool-b", email: "pool-b@example.test", isMain: false, chatgptAccountId: "acct-pool-b" }] + : []), ], activeCodexAccountId: "pool-a", } as OcxConfig); @@ -161,6 +172,15 @@ async function withPoolPassthrough( chatgptAccountId: "acct-pool-a", }); updateAccountQuota("pool-a", 10); + if (options.twoAccounts) { + saveCodexAccountCredential("pool-b", { + accessToken: "pool-b-token", + refreshToken: "pool-b-refresh", + expiresAt: Date.now() + 10 * 60_000, + chatgptAccountId: "acct-pool-b", + }); + updateAccountQuota("pool-b", 20); + } const server = startServer(0); try { @@ -171,6 +191,127 @@ async function withPoolPassthrough( } } +function installCodexTransport( + send: (init: RequestInit | undefined) => Response | Promise, +): void { + globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => { + const value = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + const url = new URL(value); + if (url.hostname === "chatgpt.com" && url.pathname.startsWith("/backend-api/codex")) { + return Promise.resolve(send(init)); + } + return originalGlobalFetch(input, init); + }) as typeof fetch; +} + +async function sendRegularPoolRequest(serverUrl: string): Promise { + return originalGlobalFetch(new URL("/v1/responses", serverUrl), { + method: "POST", + headers: { "content-type": "application/json", authorization: "Bearer inbound-token" }, + body: JSON.stringify({ model: "gpt-5.6-sol", input: "hi", stream: false }), + }); +} + +describe("regular Codex provider-host settlement (#914)", () => { + const hostKey = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com/backend-api/codex/responses")!; + + test("a bare rejection changes host health but not account health", async () => { + await withPoolPassthrough(() => new Response("unused"), async serverUrl => { + let sends = 0; + installCodexTransport(() => { + sends += 1; + throw new Error("opaque transport rejection"); + }); + const response = await sendRegularPoolRequest(serverUrl); + expect(response.status).toBe(502); + expect(sends).toBe(1); + expect(getCodexUpstreamHealth("pool-a")).toBeNull(); + expect(getCodexUpstreamHostHealth(hostKey)?.consecutiveFailures).toBe(1); + }); + }); + + test("a 503 followed by rejection records one account outcome and a terminal host failure", async () => { + await withPoolPassthrough(() => new Response("unused"), async serverUrl => { + let sends = 0; + installCodexTransport(() => { + sends += 1; + if (sends === 1) return new Response("busy", { status: 503, headers: { "retry-after": "0" } }); + throw new Error("opaque transport rejection"); + }); + const response = await sendRegularPoolRequest(serverUrl); + expect(response.status).toBe(502); + expect(sends).toBe(2); + expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(503); + expect(getCodexUpstreamHostHealth(hostKey)?.consecutiveFailures).toBe(1); + }); + }); + + test("a connect timeout changes host health without changing account health", async () => { + await withPoolPassthrough(() => new Response("unused"), async serverUrl => { + const timeout = new Error("header timeout"); + timeout.name = "TimeoutError"; + installCodexTransport(() => { throw timeout; }); + const response = await sendRegularPoolRequest(serverUrl); + expect(response.status).toBe(502); + expect(getCodexUpstreamHealth("pool-a")).toBeNull(); + expect(getCodexUpstreamHostHealth(hostKey)?.consecutiveFailures).toBe(1); + }); + }); + + test("an open host circuit returns a bounded Retry-After without another send", async () => { + await withPoolPassthrough(() => new Response("unused"), async serverUrl => { + let sends = 0; + installCodexTransport(() => { + sends += 1; + throw new Error("opaque transport rejection"); + }); + for (let attempt = 0; attempt < 3; attempt++) { + expect((await sendRegularPoolRequest(serverUrl)).status).toBe(502); + } + const blocked = await sendRegularPoolRequest(serverUrl); + expect(blocked.status).toBe(502); + expect(sends).toBe(3); + expect(Number(blocked.headers.get("retry-after"))).toBeGreaterThanOrEqual(1); + }); + }); + + test("manual redirect is credential-visible, bounded, and does not expose Location", async () => { + await withPoolPassthrough(() => new Response("unused"), async serverUrl => { + let redirectMode: RequestRedirect | undefined; + installCodexTransport(init => { + redirectMode = init?.redirect; + return new Response(null, { + status: 307, + headers: { location: "https://dead.example/private-path" }, + }); + }); + const response = await sendRegularPoolRequest(serverUrl); + expect(response.status).toBe(502); + expect(response.headers.get("location")).toBeNull(); + expect(redirectMode).toBe("manual"); + expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(502); + expect(getCodexUpstreamHostHealth(hostKey)).toBeNull(); + }); + }); + + test("alternate-account bare rejection remains one B send and host-only", async () => { + await withPoolPassthrough(() => new Response("unused"), async serverUrl => { + let sends = 0; + installCodexTransport(() => { + sends += 1; + if (sends === 1) return new Response("quota", { status: 429 }); + throw new Error("opaque alternate rejection"); + }); + const response = await sendRegularPoolRequest(serverUrl); + expect(response.status).toBe(502); + expect(sends).toBe(2); + expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(429); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(getCodexUpstreamHostHealth(hostKey)?.consecutiveFailures).toBe(1); + }, { twoAccounts: true }); + }); +}); + describe("passthrough empty 503 (#452)", () => { test("ChatGPT passthrough empty-body 503 becomes JSON Codex can parse", async () => { await withPoolPassthrough( diff --git a/tests/responses-compaction-routing.test.ts b/tests/responses-compaction-routing.test.ts index d6cd644b4b..fa1d2f2906 100644 --- a/tests/responses-compaction-routing.test.ts +++ b/tests/responses-compaction-routing.test.ts @@ -18,6 +18,13 @@ import { resolveCodexAccountForThread, } from "../src/codex/routing"; import { clearAccountQuota, updateAccountQuota } from "../src/codex/auth-api"; +import { + CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD, + canonicalCodexUpstreamHostKey, + clearCodexUpstreamHostHealth, + getCodexUpstreamHostHealth, + recordCodexUpstreamHostFailure, +} from "../src/codex/upstream-host-health"; import { MAIN_CODEX_ACCOUNT_ID } from "../src/codex/main-account"; import { releaseCodexAuthContextProbeLease, @@ -32,6 +39,7 @@ const originalFetch = globalThis.fetch; afterEach(() => { globalThis.fetch = originalFetch; + clearCodexUpstreamHostHealth(); }); function keyProviderConfig(overrides: Partial = {}): OcxConfig { @@ -387,6 +395,140 @@ describe("native Codex pool compaction", () => { else process.env.CODEX_HOME = previousCodexHome; } }); + test("a bare compact transport rejection releases its recovery probe lease", async () => { + const testDir = mkdtempSync(join(tmpdir(), "ocx-compact-host-probe-")); + const previousOpencodexHome = process.env.OPENCODEX_HOME; + const previousCodexHome = process.env.CODEX_HOME; + const originalNow = Date.now; + const now = 1_800_000_000_000; + const probeAt = now + CODEX_QUOTA_PROBE_INTERVAL_MS; + const config = nativePoolConfig(); + try { + process.env.OPENCODEX_HOME = testDir; + process.env.CODEX_HOME = testDir; + Date.now = () => now; + clearCodexUpstreamHealth(); + saveCodexAccountCredential("pool-a", { + accessToken: "pool-access-token", + refreshToken: "pool-refresh-token", + expiresAt: now + 30 * 60_000, + chatgptAccountId: "pool_acc", + }); + recordCodexUpstreamOutcome(config, "pool-a", 429, { + now, + resetAt: Math.floor((now + 4 * 24 * 60 * 60_000) / 1_000), + modelId: "gpt-5.3-codex-spark", + }); + Date.now = () => probeAt; + globalThis.fetch = (async () => { throw new Error("opaque compact rejection"); }) as typeof fetch; + const failed = await handleResponsesCompact( + compactionRequest(baseCompactionBody({ model: "gpt-5.3-codex-spark" })), + config, + { model: "", provider: "" }, + ); + expect(failed.status).toBe(502); + + Date.now = () => probeAt + CODEX_QUOTA_PROBE_INTERVAL_MS; + const nextProbe = await resolveCodexAuthContext( + new Headers({ authorization: "Bearer main-token" }), + config, + "pool", + { modelId: "gpt-5.3-codex-spark" }, + ); + expect(nextProbe).toMatchObject({ probeQuotaScope: "spark" }); + releaseCodexAuthContextProbeLease(nextProbe); + } finally { + Date.now = originalNow; + globalThis.fetch = originalFetch; + clearCodexUpstreamHealth(); + rmSync(testDir, { recursive: true, force: true }); + if (previousOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousOpencodexHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; + } + }); + + for (const endpoint of ["regular", "compact"] as const) { + test(`${endpoint} prefers an already-aborted 499 over an open host circuit and releases recovery`, async () => { + const testDir = mkdtempSync(join(tmpdir(), `ocx-${endpoint}-abort-host-probe-`)); + const previousOpencodexHome = process.env.OPENCODEX_HOME; + const previousCodexHome = process.env.CODEX_HOME; + const originalNow = Date.now; + const now = 1_810_000_000_000; + const probeAt = now + CODEX_QUOTA_PROBE_INTERVAL_MS; + const config = nativePoolConfig(); + try { + process.env.OPENCODEX_HOME = testDir; + process.env.CODEX_HOME = testDir; + Date.now = () => now; + clearCodexUpstreamHealth(); + clearCodexUpstreamHostHealth(); + saveCodexAccountCredential("pool-a", { + accessToken: "pool-access-token", + refreshToken: "pool-refresh-token", + expiresAt: now + 30 * 60_000, + chatgptAccountId: "pool_acc", + }); + recordCodexUpstreamOutcome(config, "pool-a", 429, { + now, + resetAt: Math.floor((now + 4 * 24 * 60 * 60_000) / 1_000), + modelId: "gpt-5.3-codex-spark", + }); + Date.now = () => probeAt; + const hostKey = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex/responses", + )!; + for (let attempt = 0; attempt < CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; attempt++) { + recordCodexUpstreamHostFailure(hostKey); + } + let physicalSends = 0; + globalThis.fetch = (async () => { + physicalSends += 1; + return Response.json({ id: "must-not-send", status: "completed", output: [] }); + }) as typeof fetch; + const controller = new AbortController(); + controller.abort(); + const request = compactionRequest( + endpoint === "regular" + ? { model: "gpt-5.3-codex-spark", input: "cancelled", stream: false } + : baseCompactionBody({ model: "gpt-5.3-codex-spark" }), + controller.signal, + ); + const response = endpoint === "regular" + ? await handleResponses( + request, + config, + { model: "", provider: "" }, + { abortSignal: controller.signal }, + ) + : await handleResponsesCompact(request, config, { model: "", provider: "" }); + expect(response.status).toBe(499); + expect(physicalSends).toBe(0); + + Date.now = () => probeAt + CODEX_QUOTA_PROBE_INTERVAL_MS; + const nextProbe = await resolveCodexAuthContext( + new Headers({ authorization: "Bearer main-token" }), + config, + "pool", + { modelId: "gpt-5.3-codex-spark" }, + ); + expect(nextProbe).toMatchObject({ probeQuotaScope: "spark" }); + releaseCodexAuthContextProbeLease(nextProbe); + } finally { + Date.now = originalNow; + globalThis.fetch = originalFetch; + clearCodexUpstreamHealth(); + clearCodexUpstreamHostHealth(); + rmSync(testDir, { recursive: true, force: true }); + if (previousOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousOpencodexHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; + } + }); + } }); describe("routed compaction for key-mode openai-responses (#422)", () => { @@ -571,6 +713,7 @@ describe("compact alternate-account attempt (#913)", () => { process.env.OPENCODEX_HOME = testDir; process.env.CODEX_HOME = testDir; clearCodexUpstreamHealth(); + clearCodexUpstreamHostHealth(); clearAccountQuota(); for (const id of ["pool-a", "pool-b"]) { saveCodexAccountCredential(id, { @@ -584,6 +727,7 @@ describe("compact alternate-account attempt (#913)", () => { return run(twoAccountPoolConfig()).finally(() => { globalThis.fetch = originalFetch; clearCodexUpstreamHealth(); + clearCodexUpstreamHostHealth(); clearAccountQuota(); rmSync(testDir, { recursive: true, force: true }); if (previousOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; @@ -929,4 +1073,114 @@ describe("compact alternate-account attempt (#913)", () => { expect(statuses).toEqual([402, 429]); }); }); + + test("a bare primary rejection changes host health without changing account health", async () => { + await withPoolEnv("ocx-compact-host-primary-", async config => { + let sends = 0; + globalThis.fetch = (async () => { + sends += 1; + throw new Error("opaque compact rejection"); + }) as typeof fetch; + const response = await handleResponsesCompact( + compactionRequest(baseCompactionBody({})), config, { model: "", provider: "" }, + ); + const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com/backend-api/codex/responses/compact")!; + expect(response.status).toBe(502); + expect(sends).toBe(1); + expect(getCodexUpstreamHealth("pool-a")).toBeNull(); + expect(getCodexUpstreamHostHealth(key)?.consecutiveFailures).toBe(1); + }); + }); + + test("a primary 503 followed by rejection keeps the last account evidence and records terminal host failure", async () => { + await withPoolEnv("ocx-compact-host-history-", async config => { + let sends = 0; + globalThis.fetch = (async () => { + sends += 1; + if (sends === 1) return new Response("busy", { status: 503, headers: { "retry-after": "0" } }); + throw new Error("opaque compact rejection"); + }) as typeof fetch; + const response = await handleResponsesCompact( + compactionRequest(baseCompactionBody({})), config, { model: "", provider: "" }, + ); + const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com/backend-api/codex/responses/compact")!; + expect(response.status).toBe(502); + expect(sends).toBe(2); + expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(503); + expect(getCodexUpstreamHostHealth(key)?.consecutiveFailures).toBe(1); + }); + }); + + test("a compact connect timeout changes host health without changing account health", async () => { + await withPoolEnv("ocx-compact-host-timeout-", async config => { + const timeout = new Error("header timeout"); + timeout.name = "TimeoutError"; + globalThis.fetch = (async () => { throw timeout; }) as typeof fetch; + const response = await handleResponsesCompact( + compactionRequest(baseCompactionBody({})), config, { model: "", provider: "" }, + ); + const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com/backend-api/codex/responses/compact")!; + expect(response.status).toBe(502); + expect(getCodexUpstreamHealth("pool-a")).toBeNull(); + expect(getCodexUpstreamHostHealth(key)?.consecutiveFailures).toBe(1); + }); + }); + + test("an open compact host circuit returns Retry-After without another send", async () => { + await withPoolEnv("ocx-compact-host-circuit-", async config => { + let sends = 0; + globalThis.fetch = (async () => { + sends += 1; + throw new Error("opaque compact rejection"); + }) as typeof fetch; + for (let attempt = 0; attempt < 3; attempt++) { + expect((await handleResponsesCompact( + compactionRequest(baseCompactionBody({})), config, { model: "", provider: "" }, + )).status).toBe(502); + } + const blocked = await handleResponsesCompact( + compactionRequest(baseCompactionBody({})), config, { model: "", provider: "" }, + ); + expect(blocked.status).toBe(502); + expect(sends).toBe(3); + expect(Number(blocked.headers.get("retry-after"))).toBeGreaterThanOrEqual(1); + }); + }); + + test("manual compact redirect is bounded and Location is not exposed", async () => { + await withPoolEnv("ocx-compact-host-redirect-", async config => { + let redirectMode: RequestRedirect | undefined; + globalThis.fetch = (async (_url: RequestInfo | URL, init?: RequestInit) => { + redirectMode = init?.redirect; + return new Response(null, { status: 307, headers: { location: "https://dead.example/private-path" } }); + }) as typeof fetch; + const response = await handleResponsesCompact( + compactionRequest(baseCompactionBody({})), config, { model: "", provider: "" }, + ); + expect(response.status).toBe(502); + expect(response.headers.get("location")).toBeNull(); + expect(redirectMode).toBe("manual"); + expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(502); + }); + }); + + test("alternate compact bare rejection is host-only and remains one B send", async () => { + await withPoolEnv("ocx-compact-host-alternate-", async config => { + let sends = 0; + globalThis.fetch = (async () => { + sends += 1; + if (sends === 1) return new Response("quota", { status: 429 }); + throw new Error("opaque alternate rejection"); + }) as typeof fetch; + const response = await handleResponsesCompact( + compactionRequest(baseCompactionBody({})), config, { model: "", provider: "" }, + ); + const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com/backend-api/codex/responses/compact")!; + expect(response.status).toBe(502); + expect(sends).toBe(2); + expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(429); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(getCodexUpstreamHostHealth(key)?.consecutiveFailures).toBe(1); + }); + }); }); diff --git a/tests/upstream-transient-retry.test.ts b/tests/upstream-transient-retry.test.ts index 0f63038138..6d3c9c934b 100644 --- a/tests/upstream-transient-retry.test.ts +++ b/tests/upstream-transient-retry.test.ts @@ -1,5 +1,11 @@ import { describe, expect, test } from "bun:test"; -import { fetchWithTransientRetry, isTransientUpstreamStatus } from "../src/lib/upstream-retry"; +import { + fetchWithResetRetry, + fetchWithTransientRetry, + isTransientUpstreamStatus, + lastUpstreamAttemptResponseStatus, + type UpstreamAttemptObservation, +} from "../src/lib/upstream-retry"; function bodyResponse(status: number, headers?: Record): Response { // ReadableStream body so cancel() is observable. @@ -19,6 +25,41 @@ describe("isTransientUpstreamStatus", () => { }); describe("fetchWithTransientRetry", () => { + test("preserves ordered physical evidence when a 503 is followed by rejection", async () => { + const observations: UpstreamAttemptObservation[] = []; + const rejection = new Error("opaque transport rejection"); + let calls = 0; + await expect(fetchWithTransientRetry(async () => { + calls += 1; + if (calls === 1) return bodyResponse(503, { "retry-after": "0" }); + throw rejection; + }, { + slowAttemptMs: 60_000, + onAttempt: observation => observations.push(observation), + })).rejects.toBe(rejection); + + expect(calls).toBe(2); + expect(observations).toEqual([ + { kind: "response", status: 503 }, + { kind: "rejection", recovery: "transient-5xx" }, + ]); + expect(lastUpstreamAttemptResponseStatus(observations)).toBe(503); + }); + + test("observer exceptions do not replace a successful response", async () => { + const response = await fetchWithResetRetry(async () => bodyResponse(200), { + onAttempt: () => { throw new Error("observer failed"); }, + }); + expect(response.status).toBe(200); + }); + + test("observer exceptions do not replace the original rejection", async () => { + const rejection = new Error("opaque transport rejection"); + await expect(fetchWithResetRetry(async () => { throw rejection; }, { + onAttempt: () => { throw new Error("observer failed"); }, + })).rejects.toBe(rejection); + }); + test("retries a 502 then returns the 200; failed body is cancelled", async () => { const first = bodyResponse(502) as Response & { __wasCancelled: () => boolean }; const responses = [first, bodyResponse(200)]; From 4ba9703fed4be7b52a1d4cca39dbec899e9a432b Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 07:18:41 +0900 Subject: [PATCH 02/20] fix(codex): fence host admission ownership --- src/codex/upstream-host-health.ts | 244 ++++++++++++----- src/server/responses/compact.ts | 103 ++++++-- src/server/responses/core.ts | 231 +++++++++-------- tests/codex-upstream-host-health.test.ts | 132 +++++++--- tests/responses-compaction-routing.test.ts | 288 ++++++++++++++++++++- 5 files changed, 759 insertions(+), 239 deletions(-) diff --git a/src/codex/upstream-host-health.ts b/src/codex/upstream-host-health.ts index 83c09f7d34..440f6e9a39 100644 --- a/src/codex/upstream-host-health.ts +++ b/src/codex/upstream-host-health.ts @@ -6,17 +6,27 @@ export interface CodexUpstreamHostHealthSnapshot { cooldownUntil?: number; } -export type CodexUpstreamHostProbeLease = Readonly<{ +/** Opaque capability tying one logical request to the host generation that admitted it. */ +export type CodexUpstreamHostAdmissionLease = Readonly<{ key: CodexUpstreamHostKey; leaseId: symbol; + generation: number; + halfOpen: boolean; }>; export type CodexUpstreamHostAdmission = - | { kind: "admitted"; probeLease: CodexUpstreamHostProbeLease | null } + | { kind: "admitted"; lease: CodexUpstreamHostAdmissionLease } | { kind: "blocked"; retryAfterSeconds: number }; +export interface CodexUpstreamHostFailureOptions { + /** This logical request received an HTTP response before its terminal rejection. */ + observedResponse?: boolean; +} + type CodexUpstreamHostHealth = CodexUpstreamHostHealthSnapshot & { lastTouchedAt: number; + generation: number; + activeLeaseIds: Set; halfOpenLeaseId?: symbol; }; @@ -26,6 +36,12 @@ export const CODEX_UPSTREAM_HOST_COOLDOWN_MS = 30_000; export const CODEX_UPSTREAM_HOST_MAX_ENTRIES = 128; const upstreamHostHealth = new Map(); +let nextGenerationValue = 0; + +function nextGeneration(): number { + nextGenerationValue = nextGenerationValue >= Number.MAX_SAFE_INTEGER ? 1 : nextGenerationValue + 1; + return nextGenerationValue; +} function normalizedAuthority(url: URL): string | null { if (url.protocol !== "http:" && url.protocol !== "https:") return null; @@ -59,41 +75,95 @@ function snapshot(health: CodexUpstreamHostHealth): CodexUpstreamHostHealthSnaps }; } -function removeExpiredEntries(now: number): void { +function removeExpiredNonLeasedEntries(now: number): void { for (const [key, health] of upstreamHostHealth) { - // A tripped circuit survives its cooldown so the next logical request must - // pass through the atomic half-open admission below. The bounded map still - // evicts abandoned entries when capacity is needed. - if (health.cooldownUntil === undefined - && now - health.lastFailureAt > CODEX_UPSTREAM_HOST_FAILURE_WINDOW_MS) { + if (health.activeLeaseIds.size > 0) continue; + if (health.consecutiveFailures === 0 || ( + health.cooldownUntil === undefined + && now - health.lastFailureAt > CODEX_UPSTREAM_HOST_FAILURE_WINDOW_MS + )) { upstreamHostHealth.delete(key); } } } +function oldestNonLeasedKey(): CodexUpstreamHostKey | undefined { + let oldestKey: CodexUpstreamHostKey | undefined; + let oldestAt = Number.POSITIVE_INFINITY; + for (const [key, health] of upstreamHostHealth) { + if (health.activeLeaseIds.size > 0) continue; + if (health.lastTouchedAt < oldestAt) { + oldestKey = key; + oldestAt = health.lastTouchedAt; + } + } + return oldestKey; +} + function makeRoom(now: number): void { - removeExpiredEntries(now); + removeExpiredNonLeasedEntries(now); while (upstreamHostHealth.size >= CODEX_UPSTREAM_HOST_MAX_ENTRIES) { - let oldestKey: CodexUpstreamHostKey | undefined; - let oldestAt = Number.POSITIVE_INFINITY; - for (const [key, health] of upstreamHostHealth) { - if (health.lastTouchedAt < oldestAt) { - oldestKey = key; - oldestAt = health.lastTouchedAt; - } - } - if (!oldestKey) break; + const oldestKey = oldestNonLeasedKey(); + if (!oldestKey) return; // Every entry is leased: preserve correctness with temporary overflow. upstreamHostHealth.delete(oldestKey); } } +function pruneOverflow(now: number): void { + removeExpiredNonLeasedEntries(now); + while (upstreamHostHealth.size > CODEX_UPSTREAM_HOST_MAX_ENTRIES) { + const oldestKey = oldestNonLeasedKey(); + if (!oldestKey) return; + upstreamHostHealth.delete(oldestKey); + } +} + +function newHealthyState(now: number): CodexUpstreamHostHealth { + return { + consecutiveFailures: 0, + lastFailureAt: 0, + lastTouchedAt: now, + generation: nextGeneration(), + activeLeaseIds: new Set(), + }; +} + +function advanceGeneration(health: CodexUpstreamHostHealth): void { + health.generation = nextGeneration(); + health.activeLeaseIds.clear(); + delete health.halfOpenLeaseId; +} + +function issueLease( + key: CodexUpstreamHostKey, + health: CodexUpstreamHostHealth, + halfOpen: boolean, + now: number, +): CodexUpstreamHostAdmissionLease { + const leaseId = Symbol(halfOpen ? "codex-upstream-host-half-open" : "codex-upstream-host-admission"); + health.activeLeaseIds.add(leaseId); + health.lastTouchedAt = now; + if (halfOpen) health.halfOpenLeaseId = leaseId; + return { key, leaseId, generation: health.generation, halfOpen }; +} + +function matchingHealth(lease: CodexUpstreamHostAdmissionLease): CodexUpstreamHostHealth | null { + const health = upstreamHostHealth.get(lease.key); + if (!health || health.generation !== lease.generation || !health.activeLeaseIds.has(lease.leaseId)) { + return null; + } + if (lease.halfOpen && health.halfOpenLeaseId !== lease.leaseId) return null; + return health; +} + export function getCodexUpstreamHostHealth( key: CodexUpstreamHostKey, now = Date.now(), ): CodexUpstreamHostHealthSnapshot | null { const health = upstreamHostHealth.get(key); - if (!health) return null; - if (health.cooldownUntil === undefined + if (!health || health.consecutiveFailures === 0) return null; + if (health.activeLeaseIds.size === 0 + && health.cooldownUntil === undefined && now - health.lastFailureAt > CODEX_UPSTREAM_HOST_FAILURE_WINDOW_MS) { upstreamHostHealth.delete(key); return null; @@ -106,8 +176,7 @@ export function getCodexUpstreamHostCooldownUntil( now = Date.now(), ): number | null { const health = upstreamHostHealth.get(key); - if (!health?.cooldownUntil) return null; - if (health.cooldownUntil <= now) return null; + if (!health?.cooldownUntil || health.cooldownUntil <= now) return null; return health.cooldownUntil; } @@ -115,69 +184,106 @@ export function acquireCodexUpstreamHostAdmission( key: CodexUpstreamHostKey, now = Date.now(), ): CodexUpstreamHostAdmission { - const health = upstreamHostHealth.get(key); - if (!health) return { kind: "admitted", probeLease: null }; - if (health.cooldownUntil === undefined) { - if (now - health.lastFailureAt > CODEX_UPSTREAM_HOST_FAILURE_WINDOW_MS) { - upstreamHostHealth.delete(key); - } - return { kind: "admitted", probeLease: null }; + pruneOverflow(now); + let health = upstreamHostHealth.get(key); + if (health?.activeLeaseIds.size === 0 + && health.cooldownUntil === undefined + && health.consecutiveFailures > 0 + && now - health.lastFailureAt > CODEX_UPSTREAM_HOST_FAILURE_WINDOW_MS) { + upstreamHostHealth.delete(key); + health = undefined; } - if (health.cooldownUntil > now) { - return { - kind: "blocked", - retryAfterSeconds: Math.max(1, Math.ceil((health.cooldownUntil - now) / 1_000)), - }; + if (!health) { + makeRoom(now); + health = newHealthyState(now); + upstreamHostHealth.set(key, health); } - if (health.halfOpenLeaseId !== undefined) { - return { kind: "blocked", retryAfterSeconds: 1 }; + if (health.cooldownUntil !== undefined) { + if (health.cooldownUntil > now) { + return { + kind: "blocked", + retryAfterSeconds: Math.max(1, Math.ceil((health.cooldownUntil - now) / 1_000)), + }; + } + if (health.halfOpenLeaseId !== undefined) { + return { kind: "blocked", retryAfterSeconds: 1 }; + } + advanceGeneration(health); + return { kind: "admitted", lease: issueLease(key, health, true, now) }; } - - const leaseId = Symbol("codex-upstream-host-probe"); - health.halfOpenLeaseId = leaseId; - health.lastTouchedAt = now; - return { kind: "admitted", probeLease: { key, leaseId } }; + return { kind: "admitted", lease: issueLease(key, health, false, now) }; } -/** Release a half-open probe without recording host or account evidence. */ -export function releaseCodexUpstreamHostProbeLease( - lease: CodexUpstreamHostProbeLease | null | undefined, +/** Release an admitted request without recording host or account evidence. */ +export function releaseCodexUpstreamHostAdmissionLease( + lease: CodexUpstreamHostAdmissionLease | null | undefined, + now = Date.now(), ): boolean { if (!lease) return false; - const health = upstreamHostHealth.get(lease.key); - if (!health || health.halfOpenLeaseId !== lease.leaseId) return false; - delete health.halfOpenLeaseId; + const health = matchingHealth(lease); + if (!health) return false; + health.activeLeaseIds.delete(lease.leaseId); + if (health.halfOpenLeaseId === lease.leaseId) delete health.halfOpenLeaseId; + health.lastTouchedAt = now; + if (health.activeLeaseIds.size === 0 && health.consecutiveFailures === 0) { + upstreamHostHealth.delete(lease.key); + } + pruneOverflow(now); return true; } export function recordCodexUpstreamHostFailure( - key: CodexUpstreamHostKey, + lease: CodexUpstreamHostAdmissionLease, now = Date.now(), -): CodexUpstreamHostHealthSnapshot { - const current = upstreamHostHealth.get(key); - const reopensCircuit = current?.cooldownUntil !== undefined; - const stale = !current || (!reopensCircuit - && now - current.lastFailureAt > CODEX_UPSTREAM_HOST_FAILURE_WINDOW_MS); - const consecutiveFailures = reopensCircuit + options: CodexUpstreamHostFailureOptions = {}, +): CodexUpstreamHostHealthSnapshot | null { + const current = matchingHealth(lease); + if (!current) return null; + current.activeLeaseIds.delete(lease.leaseId); + if (current.halfOpenLeaseId === lease.leaseId) delete current.halfOpenLeaseId; + + if (options.observedResponse) { + upstreamHostHealth.delete(lease.key); + makeRoom(now); + const afterResponse: CodexUpstreamHostHealth = { + consecutiveFailures: 1, + lastFailureAt: now, + lastTouchedAt: now, + generation: nextGeneration(), + activeLeaseIds: new Set(), + }; + upstreamHostHealth.set(lease.key, afterResponse); + pruneOverflow(now); + return snapshot(afterResponse); + } + + const reopensCircuit = lease.halfOpen || current.cooldownUntil !== undefined; + const stale = current.consecutiveFailures === 0 + || (!reopensCircuit && now - current.lastFailureAt > CODEX_UPSTREAM_HOST_FAILURE_WINDOW_MS); + current.consecutiveFailures = reopensCircuit ? Math.max(CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD, current.consecutiveFailures + 1) : stale ? 1 : current.consecutiveFailures + 1; - const cooldownUntil = reopensCircuit || consecutiveFailures >= CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD - ? now + CODEX_UPSTREAM_HOST_COOLDOWN_MS - : undefined; - if (!current) makeRoom(now); - const next: CodexUpstreamHostHealth = { - consecutiveFailures, - lastFailureAt: now, - lastTouchedAt: now, - ...(cooldownUntil !== undefined ? { cooldownUntil } : {}), - }; - upstreamHostHealth.set(key, next); - return snapshot(next); + current.lastFailureAt = now; + current.lastTouchedAt = now; + if (reopensCircuit || current.consecutiveFailures >= CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD) { + current.cooldownUntil = now + CODEX_UPSTREAM_HOST_COOLDOWN_MS; + advanceGeneration(current); + } else { + delete current.cooldownUntil; + } + pruneOverflow(now); + return snapshot(current); } -/** Any HTTP response proves that the configured provider host was reachable. */ -export function recordCodexUpstreamHostResponse(key: CodexUpstreamHostKey): void { - upstreamHostHealth.delete(key); +/** Any HTTP response from this admitted logical request proves the host was reachable. */ +export function recordCodexUpstreamHostResponse( + lease: CodexUpstreamHostAdmissionLease, + now = Date.now(), +): boolean { + if (!matchingHealth(lease)) return false; + upstreamHostHealth.delete(lease.key); + pruneOverflow(now); + return true; } export function isCodexUpstreamRedirectStatus(status: number): boolean { diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index 11cee901d6..65b035020c 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -76,8 +76,8 @@ import { isCodexUpstreamRedirectStatus, recordCodexUpstreamHostFailure, recordCodexUpstreamHostResponse, - releaseCodexUpstreamHostProbeLease, - type CodexUpstreamHostProbeLease, + releaseCodexUpstreamHostAdmissionLease, + type CodexUpstreamHostAdmissionLease, } from "../../codex/upstream-host-health"; import { ForwardAdmissionCredentialError, validateForwardAdmissionCredential } from "../auth-cors"; import { listOpenAiForwardSidecarCandidates, resolveFirstUsableOpenAiSidecar, type ResolvedOpenAiForwardSidecar } from "../../providers/openai-sidecar"; @@ -156,13 +156,17 @@ async function resolveAlternateCompactContext(args: { }): Promise<{ authCtx: CodexAuthContext; provider: OcxProviderConfig; headers: Headers } | null> { const { req, config, route, selectedModelId, excludeAccountId, turnAdmissionLease } = args; if (!route.codexAccountMode || !excludeAccountId) return null; + let authCtx: CodexAuthContext | undefined; try { - const authCtx = await resolveCodexAuthContext(req.headers, config, route.codexAccountMode, { + authCtx = await resolveCodexAuthContext(req.headers, config, route.codexAccountMode, { ...(selectedModelId ? { modelId: selectedModelId } : {}), excludeAccountId, beginCodexAccountSelection: codexAccountSelectionForTurn(turnAdmissionLease), }); - if (!authCtx.accountId || authCtx.accountId === excludeAccountId) return null; + if (!authCtx.accountId || authCtx.accountId === excludeAccountId) { + releaseCodexAuthContextProbeLease(authCtx); + return null; + } const provider = applyCodexAuthContextToProvider(route.provider, authCtx, route.codexAccountMode); const headers = new Headers({ "content-type": "application/json" }); const selected = headersForCodexAuthContext(req.headers, authCtx); @@ -178,6 +182,7 @@ async function resolveAlternateCompactContext(args: { if (provider.apiKey) headers.set("authorization", `Bearer ${resolveEnvValue(provider.apiKey)}`); return { authCtx, provider, headers }; } catch (err) { + releaseCodexAuthContextProbeLease(authCtx); if (err instanceof CodexMainProfileDrainingError) { // The native-main fence can start after account A has already rejected the // request. Treat the now-fenced main profile as no alternate and preserve A's @@ -365,7 +370,12 @@ export async function handleResponsesCompact( const compactHostKey = usesCodexForwardPoolAuth(authCtx, route.provider) ? canonicalCodexUpstreamHostKey(route.providerName, compactUrl) : null; - let compactHostProbeLease: CodexUpstreamHostProbeLease | null = null; + let compactHostAdmissionLease: CodexUpstreamHostAdmissionLease | null = null; + const settleObservedCompactHostResponse = (): void => { + if (!compactHostAdmissionLease) return; + recordCodexUpstreamHostResponse(compactHostAdmissionLease); + compactHostAdmissionLease = null; + }; // Takes its context explicitly: the alternate-account flow below records a rejection // against A while promoting B, then records B's own outcome. A closure over a single // `authCtx` cannot express either. @@ -401,20 +411,27 @@ export async function handleResponsesCompact( sendHeaders: Headers, recovery: "normal" | "single", attempts: UpstreamAttemptObservation[], + onSendStart?: () => void, ): Promise => { - const doFetch = (upstreamRecovery?: UpstreamSendRecovery) => fetchWithHeaderTimeout( - compactUrl, - applyUpstreamRecoveryInit({ + const body = JSON.stringify({ ...compactBody, model: route.modelId }); + const fetcher = providerFetch(sendProvider); + let executorStarted = false; + const executor = ((input: RequestInfo | URL, init?: RequestInit) => { + if (!executorStarted) { + executorStarted = true; + onSendStart?.(); + } + return fetcher(input, init); + }) as typeof globalThis.fetch; + const doFetch = (upstreamRecovery?: UpstreamSendRecovery) => { + const init = applyUpstreamRecoveryInit({ method: "POST", headers: sendHeaders, - body: JSON.stringify({ ...compactBody, model: route.modelId }), + body, ...(compactHostKey ? { redirect: "manual" as const } : {}), - }, upstreamRecovery), - req.signal, - connectMs, - false, - providerFetch(sendProvider), - ); + }, upstreamRecovery); + return fetchWithHeaderTimeout(compactUrl, init, req.signal, connectMs, false, executor); + }; if (recovery === "normal") { return fetchWithTransientRetry(doFetch, { abortSignal: req.signal, @@ -436,20 +453,29 @@ export async function handleResponsesCompact( ctx: CodexAuthContext, err: unknown, attempts: readonly UpstreamAttemptObservation[], + hostResponseObserved = false, ): Response => { + const observedStatus = lastUpstreamAttemptResponseStatus(attempts); if (req.signal.aborted) { - releaseCodexUpstreamHostProbeLease(compactHostProbeLease); + if (hostResponseObserved || observedStatus !== undefined) { + settleObservedCompactHostResponse(); + } else { + releaseCodexUpstreamHostAdmissionLease(compactHostAdmissionLease); + compactHostAdmissionLease = null; + } recordCompactPoolOutcome(ctx, 499); return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); } - const observedStatus = lastUpstreamAttemptResponseStatus(attempts); if (observedStatus !== undefined) { recordCompactPoolOutcome(ctx, observedStatus); - if (compactHostKey) recordCodexUpstreamHostResponse(compactHostKey); - } else if (compactHostKey) { + } else if (compactHostAdmissionLease) { releaseCodexAuthContextProbeLease(ctx); } - if (compactHostKey) recordCodexUpstreamHostFailure(compactHostKey); + if (compactHostAdmissionLease) { + recordCodexUpstreamHostFailure(compactHostAdmissionLease, Date.now(), { + observedResponse: hostResponseObserved || observedStatus !== undefined, + }); + } return formatErrorResponse(502, "upstream_error", "Failed to connect to compact upstream"); }; @@ -470,17 +496,17 @@ export async function handleResponsesCompact( retryAfter: String(compactHostAdmission.retryAfterSeconds), }); } - compactHostProbeLease = compactHostAdmission?.probeLease ?? null; + compactHostAdmissionLease = compactHostAdmission?.lease ?? null; const primaryAttempts: UpstreamAttemptObservation[] = []; try { // Same connect timeout + keep-alive reset + transient-5xx recovery as /v1/responses — // compact hits the same ChatGPT host and must soft-avoid / clear affinity (#186). upstream = await sendCompactAttempt(compactProvider, headers, "normal", primaryAttempts); - if (compactHostKey) recordCodexUpstreamHostResponse(compactHostKey); } catch (err) { return compactTransportFailureResponse(outcomeCtx, err, primaryAttempts); } if (compactHostKey && isCodexUpstreamRedirectStatus(upstream.status)) { + settleObservedCompactHostResponse(); recordCompactPoolOutcome(outcomeCtx, 502); await upstream.body?.cancel().catch(() => undefined); return formatErrorResponse(502, "upstream_error", "Provider returned an unsupported redirect"); @@ -490,13 +516,15 @@ export async function handleResponsesCompact( // (core.ts:319-423) and recognizes exactly 429/402. Without it a pool rejection // surfaces to the client, which retries the compact task OUTSIDE the logical request // — reporting exhausted retries while another pool account sat idle (#913). - if ( + let pendingAlternateAuthCtx: CodexAuthContext | null = null; + try { + if ( (upstream.status === 429 || upstream.status === 402) && usesCodexForwardPoolAuth(authCtx, route.provider) && !authCtx.fixedAccount && route.codexAccountMode && !req.signal.aborted - ) { + ) { const firstRetryAfter = upstream.headers.get("retry-after"); const firstResetAt = [ upstream.headers.get("x-codex-primary-reset-at"), @@ -513,13 +541,15 @@ export async function handleResponsesCompact( excludeAccountId: authCtx.accountId, turnAdmissionLease, }); + pendingAlternateAuthCtx = alternate?.authCtx ?? null; // Resolution can await a credential refresh, so the client may have gone away // while we were choosing B. Re-check before spending anything: recording A, // cancelling its body, and sending B are all observable side effects, and B's // quota is not ours to spend on a request nobody is waiting for. if (alternate && req.signal.aborted) { releaseCodexAuthContextProbeLease(alternate.authCtx); - releaseCodexUpstreamHostProbeLease(compactHostProbeLease); + pendingAlternateAuthCtx = null; + settleObservedCompactHostResponse(); recordCompactPoolOutcome(outcomeCtx, 499); return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); } @@ -544,19 +574,36 @@ export async function handleResponsesCompact( await upstream.body?.cancel().catch(() => undefined); outcomeCtx = alternate.authCtx; const alternateAttempts: UpstreamAttemptObservation[] = []; + let alternateSendBegan = false; try { - upstream = await sendCompactAttempt(alternate.provider, alternate.headers, "single", alternateAttempts); - if (compactHostKey) recordCodexUpstreamHostResponse(compactHostKey); + upstream = await sendCompactAttempt( + alternate.provider, + alternate.headers, + "single", + alternateAttempts, + () => { + alternateSendBegan = true; + pendingAlternateAuthCtx = null; + }, + ); } catch (err) { - return compactTransportFailureResponse(outcomeCtx, err, alternateAttempts); + if (!alternateSendBegan) throw err; + return compactTransportFailureResponse(outcomeCtx, err, alternateAttempts, true); } if (compactHostKey && isCodexUpstreamRedirectStatus(upstream.status)) { + settleObservedCompactHostResponse(); recordCompactPoolOutcome(outcomeCtx, 502); await upstream.body?.cancel().catch(() => undefined); return formatErrorResponse(502, "upstream_error", "Provider returned an unsupported redirect"); } } } + } catch (error) { + releaseCodexAuthContextProbeLease(pendingAlternateAuthCtx ?? undefined); + settleObservedCompactHostResponse(); + throw error; + } + settleObservedCompactHostResponse(); const retryAfter = upstream.headers.get("retry-after"); const resetAt = [ upstream.headers.get("x-codex-primary-reset-at"), diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 9c2662fd63..af41dc187d 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -101,9 +101,8 @@ import { isCodexUpstreamRedirectStatus, recordCodexUpstreamHostFailure, recordCodexUpstreamHostResponse, - releaseCodexUpstreamHostProbeLease, - type CodexUpstreamHostKey, - type CodexUpstreamHostProbeLease, + releaseCodexUpstreamHostAdmissionLease, + type CodexUpstreamHostAdmissionLease, } from "../../codex/upstream-host-health"; import { ForwardAdmissionCredentialError, validateForwardAdmissionCredential } from "../auth-cors"; import { createTranslatorBudget, isTranslatorBudgetExceededError, type TranslatorBudget } from "../../lib/translator-budget"; @@ -323,7 +322,6 @@ type CodexPoolAccountRetryResult = error: unknown; authCtx: Extract; attempts: UpstreamAttemptObservation[]; - hostKey: CodexUpstreamHostKey | null; }; function codexQuotaOutcomeMeta(response: Response): { @@ -390,54 +388,65 @@ async function retryCodexPoolOnAlternateAccount( return { kind: "no-alternate" }; } - const quotaMeta = codexQuotaOutcomeMeta(firstResponse); - if (outcomeStatus === 429 || outcomeStatus === 402) { - const { applyAccountQuotaFromUpstreamHeaders } = await import("../../codex/auth-api"); - applyAccountQuotaFromUpstreamHeaders( - firstAuthCtx.accountId, - firstResponse.headers, - firstAuthCtx.writerGeneration, - ); - } - if (!shouldDeferCodexResetDerivedCooldown(firstResponse, options.deferCodexResetDerivedCooldown)) { - recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { - ...quotaMeta, - threadId: req.headers.get("x-codex-parent-thread-id"), - modelId: route.modelId, - probeLeaseId: codexProbeLeaseId(firstAuthCtx), - probeQuotaScope: codexProbeQuotaScope(firstAuthCtx), - writerGeneration: firstAuthCtx.writerGeneration, - // Retry already advanced the RR ring via excludeAccountId — reuse for promotion. - ...(retryAuthCtx.accountId ? { promoteAccountId: retryAuthCtx.accountId } : {}), - }); - } - - const retryHeaders = headersForCodexAuthContext(req.headers, retryAuthCtx); - const retryProvider = applyCodexAuthContextToProvider( - stripCodexRuntimeProviderFields(route.provider), - retryAuthCtx, - "pool", - ); - const retryAdapter = resolveAdapter( - resolveWireProtocolOverride(route.providerName, route.modelId, retryProvider, inboundWire), - config.cacheRetention, - ); - const request = await retryAdapter.buildRequest(parsed, { - headers: retryHeaders, - translatorBudget: options.translatorBudget, - }); - recordAdapterReasoning(logCtx, request); + const prepared = await (async () => { + let request: Awaited["buildRequest"]>> | undefined; + try { + const quotaMeta = codexQuotaOutcomeMeta(firstResponse); + if (outcomeStatus === 429 || outcomeStatus === 402) { + const { applyAccountQuotaFromUpstreamHeaders } = await import("../../codex/auth-api"); + applyAccountQuotaFromUpstreamHeaders( + firstAuthCtx.accountId, + firstResponse.headers, + firstAuthCtx.writerGeneration, + ); + } + if (!shouldDeferCodexResetDerivedCooldown(firstResponse, options.deferCodexResetDerivedCooldown)) { + recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { + ...quotaMeta, + threadId: req.headers.get("x-codex-parent-thread-id"), + modelId: route.modelId, + probeLeaseId: codexProbeLeaseId(firstAuthCtx), + probeQuotaScope: codexProbeQuotaScope(firstAuthCtx), + writerGeneration: firstAuthCtx.writerGeneration, + // Retry already advanced the RR ring via excludeAccountId — reuse for promotion. + ...(retryAuthCtx.accountId ? { promoteAccountId: retryAuthCtx.accountId } : {}), + }); + } - await firstResponse.body?.cancel().catch(() => undefined); - options.onCodexAuthContextResolved?.(retryAuthCtx); - route.provider = retryProvider; - logCtx.provider = formatCodexProviderForLog( - route.providerName, - retryAuthCtx.accountId, - config, - ); + const retryHeaders = headersForCodexAuthContext(req.headers, retryAuthCtx); + const retryProvider = applyCodexAuthContextToProvider( + stripCodexRuntimeProviderFields(route.provider), + retryAuthCtx, + "pool", + ); + const retryAdapter = resolveAdapter( + resolveWireProtocolOverride(route.providerName, route.modelId, retryProvider, inboundWire), + config.cacheRetention, + ); + request = await retryAdapter.buildRequest(parsed, { + headers: retryHeaders, + translatorBudget: options.translatorBudget, + }); + recordAdapterReasoning(logCtx, request); + + await firstResponse.body?.cancel().catch(() => undefined); + options.onCodexAuthContextResolved?.(retryAuthCtx); + route.provider = retryProvider; + logCtx.provider = formatCodexProviderForLog( + route.providerName, + retryAuthCtx.accountId, + config, + ); + const fetcher = providerFetch(route.provider); + return { fetcher, request, retryHeaders }; + } catch (error) { + request?.releaseBodyObservation?.(); + releaseCodexAuthContextProbeLease(retryAuthCtx); + throw error; + } + })(); + const { fetcher, request, retryHeaders } = prepared; - const hostKey = canonicalCodexUpstreamHostKey(route.providerName, request.url); const attempts: UpstreamAttemptObservation[] = []; noteAttemptSend(logCtx.activeAttempt, passthroughEstimate); try { @@ -452,10 +461,9 @@ async function retryCodexPoolOnAlternateAccount( upstream.signal, connectMs, stream, - providerFetch(route.provider), + fetcher, ); attempts.push({ kind: "response", status: upstreamResponse.status }); - if (hostKey) recordCodexUpstreamHostResponse(hostKey); return { kind: "retried", authCtx: retryAuthCtx, @@ -465,7 +473,7 @@ async function retryCodexPoolOnAlternateAccount( }; } catch (error) { attempts.push({ kind: "rejection" }); - return { kind: "transport", error, authCtx: retryAuthCtx, attempts, hostKey }; + return { kind: "transport", error, authCtx: retryAuthCtx, attempts }; } finally { request.releaseBodyObservation?.(); } @@ -1754,7 +1762,7 @@ async function handleResponsesInner( const hostKey = tracksCodexPoolHost ? canonicalCodexUpstreamHostKey(route.providerName, request.url) : null; - let hostProbeLease: CodexUpstreamHostProbeLease | null = null; + let hostAdmissionLease: CodexUpstreamHostAdmissionLease | null = null; const attemptHistory: UpstreamAttemptObservation[] = []; const recordPoolTransportOutcome = (outcome: CodexUpstreamOutcome): void => { if (!usesCodexForwardPoolAuth(authCtx, route.provider)) return; @@ -1767,25 +1775,38 @@ async function handleResponsesInner( writerGeneration: authCtx.writerGeneration, }); }; + const settleObservedHostResponse = (): void => { + if (!hostAdmissionLease) return; + recordCodexUpstreamHostResponse(hostAdmissionLease); + hostAdmissionLease = null; + }; const transportFailureResponse = ( err: unknown, observations: readonly UpstreamAttemptObservation[] = attemptHistory, - failedHostKey: CodexUpstreamHostKey | null = hostKey, + hostResponseObserved = false, ): Response => { upstream.abort(); + const observedStatus = lastUpstreamAttemptResponseStatus(observations); if (options.abortSignal?.aborted) { releaseCodexAuthContextProbeLease(authCtx); - releaseCodexUpstreamHostProbeLease(hostProbeLease); + if (hostResponseObserved || observedStatus !== undefined) { + settleObservedHostResponse(); + } else { + releaseCodexUpstreamHostAdmissionLease(hostAdmissionLease); + hostAdmissionLease = null; + } return clientCancelledResponse(); } - const observedStatus = lastUpstreamAttemptResponseStatus(observations); if (observedStatus !== undefined) { recordPoolTransportOutcome(observedStatus); - if (failedHostKey) recordCodexUpstreamHostResponse(failedHostKey); - } else if (failedHostKey) { + } else if (hostAdmissionLease) { releaseCodexAuthContextProbeLease(authCtx); } - if (failedHostKey) recordCodexUpstreamHostFailure(failedHostKey); + if (hostAdmissionLease) { + recordCodexUpstreamHostFailure(hostAdmissionLease, Date.now(), { + observedResponse: hostResponseObserved || observedStatus !== undefined, + }); + } const msg = err instanceof Error && err.name === "TimeoutError" ? `Provider connect timeout after ${connectMs}ms` : describeUpstreamConnectFailure(err, connectMs); @@ -1803,7 +1824,7 @@ async function handleResponsesInner( retryAfter: String(hostAdmission.retryAfterSeconds), }); } - hostProbeLease = hostAdmission?.probeLease ?? null; + hostAdmissionLease = hostAdmission?.lease ?? null; // Transient-5xx pre-stream retry (devlog/_plan/260716_claudecode_hardening/010): // the ChatGPT backend emits transient 502/520s that an immediate retry absorbs. // Body is a replayable string; nothing has streamed to the client yet. @@ -1828,8 +1849,6 @@ async function handleResponsesInner( } finally { request.releaseBodyObservation?.(); } - if (hostKey) recordCodexUpstreamHostResponse(hostKey); - // Same-target 429 wait-and-retry (opt-in `retryOn429`) for key-auth providers on the // passthrough wire. This branch returns before the recovery loop below, so Responses-shaped // key-auth gateways (e.g. the built-in DeepSeek preset) would otherwise surface 429 @@ -1883,49 +1902,59 @@ async function handleResponsesInner( } } - if (usesCodexForwardPoolAuth(authCtx, route.provider) && !authCtx.fixedAccount) { - let poolRetryOutcome: number | undefined; - if (await shouldRetryCodexPoolAccountModel400( - upstreamResponse, - route.modelId, - options.abortSignal, - )) { - poolRetryOutcome = 400; - } else if (shouldRetryCodexPoolAccountQuota(upstreamResponse)) { - // Pre-stream only: once SSE has begun, mid-stream quota stays terminal. - poolRetryOutcome = upstreamResponse.status; - } + try { + if (usesCodexForwardPoolAuth(authCtx, route.provider) && !authCtx.fixedAccount) { + let poolRetryOutcome: number | undefined; + if (await shouldRetryCodexPoolAccountModel400( + upstreamResponse, + route.modelId, + options.abortSignal, + )) { + poolRetryOutcome = 400; + } else if (shouldRetryCodexPoolAccountQuota(upstreamResponse)) { + // Pre-stream only: once SSE has begun, mid-stream quota stays terminal. + poolRetryOutcome = upstreamResponse.status; - if (poolRetryOutcome !== undefined) { - const retry = await retryCodexPoolOnAlternateAccount({ - req, - config, - route, - parsed, - logCtx, - options, - firstAuthCtx: authCtx, - firstResponse: upstreamResponse, - outcomeStatus: poolRetryOutcome, - upstream, - connectMs, - passthroughEstimate, - stream: parsed.stream, - }); - if (retry.kind === "transport") { - authCtx = retry.authCtx; - return transportFailureResponse(retry.error, retry.attempts, retry.hostKey); } - if (retry.kind === "retried") { - authCtx = retry.authCtx; - request = retry.request; - upstreamResponse = retry.upstreamResponse; - selectedForwardHeaders = retry.selectedForwardHeaders; - // Keep subagent quota-failure health keyed to the account that actually served. - subagentFallbackAccountId = retry.authCtx.accountId; + + if (poolRetryOutcome !== undefined) { + const retry = await retryCodexPoolOnAlternateAccount({ + req, + config, + route, + parsed, + logCtx, + options, + firstAuthCtx: authCtx, + firstResponse: upstreamResponse, + outcomeStatus: poolRetryOutcome, + upstream, + connectMs, + passthroughEstimate, + stream: parsed.stream, + }); + if (retry.kind === "transport") { + authCtx = retry.authCtx; + return transportFailureResponse(retry.error, retry.attempts, true); + } + if (retry.kind === "retried") { + authCtx = retry.authCtx; + request = retry.request; + upstreamResponse = retry.upstreamResponse; + selectedForwardHeaders = retry.selectedForwardHeaders; + // Keep subagent quota-failure health keyed to the account that actually served. + subagentFallbackAccountId = retry.authCtx.accountId; + } } } + } catch (error) { + // The primary request produced a real HTTP response. Alternate auth/import/ + // adapter preparation can still throw before returning a typed retry result; + // settle that response so a half-open admission cannot leak indefinitely. + settleObservedHostResponse(); + throw error; } + settleObservedHostResponse(); if (hostKey && isCodexUpstreamRedirectStatus(upstreamResponse.status)) { recordPoolTransportOutcome(502); await upstreamResponse.body?.cancel().catch(() => undefined); diff --git a/tests/codex-upstream-host-health.test.ts b/tests/codex-upstream-host-health.test.ts index 7755a1cdd7..6925a3e3af 100644 --- a/tests/codex-upstream-host-health.test.ts +++ b/tests/codex-upstream-host-health.test.ts @@ -11,11 +11,27 @@ import { getCodexUpstreamHostHealth, recordCodexUpstreamHostFailure, recordCodexUpstreamHostResponse, - releaseCodexUpstreamHostProbeLease, + releaseCodexUpstreamHostAdmissionLease, + type CodexUpstreamHostAdmissionLease, + type CodexUpstreamHostHealthSnapshot, + type CodexUpstreamHostKey, } from "../src/codex/upstream-host-health"; beforeEach(() => clearCodexUpstreamHostHealth()); +function admit(key: CodexUpstreamHostKey, now: number): CodexUpstreamHostAdmissionLease { + const admission = acquireCodexUpstreamHostAdmission(key, now); + expect(admission.kind).toBe("admitted"); + if (admission.kind !== "admitted") throw new Error("expected host admission"); + return admission.lease; +} + +function fail(key: CodexUpstreamHostKey, now: number): CodexUpstreamHostHealthSnapshot { + const health = recordCodexUpstreamHostFailure(admit(key, now), now); + expect(health).not.toBeNull(); + return health!; +} + describe("Codex upstream host health (#914)", () => { test("keys normalized provider plus canonical HTTP origin only", () => { const canonical = canonicalCodexUpstreamHostKey( @@ -28,92 +44,132 @@ describe("Codex upstream host health (#914)", () => { expect(canonicalCodexUpstreamHostKey("openai", "ftp://chatgpt.com/file")).toBeNull(); }); - test("opens a fixed host cooldown only at its own threshold and clears on HTTP response", () => { + test("opens only at its threshold and a half-open HTTP response clears it", () => { const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com/backend-api/codex")!; const now = 1_900_000_000_000; for (let attempt = 1; attempt < CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; attempt++) { - const health = recordCodexUpstreamHostFailure(key, now + attempt); + const health = fail(key, now + attempt); expect(health.consecutiveFailures).toBe(attempt); expect(health.cooldownUntil).toBeUndefined(); } const trippedAt = now + CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; - const tripped = recordCodexUpstreamHostFailure(key, trippedAt); + const tripped = fail(key, trippedAt); expect(tripped.cooldownUntil).toBe(trippedAt + CODEX_UPSTREAM_HOST_COOLDOWN_MS); expect(getCodexUpstreamHostCooldownUntil(key, trippedAt)).toBe(tripped.cooldownUntil!); - recordCodexUpstreamHostResponse(key); - expect(getCodexUpstreamHostHealth(key, trippedAt)).toBeNull(); + const probeAt = tripped.cooldownUntil!; + expect(recordCodexUpstreamHostResponse(admit(key, probeAt), probeAt)).toBe(true); + expect(getCodexUpstreamHostHealth(key, probeAt)).toBeNull(); }); test("an expired window starts a fresh streak", () => { const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com")!; - recordCodexUpstreamHostFailure(key, 1_000); - const next = recordCodexUpstreamHostFailure(key, 1_000 + CODEX_UPSTREAM_HOST_FAILURE_WINDOW_MS + 1); + fail(key, 1_000); + const next = fail(key, 1_000 + CODEX_UPSTREAM_HOST_FAILURE_WINDOW_MS + 1); expect(next.consecutiveFailures).toBe(1); }); test("admits exactly one half-open logical request after cooldown", () => { const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com")!; const now = 1_900_000_000_000; + let tripped: CodexUpstreamHostHealthSnapshot | null = null; for (let attempt = 0; attempt < CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; attempt++) { - recordCodexUpstreamHostFailure(key, now + attempt); + tripped = fail(key, now + attempt); } - const cooldownUntil = now + CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD - 1 - + CODEX_UPSTREAM_HOST_COOLDOWN_MS; - expect(acquireCodexUpstreamHostAdmission(key, cooldownUntil - 1)).toMatchObject({ - kind: "blocked", - }); + const cooldownUntil = tripped!.cooldownUntil!; + expect(acquireCodexUpstreamHostAdmission(key, cooldownUntil - 1)).toMatchObject({ kind: "blocked" }); const probe = acquireCodexUpstreamHostAdmission(key, cooldownUntil); expect(probe.kind).toBe("admitted"); - if (probe.kind !== "admitted") throw new Error("expected half-open admission"); - expect(probe.probeLease).not.toBeNull(); expect(acquireCodexUpstreamHostAdmission(key, cooldownUntil)).toEqual({ kind: "blocked", retryAfterSeconds: 1, }); - - recordCodexUpstreamHostResponse(key); - expect(acquireCodexUpstreamHostAdmission(key, cooldownUntil)).toEqual({ - kind: "admitted", - probeLease: null, - }); + if (probe.kind !== "admitted") throw new Error("expected half-open admission"); + expect(recordCodexUpstreamHostResponse(probe.lease, cooldownUntil)).toBe(true); + expect(getCodexUpstreamHostHealth(key, cooldownUntil)).toBeNull(); }); test("a half-open terminal rejection immediately reopens the cooldown", () => { const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com")!; const trippedAt = 2_000; + let tripped: CodexUpstreamHostHealthSnapshot | null = null; for (let attempt = 0; attempt < CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; attempt++) { - recordCodexUpstreamHostFailure(key, trippedAt); + tripped = fail(key, trippedAt); } - const probeAt = trippedAt + CODEX_UPSTREAM_HOST_COOLDOWN_MS; - expect(acquireCodexUpstreamHostAdmission(key, probeAt).kind).toBe("admitted"); - const reopened = recordCodexUpstreamHostFailure(key, probeAt); + const probeAt = tripped!.cooldownUntil!; + const reopened = recordCodexUpstreamHostFailure(admit(key, probeAt), probeAt)!; expect(reopened.cooldownUntil).toBe(probeAt + CODEX_UPSTREAM_HOST_COOLDOWN_MS); expect(acquireCodexUpstreamHostAdmission(key, probeAt)).toMatchObject({ kind: "blocked" }); }); - test("caller abort releases a half-open probe without adding evidence", () => { + test("caller abort releases a half-open admission without adding evidence", () => { const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com")!; const trippedAt = 3_000; + let tripped: CodexUpstreamHostHealthSnapshot | null = null; + for (let attempt = 0; attempt < CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; attempt++) { + tripped = fail(key, trippedAt); + } + const probeAt = tripped!.cooldownUntil!; + const first = admit(key, probeAt); + const before = getCodexUpstreamHostHealth(key, probeAt); + expect(releaseCodexUpstreamHostAdmissionLease(first, probeAt)).toBe(true); + expect(getCodexUpstreamHostHealth(key, probeAt)).toEqual(before); + expect(admit(key, probeAt).halfOpen).toBe(true); + }); + + test("stale pre-trip success and failure cannot settle a newer half-open generation", () => { + const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com")!; + const now = 4_000; + const staleSuccess = admit(key, now); + const staleFailure = admit(key, now); + let tripped: CodexUpstreamHostHealthSnapshot | null = null; for (let attempt = 0; attempt < CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; attempt++) { - recordCodexUpstreamHostFailure(key, trippedAt); + tripped = fail(key, now + attempt); } - const probeAt = trippedAt + CODEX_UPSTREAM_HOST_COOLDOWN_MS; - const first = acquireCodexUpstreamHostAdmission(key, probeAt); - if (first.kind !== "admitted") throw new Error("expected half-open admission"); + const probeAt = tripped!.cooldownUntil!; + const halfOpen = admit(key, probeAt); const before = getCodexUpstreamHostHealth(key, probeAt); - expect(releaseCodexUpstreamHostProbeLease(first.probeLease)).toBe(true); + + expect(recordCodexUpstreamHostResponse(staleSuccess, probeAt)).toBe(false); + expect(recordCodexUpstreamHostFailure(staleFailure, probeAt)).toBeNull(); expect(getCodexUpstreamHostHealth(key, probeAt)).toEqual(before); - const replacement = acquireCodexUpstreamHostAdmission(key, probeAt); - expect(replacement.kind).toBe("admitted"); - if (replacement.kind !== "admitted") throw new Error("expected replacement probe"); - expect(replacement.probeLease).not.toBeNull(); + expect(acquireCodexUpstreamHostAdmission(key, probeAt)).toEqual({ + kind: "blocked", + retryAfterSeconds: 1, + }); + expect(recordCodexUpstreamHostResponse(halfOpen, probeAt)).toBe(true); + }); + + test("capacity pressure never evicts an active half-open admission", () => { + const protectedKey = canonicalCodexUpstreamHostKey("openai", "https://protected.example")!; + const now = 5_000; + let tripped: CodexUpstreamHostHealthSnapshot | null = null; + for (let attempt = 0; attempt < CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; attempt++) { + tripped = fail(protectedKey, now); + } + const probeAt = tripped!.cooldownUntil!; + const halfOpen = admit(protectedKey, probeAt); + const pressureLeases: CodexUpstreamHostAdmissionLease[] = []; + for (let index = 0; index < CODEX_UPSTREAM_HOST_MAX_ENTRIES + 16; index++) { + const key = canonicalCodexUpstreamHostKey(`provider-${index}`, `https://host-${index}.example`)!; + pressureLeases.push(admit(key, probeAt + index)); + } + expect(acquireCodexUpstreamHostAdmission(protectedKey, probeAt)).toEqual({ + kind: "blocked", + retryAfterSeconds: 1, + }); + for (const lease of pressureLeases) releaseCodexUpstreamHostAdmissionLease(lease, probeAt); + expect(acquireCodexUpstreamHostAdmission(protectedKey, probeAt)).toEqual({ + kind: "blocked", + retryAfterSeconds: 1, + }); + expect(recordCodexUpstreamHostResponse(halfOpen, probeAt)).toBe(true); }); - test("bounds the process-local map and evicts the oldest entry", () => { + test("bounds the process-local map and evicts the oldest non-leased entry", () => { const first = canonicalCodexUpstreamHostKey("provider-0", "https://host-0.example")!; for (let index = 0; index <= CODEX_UPSTREAM_HOST_MAX_ENTRIES; index++) { const key = canonicalCodexUpstreamHostKey(`provider-${index}`, `https://host-${index}.example`)!; - recordCodexUpstreamHostFailure(key, 10_000 + index); + fail(key, 10_000 + index); } expect(getCodexUpstreamHostHealth(first, 10_000 + CODEX_UPSTREAM_HOST_MAX_ENTRIES)).toBeNull(); }); diff --git a/tests/responses-compaction-routing.test.ts b/tests/responses-compaction-routing.test.ts index fa1d2f2906..9b70426864 100644 --- a/tests/responses-compaction-routing.test.ts +++ b/tests/responses-compaction-routing.test.ts @@ -17,13 +17,15 @@ import { recordCodexUpstreamOutcome, resolveCodexAccountForThread, } from "../src/codex/routing"; -import { clearAccountQuota, updateAccountQuota } from "../src/codex/auth-api"; +import { clearAccountNeedsReauth, clearAccountQuota, updateAccountQuota } from "../src/codex/auth-api"; import { CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD, + acquireCodexUpstreamHostAdmission, canonicalCodexUpstreamHostKey, clearCodexUpstreamHostHealth, getCodexUpstreamHostHealth, recordCodexUpstreamHostFailure, + releaseCodexUpstreamHostAdmissionLease, } from "../src/codex/upstream-host-health"; import { MAIN_CODEX_ACCOUNT_ID } from "../src/codex/main-account"; import { @@ -481,7 +483,9 @@ describe("native Codex pool compaction", () => { "https://chatgpt.com/backend-api/codex/responses", )!; for (let attempt = 0; attempt < CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; attempt++) { - recordCodexUpstreamHostFailure(hostKey); + const admission = acquireCodexUpstreamHostAdmission(hostKey); + if (admission.kind !== "admitted") throw new Error("expected host admission while seeding circuit"); + recordCodexUpstreamHostFailure(admission.lease); } let physicalSends = 0; globalThis.fetch = (async () => { @@ -716,10 +720,11 @@ describe("compact alternate-account attempt (#913)", () => { clearCodexUpstreamHostHealth(); clearAccountQuota(); for (const id of ["pool-a", "pool-b"]) { + clearAccountNeedsReauth(id); saveCodexAccountCredential(id, { accessToken: `${id}-access-token`, refreshToken: `${id}-refresh-token`, - expiresAt: Date.now() + 300_000, + expiresAt: Date.now() + 30 * 60_000, chatgptAccountId: id === "pool-a" ? "pool_acc_a" : "pool_acc_b", }); updateAccountQuota(id, id === "pool-a" ? 10 : 20); @@ -729,6 +734,8 @@ describe("compact alternate-account attempt (#913)", () => { clearCodexUpstreamHealth(); clearCodexUpstreamHostHealth(); clearAccountQuota(); + clearAccountNeedsReauth("pool-a"); + clearAccountNeedsReauth("pool-b"); rmSync(testDir, { recursive: true, force: true }); if (previousOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousOpencodexHome; @@ -737,6 +744,281 @@ describe("compact alternate-account attempt (#913)", () => { }); } + function prepareHalfOpenHost(now: number): number { + const key = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex/responses", + )!; + let cooldownUntil: number | undefined; + for (let attempt = 0; attempt < CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; attempt++) { + const admission = acquireCodexUpstreamHostAdmission(key, now + attempt); + if (admission.kind !== "admitted") throw new Error("expected host admission while preparing half-open test"); + cooldownUntil = recordCodexUpstreamHostFailure(admission.lease, now + attempt)?.cooldownUntil; + } + if (cooldownUntil === undefined) throw new Error("expected host circuit cooldown"); + return cooldownUntil; + } + + function regularBody(): Record { + return { model: "gpt-5.6-sol", input: "host settlement regression", stream: false }; + } + + function expectHostAdmissionUsable(hostKey: NonNullable>, now: number): void { + const admission = acquireCodexUpstreamHostAdmission(hostKey, now); + expect(admission.kind).toBe("admitted"); + if (admission.kind === "admitted") releaseCodexUpstreamHostAdmissionLease(admission.lease, now); + } + + test("regular settles a half-open primary response when alternate preparation throws", async () => { + await withPoolEnv("ocx-regular-host-alt-throw-", async config => { + const originalNow = Date.now; + config.accountPoolStrategy = "fill-first"; + config.activeCodexAccountId = "pool-a"; + const hostKey = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex/responses", + )!; + const probeAt = prepareHalfOpenHost(Date.now()); + Date.now = () => probeAt; + const expected = new Error("alternate resolution hook failed"); + let bCallbackReached = false; + const physicalAccounts: string[] = []; + globalThis.fetch = (async (_url: RequestInfo | URL, init?: RequestInit) => { + physicalAccounts.push(new Headers(init?.headers).get("chatgpt-account-id") ?? ""); + return Response.json({ error: { message: "A quota" } }, { status: 429 }); + }) as typeof fetch; + try { + await expect(handleResponses( + compactionRequest(regularBody()), + config, + { model: "", provider: "" }, + { + onCodexAuthContextResolved: ctx => { + if ((ctx.kind === "pool" || ctx.kind === "main-pool") && ctx.accountId === "pool-b") { + bCallbackReached = true; + throw expected; + } + }, + }, + )).rejects.toBe(expected); + expect(bCallbackReached).toBe(true); + expect(physicalAccounts).toEqual(["pool_acc_a"]); + expect(getCodexUpstreamHostHealth(hostKey, probeAt)).toBeNull(); + expectHostAdmissionUsable(hostKey, probeAt); + } finally { + Date.now = originalNow; + } + }); + }); + + test("compact releases B recovery ownership when alternate header construction fails", async () => { + await withPoolEnv("ocx-compact-b-header-prep-", async config => { + config.accountPoolStrategy = "fill-first"; + config.activeCodexAccountId = "pool-a"; + const NativeHeaders = globalThis.Headers; + let bHeaderAttempted = false; + const physicalAccounts: string[] = []; + class ThrowOnBHeaders extends NativeHeaders { + override set(name: string, value: string): void { + if (name.toLowerCase() === "chatgpt-account-id" && value === "pool_acc_b") { + bHeaderAttempted = true; + throw new Error("B header construction failed"); + } + super.set(name, value); + } + } + globalThis.Headers = ThrowOnBHeaders as typeof Headers; + globalThis.fetch = (async (_url: RequestInfo | URL, init?: RequestInit) => { + physicalAccounts.push(new NativeHeaders(init?.headers).get("chatgpt-account-id") ?? ""); + return Response.json({ error: { message: "A quota" } }, { status: 429 }); + }) as typeof fetch; + try { + const response = await handleResponsesCompact( + compactionRequest(baseCompactionBody({})), + config, + { model: "", provider: "" }, + ); + expect(response.status).toBe(429); + expect(bHeaderAttempted).toBe(true); + expect(physicalAccounts).toEqual(["pool_acc_a"]); + } finally { + globalThis.Headers = NativeHeaders; + } + }); + }); + + test("compact settles half-open primary response and releases B when caller preparation throws", async () => { + await withPoolEnv("ocx-compact-host-alt-throw-", async config => { + const originalNow = Date.now; + config.accountPoolStrategy = "fill-first"; + config.activeCodexAccountId = "pool-a"; + const NativeHeaders = globalThis.Headers; + const probeAt = prepareHalfOpenHost(Date.now()); + const hostKey = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex/responses", + )!; + const expected = new Error("post-B compact preparation failed"); + let bPrepared = false; + const physicalAccounts: string[] = []; + class TrackBHeaders extends NativeHeaders { + override set(name: string, value: string): void { + if (name.toLowerCase() === "chatgpt-account-id" && value === "pool_acc_b") bPrepared = true; + super.set(name, value); + } + } + Date.now = () => probeAt; + globalThis.Headers = TrackBHeaders as typeof Headers; + globalThis.fetch = (async (_url: RequestInfo | URL, init?: RequestInit) => { + physicalAccounts.push(new NativeHeaders(init?.headers).get("chatgpt-account-id") ?? ""); + const response = new Response( + JSON.stringify({ error: { message: "A quota" } }), + { status: 429, headers: { "content-type": "application/json" } }, + ); + const responseHeaders = response.headers; + Object.defineProperty(response, "headers", { + configurable: true, + get: () => { + if (bPrepared) throw expected; + return responseHeaders; + }, + }); + return response; + }) as typeof fetch; + try { + await expect(handleResponsesCompact( + compactionRequest(baseCompactionBody({})), + config, + { model: "", provider: "" }, + )).rejects.toBe(expected); + expect(bPrepared).toBe(true); + expect(physicalAccounts).toEqual(["pool_acc_a"]); + expect(getCodexUpstreamHostHealth(hostKey, probeAt)).toBeNull(); + expectHostAdmissionUsable(hostKey, probeAt); + } finally { + globalThis.Headers = NativeHeaders; + Date.now = originalNow; + } + }); + }); + + test("compact keeps B ownership when header-timeout setup fails before provider execution", async () => { + await withPoolEnv("ocx-compact-b-pre-executor-", async config => { + const originalNow = Date.now; + config.accountPoolStrategy = "fill-first"; + config.activeCodexAccountId = "pool-a"; + config.connectTimeoutMs = 25; + const NativeHeaders = globalThis.Headers; + const probeAt = prepareHalfOpenHost(Date.now()); + const hostKey = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex/responses", + )!; + const expected = new Error("B header clone failed before executor"); + const bHeaderInstances = new WeakSet(); + const physicalAccounts: string[] = []; + let bPrepared = false; + let preExecutorCloneThrew = false; + class ThrowOnPreparedBClone extends NativeHeaders { + constructor(init?: HeadersInit) { + const shouldThrow = typeof init === "object" + && init !== null + && bHeaderInstances.has(init as object); + super(shouldThrow ? undefined : init); + if (shouldThrow) { + preExecutorCloneThrew = true; + throw expected; + } + } + + override set(name: string, value: string): void { + super.set(name, value); + if (name.toLowerCase() === "chatgpt-account-id" && value === "pool_acc_b") { + bPrepared = true; + bHeaderInstances.add(this); + } + } + } + Date.now = () => probeAt; + globalThis.Headers = ThrowOnPreparedBClone as typeof Headers; + globalThis.fetch = (async (_url: RequestInfo | URL, init?: RequestInit) => { + physicalAccounts.push(new NativeHeaders(init?.headers).get("chatgpt-account-id") ?? ""); + return Response.json({ error: { message: "A quota" } }, { status: 429 }); + }) as typeof fetch; + try { + await expect(handleResponsesCompact( + compactionRequest(baseCompactionBody({})), + config, + { model: "", provider: "" }, + )).rejects.toBe(expected); + expect(bPrepared).toBe(true); + expect(preExecutorCloneThrew).toBe(true); + expect(physicalAccounts).toEqual(["pool_acc_a"]); + expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(429); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(getCodexUpstreamHostHealth(hostKey, probeAt)).toBeNull(); + expectHostAdmissionUsable(hostKey, probeAt); + } finally { + globalThis.Headers = NativeHeaders; + Date.now = originalNow; + } + }); + }); + + for (const endpoint of ["regular", "compact"] as const) { + for (const scenario of ["503-retry", "429-alternate"] as const) { + test(`${endpoint} ${scenario} abort clears observed half-open host response`, async () => { + await withPoolEnv(`ocx-${endpoint}-${scenario}-abort-host-`, async config => { + const originalNow = Date.now; + const hostKey = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex/responses", + )!; + const probeAt = prepareHalfOpenHost(Date.now()); + Date.now = () => probeAt; + const abort = new AbortController(); + let sends = 0; + globalThis.fetch = (async () => { + sends += 1; + if (sends === 1) { + return Response.json({ error: { message: scenario } }, { + status: scenario === "503-retry" ? 503 : 429, + headers: { "retry-after": "0" }, + }); + } + abort.abort(); + throw new DOMException("aborted alternate", "AbortError"); + }) as typeof fetch; + try { + const request = compactionRequest( + endpoint === "regular" ? regularBody() : baseCompactionBody({}), + abort.signal, + ); + const response = endpoint === "regular" + ? await handleResponses( + request, + config, + { model: "", provider: "" }, + { abortSignal: abort.signal }, + ) + : await handleResponsesCompact(request, config, { model: "", provider: "" }); + expect(response.status).toBe(499); + expect(sends).toBe(2); + expect(getCodexUpstreamHostHealth(hostKey, probeAt)).toBeNull(); + if (scenario === "503-retry") { + expect(getCodexUpstreamHealth("pool-a")).toBeNull(); + } else { + expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(429); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + } + } finally { + Date.now = originalNow; + } + }); + }); + } + } + for (const rejection of [429, 402] as const) { test(`a pre-body ${rejection} tries exactly one alternate account`, async () => { await withPoolEnv(`ocx-compact-alt-${rejection}-`, async config => { From 11771d105c865964444b04305fb95a9dba28c48c Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 09:48:35 +0900 Subject: [PATCH 03/20] fix(codex): distinguish local setup failures --- src/server/responses/compact.ts | 47 ++++-- src/server/responses/core.ts | 49 +++++- tests/codex-upstream-host-health.test.ts | 19 +++ tests/responses-compaction-routing.test.ts | 176 ++++++++++++++++++--- 4 files changed, 248 insertions(+), 43 deletions(-) diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index 65b035020c..d794a038f0 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -412,31 +412,38 @@ export async function handleResponsesCompact( recovery: "normal" | "single", attempts: UpstreamAttemptObservation[], onSendStart?: () => void, + attemptBoundary?: { executorStarted: boolean }, ): Promise => { const body = JSON.stringify({ ...compactBody, model: route.modelId }); const fetcher = providerFetch(sendProvider); - let executorStarted = false; - const executor = ((input: RequestInfo | URL, init?: RequestInit) => { - if (!executorStarted) { - executorStarted = true; - onSendStart?.(); - } - return fetcher(input, init); - }) as typeof globalThis.fetch; const doFetch = (upstreamRecovery?: UpstreamSendRecovery) => { + if (attemptBoundary) attemptBoundary.executorStarted = false; const init = applyUpstreamRecoveryInit({ method: "POST", headers: sendHeaders, body, ...(compactHostKey ? { redirect: "manual" as const } : {}), }, upstreamRecovery); + let executorStarted = false; + const executor = ((input: RequestInfo | URL, fetchInit?: RequestInit) => { + if (!executorStarted) { + executorStarted = true; + if (attemptBoundary) attemptBoundary.executorStarted = true; + onSendStart?.(); + } + return fetcher(input, fetchInit); + }) as typeof globalThis.fetch; return fetchWithHeaderTimeout(compactUrl, init, req.signal, connectMs, false, executor); }; if (recovery === "normal") { return fetchWithTransientRetry(doFetch, { abortSignal: req.signal, label: safeHostLabel(compactUrl), - onAttempt: observation => attempts.push(observation), + onAttempt: observation => { + if (observation.kind === "response" || attemptBoundary?.executorStarted !== false) { + attempts.push(observation); + } + }, }); } try { @@ -444,7 +451,7 @@ export async function handleResponsesCompact( attempts.push({ kind: "response", status: response.status }); return response; } catch (error) { - attempts.push({ kind: "rejection" }); + if (attemptBoundary?.executorStarted !== false) attempts.push({ kind: "rejection" }); throw error; } }; @@ -498,11 +505,29 @@ export async function handleResponsesCompact( } compactHostAdmissionLease = compactHostAdmission?.lease ?? null; const primaryAttempts: UpstreamAttemptObservation[] = []; + const primaryAttemptBoundary = { executorStarted: false }; try { // Same connect timeout + keep-alive reset + transient-5xx recovery as /v1/responses — // compact hits the same ChatGPT host and must soft-avoid / clear affinity (#186). - upstream = await sendCompactAttempt(compactProvider, headers, "normal", primaryAttempts); + upstream = await sendCompactAttempt( + compactProvider, + headers, + "normal", + primaryAttempts, + undefined, + primaryAttemptBoundary, + ); } catch (err) { + if (!primaryAttemptBoundary.executorStarted && !req.signal.aborted) { + releaseCodexAuthContextProbeLease(outcomeCtx); + if (lastUpstreamAttemptResponseStatus(primaryAttempts) !== undefined) { + settleObservedCompactHostResponse(); + } else { + releaseCodexUpstreamHostAdmissionLease(compactHostAdmissionLease); + compactHostAdmissionLease = null; + } + throw err; + } return compactTransportFailureResponse(outcomeCtx, err, primaryAttempts); } if (compactHostKey && isCodexUpstreamRedirectStatus(upstream.status)) { diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index af41dc187d..a2ba17f966 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -448,7 +448,14 @@ async function retryCodexPoolOnAlternateAccount( const { fetcher, request, retryHeaders } = prepared; const attempts: UpstreamAttemptObservation[] = []; - noteAttemptSend(logCtx.activeAttempt, passthroughEstimate); + let executorStarted = false; + const executor = ((input: RequestInfo | URL, init?: RequestInit) => { + if (!executorStarted) { + executorStarted = true; + noteAttemptSend(logCtx.activeAttempt, passthroughEstimate); + } + return fetcher(input, init); + }) as typeof globalThis.fetch; try { const upstreamResponse = await fetchWithHeaderTimeout( request.url, @@ -461,7 +468,7 @@ async function retryCodexPoolOnAlternateAccount( upstream.signal, connectMs, stream, - fetcher, + executor, ); attempts.push({ kind: "response", status: upstreamResponse.status }); return { @@ -472,6 +479,10 @@ async function retryCodexPoolOnAlternateAccount( selectedForwardHeaders: retryHeaders, }; } catch (error) { + if (!executorStarted) { + releaseCodexAuthContextProbeLease(retryAuthCtx); + throw error; + } attempts.push({ kind: "rejection" }); return { kind: "transport", error, authCtx: retryAuthCtx, attempts }; } finally { @@ -1764,6 +1775,7 @@ async function handleResponsesInner( : null; let hostAdmissionLease: CodexUpstreamHostAdmissionLease | null = null; const attemptHistory: UpstreamAttemptObservation[] = []; + let primaryAttemptExecutorStarted = false; const recordPoolTransportOutcome = (outcome: CodexUpstreamOutcome): void => { if (!usesCodexForwardPoolAuth(authCtx, route.provider)) return; recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, { @@ -1830,21 +1842,46 @@ async function handleResponsesInner( // Body is a replayable string; nothing has streamed to the client yet. upstreamResponse = await fetchWithTransientRetry( recovery => { - noteAttemptSend(logCtx.activeAttempt, passthroughEstimate, recovery); - return fetchWithHeaderTimeout(request.url, applyUpstreamRecoveryInit({ + primaryAttemptExecutorStarted = false; + const init = applyUpstreamRecoveryInit({ method: request.method, headers: request.headers, body: request.body, ...(hostKey ? { redirect: "manual" as const } : {}), - }, recovery), upstream.signal, connectMs, parsed.stream, providerFetch(route.provider)); + }, recovery); + const fetcher = providerFetch(route.provider); + let executorStarted = false; + const executor = ((input: RequestInfo | URL, fetchInit?: RequestInit) => { + if (!executorStarted) { + executorStarted = true; + primaryAttemptExecutorStarted = true; + noteAttemptSend(logCtx.activeAttempt, passthroughEstimate, recovery); + } + return fetcher(input, fetchInit); + }) as typeof globalThis.fetch; + return fetchWithHeaderTimeout(request.url, init, upstream.signal, connectMs, parsed.stream, executor); }, { abortSignal: upstream.signal, label: safeHostLabel(request.url), - onAttempt: observation => attemptHistory.push(observation), + onAttempt: observation => { + if (observation.kind === "response" || primaryAttemptExecutorStarted) { + attemptHistory.push(observation); + } + }, }, ); } catch (err) { + if (!primaryAttemptExecutorStarted && !options.abortSignal?.aborted) { + releaseCodexAuthContextProbeLease(authCtx); + if (lastUpstreamAttemptResponseStatus(attemptHistory) !== undefined) { + settleObservedHostResponse(); + } else { + releaseCodexUpstreamHostAdmissionLease(hostAdmissionLease); + hostAdmissionLease = null; + } + throw err; + } return transportFailureResponse(err); } finally { request.releaseBodyObservation?.(); diff --git a/tests/codex-upstream-host-health.test.ts b/tests/codex-upstream-host-health.test.ts index 6925a3e3af..ab2a204976 100644 --- a/tests/codex-upstream-host-health.test.ts +++ b/tests/codex-upstream-host-health.test.ts @@ -101,6 +101,25 @@ describe("Codex upstream host health (#914)", () => { expect(acquireCodexUpstreamHostAdmission(key, probeAt)).toMatchObject({ kind: "blocked" }); }); + test("a half-open rejection after an observed response starts a fresh streak", () => { + const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com")!; + const trippedAt = 2_500; + let tripped: CodexUpstreamHostHealthSnapshot | null = null; + for (let attempt = 0; attempt < CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; attempt++) { + tripped = fail(key, trippedAt); + } + const probeAt = tripped!.cooldownUntil!; + const afterResponse = recordCodexUpstreamHostFailure(admit(key, probeAt), probeAt, { + observedResponse: true, + })!; + + expect(afterResponse.consecutiveFailures).toBe(1); + expect(afterResponse.cooldownUntil).toBeUndefined(); + const next = admit(key, probeAt); + expect(next.halfOpen).toBe(false); + expect(releaseCodexUpstreamHostAdmissionLease(next, probeAt)).toBe(true); + }); + test("caller abort releases a half-open admission without adding evidence", () => { const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com")!; const trippedAt = 3_000; diff --git a/tests/responses-compaction-routing.test.ts b/tests/responses-compaction-routing.test.ts index 9b70426864..d00e91ad1a 100644 --- a/tests/responses-compaction-routing.test.ts +++ b/tests/responses-compaction-routing.test.ts @@ -769,6 +769,153 @@ describe("compact alternate-account attempt (#913)", () => { if (admission.kind === "admitted") releaseCodexUpstreamHostAdmissionLease(admission.lease, now); } + function throwingPreExecutorHeaders( + NativeHeaders: typeof Headers, + targetAccountId: string, + expected: Error, + state: { prepared: boolean; cloneThrew: boolean }, + ): typeof Headers { + return class ThrowInFetchWithHeaderTimeout extends NativeHeaders { + constructor(init?: HeadersInit) { + super(init); + if ( + this.get("chatgpt-account-id") === targetAccountId + && new Error().stack?.includes("fetchWithHeaderTimeout") + ) { + state.cloneThrew = true; + throw expected; + } + } + + override set(name: string, value: string): void { + super.set(name, value); + if (name.toLowerCase() === "chatgpt-account-id" && value === targetAccountId) { + state.prepared = true; + } + } + } as typeof Headers; + } + + test("regular primary local header setup failure never becomes a physical rejection", async () => { + await withPoolEnv("ocx-regular-primary-pre-executor-", async config => { + const originalNow = Date.now; + const NativeHeaders = globalThis.Headers; + config.accountPoolStrategy = "fill-first"; + config.activeCodexAccountId = "pool-a"; + config.connectTimeoutMs = 25; + const probeAt = prepareHalfOpenHost(Date.now()); + const hostKey = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex/responses", + )!; + const expected = new Error("regular primary header clone failed before executor"); + const state = { prepared: false, cloneThrew: false }; + const physicalAccounts: string[] = []; + Date.now = () => probeAt; + globalThis.Headers = throwingPreExecutorHeaders(NativeHeaders, "pool_acc_a", expected, state); + globalThis.fetch = (async (_url: RequestInfo | URL, init?: RequestInit) => { + physicalAccounts.push(new NativeHeaders(init?.headers).get("chatgpt-account-id") ?? ""); + return Response.json({ id: "must-not-run", status: "completed", output: [] }); + }) as typeof fetch; + try { + await expect(handleResponses( + compactionRequest(regularBody()), + config, + { model: "", provider: "" }, + )).rejects.toBe(expected); + expect(state).toEqual({ prepared: true, cloneThrew: true }); + expect(physicalAccounts).toEqual([]); + expect(getCodexUpstreamHealth("pool-a")).toBeNull(); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(getCodexUpstreamHostHealth(hostKey, probeAt)).not.toBeNull(); + expectHostAdmissionUsable(hostKey, probeAt); + } finally { + globalThis.Headers = NativeHeaders; + Date.now = originalNow; + } + }); + }); + + test("regular B local header setup failure preserves A response without a B send", async () => { + await withPoolEnv("ocx-regular-b-pre-executor-", async config => { + const originalNow = Date.now; + const NativeHeaders = globalThis.Headers; + config.accountPoolStrategy = "fill-first"; + config.activeCodexAccountId = "pool-a"; + config.connectTimeoutMs = 25; + const probeAt = prepareHalfOpenHost(Date.now()); + const hostKey = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex/responses", + )!; + const expected = new Error("regular B header clone failed before executor"); + const state = { prepared: false, cloneThrew: false }; + const physicalAccounts: string[] = []; + Date.now = () => probeAt; + globalThis.Headers = throwingPreExecutorHeaders(NativeHeaders, "pool_acc_b", expected, state); + globalThis.fetch = (async (_url: RequestInfo | URL, init?: RequestInit) => { + physicalAccounts.push(new NativeHeaders(init?.headers).get("chatgpt-account-id") ?? ""); + return Response.json({ error: { message: "A quota" } }, { status: 429 }); + }) as typeof fetch; + try { + await expect(handleResponses( + compactionRequest(regularBody()), + config, + { model: "", provider: "" }, + )).rejects.toBe(expected); + expect(state).toEqual({ prepared: true, cloneThrew: true }); + expect(physicalAccounts).toEqual(["pool_acc_a"]); + expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(429); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(getCodexUpstreamHostHealth(hostKey, probeAt)).toBeNull(); + expectHostAdmissionUsable(hostKey, probeAt); + } finally { + globalThis.Headers = NativeHeaders; + Date.now = originalNow; + } + }); + }); + + test("compact primary local header setup failure never becomes a physical rejection", async () => { + await withPoolEnv("ocx-compact-primary-pre-executor-", async config => { + const originalNow = Date.now; + const NativeHeaders = globalThis.Headers; + config.accountPoolStrategy = "fill-first"; + config.activeCodexAccountId = "pool-a"; + config.connectTimeoutMs = 25; + const probeAt = prepareHalfOpenHost(Date.now()); + const hostKey = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex/responses", + )!; + const expected = new Error("compact primary header clone failed before executor"); + const state = { prepared: false, cloneThrew: false }; + const physicalAccounts: string[] = []; + Date.now = () => probeAt; + globalThis.Headers = throwingPreExecutorHeaders(NativeHeaders, "pool_acc_a", expected, state); + globalThis.fetch = (async (_url: RequestInfo | URL, init?: RequestInit) => { + physicalAccounts.push(new NativeHeaders(init?.headers).get("chatgpt-account-id") ?? ""); + return Response.json({ id: "must-not-run", status: "completed", output: [] }); + }) as typeof fetch; + try { + await expect(handleResponsesCompact( + compactionRequest(baseCompactionBody({})), + config, + { model: "", provider: "" }, + )).rejects.toBe(expected); + expect(state).toEqual({ prepared: true, cloneThrew: true }); + expect(physicalAccounts).toEqual([]); + expect(getCodexUpstreamHealth("pool-a")).toBeNull(); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(getCodexUpstreamHostHealth(hostKey, probeAt)).not.toBeNull(); + expectHostAdmissionUsable(hostKey, probeAt); + } finally { + globalThis.Headers = NativeHeaders; + Date.now = originalNow; + } + }); + }); + test("regular settles a half-open primary response when alternate preparation throws", async () => { await withPoolEnv("ocx-regular-host-alt-throw-", async config => { const originalNow = Date.now; @@ -915,32 +1062,10 @@ describe("compact alternate-account attempt (#913)", () => { "https://chatgpt.com/backend-api/codex/responses", )!; const expected = new Error("B header clone failed before executor"); - const bHeaderInstances = new WeakSet(); const physicalAccounts: string[] = []; - let bPrepared = false; - let preExecutorCloneThrew = false; - class ThrowOnPreparedBClone extends NativeHeaders { - constructor(init?: HeadersInit) { - const shouldThrow = typeof init === "object" - && init !== null - && bHeaderInstances.has(init as object); - super(shouldThrow ? undefined : init); - if (shouldThrow) { - preExecutorCloneThrew = true; - throw expected; - } - } - - override set(name: string, value: string): void { - super.set(name, value); - if (name.toLowerCase() === "chatgpt-account-id" && value === "pool_acc_b") { - bPrepared = true; - bHeaderInstances.add(this); - } - } - } + const state = { prepared: false, cloneThrew: false }; Date.now = () => probeAt; - globalThis.Headers = ThrowOnPreparedBClone as typeof Headers; + globalThis.Headers = throwingPreExecutorHeaders(NativeHeaders, "pool_acc_b", expected, state); globalThis.fetch = (async (_url: RequestInfo | URL, init?: RequestInit) => { physicalAccounts.push(new NativeHeaders(init?.headers).get("chatgpt-account-id") ?? ""); return Response.json({ error: { message: "A quota" } }, { status: 429 }); @@ -951,8 +1076,7 @@ describe("compact alternate-account attempt (#913)", () => { config, { model: "", provider: "" }, )).rejects.toBe(expected); - expect(bPrepared).toBe(true); - expect(preExecutorCloneThrew).toBe(true); + expect(state).toEqual({ prepared: true, cloneThrew: true }); expect(physicalAccounts).toEqual(["pool_acc_a"]); expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(429); expect(getCodexUpstreamHealth("pool-b")).toBeNull(); From 625400b8842bc58c73f1615b9624c7e2eba62fa0 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 10:05:14 +0900 Subject: [PATCH 04/20] fix(codex): preserve observed retry outcomes --- src/server/responses/compact.ts | 6 +- src/server/responses/core.ts | 6 +- tests/responses-compaction-routing.test.ts | 93 +++++++++++++++++++++- 3 files changed, 99 insertions(+), 6 deletions(-) diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index d794a038f0..4c0ecfd68b 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -519,10 +519,12 @@ export async function handleResponsesCompact( ); } catch (err) { if (!primaryAttemptBoundary.executorStarted && !req.signal.aborted) { - releaseCodexAuthContextProbeLease(outcomeCtx); - if (lastUpstreamAttemptResponseStatus(primaryAttempts) !== undefined) { + const observedStatus = lastUpstreamAttemptResponseStatus(primaryAttempts); + if (observedStatus !== undefined) { + recordCompactPoolOutcome(outcomeCtx, observedStatus); settleObservedCompactHostResponse(); } else { + releaseCodexAuthContextProbeLease(outcomeCtx); releaseCodexUpstreamHostAdmissionLease(compactHostAdmissionLease); compactHostAdmissionLease = null; } diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index a2ba17f966..fc123175e0 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -1873,10 +1873,12 @@ async function handleResponsesInner( ); } catch (err) { if (!primaryAttemptExecutorStarted && !options.abortSignal?.aborted) { - releaseCodexAuthContextProbeLease(authCtx); - if (lastUpstreamAttemptResponseStatus(attemptHistory) !== undefined) { + const observedStatus = lastUpstreamAttemptResponseStatus(attemptHistory); + if (observedStatus !== undefined) { + recordPoolTransportOutcome(observedStatus); settleObservedHostResponse(); } else { + releaseCodexAuthContextProbeLease(authCtx); releaseCodexUpstreamHostAdmissionLease(hostAdmissionLease); hostAdmissionLease = null; } diff --git a/tests/responses-compaction-routing.test.ts b/tests/responses-compaction-routing.test.ts index d00e91ad1a..bd40098e03 100644 --- a/tests/responses-compaction-routing.test.ts +++ b/tests/responses-compaction-routing.test.ts @@ -774,7 +774,9 @@ describe("compact alternate-account attempt (#913)", () => { targetAccountId: string, expected: Error, state: { prepared: boolean; cloneThrew: boolean }, + throwOnBoundaryClone = 1, ): typeof Headers { + let boundaryCloneCount = 0; return class ThrowInFetchWithHeaderTimeout extends NativeHeaders { constructor(init?: HeadersInit) { super(init); @@ -782,8 +784,11 @@ describe("compact alternate-account attempt (#913)", () => { this.get("chatgpt-account-id") === targetAccountId && new Error().stack?.includes("fetchWithHeaderTimeout") ) { - state.cloneThrew = true; - throw expected; + boundaryCloneCount += 1; + if (boundaryCloneCount === throwOnBoundaryClone) { + state.cloneThrew = true; + throw expected; + } } } @@ -876,6 +881,48 @@ describe("compact alternate-account attempt (#913)", () => { }); }); + test("regular preserves a prior 503 when retry setup fails before provider execution", async () => { + await withPoolEnv("ocx-regular-retry-pre-executor-", async config => { + const originalNow = Date.now; + const NativeHeaders = globalThis.Headers; + config.accountPoolStrategy = "fill-first"; + config.activeCodexAccountId = "pool-a"; + config.connectTimeoutMs = 25; + const probeAt = prepareHalfOpenHost(Date.now()); + const hostKey = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex/responses", + )!; + const expected = new Error("regular retry header clone failed before executor"); + const state = { prepared: false, cloneThrew: false }; + const physicalAccounts: string[] = []; + Date.now = () => probeAt; + globalThis.Headers = throwingPreExecutorHeaders(NativeHeaders, "pool_acc_a", expected, state, 2); + globalThis.fetch = (async (_url: RequestInfo | URL, init?: RequestInit) => { + physicalAccounts.push(new NativeHeaders(init?.headers).get("chatgpt-account-id") ?? ""); + return Response.json({ error: { message: "temporarily unavailable" } }, { + status: 503, + headers: { "retry-after": "0" }, + }); + }) as typeof fetch; + try { + await expect(handleResponses( + compactionRequest(regularBody()), + config, + { model: "", provider: "" }, + )).rejects.toBe(expected); + expect(state).toEqual({ prepared: true, cloneThrew: true }); + expect(physicalAccounts).toEqual(["pool_acc_a"]); + expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(503); + expect(getCodexUpstreamHostHealth(hostKey, probeAt)).toBeNull(); + expectHostAdmissionUsable(hostKey, probeAt); + } finally { + globalThis.Headers = NativeHeaders; + Date.now = originalNow; + } + }); + }); + test("compact primary local header setup failure never becomes a physical rejection", async () => { await withPoolEnv("ocx-compact-primary-pre-executor-", async config => { const originalNow = Date.now; @@ -916,6 +963,48 @@ describe("compact alternate-account attempt (#913)", () => { }); }); + test("compact preserves a prior 503 when retry setup fails before provider execution", async () => { + await withPoolEnv("ocx-compact-retry-pre-executor-", async config => { + const originalNow = Date.now; + const NativeHeaders = globalThis.Headers; + config.accountPoolStrategy = "fill-first"; + config.activeCodexAccountId = "pool-a"; + config.connectTimeoutMs = 25; + const probeAt = prepareHalfOpenHost(Date.now()); + const hostKey = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex/responses", + )!; + const expected = new Error("compact retry header clone failed before executor"); + const state = { prepared: false, cloneThrew: false }; + const physicalAccounts: string[] = []; + Date.now = () => probeAt; + globalThis.Headers = throwingPreExecutorHeaders(NativeHeaders, "pool_acc_a", expected, state, 2); + globalThis.fetch = (async (_url: RequestInfo | URL, init?: RequestInit) => { + physicalAccounts.push(new NativeHeaders(init?.headers).get("chatgpt-account-id") ?? ""); + return Response.json({ error: { message: "temporarily unavailable" } }, { + status: 503, + headers: { "retry-after": "0" }, + }); + }) as typeof fetch; + try { + await expect(handleResponsesCompact( + compactionRequest(baseCompactionBody({})), + config, + { model: "", provider: "" }, + )).rejects.toBe(expected); + expect(state).toEqual({ prepared: true, cloneThrew: true }); + expect(physicalAccounts).toEqual(["pool_acc_a"]); + expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(503); + expect(getCodexUpstreamHostHealth(hostKey, probeAt)).toBeNull(); + expectHostAdmissionUsable(hostKey, probeAt); + } finally { + globalThis.Headers = NativeHeaders; + Date.now = originalNow; + } + }); + }); + test("regular settles a half-open primary response when alternate preparation throws", async () => { await withPoolEnv("ocx-regular-host-alt-throw-", async config => { const originalNow = Date.now; From ac54369579e215872a45de7e824456bc97953eab Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:15:38 +0900 Subject: [PATCH 05/20] test(codex): assert host-only transport failures --- tests/server-auth.test.ts | 56 ++++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/tests/server-auth.test.ts b/tests/server-auth.test.ts index 60d6736187..4100b0ac4a 100644 --- a/tests/server-auth.test.ts +++ b/tests/server-auth.test.ts @@ -15,7 +15,13 @@ import { clearThreadAccountMap, getCodexUpstreamHealth, recordCodexUpstreamOutcome, + resolveCodexAccountForThread, } from "../src/codex/routing"; +import { + canonicalCodexUpstreamHostKey, + clearCodexUpstreamHostHealth, + getCodexUpstreamHostHealth, +} from "../src/codex/upstream-host-health"; import { loadConfig, saveConfig } from "../src/config"; import { deriveProviderPresets } from "../src/providers/derive"; import { MAIN_CODEX_ACCOUNT_ID } from "../src/codex/main-account"; @@ -129,6 +135,7 @@ afterEach(() => { isolatedCodexHome?.restore(); isolatedCodexHome = null; clearCodexUpstreamHealth(); + clearCodexUpstreamHostHealth(); clearThreadAccountMap(); clearAccountNeedsReauth("pool-a"); clearAccountNeedsReauth("pool-b"); @@ -197,6 +204,7 @@ async function startPoolRetryHarness( mkdirSync(TEST_DIR, { recursive: true }); process.env.OPENCODEX_HOME = TEST_DIR; clearCodexUpstreamHealth(); + clearCodexUpstreamHostHealth(); clearThreadAccountMap(); clearAccountQuota(); clearRequestLogsForTests(); @@ -2561,8 +2569,14 @@ describe("server local API auth", () => { } }); - test("retry-dispatch transport failure records only B and never triple-dispatches", async () => { + test("retry-dispatch transport failure is host-only and never triple-dispatches", async () => { const harness = await startPoolRetryHarness(() => rejectionResponse(unsupportedModelBody())); + const affinityThread = "retry-dispatch-host-only-affinity"; + const hostKey = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex/responses", + )!; + expect(resolveCodexAccountForThread(affinityThread, harness.config)).toBe("pool-a"); const redirectedFetch = globalThis.fetch; globalThis.fetch = (async (input, init) => { const accountId = new Headers(init?.headers).get("chatgpt-account-id"); @@ -2575,12 +2589,20 @@ describe("server local API auth", () => { try { const response = await harness.request(); expect(response.status).toBe(502); + expect(await response.json()).toEqual({ + error: { + message: "Provider unreachable: synthetic retry connect failure", + type: "server_error", + code: "upstream_server_error", + }, + }); expect(harness.dispatches).toEqual(["acct-pool-a", "acct-pool-b"]); expect(getCodexUpstreamHealth("pool-a")).toBeNull(); - expect(getCodexUpstreamHealth("pool-b")).toMatchObject({ - consecutiveFailures: 1, - lastFailureStatus: 0, - }); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(getCodexUpstreamHostHealth(hostKey)).toMatchObject({ consecutiveFailures: 1 }); + const persisted = loadConfig(); + expect(persisted.activeCodexAccountId).toBe("pool-a"); + expect(resolveCodexAccountForThread(affinityThread, persisted)).toBe("pool-a"); } finally { await stopPoolRetryHarness(harness); } @@ -2634,11 +2656,12 @@ describe("server local API auth", () => { } }, { timeout: 30_000 }); - test("passthrough connect failure records selected pool account health", async () => { + test("passthrough connect failure leaves selected pool account health clear", async () => { if (existsSync(TEST_DIR)) rmSync(TEST_DIR, { recursive: true }); mkdirSync(TEST_DIR, { recursive: true }); process.env.OPENCODEX_HOME = TEST_DIR; clearCodexUpstreamHealth(); + clearCodexUpstreamHostHealth(); clearThreadAccountMap(); clearAccountNeedsReauth("pool-a"); @@ -2665,6 +2688,12 @@ describe("server local API auth", () => { // Known low quota keeps "pool-a" the deterministic active (this case tests // failure-health recording, not the all-unknown rotation added in Phase 10). updateAccountQuota("pool-a", 10, 5); + const affinityThread = "passthrough-connect-host-only-affinity"; + const hostKey = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex/responses", + )!; + expect(resolveCodexAccountForThread(affinityThread, loadConfig())).toBe("pool-a"); const server = startServer(0); try { @@ -2678,10 +2707,19 @@ describe("server local API auth", () => { }); expect(response.status).toBe(502); - expect(getCodexUpstreamHealth("pool-a")).toMatchObject({ - consecutiveFailures: 1, - lastFailureStatus: 0, + const error = await response.json() as { + error: { message: string; type: string; code: string }; + }; + expect(error.error).toMatchObject({ + type: "server_error", + code: "upstream_server_error", }); + expect(error.error.message).toMatch(/^Provider unreachable:/); + expect(getCodexUpstreamHealth("pool-a")).toBeNull(); + expect(getCodexUpstreamHostHealth(hostKey)).toMatchObject({ consecutiveFailures: 1 }); + const persisted = loadConfig(); + expect(persisted.activeCodexAccountId).toBe("pool-a"); + expect(resolveCodexAccountForThread(affinityThread, persisted)).toBe("pool-a"); } finally { await server.stop(true); } From 9a26f939f8e69d1e8a3fd2c498f2ecf9602b30b6 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:47:36 +0900 Subject: [PATCH 06/20] test(codex): bind host-only affinity assertions --- tests/server-auth.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/server-auth.test.ts b/tests/server-auth.test.ts index 4100b0ac4a..a1f462ecd6 100644 --- a/tests/server-auth.test.ts +++ b/tests/server-auth.test.ts @@ -160,6 +160,7 @@ type PoolRetryHarness = { model?: string; path?: "/v1/responses" | "/v1/responses/compact"; callerBearer?: boolean; + threadId?: string; }) => Promise; restoreFetch: () => void; server: ReturnType; @@ -294,11 +295,13 @@ async function startPoolRetryHarness( model = POOL_RETRY_MODEL, path = "/v1/responses", callerBearer = true, + threadId, } = {}) => originalGlobalFetch(new URL(path, server.url), { method: "POST", headers: { "content-type": "application/json", ...(callerBearer ? { authorization: "Bearer inbound-token" } : {}), + ...(threadId ? { "x-codex-parent-thread-id": threadId } : {}), }, body: JSON.stringify({ model, input: path.endsWith("/compact") ? [] : "hello", stream }), signal, @@ -1860,7 +1863,7 @@ describe("server local API auth", () => { ? rejectionResponse(unsupportedModelBody()) : Response.json({ id: "retry-success", status: "completed", output: [] })); try { - const response = await harness.request(); + const response = await harness.request({ threadId: affinityThread }); expect(response.status).toBe(200); expect((await response.json() as { id: string }).id).toBe("retry-success"); expect(harness.dispatches).toEqual(["acct-pool-a", "acct-pool-b"]); @@ -2702,6 +2705,7 @@ describe("server local API auth", () => { headers: { "content-type": "application/json", authorization: "Bearer inbound-main-token", + "x-codex-parent-thread-id": affinityThread, }, body: JSON.stringify({ model: "gpt-test", input: "hello", stream: false }), }); From 2876c0cf36c818ce9c1ec936489db1f92ea260c9 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:56:38 +0900 Subject: [PATCH 07/20] test(codex): correct affinity request binding --- tests/server-auth.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/server-auth.test.ts b/tests/server-auth.test.ts index a1f462ecd6..d211e864d9 100644 --- a/tests/server-auth.test.ts +++ b/tests/server-auth.test.ts @@ -1863,7 +1863,7 @@ describe("server local API auth", () => { ? rejectionResponse(unsupportedModelBody()) : Response.json({ id: "retry-success", status: "completed", output: [] })); try { - const response = await harness.request({ threadId: affinityThread }); + const response = await harness.request(); expect(response.status).toBe(200); expect((await response.json() as { id: string }).id).toBe("retry-success"); expect(harness.dispatches).toEqual(["acct-pool-a", "acct-pool-b"]); @@ -2590,7 +2590,7 @@ describe("server local API auth", () => { return redirectedFetch(input, init); }) as typeof fetch; try { - const response = await harness.request(); + const response = await harness.request({ threadId: affinityThread }); expect(response.status).toBe(502); expect(await response.json()).toEqual({ error: { From c6638e209cd00190ced353d0ca2a21f47e3f8146 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 12:38:18 +0900 Subject: [PATCH 08/20] fix(codex): address host-health review --- .../ja/reference/configuration/providers.md | 2 +- .../ko/reference/configuration/providers.md | 2 +- .../docs/reference/configuration/providers.md | 2 +- .../ru/reference/configuration/providers.md | 2 +- .../reference/configuration/providers.md | 2 +- src/codex/upstream-host-health.ts | 43 ++++++------ src/server/responses/compact.ts | 2 +- src/server/responses/core.ts | 2 +- tests/codex-host-health-runtime.test.ts | 3 + tests/codex-upstream-host-health.test.ts | 65 +++++++++++++++++ tests/issue-452-empty-503.test.ts | 9 +-- tests/server-auth.test.ts | 69 +++++++++++++++++++ 12 files changed, 172 insertions(+), 31 deletions(-) diff --git a/docs-site/src/content/docs/ja/reference/configuration/providers.md b/docs-site/src/content/docs/ja/reference/configuration/providers.md index 8c1172e4fd..3c6857ed77 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ja/reference/configuration/providers.md @@ -21,7 +21,7 @@ description: プロバイダー エントリ、認証、エンドポイント、 | `autoSwitchThreshold?` | `number` | `80` | 使用量ベースのプロアクティブ切り替えしきい値。`quota` は紐付け済み/未紐付けタスクの次のリクエストを再評価でき、`fill-first` は未紐付け割り当ての使い切り基準としてのみ使用し、通常の `round-robin` 選択は使用しません。既知の 5 時間、週次、30 日 quota window の最大スコアを使います。`0` は使用量ベースの切り替えだけを無効にし、未紐付け割り当てや障害回復は無効にしません。 | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 新規/未紐付け Codex リクエストの割り当て戦略。live な `(parent thread id, quota scope)` affinity がなければ未紐付けで、プロキシ再起動や affinity リセット後は既存の表示タスクも未紐付けになり得ます。`quota` はアクティブアカウントがなければ既知 usage 最小の適格アカウントを選び、適格なアクティブアカウントが `autoSwitchThreshold` 未満なら維持します。しきい値到達後は、未紐付けリクエストまたは紐付け済みタスクの次のリクエストを usage の低い適格アカウントへ移せます。`round-robin` は未紐付けリクエストを均等分散し、`fill-first` は cooldown、使用不可、または drain threshold までアクティブアカウントへ割り当てます。 | | `accountPoolStickyLimit?` | `number` | `1` | 1 回の round-robin 選択で次へ進む前に保持する新規/未紐付けタスク割り当て数。カウンターは上流の成功後ではなくタスクの紐付け時に増えます。範囲 1–100。`accountPoolStrategy` が `round-robin` のときのみ。 | -| `upstreamFailoverThreshold?` | `number` | `3` | 将来の新規セッションでアカウントの failover を行えるようになるまでに必要な、アカウント単位の一時的な HTTP または意味上の失敗結果の連続回数です。`0` はアカウント failover だけを無効にします。HTTP 応答を受け取らない rejection、接続/ヘッダーの `TimeoutError`、および保守的に判定される read-then-close は、アカウントの health/affinity を変更せず、`(provider, canonical HTTP(S) origin)` をキーとするプロセスローカルな host health だけを更新します。5 分以内に論理リクエスト単位の終端 host failure が 3 回発生すると host circuit が 30 秒間開き、その後は正確に 1 件の half-open 論理リクエストだけが許可され、同時リクエストは引き続き拒否されます。実際の HTTP 応答を受け取ると以前の host state は消去されます。`503` の後に rejection が起きた場合、`503` はアカウント evidence として保持され、後の failure は host failure として記録されます。プールされた通常の Responses と native compact リクエストの Codex bearer redirect は手動処理され、追従せず、`Location` を公開せず、サイズを制限したアカウント単位の `502` に変換されます。資格情報が見える read-then-close failure では peer がリクエストを消費済みの可能性があるため別の資格情報では意図的に再送せず、正常な代替アカウントが一時的にブロックされることがあります。`200` 後の body/stream 処理は変更されず、この設定の範囲外です。 | +| `upstreamFailoverThreshold?` | `number` | `3` | 将来の新規セッションでアカウントの failover を行えるようになるまでに必要な、アカウント単位の一時的な HTTP または意味上の失敗結果の連続回数です。`0` はアカウント failover だけを無効にします。HTTP status が観測される前に rejection で終了した論理リクエストは、`(provider, canonical HTTP(S) origin)` をキーとするプロセスローカルな host health だけを更新して account probe lease を解放します。アカウントの quarantine、cooldown/failure streak、affinity、pool rotation、active account selection は変更しません。5 分以内に論理リクエスト単位の終端 host failure が 3 回発生すると host circuit が 30 秒間開き、その後は正確に 1 件の half-open 論理リクエストだけが許可され、同時リクエストは引き続き拒否されます。観測された HTTP failure status はアカウント evidence として残ります。`503` response の後に rejection が起きた場合、順序付き evidence は `503` をアカウント用に、後の rejection を host 用に保持します。プールされた通常の Responses と native compact リクエストの Codex bearer redirect は手動処理され、追従せず、`Location` を公開せず、サイズを制限したアカウント単位の `502` に変換されます。peer が資格情報を伴うリクエストを消費した可能性がある後で response body の read/close に失敗した場合、そのリクエストを別の資格情報では再送しません。その failure は、正常な代替アカウントを一時的にブロックする host circuit の成立に寄与する可能性があります。 | | `modelCacheTtlMs?` | `number` | `300000` |プロバイダーごとの `/models` キャッシュの鮮度ウィンドウ。 | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic プロンプト キャッシュ ポリシー: 無効、5 分間の一時的、または 1 時間の延長。 | | `tokenGuardian?` | `OcxTokenGuardianConfig` |オフ |オプションのプロアクティブな OAuth 更新および Codex アカウントのウォームアップ ポリシー。 | diff --git a/docs-site/src/content/docs/ko/reference/configuration/providers.md b/docs-site/src/content/docs/ko/reference/configuration/providers.md index d9cad66806..91ef6b5e13 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ko/reference/configuration/providers.md @@ -21,7 +21,7 @@ description: 공급자 항목, 인증, 엔드포인트, 모델 카탈로그, 할 | `autoSwitchThreshold?` | `number` | `80` | 사용량 기반 선제 전환 임계값입니다. `quota`는 바인딩된 작업과 바인딩 없는 작업의 다음 요청을 모두 재평가할 수 있고, `fill-first`는 바인딩 없는 작업 배정의 소진 기준으로만 사용하며, 기본 `round-robin` 선택은 이 값을 사용하지 않습니다. 알려진 5시간, 주간, 30일 quota window 중 가장 높은 점수를 씁니다. `0`은 사용량 기반 전환만 끄며 바인딩 없는 작업 배정이나 실패 복구는 끄지 않습니다. | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 새 작업/바인딩 없는 Codex 요청의 계정 배정 전략입니다. `(parent thread id, quota scope)`의 live affinity가 없으면 바인딩 없는 요청이며, 프록시 재시작이나 affinity 초기화 뒤에는 기존에 보이던 작업도 바인딩이 없어질 수 있습니다. `quota`는 활성 계정이 없을 때 알려진 usage가 가장 낮은 적격 계정을 선택하고, 적격 활성 계정이 `autoSwitchThreshold` 미만이면 유지합니다. 임계값 도달 뒤에는 바인딩 없는 요청이나 바인딩된 작업의 다음 요청을 usage가 더 낮은 적격 계정으로 옮길 수 있습니다. `round-robin`은 바인딩 없는 요청을 균등 분배하고, `fill-first`는 cooldown, 사용 불가 또는 drain threshold까지 활성 계정에 배정합니다. | | `accountPoolStickyLimit?` | `number` | `1` | 한 round-robin 선택이 다음으로 넘어가기 전에 유지하는 새 작업/바인딩 없는 작업 배정 수입니다. 카운터는 업스트림 성공 뒤가 아니라 작업을 바인딩할 때 증가합니다. 범위 1–100이며 `accountPoolStrategy`가 `round-robin`일 때만 적용됩니다. | -| `upstreamFailoverThreshold?` | `number` | `3` | 이후 새 세션이 계정 failover될 수 있기 전에 필요한 연속 계정 범위의 일시적 HTTP 또는 의미상 실패 결과 수입니다. `0`은 계정 failover만 비활성화합니다. HTTP 응답이 없는 거부, 연결/헤더 `TimeoutError`, 보수적으로 분류되는 read-then-close 실패는 계정 상태나 affinity를 바꾸지 않고 `(provider, canonical HTTP(S) origin)` 키의 프로세스 로컬 host health만 갱신합니다. 5분 안에 논리 요청 단위의 최종 host failure가 3회 발생하면 host circuit이 30초 동안 열리고, 이후 정확히 하나의 half-open 논리 요청만 허용되며 동시 요청은 계속 차단됩니다. 실제 HTTP 응답은 이전 host state를 지웁니다. `503` 후 거부가 발생하면 `503`은 계정 근거로 보존되고 뒤의 실패는 host failure로 기록됩니다. 풀의 일반 Responses 및 native compact 요청에 대한 Codex bearer redirect는 수동으로 처리되어 따라가지 않고 `Location`을 노출하지 않으며, 크기가 제한된 계정 범위 `502`로 변환됩니다. 자격 증명이 보이는 read-then-close 실패는 peer가 요청을 이미 소비했을 수 있어 다른 자격 증명으로 의도적으로 재전송하지 않으므로, 정상인 대체 계정을 일시적으로 차단할 수 있습니다. `200` 이후 본문/스트림 처리는 변경되지 않으며 이 설정의 범위 밖입니다. | +| `upstreamFailoverThreshold?` | `number` | `3` | 이후 새 세션이 계정 failover될 수 있기 전에 필요한 연속 계정 범위의 일시적 HTTP 또는 의미상 실패 결과 수입니다. `0`은 계정 failover만 비활성화합니다. HTTP 상태가 관측되기 전에 최종적으로 거부된 논리 요청은 `(provider, canonical HTTP(S) origin)` 키의 프로세스 로컬 host health만 갱신하고 계정 probe lease를 해제합니다. 계정 quarantine, cooldown/failure streak, affinity, pool rotation, active account selection은 변경하지 않습니다. 5분 안에 논리 요청 단위의 최종 host failure가 3회 발생하면 host circuit이 30초 동안 열리고, 이후 정확히 하나의 half-open 논리 요청만 허용되며 동시 요청은 계속 차단됩니다. 관측된 HTTP 실패 상태는 계정 근거로 남습니다. `503` 응답 후 거부가 발생하면 순서가 보존된 근거는 `503`을 계정용으로, 뒤의 거부를 host용으로 유지합니다. 풀의 일반 Responses 및 native compact 요청에 대한 Codex bearer redirect는 수동으로 처리되어 따라가지 않고 `Location`을 노출하지 않으며, 크기가 제한된 계정 범위 `502`로 변환됩니다. peer가 자격 증명을 실어 보낸 요청을 소비했을 수 있는 상태에서 응답 본문 read/close가 실패하면 그 요청을 다른 자격 증명으로 재전송하지 않습니다. 따라서 그 failure는 정상인 대체 계정을 일시적으로 차단하는 host circuit의 성립에 기여할 수 있습니다. | | `modelCacheTtlMs?` | `number` | `300000` | 공급자별 `/models` 캐시의 최신성 창입니다. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic 프롬프트 캐시 정책입니다. 비활성, 5분짜리 임시, 1시간짜리 확장 중 하나입니다. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | 꺼짐 | 선택적 선제 OAuth 갱신과 Codex 계정 워밍업 정책입니다. | diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 2bc4a18df8..cc171e5d48 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -22,7 +22,7 @@ authenticated. | `autoSwitchThreshold?` | `number` | `80` | Usage threshold for proactive switching. `quota` can re-evaluate both bound and unbound tasks on their next request; `fill-first` uses it only as the drain point for unbound assignment; normal `round-robin` selection does not use it. The score uses the hottest known 5h, weekly, or 30d quota window. `0` disables usage-based proactive switching only, not unbound assignment or failure recovery. | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | Assignment strategy for new/unbound Codex requests. A request is unbound when it has no live (parent thread id, quota scope) affinity; a visible existing task can become unbound after proxy restart or affinity reset. `quota` picks the lowest-usage eligible account when no active account exists, keeps an eligible active account below `autoSwitchThreshold`, and after the threshold may move an unbound request or proactively rebind a bound task to a lower-usage eligible account. `round-robin` distributes unbound requests evenly; `fill-first` keeps assigning unbound requests to the active account until cooldown, unavailability, or the configured drain threshold. | | `accountPoolStickyLimit?` | `number` | `1` | New/unbound task assignments retained on one round-robin selection before advancing; the counter advances when a task is bound, not after an upstream success. Range 1–100. | -| `upstreamFailoverThreshold?` | `number` | `3` | Consecutive account-scoped transient HTTP or semantic failure outcomes required before future new sessions may fail over; `0` disables only account failover. Rejections with no HTTP response, connect/header `TimeoutError`s, and conservatively classified read-then-close failures instead update process-local host health keyed by `(provider, canonical HTTP(S) origin)` without changing account health or affinity. Three terminal logical host failures within five minutes open the host circuit for 30 seconds; then exactly one half-open logical request is admitted while concurrent requests remain blocked. Any actual HTTP response clears prior host state. A `503` followed by a rejection retains the `503` as account evidence and records the later host failure. Codex bearer redirects for pooled regular Responses and native compact requests use manual redirect handling: they are not followed, `Location` is not exposed, and they become a bounded account-scoped `502`. A conservative credential-visible read-then-close failure can temporarily block an otherwise healthy alternate because a request the peer may have consumed is intentionally not replayed under another credential. Post-`200` body/stream handling is unchanged and outside this setting. | +| `upstreamFailoverThreshold?` | `number` | `3` | Consecutive account-scoped transient HTTP or semantic failure outcomes required before future new sessions may fail over; `0` disables only account failover. A terminal logical request that rejects before any HTTP status is observed updates only process-local host health keyed by `(provider, canonical HTTP(S) origin)` and releases any account probe lease; it does not change account quarantine, account cooldown/failure streak, affinity, pool rotation, or active account selection. Three terminal logical host failures within five minutes open the host circuit for 30 seconds; then exactly one half-open logical request is admitted while concurrent requests remain blocked. An observed HTTP failure status remains account evidence. If a `503` response is followed by a rejection, ordered evidence retains the `503` for the account and the later rejection for the host. Codex bearer redirects for pooled regular Responses and native compact requests are handled manually: they are not followed, `Location` is not exposed, and they become a bounded account-scoped `502`. If response-body read/close fails after the peer may have consumed a credential-bearing request, that request is not replayed under another credential; the failure can therefore contribute to a host circuit that temporarily blocks an otherwise healthy alternate. | | `modelCacheTtlMs?` | `number` | `300000` | Freshness window for the per-provider `/models` cache. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic prompt-cache policy: disabled, 5-minute ephemeral, or 1-hour extended. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | off | Optional proactive OAuth refresh and Codex-account warmup policy. | diff --git a/docs-site/src/content/docs/ru/reference/configuration/providers.md b/docs-site/src/content/docs/ru/reference/configuration/providers.md index 24f7fd4ac5..b19d5809fe 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ru/reference/configuration/providers.md @@ -22,7 +22,7 @@ description: Записи провайдеров, аутентификация, | `autoSwitchThreshold?` | `number` | `80` | Порог проактивного переключения по использованию. `quota` может повторно оценить следующий запрос как привязанной, так и непривязанной задачи; `fill-first` использует его только как точку исчерпания для непривязанных назначений; обычный `round-robin` его не использует. Оценка берёт самое горячее из окон 5 часов, недели и 30 дней. `0` отключает только переключение по использованию, но не назначение непривязанных задач и не восстановление после сбоев. | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | Стратегия назначения для новых/непривязанных запросов Codex. Запрос непривязан, если у него нет live affinity `(parent thread id, quota scope)`; видимая существующая задача может стать непривязанной после перезапуска прокси или сброса affinity. `quota` выбирает подходящий аккаунт с наименьшим известным usage, когда активного аккаунта нет, сохраняет подходящий активный аккаунт ниже `autoSwitchThreshold`, а после порога может перевести непривязанный запрос или следующий запрос привязанной задачи на подходящий аккаунт с меньшим usage. `round-robin` равномерно распределяет непривязанные запросы; `fill-first` назначает их активному аккаунту до cooldown, недоступности или порога исчерпания. | | `accountPoolStickyLimit?` | `number` | `1` | Число назначений новых/непривязанных задач на одном выборе round-robin перед переходом дальше. Счётчик растёт при привязке задачи, а не после успеха upstream. Диапазон 1–100; только при `accountPoolStrategy` = `round-robin`. | -| `upstreamFailoverThreshold?` | `number` | `3` | Количество последовательных временных HTTP- или семантических результатов сбоя, относящихся к аккаунту, после которого новые сессии могут выполнить failover; `0` отключает только failover аккаунта. Отклонения без HTTP-ответа, `TimeoutError` при подключении/ожидании заголовков и консервативно классифицированные сбои read-then-close обновляют только локальное для процесса состояние хоста с ключом `(provider, canonical HTTP(S) origin)`, не меняя состояние аккаунта и affinity. Три конечных сбоя хоста на уровне логического запроса за пять минут открывают circuit хоста на 30 секунд; затем допускается ровно один half-open логический запрос, а параллельные запросы остаются заблокированными. Любой фактический HTTP-ответ очищает предыдущее состояние хоста. При `503` с последующим отклонением `503` сохраняется как свидетельство для аккаунта, а последующее событие записывается как сбой хоста. Перенаправления bearer-запросов Codex для обычных Responses из пула и native compact обрабатываются вручную: переход не выполняется, `Location` не раскрывается, а результат преобразуется в ограниченный по размеру ответ `502`, учитываемый для аккаунта. При read-then-close с видимыми учетными данными исправный альтернативный аккаунт может временно блокироваться, поскольку запрос, уже потенциально принятый peer, намеренно не повторяется с другими учетными данными. Обработка тела/потока после `200` не изменяется и находится вне области этой настройки. | +| `upstreamFailoverThreshold?` | `number` | `3` | Количество последовательных временных HTTP- или семантических результатов сбоя, относящихся к аккаунту, после которого новые сессии могут выполнить failover; `0` отключает только failover аккаунта. Логический запрос, окончательно отклоненный до наблюдения какого-либо HTTP-статуса, обновляет только локальное для процесса состояние хоста с ключом `(provider, canonical HTTP(S) origin)` и освобождает lease account probe; он не меняет quarantine аккаунта, его cooldown/failure streak, affinity, pool rotation или выбор активного аккаунта. Три конечных сбоя хоста на уровне логического запроса за пять минут открывают circuit хоста на 30 секунд; затем допускается ровно один half-open логический запрос, а параллельные запросы остаются заблокированными. Наблюдаемый HTTP-статус сбоя остается свидетельством для аккаунта. Если за ответом `503` следует отклонение, упорядоченные свидетельства сохраняют `503` для аккаунта, а последующее отклонение для хоста. Перенаправления bearer-запросов Codex для обычных Responses из пула и native compact обрабатываются вручную: переход не выполняется, `Location` не раскрывается, а результат преобразуется в ограниченный по размеру ответ `502`, учитываемый для аккаунта. Если чтение/закрытие тела ответа завершается ошибкой после того, как peer мог принять запрос с учетными данными, запрос не повторяется с другими учетными данными. Такой сбой может способствовать открытию circuit хоста, который временно заблокирует исправный альтернативный аккаунт. | | `modelCacheTtlMs?` | `number` | `300000` | Окно свежести для кэша `/models` на уровне провайдера. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Политика prompt-cache Anthropic: отключено, 5-минутный ephemeral или 1-часовой extended. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | off | Необязательная политика proactive OAuth refresh и warmup'а аккаунтов Codex. | diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md index d4e1e57744..0220f4707b 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md @@ -21,7 +21,7 @@ description: 提供者条目、身份验证、端点、模型目录、配额、 | `autoSwitchThreshold?` | `number` | `80` | 基于用量的主动切换阈值。`quota` 可在下一次请求中重新评估已绑定和未绑定任务；`fill-first` 仅把它用作未绑定分配的耗尽点；正常 `round-robin` 不使用它。分数取已知 5 小时、周或 30 天 quota window 的最高值。`0` 只关闭基于用量的主动切换，不关闭未绑定任务分配或故障恢复。 | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 新建/未绑定 Codex 请求的分配策略。没有 live `(parent thread id, quota scope)` affinity 的请求属于未绑定；代理重启或 affinity 重置后，已有可见任务也可能未绑定。`quota` 在没有活跃账号时选择已知 usage 最低的合格账号；活跃账号合格且低于 `autoSwitchThreshold` 时继续使用；达到阈值后，可把未绑定请求或已绑定任务的下一次请求切换到 usage 更低的合格账号。`round-robin` 均匀分配未绑定请求；`fill-first` 在 cooldown、不可用或耗尽阈值前持续分配给活跃账号。 | | `accountPoolStickyLimit?` | `number` | `1` | 一次 round-robin 选择在推进前保留的新建/未绑定任务分配数。计数在任务绑定时增加，而不是在上游成功后增加。范围 1–100；仅当 `accountPoolStrategy` 为 `round-robin` 时生效。 | -| `upstreamFailoverThreshold?` | `number` | `3` | 新会话允许执行账户故障转移前所需的连续账户级瞬态 HTTP 或语义失败结果数；`0` 仅禁用账户故障转移。未收到 HTTP 响应的拒绝、连接/响应头 `TimeoutError`，以及保守判定的 read-then-close 故障，只更新以 `(provider, canonical HTTP(S) origin)` 为键的进程内主机健康状态，不改变账户健康状态或 affinity。五分钟内出现三次逻辑请求级最终主机故障时，主机熔断器打开 30 秒；之后只允许一个 half-open 逻辑请求，并继续阻止并发请求。任何实际 HTTP 响应都会清除先前主机状态。若先收到 `503` 后发生拒绝，`503` 保留为账户证据，后续事件记录为主机故障。池化普通 Responses 和 native compact 请求的 Codex bearer 重定向采用手动处理：不跟随、不暴露 `Location`，并转换为大小受限的账户级 `502`。对于凭据可见的 read-then-close 故障，peer 可能已经消费请求，因此不会使用其他凭据重放，这可能会暂时阻止原本健康的备用账户。`200` 之后的响应体/流处理保持不变，不属于此设置的范围。 | +| `upstreamFailoverThreshold?` | `number` | `3` | 新会话允许执行账户故障转移前所需的连续账户级瞬态 HTTP 或语义失败结果数；`0` 仅禁用账户故障转移。在尚未观测到任何 HTTP 状态时以拒绝结束的终端逻辑请求，只更新以 `(provider, canonical HTTP(S) origin)` 为键的进程内主机健康状态并释放账户探测 lease；它不会改变账户 quarantine、cooldown/failure streak、affinity、pool rotation 或活跃账户选择。五分钟内出现三次逻辑请求级最终主机故障时，主机熔断器打开 30 秒；之后只允许一个 half-open 逻辑请求，并继续阻止并发请求。已观测到的 HTTP 失败状态仍是账户证据。若 `503` 响应后发生拒绝，有序证据会为账户保留 `503`，并为主机保留后续拒绝。池化普通 Responses 和 native compact 请求的 Codex bearer 重定向采用手动处理：不跟随、不暴露 `Location`，并转换为大小受限的账户级 `502`。如果 peer 可能已消费携带凭据的请求，随后读取/关闭响应体又失败，则不会使用其他凭据重放该请求；这一故障因此可能促成主机熔断，并暂时阻止原本健康的备用账户。 | | `modelCacheTtlMs?` | `number` | `300000` | 每个提供者 `/models` 缓存的新鲜度窗口。 | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic 提示缓存策略：禁用、5 分钟临时缓存，或 1 小时扩展缓存。 | | `tokenGuardian?` | `OcxTokenGuardianConfig` | 关闭 | 可选的主动 OAuth 刷新与 Codex 账户预热策略。 | diff --git a/src/codex/upstream-host-health.ts b/src/codex/upstream-host-health.ts index 440f6e9a39..79690207af 100644 --- a/src/codex/upstream-host-health.ts +++ b/src/codex/upstream-host-health.ts @@ -87,23 +87,32 @@ function removeExpiredNonLeasedEntries(now: number): void { } } -function oldestNonLeasedKey(): CodexUpstreamHostKey | undefined { - let oldestKey: CodexUpstreamHostKey | undefined; - let oldestAt = Number.POSITIVE_INFINITY; +function oldestNonLeasedKey(now: number): CodexUpstreamHostKey | undefined { + let oldestPreferredKey: CodexUpstreamHostKey | undefined; + let oldestPreferredAt = Number.POSITIVE_INFINITY; + let oldestCooldownKey: CodexUpstreamHostKey | undefined; + let oldestCooldownAt = Number.POSITIVE_INFINITY; for (const [key, health] of upstreamHostHealth) { if (health.activeLeaseIds.size > 0) continue; - if (health.lastTouchedAt < oldestAt) { - oldestKey = key; - oldestAt = health.lastTouchedAt; + if (health.cooldownUntil !== undefined && health.cooldownUntil > now) { + if (health.lastTouchedAt < oldestCooldownAt) { + oldestCooldownKey = key; + oldestCooldownAt = health.lastTouchedAt; + } + continue; + } + if (health.lastTouchedAt < oldestPreferredAt) { + oldestPreferredKey = key; + oldestPreferredAt = health.lastTouchedAt; } } - return oldestKey; + return oldestPreferredKey ?? oldestCooldownKey; } function makeRoom(now: number): void { removeExpiredNonLeasedEntries(now); while (upstreamHostHealth.size >= CODEX_UPSTREAM_HOST_MAX_ENTRIES) { - const oldestKey = oldestNonLeasedKey(); + const oldestKey = oldestNonLeasedKey(now); if (!oldestKey) return; // Every entry is leased: preserve correctness with temporary overflow. upstreamHostHealth.delete(oldestKey); } @@ -112,7 +121,7 @@ function makeRoom(now: number): void { function pruneOverflow(now: number): void { removeExpiredNonLeasedEntries(now); while (upstreamHostHealth.size > CODEX_UPSTREAM_HOST_MAX_ENTRIES) { - const oldestKey = oldestNonLeasedKey(); + const oldestKey = oldestNonLeasedKey(now); if (!oldestKey) return; upstreamHostHealth.delete(oldestKey); } @@ -243,18 +252,12 @@ export function recordCodexUpstreamHostFailure( if (current.halfOpenLeaseId === lease.leaseId) delete current.halfOpenLeaseId; if (options.observedResponse) { - upstreamHostHealth.delete(lease.key); - makeRoom(now); - const afterResponse: CodexUpstreamHostHealth = { - consecutiveFailures: 1, - lastFailureAt: now, - lastTouchedAt: now, - generation: nextGeneration(), - activeLeaseIds: new Set(), - }; - upstreamHostHealth.set(lease.key, afterResponse); + current.consecutiveFailures = 1; + current.lastFailureAt = now; + current.lastTouchedAt = now; + delete current.cooldownUntil; pruneOverflow(now); - return snapshot(afterResponse); + return snapshot(current); } const reopensCircuit = lease.halfOpen || current.cooldownUntil !== undefined; diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index 4c0ecfd68b..5f75ee1f12 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -475,7 +475,7 @@ export async function handleResponsesCompact( } if (observedStatus !== undefined) { recordCompactPoolOutcome(ctx, observedStatus); - } else if (compactHostAdmissionLease) { + } else { releaseCodexAuthContextProbeLease(ctx); } if (compactHostAdmissionLease) { diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index fc123175e0..ce38004db1 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -1811,7 +1811,7 @@ async function handleResponsesInner( } if (observedStatus !== undefined) { recordPoolTransportOutcome(observedStatus); - } else if (hostAdmissionLease) { + } else { releaseCodexAuthContextProbeLease(authCtx); } if (hostAdmissionLease) { diff --git a/tests/codex-host-health-runtime.test.ts b/tests/codex-host-health-runtime.test.ts index 60952833ea..057ae70a46 100644 --- a/tests/codex-host-health-runtime.test.ts +++ b/tests/codex-host-health-runtime.test.ts @@ -141,6 +141,9 @@ async function startCredentialDependentReadThenCloseServer(): Promise<{ const successfulBRequests: string[] = []; const server = createServer(socket => { trackedSockets.add(socket); + socket.on("error", () => { + // Expected when this fixture deliberately severs a credential-bearing request. + }); let bytes = Buffer.alloc(0); socket.on("data", chunk => { bytes = Buffer.concat([bytes, chunk]); diff --git a/tests/codex-upstream-host-health.test.ts b/tests/codex-upstream-host-health.test.ts index ab2a204976..414d84f46f 100644 --- a/tests/codex-upstream-host-health.test.ts +++ b/tests/codex-upstream-host-health.test.ts @@ -9,6 +9,7 @@ import { clearCodexUpstreamHostHealth, getCodexUpstreamHostCooldownUntil, getCodexUpstreamHostHealth, + isCodexUpstreamRedirectStatus, recordCodexUpstreamHostFailure, recordCodexUpstreamHostResponse, releaseCodexUpstreamHostAdmissionLease, @@ -44,6 +45,24 @@ describe("Codex upstream host health (#914)", () => { expect(canonicalCodexUpstreamHostKey("openai", "ftp://chatgpt.com/file")).toBeNull(); }); + test("normalizes trailing dots, IPv6 brackets, and the implicit HTTP port", () => { + expect(canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com./path")) + .toBe(canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com/other")); + expect(canonicalCodexUpstreamHostKey(" OpenAI ", "https://[2001:DB8::1]/path")) + .toBe("openai\u0000https://[2001:db8::1]:443"); + expect(canonicalCodexUpstreamHostKey("openai", "http://chatgpt.com/path")) + .toBe(canonicalCodexUpstreamHostKey("openai", "http://chatgpt.com:80/other")); + }); + + test("identifies only supported upstream redirect statuses", () => { + for (const status of [300, 301, 302, 303, 307, 308]) { + expect(isCodexUpstreamRedirectStatus(status)).toBe(true); + } + for (const status of [299, 304, 305, 306, 309]) { + expect(isCodexUpstreamRedirectStatus(status)).toBe(false); + } + }); + test("opens only at its threshold and a half-open HTTP response clears it", () => { const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com/backend-api/codex")!; const now = 1_900_000_000_000; @@ -120,6 +139,23 @@ describe("Codex upstream host health (#914)", () => { expect(releaseCodexUpstreamHostAdmissionLease(next, probeAt)).toBe(true); }); + test("an observed response preserves a concurrent lease and its later failure", () => { + const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com")!; + const now = 2_750; + const observedResponse = admit(key, now); + const concurrentFailure = admit(key, now); + + const reset = recordCodexUpstreamHostFailure(observedResponse, now, { + observedResponse: true, + })!; + expect(reset.consecutiveFailures).toBe(1); + expect(reset.cooldownUntil).toBeUndefined(); + + const afterConcurrentFailure = recordCodexUpstreamHostFailure(concurrentFailure, now + 1); + expect(afterConcurrentFailure?.consecutiveFailures).toBe(2); + expect(afterConcurrentFailure?.cooldownUntil).toBeUndefined(); + }); + test("caller abort releases a half-open admission without adding evidence", () => { const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com")!; const trippedAt = 3_000; @@ -184,6 +220,35 @@ describe("Codex upstream host health (#914)", () => { expect(recordCodexUpstreamHostResponse(halfOpen, probeAt)).toBe(true); }); + test("capacity pressure evicts a non-cooldown entry before an active cooldown", () => { + const cooldownKey = canonicalCodexUpstreamHostKey("openai", "https://cooldown.example")!; + const trippedAt = 6_000; + let tripped: CodexUpstreamHostHealthSnapshot | null = null; + for (let attempt = 0; attempt < CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; attempt++) { + tripped = fail(cooldownKey, trippedAt + attempt); + } + const cooldownUntil = tripped!.cooldownUntil!; + const oldestOrdinaryKey = canonicalCodexUpstreamHostKey( + "ordinary-0", + "https://ordinary-0.example", + )!; + for (let index = 0; index < CODEX_UPSTREAM_HOST_MAX_ENTRIES - 1; index++) { + const key = canonicalCodexUpstreamHostKey( + `ordinary-${index}`, + `https://ordinary-${index}.example`, + )!; + fail(key, trippedAt + 100 + index); + } + + const newcomerKey = canonicalCodexUpstreamHostKey("newcomer", "https://newcomer.example")!; + const pressureAt = trippedAt + 100 + CODEX_UPSTREAM_HOST_MAX_ENTRIES; + fail(newcomerKey, pressureAt); + + expect(getCodexUpstreamHostCooldownUntil(cooldownKey, pressureAt)).toBe(cooldownUntil); + expect(getCodexUpstreamHostHealth(oldestOrdinaryKey, pressureAt)).toBeNull(); + expect(getCodexUpstreamHostHealth(newcomerKey, pressureAt)).not.toBeNull(); + }); + test("bounds the process-local map and evicts the oldest non-leased entry", () => { const first = canonicalCodexUpstreamHostKey("provider-0", "https://host-0.example")!; for (let index = 0; index <= CODEX_UPSTREAM_HOST_MAX_ENTRIES; index++) { diff --git a/tests/issue-452-empty-503.test.ts b/tests/issue-452-empty-503.test.ts index b971a2fff2..4091d1c2d7 100644 --- a/tests/issue-452-empty-503.test.ts +++ b/tests/issue-452-empty-503.test.ts @@ -7,6 +7,7 @@ import { clearCodexUpstreamHealth, clearThreadAccountMap, getCodexUpstreamHealth import { canonicalCodexUpstreamHostKey, clearCodexUpstreamHostHealth, + CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD, getCodexUpstreamHostHealth, } from "../src/codex/upstream-host-health"; import { saveConfig } from "../src/config"; @@ -194,11 +195,11 @@ async function withPoolPassthrough( function installCodexTransport( send: (init: RequestInit | undefined) => Response | Promise, ): void { - globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => { + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { const value = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; const url = new URL(value); if (url.hostname === "chatgpt.com" && url.pathname.startsWith("/backend-api/codex")) { - return Promise.resolve(send(init)); + return send(init); } return originalGlobalFetch(input, init); }) as typeof fetch; @@ -265,12 +266,12 @@ describe("regular Codex provider-host settlement (#914)", () => { sends += 1; throw new Error("opaque transport rejection"); }); - for (let attempt = 0; attempt < 3; attempt++) { + for (let attempt = 0; attempt < CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; attempt++) { expect((await sendRegularPoolRequest(serverUrl)).status).toBe(502); } const blocked = await sendRegularPoolRequest(serverUrl); expect(blocked.status).toBe(502); - expect(sends).toBe(3); + expect(sends).toBe(CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD); expect(Number(blocked.headers.get("retry-after"))).toBeGreaterThanOrEqual(1); }); }); diff --git a/tests/server-auth.test.ts b/tests/server-auth.test.ts index d211e864d9..9a564a7637 100644 --- a/tests/server-auth.test.ts +++ b/tests/server-auth.test.ts @@ -10,6 +10,7 @@ import { clearCodexWebSocketRegistry, getTrackedCodexWebSocketCountForAccount } import { INTERNAL_DEADLINE_MS, SERVER_BUDGET_MS } from "./helpers/test-budget"; import { clearAccountNeedsReauth, clearAccountQuota, getAccountQuota, isAccountNeedsReauth, markAccountNeedsReauth, updateAccountQuota } from "../src/codex/auth-api"; import { + CODEX_QUOTA_PROBE_INTERVAL_MS, CODEX_THREAD_AFFINITY_IDLE_TTL_MS, clearCodexUpstreamHealth, clearThreadAccountMap, @@ -2659,6 +2660,74 @@ describe("server local API auth", () => { } }, { timeout: 30_000 }); + for (const path of ["/v1/responses", "/v1/responses/compact"] as const) { + test(`${path} releases a pool probe after a pre-response transport failure with no host key`, async () => { + const harness = await startPoolRetryHarness(() => new Response("unused"), { secondAccount: false }); + const upstreamUrl = `https://chatgpt.com/backend-api/codex${path.slice(3)}`; + const redirectedFetch = globalThis.fetch; + const NativeURL = globalThis.URL; + const observedPoolAuth: Array<{ authorization: string | null; accountId: string | null }> = []; + const observedProbeLeaseIds: Array = []; + let forcedNullHostKey = false; + + class NullHostKeyURL extends NativeURL { + constructor(input: string | URL, base?: string | URL) { + const value = typeof input === "string" ? input : input.toString(); + if (!forcedNullHostKey && base === undefined && value === upstreamUrl) { + forcedNullHostKey = true; + throw new TypeError("synthetic null canonical host key"); + } + super(input, base); + } + } + + globalThis.URL = NullHostKeyURL as typeof URL; + globalThis.fetch = (async (input, init) => { + const requestUrl = typeof input === "string" + ? input + : input instanceof NativeURL + ? input.toString() + : input.url; + if (requestUrl === upstreamUrl) { + const headers = new Headers(init?.headers); + observedPoolAuth.push({ + authorization: headers.get("authorization"), + accountId: headers.get("chatgpt-account-id"), + }); + observedProbeLeaseIds.push(getCodexUpstreamHealth("pool-a")?.probeLeaseId); + throw new Error("synthetic null-host transport failure"); + } + return redirectedFetch(input, init); + }) as typeof fetch; + + const now = Date.now(); + recordCodexUpstreamOutcome(harness.config, "pool-a", 429, { + now: now - CODEX_QUOTA_PROBE_INTERVAL_MS, + resetAt: Math.floor((now + 60 * 60_000) / 1000), + }); + + try { + const response = await harness.request({ path }); + + expect(response.status).toBe(502); + expect(forcedNullHostKey).toBe(true); + expect(observedPoolAuth.length).toBeGreaterThan(0); + expect(observedPoolAuth).toEqual(observedPoolAuth.map(() => ({ + authorization: "Bearer pool-a-token", + accountId: "acct-pool-a", + }))); + expect(observedProbeLeaseIds.length).toBeGreaterThan(0); + expect(observedProbeLeaseIds.every(Boolean)).toBe(true); + expect(getCodexUpstreamHealth("pool-a")?.cooldownUntil).toEqual(expect.any(Number)); + expect(getCodexUpstreamHealth("pool-a")?.probeLeaseId).toBeUndefined(); + } finally { + globalThis.URL = NativeURL; + globalThis.fetch = redirectedFetch; + await stopPoolRetryHarness(harness); + } + }); + } + test("passthrough connect failure leaves selected pool account health clear", async () => { if (existsSync(TEST_DIR)) rmSync(TEST_DIR, { recursive: true }); mkdirSync(TEST_DIR, { recursive: true }); From 9bfe1de981ed6cbbdad30efa82007cb7f4a7057d Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:11:52 +0900 Subject: [PATCH 09/20] fix(codex): settle recovery probes on local failures --- .../ja/reference/configuration/providers.md | 2 +- .../ko/reference/configuration/providers.md | 2 +- .../docs/reference/configuration/providers.md | 2 +- .../ru/reference/configuration/providers.md | 2 +- .../reference/configuration/providers.md | 2 +- src/server/responses/compact.ts | 43 +++++++++++-- src/server/responses/core.ts | 3 + tests/responses-compaction-routing.test.ts | 51 +++++++++++++++ tests/server-auth.test.ts | 63 ++++++++++++++++++- 9 files changed, 159 insertions(+), 11 deletions(-) diff --git a/docs-site/src/content/docs/ja/reference/configuration/providers.md b/docs-site/src/content/docs/ja/reference/configuration/providers.md index 3c6857ed77..d6615e18be 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ja/reference/configuration/providers.md @@ -21,7 +21,7 @@ description: プロバイダー エントリ、認証、エンドポイント、 | `autoSwitchThreshold?` | `number` | `80` | 使用量ベースのプロアクティブ切り替えしきい値。`quota` は紐付け済み/未紐付けタスクの次のリクエストを再評価でき、`fill-first` は未紐付け割り当ての使い切り基準としてのみ使用し、通常の `round-robin` 選択は使用しません。既知の 5 時間、週次、30 日 quota window の最大スコアを使います。`0` は使用量ベースの切り替えだけを無効にし、未紐付け割り当てや障害回復は無効にしません。 | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 新規/未紐付け Codex リクエストの割り当て戦略。live な `(parent thread id, quota scope)` affinity がなければ未紐付けで、プロキシ再起動や affinity リセット後は既存の表示タスクも未紐付けになり得ます。`quota` はアクティブアカウントがなければ既知 usage 最小の適格アカウントを選び、適格なアクティブアカウントが `autoSwitchThreshold` 未満なら維持します。しきい値到達後は、未紐付けリクエストまたは紐付け済みタスクの次のリクエストを usage の低い適格アカウントへ移せます。`round-robin` は未紐付けリクエストを均等分散し、`fill-first` は cooldown、使用不可、または drain threshold までアクティブアカウントへ割り当てます。 | | `accountPoolStickyLimit?` | `number` | `1` | 1 回の round-robin 選択で次へ進む前に保持する新規/未紐付けタスク割り当て数。カウンターは上流の成功後ではなくタスクの紐付け時に増えます。範囲 1–100。`accountPoolStrategy` が `round-robin` のときのみ。 | -| `upstreamFailoverThreshold?` | `number` | `3` | 将来の新規セッションでアカウントの failover を行えるようになるまでに必要な、アカウント単位の一時的な HTTP または意味上の失敗結果の連続回数です。`0` はアカウント failover だけを無効にします。HTTP status が観測される前に rejection で終了した論理リクエストは、`(provider, canonical HTTP(S) origin)` をキーとするプロセスローカルな host health だけを更新して account probe lease を解放します。アカウントの quarantine、cooldown/failure streak、affinity、pool rotation、active account selection は変更しません。5 分以内に論理リクエスト単位の終端 host failure が 3 回発生すると host circuit が 30 秒間開き、その後は正確に 1 件の half-open 論理リクエストだけが許可され、同時リクエストは引き続き拒否されます。観測された HTTP failure status はアカウント evidence として残ります。`503` response の後に rejection が起きた場合、順序付き evidence は `503` をアカウント用に、後の rejection を host 用に保持します。プールされた通常の Responses と native compact リクエストの Codex bearer redirect は手動処理され、追従せず、`Location` を公開せず、サイズを制限したアカウント単位の `502` に変換されます。peer が資格情報を伴うリクエストを消費した可能性がある後で response body の read/close に失敗した場合、そのリクエストを別の資格情報では再送しません。その failure は、正常な代替アカウントを一時的にブロックする host circuit の成立に寄与する可能性があります。 | +| `upstreamFailoverThreshold?` | `number` | `3` | 将来の新規セッションでアカウントの failover を行えるようになるまでに必要な、アカウント単位の一時的な HTTP または意味上の失敗結果の連続回数です。`0` はアカウント failover だけを無効にします。HTTP status が観測される前に rejection で終了した論理リクエストは、`(provider, canonical HTTP(S) origin)` をキーとするプロセスローカルな host health だけを更新して account probe lease を解放します。アカウントの quarantine、cooldown/failure streak、affinity、pool rotation、active account selection は変更しません。5 分以内に論理リクエスト単位の終端 host failure が 3 回発生すると host circuit が 30 秒間開き、その後は正確に 1 件の half-open 論理リクエストだけが許可され、同時リクエストは引き続き拒否されます。観測された HTTP failure status はアカウント evidence として残ります。`503` response の後に rejection が起きた場合、順序付き evidence は `503` をアカウント用に、後の rejection を host 用に保持します。プールされた通常の Responses と native compact リクエストの Codex bearer redirect は手動処理され、追従せず、`Location` を公開せず、サイズを制限したアカウント単位の `502` に変換されます。peer が資格情報を伴うリクエストを消費した可能性があっても、HTTP status が観測される前に transport rejection が発生した場合、そのリクエストを別の資格情報では再送しません。その終端 rejection は、正常な代替アカウントを一時的にブロックする host circuit の成立に寄与する可能性があります。 | | `modelCacheTtlMs?` | `number` | `300000` |プロバイダーごとの `/models` キャッシュの鮮度ウィンドウ。 | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic プロンプト キャッシュ ポリシー: 無効、5 分間の一時的、または 1 時間の延長。 | | `tokenGuardian?` | `OcxTokenGuardianConfig` |オフ |オプションのプロアクティブな OAuth 更新および Codex アカウントのウォームアップ ポリシー。 | diff --git a/docs-site/src/content/docs/ko/reference/configuration/providers.md b/docs-site/src/content/docs/ko/reference/configuration/providers.md index 91ef6b5e13..393d4bb542 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ko/reference/configuration/providers.md @@ -21,7 +21,7 @@ description: 공급자 항목, 인증, 엔드포인트, 모델 카탈로그, 할 | `autoSwitchThreshold?` | `number` | `80` | 사용량 기반 선제 전환 임계값입니다. `quota`는 바인딩된 작업과 바인딩 없는 작업의 다음 요청을 모두 재평가할 수 있고, `fill-first`는 바인딩 없는 작업 배정의 소진 기준으로만 사용하며, 기본 `round-robin` 선택은 이 값을 사용하지 않습니다. 알려진 5시간, 주간, 30일 quota window 중 가장 높은 점수를 씁니다. `0`은 사용량 기반 전환만 끄며 바인딩 없는 작업 배정이나 실패 복구는 끄지 않습니다. | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 새 작업/바인딩 없는 Codex 요청의 계정 배정 전략입니다. `(parent thread id, quota scope)`의 live affinity가 없으면 바인딩 없는 요청이며, 프록시 재시작이나 affinity 초기화 뒤에는 기존에 보이던 작업도 바인딩이 없어질 수 있습니다. `quota`는 활성 계정이 없을 때 알려진 usage가 가장 낮은 적격 계정을 선택하고, 적격 활성 계정이 `autoSwitchThreshold` 미만이면 유지합니다. 임계값 도달 뒤에는 바인딩 없는 요청이나 바인딩된 작업의 다음 요청을 usage가 더 낮은 적격 계정으로 옮길 수 있습니다. `round-robin`은 바인딩 없는 요청을 균등 분배하고, `fill-first`는 cooldown, 사용 불가 또는 drain threshold까지 활성 계정에 배정합니다. | | `accountPoolStickyLimit?` | `number` | `1` | 한 round-robin 선택이 다음으로 넘어가기 전에 유지하는 새 작업/바인딩 없는 작업 배정 수입니다. 카운터는 업스트림 성공 뒤가 아니라 작업을 바인딩할 때 증가합니다. 범위 1–100이며 `accountPoolStrategy`가 `round-robin`일 때만 적용됩니다. | -| `upstreamFailoverThreshold?` | `number` | `3` | 이후 새 세션이 계정 failover될 수 있기 전에 필요한 연속 계정 범위의 일시적 HTTP 또는 의미상 실패 결과 수입니다. `0`은 계정 failover만 비활성화합니다. HTTP 상태가 관측되기 전에 최종적으로 거부된 논리 요청은 `(provider, canonical HTTP(S) origin)` 키의 프로세스 로컬 host health만 갱신하고 계정 probe lease를 해제합니다. 계정 quarantine, cooldown/failure streak, affinity, pool rotation, active account selection은 변경하지 않습니다. 5분 안에 논리 요청 단위의 최종 host failure가 3회 발생하면 host circuit이 30초 동안 열리고, 이후 정확히 하나의 half-open 논리 요청만 허용되며 동시 요청은 계속 차단됩니다. 관측된 HTTP 실패 상태는 계정 근거로 남습니다. `503` 응답 후 거부가 발생하면 순서가 보존된 근거는 `503`을 계정용으로, 뒤의 거부를 host용으로 유지합니다. 풀의 일반 Responses 및 native compact 요청에 대한 Codex bearer redirect는 수동으로 처리되어 따라가지 않고 `Location`을 노출하지 않으며, 크기가 제한된 계정 범위 `502`로 변환됩니다. peer가 자격 증명을 실어 보낸 요청을 소비했을 수 있는 상태에서 응답 본문 read/close가 실패하면 그 요청을 다른 자격 증명으로 재전송하지 않습니다. 따라서 그 failure는 정상인 대체 계정을 일시적으로 차단하는 host circuit의 성립에 기여할 수 있습니다. | +| `upstreamFailoverThreshold?` | `number` | `3` | 이후 새 세션이 계정 failover될 수 있기 전에 필요한 연속 계정 범위의 일시적 HTTP 또는 의미상 실패 결과 수입니다. `0`은 계정 failover만 비활성화합니다. HTTP 상태가 관측되기 전에 최종적으로 거부된 논리 요청은 `(provider, canonical HTTP(S) origin)` 키의 프로세스 로컬 host health만 갱신하고 계정 probe lease를 해제합니다. 계정 quarantine, cooldown/failure streak, affinity, pool rotation, active account selection은 변경하지 않습니다. 5분 안에 논리 요청 단위의 최종 host failure가 3회 발생하면 host circuit이 30초 동안 열리고, 이후 정확히 하나의 half-open 논리 요청만 허용되며 동시 요청은 계속 차단됩니다. 관측된 HTTP 실패 상태는 계정 근거로 남습니다. `503` 응답 후 거부가 발생하면 순서가 보존된 근거는 `503`을 계정용으로, 뒤의 거부를 host용으로 유지합니다. 풀의 일반 Responses 및 native compact 요청에 대한 Codex bearer redirect는 수동으로 처리되어 따라가지 않고 `Location`을 노출하지 않으며, 크기가 제한된 계정 범위 `502`로 변환됩니다. peer가 자격 증명을 실어 보낸 요청을 소비했을 수 있더라도 HTTP 상태가 관측되기 전에 transport rejection이 발생하면 그 요청을 다른 자격 증명으로 재전송하지 않습니다. 이 최종 rejection은 정상인 대체 계정을 일시적으로 차단하는 host circuit의 성립에 기여할 수 있습니다. | | `modelCacheTtlMs?` | `number` | `300000` | 공급자별 `/models` 캐시의 최신성 창입니다. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic 프롬프트 캐시 정책입니다. 비활성, 5분짜리 임시, 1시간짜리 확장 중 하나입니다. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | 꺼짐 | 선택적 선제 OAuth 갱신과 Codex 계정 워밍업 정책입니다. | diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index cc171e5d48..6fc36c32c1 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -22,7 +22,7 @@ authenticated. | `autoSwitchThreshold?` | `number` | `80` | Usage threshold for proactive switching. `quota` can re-evaluate both bound and unbound tasks on their next request; `fill-first` uses it only as the drain point for unbound assignment; normal `round-robin` selection does not use it. The score uses the hottest known 5h, weekly, or 30d quota window. `0` disables usage-based proactive switching only, not unbound assignment or failure recovery. | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | Assignment strategy for new/unbound Codex requests. A request is unbound when it has no live (parent thread id, quota scope) affinity; a visible existing task can become unbound after proxy restart or affinity reset. `quota` picks the lowest-usage eligible account when no active account exists, keeps an eligible active account below `autoSwitchThreshold`, and after the threshold may move an unbound request or proactively rebind a bound task to a lower-usage eligible account. `round-robin` distributes unbound requests evenly; `fill-first` keeps assigning unbound requests to the active account until cooldown, unavailability, or the configured drain threshold. | | `accountPoolStickyLimit?` | `number` | `1` | New/unbound task assignments retained on one round-robin selection before advancing; the counter advances when a task is bound, not after an upstream success. Range 1–100. | -| `upstreamFailoverThreshold?` | `number` | `3` | Consecutive account-scoped transient HTTP or semantic failure outcomes required before future new sessions may fail over; `0` disables only account failover. A terminal logical request that rejects before any HTTP status is observed updates only process-local host health keyed by `(provider, canonical HTTP(S) origin)` and releases any account probe lease; it does not change account quarantine, account cooldown/failure streak, affinity, pool rotation, or active account selection. Three terminal logical host failures within five minutes open the host circuit for 30 seconds; then exactly one half-open logical request is admitted while concurrent requests remain blocked. An observed HTTP failure status remains account evidence. If a `503` response is followed by a rejection, ordered evidence retains the `503` for the account and the later rejection for the host. Codex bearer redirects for pooled regular Responses and native compact requests are handled manually: they are not followed, `Location` is not exposed, and they become a bounded account-scoped `502`. If response-body read/close fails after the peer may have consumed a credential-bearing request, that request is not replayed under another credential; the failure can therefore contribute to a host circuit that temporarily blocks an otherwise healthy alternate. | +| `upstreamFailoverThreshold?` | `number` | `3` | Consecutive account-scoped transient HTTP or semantic failure outcomes required before future new sessions may fail over; `0` disables only account failover. A terminal logical request that rejects before any HTTP status is observed updates only process-local host health keyed by `(provider, canonical HTTP(S) origin)` and releases any account probe lease; it does not change account quarantine, account cooldown/failure streak, affinity, pool rotation, or active account selection. Three terminal logical host failures within five minutes open the host circuit for 30 seconds; then exactly one half-open logical request is admitted while concurrent requests remain blocked. An observed HTTP failure status remains account evidence. If a `503` response is followed by a rejection, ordered evidence retains the `503` for the account and the later rejection for the host. Codex bearer redirects for pooled regular Responses and native compact requests are handled manually: they are not followed, `Location` is not exposed, and they become a bounded account-scoped `502`. If the peer may have consumed a credential-bearing request but the transport rejects before any HTTP status is observed, the request is not replayed under another credential; that terminal rejection can contribute to a host circuit that temporarily blocks an otherwise healthy alternate. | | `modelCacheTtlMs?` | `number` | `300000` | Freshness window for the per-provider `/models` cache. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic prompt-cache policy: disabled, 5-minute ephemeral, or 1-hour extended. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | off | Optional proactive OAuth refresh and Codex-account warmup policy. | diff --git a/docs-site/src/content/docs/ru/reference/configuration/providers.md b/docs-site/src/content/docs/ru/reference/configuration/providers.md index b19d5809fe..a1f8915b5e 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ru/reference/configuration/providers.md @@ -22,7 +22,7 @@ description: Записи провайдеров, аутентификация, | `autoSwitchThreshold?` | `number` | `80` | Порог проактивного переключения по использованию. `quota` может повторно оценить следующий запрос как привязанной, так и непривязанной задачи; `fill-first` использует его только как точку исчерпания для непривязанных назначений; обычный `round-robin` его не использует. Оценка берёт самое горячее из окон 5 часов, недели и 30 дней. `0` отключает только переключение по использованию, но не назначение непривязанных задач и не восстановление после сбоев. | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | Стратегия назначения для новых/непривязанных запросов Codex. Запрос непривязан, если у него нет live affinity `(parent thread id, quota scope)`; видимая существующая задача может стать непривязанной после перезапуска прокси или сброса affinity. `quota` выбирает подходящий аккаунт с наименьшим известным usage, когда активного аккаунта нет, сохраняет подходящий активный аккаунт ниже `autoSwitchThreshold`, а после порога может перевести непривязанный запрос или следующий запрос привязанной задачи на подходящий аккаунт с меньшим usage. `round-robin` равномерно распределяет непривязанные запросы; `fill-first` назначает их активному аккаунту до cooldown, недоступности или порога исчерпания. | | `accountPoolStickyLimit?` | `number` | `1` | Число назначений новых/непривязанных задач на одном выборе round-robin перед переходом дальше. Счётчик растёт при привязке задачи, а не после успеха upstream. Диапазон 1–100; только при `accountPoolStrategy` = `round-robin`. | -| `upstreamFailoverThreshold?` | `number` | `3` | Количество последовательных временных HTTP- или семантических результатов сбоя, относящихся к аккаунту, после которого новые сессии могут выполнить failover; `0` отключает только failover аккаунта. Логический запрос, окончательно отклоненный до наблюдения какого-либо HTTP-статуса, обновляет только локальное для процесса состояние хоста с ключом `(provider, canonical HTTP(S) origin)` и освобождает lease account probe; он не меняет quarantine аккаунта, его cooldown/failure streak, affinity, pool rotation или выбор активного аккаунта. Три конечных сбоя хоста на уровне логического запроса за пять минут открывают circuit хоста на 30 секунд; затем допускается ровно один half-open логический запрос, а параллельные запросы остаются заблокированными. Наблюдаемый HTTP-статус сбоя остается свидетельством для аккаунта. Если за ответом `503` следует отклонение, упорядоченные свидетельства сохраняют `503` для аккаунта, а последующее отклонение для хоста. Перенаправления bearer-запросов Codex для обычных Responses из пула и native compact обрабатываются вручную: переход не выполняется, `Location` не раскрывается, а результат преобразуется в ограниченный по размеру ответ `502`, учитываемый для аккаунта. Если чтение/закрытие тела ответа завершается ошибкой после того, как peer мог принять запрос с учетными данными, запрос не повторяется с другими учетными данными. Такой сбой может способствовать открытию circuit хоста, который временно заблокирует исправный альтернативный аккаунт. | +| `upstreamFailoverThreshold?` | `number` | `3` | Количество последовательных временных HTTP- или семантических результатов сбоя, относящихся к аккаунту, после которого новые сессии могут выполнить failover; `0` отключает только failover аккаунта. Логический запрос, окончательно отклоненный до наблюдения какого-либо HTTP-статуса, обновляет только локальное для процесса состояние хоста с ключом `(provider, canonical HTTP(S) origin)` и освобождает lease account probe; он не меняет quarantine аккаунта, его cooldown/failure streak, affinity, pool rotation или выбор активного аккаунта. Три конечных сбоя хоста на уровне логического запроса за пять минут открывают circuit хоста на 30 секунд; затем допускается ровно один half-open логический запрос, а параллельные запросы остаются заблокированными. Наблюдаемый HTTP-статус сбоя остается свидетельством для аккаунта. Если за ответом `503` следует отклонение, упорядоченные свидетельства сохраняют `503` для аккаунта, а последующее отклонение для хоста. Перенаправления bearer-запросов Codex для обычных Responses из пула и native compact обрабатываются вручную: переход не выполняется, `Location` не раскрывается, а результат преобразуется в ограниченный по размеру ответ `502`, учитываемый для аккаунта. Если peer мог принять запрос с учетными данными, но до наблюдения какого-либо HTTP-статуса происходит transport rejection, запрос не повторяется с другими учетными данными. Такое конечное отклонение может способствовать открытию circuit хоста, который временно заблокирует исправный альтернативный аккаунт. | | `modelCacheTtlMs?` | `number` | `300000` | Окно свежести для кэша `/models` на уровне провайдера. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Политика prompt-cache Anthropic: отключено, 5-минутный ephemeral или 1-часовой extended. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | off | Необязательная политика proactive OAuth refresh и warmup'а аккаунтов Codex. | diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md index 0220f4707b..3c14c90def 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md @@ -21,7 +21,7 @@ description: 提供者条目、身份验证、端点、模型目录、配额、 | `autoSwitchThreshold?` | `number` | `80` | 基于用量的主动切换阈值。`quota` 可在下一次请求中重新评估已绑定和未绑定任务；`fill-first` 仅把它用作未绑定分配的耗尽点；正常 `round-robin` 不使用它。分数取已知 5 小时、周或 30 天 quota window 的最高值。`0` 只关闭基于用量的主动切换，不关闭未绑定任务分配或故障恢复。 | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 新建/未绑定 Codex 请求的分配策略。没有 live `(parent thread id, quota scope)` affinity 的请求属于未绑定；代理重启或 affinity 重置后，已有可见任务也可能未绑定。`quota` 在没有活跃账号时选择已知 usage 最低的合格账号；活跃账号合格且低于 `autoSwitchThreshold` 时继续使用；达到阈值后，可把未绑定请求或已绑定任务的下一次请求切换到 usage 更低的合格账号。`round-robin` 均匀分配未绑定请求；`fill-first` 在 cooldown、不可用或耗尽阈值前持续分配给活跃账号。 | | `accountPoolStickyLimit?` | `number` | `1` | 一次 round-robin 选择在推进前保留的新建/未绑定任务分配数。计数在任务绑定时增加，而不是在上游成功后增加。范围 1–100；仅当 `accountPoolStrategy` 为 `round-robin` 时生效。 | -| `upstreamFailoverThreshold?` | `number` | `3` | 新会话允许执行账户故障转移前所需的连续账户级瞬态 HTTP 或语义失败结果数；`0` 仅禁用账户故障转移。在尚未观测到任何 HTTP 状态时以拒绝结束的终端逻辑请求，只更新以 `(provider, canonical HTTP(S) origin)` 为键的进程内主机健康状态并释放账户探测 lease；它不会改变账户 quarantine、cooldown/failure streak、affinity、pool rotation 或活跃账户选择。五分钟内出现三次逻辑请求级最终主机故障时，主机熔断器打开 30 秒；之后只允许一个 half-open 逻辑请求，并继续阻止并发请求。已观测到的 HTTP 失败状态仍是账户证据。若 `503` 响应后发生拒绝，有序证据会为账户保留 `503`，并为主机保留后续拒绝。池化普通 Responses 和 native compact 请求的 Codex bearer 重定向采用手动处理：不跟随、不暴露 `Location`，并转换为大小受限的账户级 `502`。如果 peer 可能已消费携带凭据的请求，随后读取/关闭响应体又失败，则不会使用其他凭据重放该请求；这一故障因此可能促成主机熔断，并暂时阻止原本健康的备用账户。 | +| `upstreamFailoverThreshold?` | `number` | `3` | 新会话允许执行账户故障转移前所需的连续账户级瞬态 HTTP 或语义失败结果数；`0` 仅禁用账户故障转移。在尚未观测到任何 HTTP 状态时以拒绝结束的终端逻辑请求，只更新以 `(provider, canonical HTTP(S) origin)` 为键的进程内主机健康状态并释放账户探测 lease；它不会改变账户 quarantine、cooldown/failure streak、affinity、pool rotation 或活跃账户选择。五分钟内出现三次逻辑请求级最终主机故障时，主机熔断器打开 30 秒；之后只允许一个 half-open 逻辑请求，并继续阻止并发请求。已观测到的 HTTP 失败状态仍是账户证据。若 `503` 响应后发生拒绝，有序证据会为账户保留 `503`，并为主机保留后续拒绝。池化普通 Responses 和 native compact 请求的 Codex bearer 重定向采用手动处理：不跟随、不暴露 `Location`，并转换为大小受限的账户级 `502`。即使 peer 可能已消费携带凭据的请求，只要在观测到任何 HTTP 状态之前发生 transport rejection，就不会使用其他凭据重放该请求；这一终端拒绝可能促成主机熔断，并暂时阻止原本健康的备用账户。 | | `modelCacheTtlMs?` | `number` | `300000` | 每个提供者 `/models` 缓存的新鲜度窗口。 | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic 提示缓存策略：禁用、5 分钟临时缓存，或 1 小时扩展缓存。 | | `tokenGuardian?` | `OcxTokenGuardianConfig` | 关闭 | 可选的主动 OAuth 刷新与 Codex 账户预热策略。 | diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index 5f75ee1f12..c0aeb7b018 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -544,6 +544,29 @@ export async function handleResponsesCompact( // surfaces to the client, which retries the compact task OUTSIDE the logical request // — reporting exhausted retries while another pool account sat idle (#913). let pendingAlternateAuthCtx: CodexAuthContext | null = null; + type PendingPrimaryPoolOutcome = { + outcome: CodexUpstreamOutcome; + retryAfter?: string | null; + resetAt?: unknown | unknown[]; + promoteAccountId?: string; + }; + let pendingPrimaryPoolOutcome: PendingPrimaryPoolOutcome | null = null; + let primaryPoolOutcomeSettled = false; + const settlePendingPrimaryPoolOutcome = (): void => { + if (!pendingPrimaryPoolOutcome || primaryPoolOutcomeSettled) return; + const { outcome, ...meta } = pendingPrimaryPoolOutcome; + try { + recordCompactPoolOutcome(authCtx, outcome, meta); + } catch (error) { + // A local post-response failure must not strand a half-open account probe. + // Preserve the caller's original error after releasing ownership if the + // account recorder itself cannot finish. + releaseCodexAuthContextProbeLease(authCtx); + primaryPoolOutcomeSettled = true; + throw error; + } + primaryPoolOutcomeSettled = true; + }; try { if ( (upstream.status === 429 || upstream.status === 402) @@ -552,12 +575,18 @@ export async function handleResponsesCompact( && route.codexAccountMode && !req.signal.aborted ) { + const primaryPoolOutcome: PendingPrimaryPoolOutcome = { + outcome: upstream.status, + }; + pendingPrimaryPoolOutcome = primaryPoolOutcome; const firstRetryAfter = upstream.headers.get("retry-after"); const firstResetAt = [ upstream.headers.get("x-codex-primary-reset-at"), upstream.headers.get("x-codex-secondary-reset-at"), upstream.headers.get("x-codex-tertiary-reset-at"), ].filter(Boolean); + primaryPoolOutcome.retryAfter = firstRetryAfter; + primaryPoolOutcome.resetAt = firstResetAt; // Build the alternate COMPLETELY before cancelling the first body: if construction // throws, the first rejection is still intact and can be returned to the client. const alternate = await resolveAlternateCompactContext({ @@ -593,11 +622,10 @@ export async function handleResponsesCompact( authCtx.writerGeneration, ); } - recordCompactPoolOutcome(authCtx, upstream.status, { - retryAfter: firstRetryAfter, - resetAt: firstResetAt, - ...(alternate.authCtx.accountId ? { promoteAccountId: alternate.authCtx.accountId } : {}), - }); + if (alternate.authCtx.accountId) { + primaryPoolOutcome.promoteAccountId = alternate.authCtx.accountId; + } + settlePendingPrimaryPoolOutcome(); await upstream.body?.cancel().catch(() => undefined); outcomeCtx = alternate.authCtx; const alternateAttempts: UpstreamAttemptObservation[] = []; @@ -627,6 +655,11 @@ export async function handleResponsesCompact( } } catch (error) { releaseCodexAuthContextProbeLease(pendingAlternateAuthCtx ?? undefined); + try { + settlePendingPrimaryPoolOutcome(); + } catch { + // The helper already released A's probe; keep the original local error. + } settleObservedCompactHostResponse(); throw error; } diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index ce38004db1..cf4c35ff44 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -390,6 +390,7 @@ async function retryCodexPoolOnAlternateAccount( const prepared = await (async () => { let request: Awaited["buildRequest"]>> | undefined; + let firstOutcomeSettled = false; try { const quotaMeta = codexQuotaOutcomeMeta(firstResponse); if (outcomeStatus === 429 || outcomeStatus === 402) { @@ -411,6 +412,7 @@ async function retryCodexPoolOnAlternateAccount( // Retry already advanced the RR ring via excludeAccountId — reuse for promotion. ...(retryAuthCtx.accountId ? { promoteAccountId: retryAuthCtx.accountId } : {}), }); + firstOutcomeSettled = true; } const retryHeaders = headersForCodexAuthContext(req.headers, retryAuthCtx); @@ -441,6 +443,7 @@ async function retryCodexPoolOnAlternateAccount( return { fetcher, request, retryHeaders }; } catch (error) { request?.releaseBodyObservation?.(); + if (!firstOutcomeSettled) releaseCodexAuthContextProbeLease(firstAuthCtx); releaseCodexAuthContextProbeLease(retryAuthCtx); throw error; } diff --git a/tests/responses-compaction-routing.test.ts b/tests/responses-compaction-routing.test.ts index bd40098e03..c556bff929 100644 --- a/tests/responses-compaction-routing.test.ts +++ b/tests/responses-compaction-routing.test.ts @@ -1138,6 +1138,57 @@ describe("compact alternate-account attempt (#913)", () => { }); }); + test("compact settles A's half-open account probe when response header processing throws", async () => { + const originalNow = Date.now; + const now = 1_800_000_000_000; + const probeAt = now + CODEX_QUOTA_PROBE_INTERVAL_MS; + const config = nativePoolConfig(); + const expected = new Error("compact response header processing failed"); + try { + Date.now = () => now; + clearCodexUpstreamHealth(); + saveCodexAccountCredential("pool-a", { + accessToken: "pool-access-token", + refreshToken: "pool-refresh-token", + expiresAt: now + 30 * 60_000, + chatgptAccountId: "pool_acc", + }); + recordCodexUpstreamOutcome(config, "pool-a", 429, { + now, + resetAt: Math.floor((now + 4 * 24 * 60 * 60_000) / 1_000), + modelId: "gpt-5.3-codex-spark", + }); + Date.now = () => probeAt; + globalThis.fetch = (async () => { + const response = Response.json({ error: { message: "A quota" } }, { status: 429 }); + Object.defineProperty(response, "headers", { + configurable: true, + get: () => { throw expected; }, + }); + return response; + }) as typeof fetch; + + await expect(handleResponsesCompact( + compactionRequest(baseCompactionBody({ model: "gpt-5.3-codex-spark" })), + config, + { model: "", provider: "" }, + )).rejects.toBe(expected); + + Date.now = () => probeAt + CODEX_QUOTA_PROBE_INTERVAL_MS; + const nextProbe = await resolveCodexAuthContext( + new Headers({ authorization: "Bearer main-token" }), + config, + "pool", + { modelId: "gpt-5.3-codex-spark" }, + ); + expect(nextProbe).toMatchObject({ accountId: "pool-a", probeQuotaScope: "spark" }); + releaseCodexAuthContextProbeLease(nextProbe); + } finally { + Date.now = originalNow; + clearCodexUpstreamHealth(); + } + }); + test("compact keeps B ownership when header-timeout setup fails before provider execution", async () => { await withPoolEnv("ocx-compact-b-pre-executor-", async config => { const originalNow = Date.now; diff --git a/tests/server-auth.test.ts b/tests/server-auth.test.ts index 9a564a7637..f0644a7236 100644 --- a/tests/server-auth.test.ts +++ b/tests/server-auth.test.ts @@ -38,8 +38,9 @@ import { safeConfigDTO, startServer, } from "../src/server"; -import { clearRequestLogsForTests, getRequestLogEntries } from "../src/server/request-log"; +import { clearRequestLogsForTests, getRequestLogEntries, type RequestLogContext } from "../src/server/request-log"; import { handleManagementAPI } from "../src/server/management-api"; +import { handleResponses } from "../src/server/responses/core"; import type { OcxConfig } from "../src/types"; import { fakeChatGptJwt } from "./helpers/fake-chatgpt-jwt"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "./helpers/isolated-codex-home"; @@ -2573,6 +2574,66 @@ describe("server local API auth", () => { } }); + test("alternate preparation failure releases A's recovery probe", async () => { + const harness = await startPoolRetryHarness(() => new Response("unused"), { + omitCredentialAccountIds: ["pool-b"], + }); + const redirectedFetch = globalThis.fetch; + const now = Date.now(); + recordCodexUpstreamOutcome(harness.config, "pool-a", 429, { + now: now - CODEX_QUOTA_PROBE_INTERVAL_MS, + resetAt: Math.floor((now + 60 * 60_000) / 1000), + }); + let sends = 0; + let headerReads = 0; + let observedProbeLeaseId: string | undefined; + + globalThis.fetch = (async (input, init) => { + const accountId = new Headers(init?.headers).get("chatgpt-account-id"); + if (accountId !== "acct-pool-a") return redirectedFetch(input, init); + sends += 1; + observedProbeLeaseId = getCodexUpstreamHealth("pool-a")?.probeLeaseId; + saveCodexAccountCredential("pool-b", { + accessToken: "pool-b-token", + refreshToken: "pool-b-refresh", + expiresAt: Date.now() + 10 * 60_000, + chatgptAccountId: "acct-pool-b", + }); + clearAccountNeedsReauth("pool-b"); + const response = new Response("quota", { status: 429 }); + return new Proxy(response, { + get(target, property) { + if (property === "headers") { + headerReads += 1; + if (headerReads === 5) throw new Error("synthetic alternate preparation failure"); + } + const value = Reflect.get(target, property, target) as unknown; + return typeof value === "function" ? value.bind(target) : value; + }, + }); + }) as typeof fetch; + + try { + const request = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json", authorization: "Bearer inbound-token" }, + body: JSON.stringify({ model: POOL_RETRY_MODEL, input: "hello", stream: false }), + }); + const logCtx: RequestLogContext = { model: POOL_RETRY_MODEL, provider: "openai" }; + await expect(handleResponses(request, harness.config, logCtx)).rejects.toThrow( + "synthetic alternate preparation failure", + ); + expect(sends).toBe(1); + expect(headerReads).toBe(5); + expect(observedProbeLeaseId).toEqual(expect.any(String)); + expect(getCodexUpstreamHealth("pool-a")?.cooldownUntil).toEqual(expect.any(Number)); + expect(getCodexUpstreamHealth("pool-a")?.probeLeaseId).toBeUndefined(); + } finally { + globalThis.fetch = redirectedFetch; + await stopPoolRetryHarness(harness); + } + }); + test("retry-dispatch transport failure is host-only and never triple-dispatches", async () => { const harness = await startPoolRetryHarness(() => rejectionResponse(unsupportedModelBody())); const affinityThread = "retry-dispatch-host-only-affinity"; From 140c130b64178e981ec3717373ca6b79271aa156 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:23:15 +0900 Subject: [PATCH 10/20] fix(codex): release probes on resolver errors --- src/server/responses/core.ts | 5 ++- tests/server-auth.test.ts | 63 ++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index cf4c35ff44..d1e9c54e62 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -382,7 +382,10 @@ async function retryCodexPoolOnAlternateAccount( && !(error instanceof CodexAuthContextError) && !(error instanceof CodexAccountCooldownError) && !(error instanceof CodexMainProfileDrainingError) - ) throw error; + ) { + releaseCodexAuthContextProbeLease(firstAuthCtx); + throw error; + } } if (retryAuthCtx?.kind !== "pool" && retryAuthCtx?.kind !== "main-pool") { return { kind: "no-alternate" }; diff --git a/tests/server-auth.test.ts b/tests/server-auth.test.ts index f0644a7236..ac698ad7ba 100644 --- a/tests/server-auth.test.ts +++ b/tests/server-auth.test.ts @@ -2634,6 +2634,69 @@ describe("server local API auth", () => { } }); + test("unexpected alternate resolver failure releases A's recovery probe and preserves the error", async () => { + const harness = await startPoolRetryHarness(() => new Response("unused"), { + omitCredentialAccountIds: ["pool-b"], + }); + const redirectedFetch = globalThis.fetch; + const resolverError = new Error("synthetic unexpected alternate resolver failure"); + const now = Date.now(); + recordCodexUpstreamOutcome(harness.config, "pool-a", 429, { + now: now - CODEX_QUOTA_PROBE_INTERVAL_MS, + resetAt: Math.floor((now + 60 * 60_000) / 1000), + }); + let sends = 0; + let selectionCalls = 0; + let observedProbeLeaseId: string | undefined; + const turnAdmissionLease = { + release() {}, + beginCodexAccountSelection() { + selectionCalls += 1; + if (selectionCalls === 2) throw resolverError; + return { + mainProfileDraining: false, + claimMainProfile: () => false, + release() {}, + }; + }, + }; + + globalThis.fetch = (async (input, init) => { + const accountId = new Headers(init?.headers).get("chatgpt-account-id"); + if (accountId !== "acct-pool-a") return redirectedFetch(input, init); + sends += 1; + observedProbeLeaseId = getCodexUpstreamHealth("pool-a")?.probeLeaseId; + saveCodexAccountCredential("pool-b", { + accessToken: "pool-b-token", + refreshToken: "pool-b-refresh", + expiresAt: Date.now() + 10 * 60_000, + chatgptAccountId: "acct-pool-b", + }); + clearAccountNeedsReauth("pool-b"); + return rejectionResponse(unsupportedModelBody()); + }) as typeof fetch; + + try { + const request = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json", authorization: "Bearer inbound-token" }, + body: JSON.stringify({ model: POOL_RETRY_MODEL, input: "hello", stream: false }), + }); + const logCtx: RequestLogContext = { model: POOL_RETRY_MODEL, provider: "openai" }; + await expect(handleResponses(request, harness.config, logCtx, { turnAdmissionLease })).rejects.toBe( + resolverError, + ); + expect(sends).toBe(1); + expect(selectionCalls).toBe(2); + expect(observedProbeLeaseId).toEqual(expect.any(String)); + expect(getCodexUpstreamHealth("pool-a")?.cooldownUntil).toEqual(expect.any(Number)); + expect(getCodexUpstreamHealth("pool-a")?.probeLeaseId).toBeUndefined(); + } finally { + globalThis.fetch = redirectedFetch; + await stopPoolRetryHarness(harness); + } + }); + test("retry-dispatch transport failure is host-only and never triple-dispatches", async () => { const harness = await startPoolRetryHarness(() => rejectionResponse(unsupportedModelBody())); const affinityThread = "retry-dispatch-host-only-affinity"; From ab7c01c13ad2c9b5926fdbf05522aa016c4347d0 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 16:27:40 +0900 Subject: [PATCH 11/20] fix(proxy): preserve host evidence across retry setup failures --- .../ja/reference/configuration/providers.md | 2 +- .../ko/reference/configuration/providers.md | 2 +- .../docs/reference/configuration/providers.md | 2 +- .../ru/reference/configuration/providers.md | 2 +- .../reference/configuration/providers.md | 2 +- src/server/responses/compact.ts | 2 +- src/server/responses/core.ts | 8 +- tests/codex-host-health-runtime.test.ts | 9 +- tests/responses-compaction-routing.test.ts | 292 +++++++++++++----- tests/server-auth.test.ts | 53 +++- 10 files changed, 284 insertions(+), 90 deletions(-) diff --git a/docs-site/src/content/docs/ja/reference/configuration/providers.md b/docs-site/src/content/docs/ja/reference/configuration/providers.md index d6615e18be..4ba9652317 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ja/reference/configuration/providers.md @@ -21,7 +21,7 @@ description: プロバイダー エントリ、認証、エンドポイント、 | `autoSwitchThreshold?` | `number` | `80` | 使用量ベースのプロアクティブ切り替えしきい値。`quota` は紐付け済み/未紐付けタスクの次のリクエストを再評価でき、`fill-first` は未紐付け割り当ての使い切り基準としてのみ使用し、通常の `round-robin` 選択は使用しません。既知の 5 時間、週次、30 日 quota window の最大スコアを使います。`0` は使用量ベースの切り替えだけを無効にし、未紐付け割り当てや障害回復は無効にしません。 | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 新規/未紐付け Codex リクエストの割り当て戦略。live な `(parent thread id, quota scope)` affinity がなければ未紐付けで、プロキシ再起動や affinity リセット後は既存の表示タスクも未紐付けになり得ます。`quota` はアクティブアカウントがなければ既知 usage 最小の適格アカウントを選び、適格なアクティブアカウントが `autoSwitchThreshold` 未満なら維持します。しきい値到達後は、未紐付けリクエストまたは紐付け済みタスクの次のリクエストを usage の低い適格アカウントへ移せます。`round-robin` は未紐付けリクエストを均等分散し、`fill-first` は cooldown、使用不可、または drain threshold までアクティブアカウントへ割り当てます。 | | `accountPoolStickyLimit?` | `number` | `1` | 1 回の round-robin 選択で次へ進む前に保持する新規/未紐付けタスク割り当て数。カウンターは上流の成功後ではなくタスクの紐付け時に増えます。範囲 1–100。`accountPoolStrategy` が `round-robin` のときのみ。 | -| `upstreamFailoverThreshold?` | `number` | `3` | 将来の新規セッションでアカウントの failover を行えるようになるまでに必要な、アカウント単位の一時的な HTTP または意味上の失敗結果の連続回数です。`0` はアカウント failover だけを無効にします。HTTP status が観測される前に rejection で終了した論理リクエストは、`(provider, canonical HTTP(S) origin)` をキーとするプロセスローカルな host health だけを更新して account probe lease を解放します。アカウントの quarantine、cooldown/failure streak、affinity、pool rotation、active account selection は変更しません。5 分以内に論理リクエスト単位の終端 host failure が 3 回発生すると host circuit が 30 秒間開き、その後は正確に 1 件の half-open 論理リクエストだけが許可され、同時リクエストは引き続き拒否されます。観測された HTTP failure status はアカウント evidence として残ります。`503` response の後に rejection が起きた場合、順序付き evidence は `503` をアカウント用に、後の rejection を host 用に保持します。プールされた通常の Responses と native compact リクエストの Codex bearer redirect は手動処理され、追従せず、`Location` を公開せず、サイズを制限したアカウント単位の `502` に変換されます。peer が資格情報を伴うリクエストを消費した可能性があっても、HTTP status が観測される前に transport rejection が発生した場合、そのリクエストを別の資格情報では再送しません。その終端 rejection は、正常な代替アカウントを一時的にブロックする host circuit の成立に寄与する可能性があります。 | +| `upstreamFailoverThreshold?` | `number` | `3` | 既存の紐付け済みタスクを含む後続リクエストでアカウントの failover を行えるようになるまでに必要な、アカウント単位の一時的な HTTP または意味上の失敗結果の連続回数です。`0` はアカウント failover だけを無効にします。HTTP status が観測される前に rejection で終了した論理リクエストは、`(provider, canonical HTTP(S) origin)` をキーとするプロセスローカルな host health だけを更新して account probe lease を解放します。アカウントの quarantine、cooldown/failure streak、affinity、pool rotation、active account selection は変更しません。5 分以内に論理リクエスト単位の終端 host failure が 3 回発生すると host circuit が 30 秒間開き、その後は正確に 1 件の half-open 論理リクエストだけが許可され、同時リクエストは引き続き拒否されます。観測された HTTP failure status はアカウント evidence として残ります。`503` response の後に rejection が起きた場合、順序付き evidence は `503` をアカウント用に、後の rejection を host 用に保持します。プールされた通常の Responses と native compact リクエストの Codex bearer redirect は手動処理され、追従せず、`Location` を公開せず、サイズを制限したアカウント単位の `502` に変換されます。peer が資格情報を伴うリクエストを消費した可能性があっても、HTTP status が観測される前に transport rejection が発生した場合、そのリクエストを別の資格情報では再送しません。その終端 rejection は、正常な代替アカウントを一時的にブロックする host circuit の成立に寄与する可能性があります。 | | `modelCacheTtlMs?` | `number` | `300000` |プロバイダーごとの `/models` キャッシュの鮮度ウィンドウ。 | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic プロンプト キャッシュ ポリシー: 無効、5 分間の一時的、または 1 時間の延長。 | | `tokenGuardian?` | `OcxTokenGuardianConfig` |オフ |オプションのプロアクティブな OAuth 更新および Codex アカウントのウォームアップ ポリシー。 | diff --git a/docs-site/src/content/docs/ko/reference/configuration/providers.md b/docs-site/src/content/docs/ko/reference/configuration/providers.md index 393d4bb542..6ff93e4dd5 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ko/reference/configuration/providers.md @@ -21,7 +21,7 @@ description: 공급자 항목, 인증, 엔드포인트, 모델 카탈로그, 할 | `autoSwitchThreshold?` | `number` | `80` | 사용량 기반 선제 전환 임계값입니다. `quota`는 바인딩된 작업과 바인딩 없는 작업의 다음 요청을 모두 재평가할 수 있고, `fill-first`는 바인딩 없는 작업 배정의 소진 기준으로만 사용하며, 기본 `round-robin` 선택은 이 값을 사용하지 않습니다. 알려진 5시간, 주간, 30일 quota window 중 가장 높은 점수를 씁니다. `0`은 사용량 기반 전환만 끄며 바인딩 없는 작업 배정이나 실패 복구는 끄지 않습니다. | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 새 작업/바인딩 없는 Codex 요청의 계정 배정 전략입니다. `(parent thread id, quota scope)`의 live affinity가 없으면 바인딩 없는 요청이며, 프록시 재시작이나 affinity 초기화 뒤에는 기존에 보이던 작업도 바인딩이 없어질 수 있습니다. `quota`는 활성 계정이 없을 때 알려진 usage가 가장 낮은 적격 계정을 선택하고, 적격 활성 계정이 `autoSwitchThreshold` 미만이면 유지합니다. 임계값 도달 뒤에는 바인딩 없는 요청이나 바인딩된 작업의 다음 요청을 usage가 더 낮은 적격 계정으로 옮길 수 있습니다. `round-robin`은 바인딩 없는 요청을 균등 분배하고, `fill-first`는 cooldown, 사용 불가 또는 drain threshold까지 활성 계정에 배정합니다. | | `accountPoolStickyLimit?` | `number` | `1` | 한 round-robin 선택이 다음으로 넘어가기 전에 유지하는 새 작업/바인딩 없는 작업 배정 수입니다. 카운터는 업스트림 성공 뒤가 아니라 작업을 바인딩할 때 증가합니다. 범위 1–100이며 `accountPoolStrategy`가 `round-robin`일 때만 적용됩니다. | -| `upstreamFailoverThreshold?` | `number` | `3` | 이후 새 세션이 계정 failover될 수 있기 전에 필요한 연속 계정 범위의 일시적 HTTP 또는 의미상 실패 결과 수입니다. `0`은 계정 failover만 비활성화합니다. HTTP 상태가 관측되기 전에 최종적으로 거부된 논리 요청은 `(provider, canonical HTTP(S) origin)` 키의 프로세스 로컬 host health만 갱신하고 계정 probe lease를 해제합니다. 계정 quarantine, cooldown/failure streak, affinity, pool rotation, active account selection은 변경하지 않습니다. 5분 안에 논리 요청 단위의 최종 host failure가 3회 발생하면 host circuit이 30초 동안 열리고, 이후 정확히 하나의 half-open 논리 요청만 허용되며 동시 요청은 계속 차단됩니다. 관측된 HTTP 실패 상태는 계정 근거로 남습니다. `503` 응답 후 거부가 발생하면 순서가 보존된 근거는 `503`을 계정용으로, 뒤의 거부를 host용으로 유지합니다. 풀의 일반 Responses 및 native compact 요청에 대한 Codex bearer redirect는 수동으로 처리되어 따라가지 않고 `Location`을 노출하지 않으며, 크기가 제한된 계정 범위 `502`로 변환됩니다. peer가 자격 증명을 실어 보낸 요청을 소비했을 수 있더라도 HTTP 상태가 관측되기 전에 transport rejection이 발생하면 그 요청을 다른 자격 증명으로 재전송하지 않습니다. 이 최종 rejection은 정상인 대체 계정을 일시적으로 차단하는 host circuit의 성립에 기여할 수 있습니다. | +| `upstreamFailoverThreshold?` | `number` | `3` | 기존 바인딩된 작업을 포함한 후속 요청이 계정 failover될 수 있기 전에 필요한 연속 계정 범위의 일시적 HTTP 또는 의미상 실패 결과 수입니다. `0`은 계정 failover만 비활성화합니다. HTTP 상태가 관측되기 전에 최종적으로 거부된 논리 요청은 `(provider, canonical HTTP(S) origin)` 키의 프로세스 로컬 host health만 갱신하고 계정 probe lease를 해제합니다. 계정 quarantine, cooldown/failure streak, affinity, pool rotation, active account selection은 변경하지 않습니다. 5분 안에 논리 요청 단위의 최종 host failure가 3회 발생하면 host circuit이 30초 동안 열리고, 이후 정확히 하나의 half-open 논리 요청만 허용되며 동시 요청은 계속 차단됩니다. 관측된 HTTP 실패 상태는 계정 근거로 남습니다. `503` 응답 후 거부가 발생하면 순서가 보존된 근거는 `503`을 계정용으로, 뒤의 거부를 host용으로 유지합니다. 풀의 일반 Responses 및 native compact 요청에 대한 Codex bearer redirect는 수동으로 처리되어 따라가지 않고 `Location`을 노출하지 않으며, 크기가 제한된 계정 범위 `502`로 변환됩니다. peer가 자격 증명을 실어 보낸 요청을 소비했을 수 있더라도 HTTP 상태가 관측되기 전에 transport rejection이 발생하면 그 요청을 다른 자격 증명으로 재전송하지 않습니다. 이 최종 rejection은 정상인 대체 계정을 일시적으로 차단하는 host circuit의 성립에 기여할 수 있습니다. | | `modelCacheTtlMs?` | `number` | `300000` | 공급자별 `/models` 캐시의 최신성 창입니다. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic 프롬프트 캐시 정책입니다. 비활성, 5분짜리 임시, 1시간짜리 확장 중 하나입니다. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | 꺼짐 | 선택적 선제 OAuth 갱신과 Codex 계정 워밍업 정책입니다. | diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 6fc36c32c1..6b844644d5 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -22,7 +22,7 @@ authenticated. | `autoSwitchThreshold?` | `number` | `80` | Usage threshold for proactive switching. `quota` can re-evaluate both bound and unbound tasks on their next request; `fill-first` uses it only as the drain point for unbound assignment; normal `round-robin` selection does not use it. The score uses the hottest known 5h, weekly, or 30d quota window. `0` disables usage-based proactive switching only, not unbound assignment or failure recovery. | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | Assignment strategy for new/unbound Codex requests. A request is unbound when it has no live (parent thread id, quota scope) affinity; a visible existing task can become unbound after proxy restart or affinity reset. `quota` picks the lowest-usage eligible account when no active account exists, keeps an eligible active account below `autoSwitchThreshold`, and after the threshold may move an unbound request or proactively rebind a bound task to a lower-usage eligible account. `round-robin` distributes unbound requests evenly; `fill-first` keeps assigning unbound requests to the active account until cooldown, unavailability, or the configured drain threshold. | | `accountPoolStickyLimit?` | `number` | `1` | New/unbound task assignments retained on one round-robin selection before advancing; the counter advances when a task is bound, not after an upstream success. Range 1–100. | -| `upstreamFailoverThreshold?` | `number` | `3` | Consecutive account-scoped transient HTTP or semantic failure outcomes required before future new sessions may fail over; `0` disables only account failover. A terminal logical request that rejects before any HTTP status is observed updates only process-local host health keyed by `(provider, canonical HTTP(S) origin)` and releases any account probe lease; it does not change account quarantine, account cooldown/failure streak, affinity, pool rotation, or active account selection. Three terminal logical host failures within five minutes open the host circuit for 30 seconds; then exactly one half-open logical request is admitted while concurrent requests remain blocked. An observed HTTP failure status remains account evidence. If a `503` response is followed by a rejection, ordered evidence retains the `503` for the account and the later rejection for the host. Codex bearer redirects for pooled regular Responses and native compact requests are handled manually: they are not followed, `Location` is not exposed, and they become a bounded account-scoped `502`. If the peer may have consumed a credential-bearing request but the transport rejects before any HTTP status is observed, the request is not replayed under another credential; that terminal rejection can contribute to a host circuit that temporarily blocks an otherwise healthy alternate. | +| `upstreamFailoverThreshold?` | `number` | `3` | Consecutive account-scoped transient HTTP or semantic failure outcomes required before subsequent requests, including existing bound tasks, may fail over; `0` disables only account failover. A terminal logical request that rejects before any HTTP status is observed updates only process-local host health keyed by `(provider, canonical HTTP(S) origin)` and releases any account probe lease; it does not change account quarantine, account cooldown/failure streak, affinity, pool rotation, or active account selection. Three terminal logical host failures within five minutes open the host circuit for 30 seconds; then exactly one half-open logical request is admitted while concurrent requests remain blocked. An observed HTTP failure status remains account evidence. If a `503` response is followed by a rejection, ordered evidence retains the `503` for the account and the later rejection for the host. Codex bearer redirects for pooled regular Responses and native compact requests are handled manually: they are not followed, `Location` is not exposed, and they become a bounded account-scoped `502`. If the peer may have consumed a credential-bearing request but the transport rejects before any HTTP status is observed, the request is not replayed under another credential; that terminal rejection can contribute to a host circuit that temporarily blocks an otherwise healthy alternate. | | `modelCacheTtlMs?` | `number` | `300000` | Freshness window for the per-provider `/models` cache. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic prompt-cache policy: disabled, 5-minute ephemeral, or 1-hour extended. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | off | Optional proactive OAuth refresh and Codex-account warmup policy. | diff --git a/docs-site/src/content/docs/ru/reference/configuration/providers.md b/docs-site/src/content/docs/ru/reference/configuration/providers.md index a1f8915b5e..35981419f0 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ru/reference/configuration/providers.md @@ -22,7 +22,7 @@ description: Записи провайдеров, аутентификация, | `autoSwitchThreshold?` | `number` | `80` | Порог проактивного переключения по использованию. `quota` может повторно оценить следующий запрос как привязанной, так и непривязанной задачи; `fill-first` использует его только как точку исчерпания для непривязанных назначений; обычный `round-robin` его не использует. Оценка берёт самое горячее из окон 5 часов, недели и 30 дней. `0` отключает только переключение по использованию, но не назначение непривязанных задач и не восстановление после сбоев. | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | Стратегия назначения для новых/непривязанных запросов Codex. Запрос непривязан, если у него нет live affinity `(parent thread id, quota scope)`; видимая существующая задача может стать непривязанной после перезапуска прокси или сброса affinity. `quota` выбирает подходящий аккаунт с наименьшим известным usage, когда активного аккаунта нет, сохраняет подходящий активный аккаунт ниже `autoSwitchThreshold`, а после порога может перевести непривязанный запрос или следующий запрос привязанной задачи на подходящий аккаунт с меньшим usage. `round-robin` равномерно распределяет непривязанные запросы; `fill-first` назначает их активному аккаунту до cooldown, недоступности или порога исчерпания. | | `accountPoolStickyLimit?` | `number` | `1` | Число назначений новых/непривязанных задач на одном выборе round-robin перед переходом дальше. Счётчик растёт при привязке задачи, а не после успеха upstream. Диапазон 1–100; только при `accountPoolStrategy` = `round-robin`. | -| `upstreamFailoverThreshold?` | `number` | `3` | Количество последовательных временных HTTP- или семантических результатов сбоя, относящихся к аккаунту, после которого новые сессии могут выполнить failover; `0` отключает только failover аккаунта. Логический запрос, окончательно отклоненный до наблюдения какого-либо HTTP-статуса, обновляет только локальное для процесса состояние хоста с ключом `(provider, canonical HTTP(S) origin)` и освобождает lease account probe; он не меняет quarantine аккаунта, его cooldown/failure streak, affinity, pool rotation или выбор активного аккаунта. Три конечных сбоя хоста на уровне логического запроса за пять минут открывают circuit хоста на 30 секунд; затем допускается ровно один half-open логический запрос, а параллельные запросы остаются заблокированными. Наблюдаемый HTTP-статус сбоя остается свидетельством для аккаунта. Если за ответом `503` следует отклонение, упорядоченные свидетельства сохраняют `503` для аккаунта, а последующее отклонение для хоста. Перенаправления bearer-запросов Codex для обычных Responses из пула и native compact обрабатываются вручную: переход не выполняется, `Location` не раскрывается, а результат преобразуется в ограниченный по размеру ответ `502`, учитываемый для аккаунта. Если peer мог принять запрос с учетными данными, но до наблюдения какого-либо HTTP-статуса происходит transport rejection, запрос не повторяется с другими учетными данными. Такое конечное отклонение может способствовать открытию circuit хоста, который временно заблокирует исправный альтернативный аккаунт. | +| `upstreamFailoverThreshold?` | `number` | `3` | Количество последовательных временных HTTP- или семантических результатов сбоя, относящихся к аккаунту, после которых последующие запросы, включая существующие привязанные задачи, могут выполнить failover; `0` отключает только failover аккаунта. Логический запрос, окончательно отклоненный до наблюдения какого-либо HTTP-статуса, обновляет только локальное для процесса состояние хоста с ключом `(provider, canonical HTTP(S) origin)` и освобождает lease account probe; он не меняет quarantine аккаунта, его cooldown/failure streak, affinity, pool rotation или выбор активного аккаунта. Три конечных сбоя хоста на уровне логического запроса за пять минут открывают circuit хоста на 30 секунд; затем допускается ровно один half-open логический запрос, а параллельные запросы остаются заблокированными. Наблюдаемый HTTP-статус сбоя остается свидетельством для аккаунта. Если за ответом `503` следует отклонение, упорядоченные свидетельства сохраняют `503` для аккаунта, а последующее отклонение для хоста. Перенаправления bearer-запросов Codex для обычных Responses из пула и native compact обрабатываются вручную: переход не выполняется, `Location` не раскрывается, а результат преобразуется в ограниченный по размеру ответ `502`, учитываемый для аккаунта. Если peer мог принять запрос с учетными данными, но до наблюдения какого-либо HTTP-статуса происходит transport rejection, запрос не повторяется с другими учетными данными. Такое конечное отклонение может способствовать открытию circuit хоста, который временно заблокирует исправный альтернативный аккаунт. | | `modelCacheTtlMs?` | `number` | `300000` | Окно свежести для кэша `/models` на уровне провайдера. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Политика prompt-cache Anthropic: отключено, 5-минутный ephemeral или 1-часовой extended. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | off | Необязательная политика proactive OAuth refresh и warmup'а аккаунтов Codex. | diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md index 3c14c90def..03ba1a1756 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md @@ -21,7 +21,7 @@ description: 提供者条目、身份验证、端点、模型目录、配额、 | `autoSwitchThreshold?` | `number` | `80` | 基于用量的主动切换阈值。`quota` 可在下一次请求中重新评估已绑定和未绑定任务；`fill-first` 仅把它用作未绑定分配的耗尽点；正常 `round-robin` 不使用它。分数取已知 5 小时、周或 30 天 quota window 的最高值。`0` 只关闭基于用量的主动切换，不关闭未绑定任务分配或故障恢复。 | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 新建/未绑定 Codex 请求的分配策略。没有 live `(parent thread id, quota scope)` affinity 的请求属于未绑定；代理重启或 affinity 重置后，已有可见任务也可能未绑定。`quota` 在没有活跃账号时选择已知 usage 最低的合格账号；活跃账号合格且低于 `autoSwitchThreshold` 时继续使用；达到阈值后，可把未绑定请求或已绑定任务的下一次请求切换到 usage 更低的合格账号。`round-robin` 均匀分配未绑定请求；`fill-first` 在 cooldown、不可用或耗尽阈值前持续分配给活跃账号。 | | `accountPoolStickyLimit?` | `number` | `1` | 一次 round-robin 选择在推进前保留的新建/未绑定任务分配数。计数在任务绑定时增加，而不是在上游成功后增加。范围 1–100；仅当 `accountPoolStrategy` 为 `round-robin` 时生效。 | -| `upstreamFailoverThreshold?` | `number` | `3` | 新会话允许执行账户故障转移前所需的连续账户级瞬态 HTTP 或语义失败结果数；`0` 仅禁用账户故障转移。在尚未观测到任何 HTTP 状态时以拒绝结束的终端逻辑请求，只更新以 `(provider, canonical HTTP(S) origin)` 为键的进程内主机健康状态并释放账户探测 lease；它不会改变账户 quarantine、cooldown/failure streak、affinity、pool rotation 或活跃账户选择。五分钟内出现三次逻辑请求级最终主机故障时，主机熔断器打开 30 秒；之后只允许一个 half-open 逻辑请求，并继续阻止并发请求。已观测到的 HTTP 失败状态仍是账户证据。若 `503` 响应后发生拒绝，有序证据会为账户保留 `503`，并为主机保留后续拒绝。池化普通 Responses 和 native compact 请求的 Codex bearer 重定向采用手动处理：不跟随、不暴露 `Location`，并转换为大小受限的账户级 `502`。即使 peer 可能已消费携带凭据的请求，只要在观测到任何 HTTP 状态之前发生 transport rejection，就不会使用其他凭据重放该请求；这一终端拒绝可能促成主机熔断，并暂时阻止原本健康的备用账户。 | +| `upstreamFailoverThreshold?` | `number` | `3` | 后续请求（包括现有已绑定任务）允许执行账户故障转移前所需的连续账户级瞬态 HTTP 或语义失败结果数；`0` 仅禁用账户故障转移。在尚未观测到任何 HTTP 状态时以拒绝结束的终端逻辑请求，只更新以 `(provider, canonical HTTP(S) origin)` 为键的进程内主机健康状态并释放账户探测 lease；它不会改变账户 quarantine、cooldown/failure streak、affinity、pool rotation 或活跃账户选择。五分钟内出现三次逻辑请求级最终主机故障时，主机熔断器打开 30 秒；之后只允许一个 half-open 逻辑请求，并继续阻止并发请求。已观测到的 HTTP 失败状态仍是账户证据。若 `503` 响应后发生拒绝，有序证据会为账户保留 `503`，并为主机保留后续拒绝。池化普通 Responses 和 native compact 请求的 Codex bearer 重定向采用手动处理：不跟随、不暴露 `Location`，并转换为大小受限的账户级 `502`。即使 peer 可能已消费携带凭据的请求，只要在观测到任何 HTTP 状态之前发生 transport rejection，就不会使用其他凭据重放该请求；这一终端拒绝可能促成主机熔断，并暂时阻止原本健康的备用账户。 | | `modelCacheTtlMs?` | `number` | `300000` | 每个提供者 `/models` 缓存的新鲜度窗口。 | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic 提示缓存策略：禁用、5 分钟临时缓存，或 1 小时扩展缓存。 | | `tokenGuardian?` | `OcxTokenGuardianConfig` | 关闭 | 可选的主动 OAuth 刷新与 Codex 账户预热策略。 | diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index c0aeb7b018..be30e4fc00 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -518,7 +518,7 @@ export async function handleResponsesCompact( primaryAttemptBoundary, ); } catch (err) { - if (!primaryAttemptBoundary.executorStarted && !req.signal.aborted) { + if (!primaryAttemptBoundary.executorStarted && primaryAttempts.length === 0 && !req.signal.aborted) { const observedStatus = lastUpstreamAttemptResponseStatus(primaryAttempts); if (observedStatus !== undefined) { recordCompactPoolOutcome(outcomeCtx, observedStatus); diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index d1e9c54e62..ed6e730b11 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -306,6 +306,7 @@ interface CodexPoolAccountRetryArgs { connectMs: number; passthroughEstimate?: number; stream: boolean; + tracksCodexUpstreamHost: boolean; } type CodexPoolAccountRetryResult = @@ -358,7 +359,7 @@ async function retryCodexPoolOnAlternateAccount( ): Promise { const { req, config, route, parsed, logCtx, options, firstAuthCtx, firstResponse, - outcomeStatus, upstream, connectMs, passthroughEstimate, stream, + outcomeStatus, upstream, connectMs, passthroughEstimate, stream, tracksCodexUpstreamHost, } = args; // Defense in depth: exact account selectors must never reach alternate-account resolution, // even if a future caller forgets to guard this helper. @@ -469,7 +470,7 @@ async function retryCodexPoolOnAlternateAccount( method: request.method, headers: request.headers, body: request.body, - redirect: "manual", + ...(tracksCodexUpstreamHost ? { redirect: "manual" as const } : {}), }, upstream.signal, connectMs, @@ -1878,7 +1879,7 @@ async function handleResponsesInner( }, ); } catch (err) { - if (!primaryAttemptExecutorStarted && !options.abortSignal?.aborted) { + if (!primaryAttemptExecutorStarted && attemptHistory.length === 0 && !options.abortSignal?.aborted) { const observedStatus = lastUpstreamAttemptResponseStatus(attemptHistory); if (observedStatus !== undefined) { recordPoolTransportOutcome(observedStatus); @@ -1977,6 +1978,7 @@ async function handleResponsesInner( connectMs, passthroughEstimate, stream: parsed.stream, + tracksCodexUpstreamHost: hostKey !== null, }); if (retry.kind === "transport") { authCtx = retry.authCtx; diff --git a/tests/codex-host-health-runtime.test.ts b/tests/codex-host-health-runtime.test.ts index 057ae70a46..136ef0d6f8 100644 --- a/tests/codex-host-health-runtime.test.ts +++ b/tests/codex-host-health-runtime.test.ts @@ -404,7 +404,7 @@ describe("Codex host-health actual Bun runtime (#914/#922)", () => { test("credential-visible 307 is manual, bounded, and never exposes Location", async () => { const harness = await startHarness({ twoAccounts: true }); - const deadPort = await closedEphemeralPort(); + let deadPort = 0; const seen: Array<{ authorization: string | null; accountId: string | null; body: string }> = []; const redirect = serve(async request => { seen.push({ @@ -417,6 +417,7 @@ describe("Codex host-health actual Bun runtime (#914/#922)", () => { headers: { location: `http://127.0.0.1:${deadPort}/credential-leak-target` }, }); }); + deadPort = await closedEphemeralPort(); const redirectModes: Array = []; installCanonicalRouter(call => { redirectModes.push(call.init?.redirect); @@ -470,7 +471,7 @@ describe("Codex host-health actual Bun runtime (#914/#922)", () => { test("a real 503 followed by a real refused retry preserves both attribution layers", async () => { const harness = await startHarness({ twoAccounts: true }); - const refusedPort = await closedEphemeralPort(); + let refusedPort = 0; let upstream503Hits = 0; const upstream503 = serve(async request => { upstream503Hits += 1; @@ -480,6 +481,7 @@ describe("Codex host-health actual Bun runtime (#914/#922)", () => { headers: { "retry-after": "0" }, }); }); + refusedPort = await closedEphemeralPort(); let physicalSends = 0; installCanonicalRouter(call => { physicalSends += 1; @@ -500,7 +502,7 @@ describe("Codex host-health actual Bun runtime (#914/#922)", () => { for (const path of ["/v1/responses", "/v1/responses/compact"] as const) { test(`${path} keeps A=429 and one real B rejection separately attributed`, async () => { const harness = await startHarness({ twoAccounts: true }); - const refusedPort = await closedEphemeralPort(); + let refusedPort = 0; const aSeen: Array<{ authorization: string | null; accountId: string | null; body: string }> = []; const aQuota = serve(async request => { aSeen.push({ @@ -513,6 +515,7 @@ describe("Codex host-health actual Bun runtime (#914/#922)", () => { headers: { "retry-after": "60" }, }); }); + refusedPort = await closedEphemeralPort(); const counts = new Map(); const bHeaders: Headers[] = []; installCanonicalRouter(call => { diff --git a/tests/responses-compaction-routing.test.ts b/tests/responses-compaction-routing.test.ts index c556bff929..39ce6ba997 100644 --- a/tests/responses-compaction-routing.test.ts +++ b/tests/responses-compaction-routing.test.ts @@ -769,11 +769,34 @@ describe("compact alternate-account attempt (#913)", () => { if (admission.kind === "admitted") releaseCodexUpstreamHostAdmissionLease(admission.lease, now); } + type PreExecutorBoundaryState = { + prepared: boolean; + cloneThrew: boolean; + boundaryFrameObserved: boolean; + }; + + const BOUNDARY_FRAME = "fetchWithHeaderTimeout"; + + async function expectPreExecutorBoundaryRejection( + run: () => Promise, + state: PreExecutorBoundaryState, + expected: Error, + ): Promise { + let rejection: unknown; + try { + await run(); + } catch (error) { + rejection = error; + } + expect(state.boundaryFrameObserved).toBe(true); + expect(rejection).toBe(expected); + } + function throwingPreExecutorHeaders( NativeHeaders: typeof Headers, targetAccountId: string, expected: Error, - state: { prepared: boolean; cloneThrew: boolean }, + state: PreExecutorBoundaryState, throwOnBoundaryClone = 1, ): typeof Headers { let boundaryCloneCount = 0; @@ -782,8 +805,9 @@ describe("compact alternate-account attempt (#913)", () => { super(init); if ( this.get("chatgpt-account-id") === targetAccountId - && new Error().stack?.includes("fetchWithHeaderTimeout") + && new Error().stack?.includes(BOUNDARY_FRAME) ) { + state.boundaryFrameObserved = true; boundaryCloneCount += 1; if (boundaryCloneCount === throwOnBoundaryClone) { state.cloneThrew = true; @@ -805,16 +829,18 @@ describe("compact alternate-account attempt (#913)", () => { await withPoolEnv("ocx-regular-primary-pre-executor-", async config => { const originalNow = Date.now; const NativeHeaders = globalThis.Headers; + const threadId = "regular-retry-pre-executor-thread"; config.accountPoolStrategy = "fill-first"; config.activeCodexAccountId = "pool-a"; config.connectTimeoutMs = 25; + const affinedAccount = resolveCodexAccountForThread(threadId, config); const probeAt = prepareHalfOpenHost(Date.now()); const hostKey = canonicalCodexUpstreamHostKey( "openai", "https://chatgpt.com/backend-api/codex/responses", )!; const expected = new Error("regular primary header clone failed before executor"); - const state = { prepared: false, cloneThrew: false }; + const state = { prepared: false, cloneThrew: false, boundaryFrameObserved: false }; const physicalAccounts: string[] = []; Date.now = () => probeAt; globalThis.Headers = throwingPreExecutorHeaders(NativeHeaders, "pool_acc_a", expected, state); @@ -823,12 +849,12 @@ describe("compact alternate-account attempt (#913)", () => { return Response.json({ id: "must-not-run", status: "completed", output: [] }); }) as typeof fetch; try { - await expect(handleResponses( + await expectPreExecutorBoundaryRejection(() => handleResponses( compactionRequest(regularBody()), config, { model: "", provider: "" }, - )).rejects.toBe(expected); - expect(state).toEqual({ prepared: true, cloneThrew: true }); + ), state, expected); + expect(state).toEqual({ prepared: true, cloneThrew: true, boundaryFrameObserved: true }); expect(physicalAccounts).toEqual([]); expect(getCodexUpstreamHealth("pool-a")).toBeNull(); expect(getCodexUpstreamHealth("pool-b")).toBeNull(); @@ -854,7 +880,7 @@ describe("compact alternate-account attempt (#913)", () => { "https://chatgpt.com/backend-api/codex/responses", )!; const expected = new Error("regular B header clone failed before executor"); - const state = { prepared: false, cloneThrew: false }; + const state = { prepared: false, cloneThrew: false, boundaryFrameObserved: false }; const physicalAccounts: string[] = []; Date.now = () => probeAt; globalThis.Headers = throwingPreExecutorHeaders(NativeHeaders, "pool_acc_b", expected, state); @@ -863,12 +889,12 @@ describe("compact alternate-account attempt (#913)", () => { return Response.json({ error: { message: "A quota" } }, { status: 429 }); }) as typeof fetch; try { - await expect(handleResponses( + await expectPreExecutorBoundaryRejection(() => handleResponses( compactionRequest(regularBody()), config, { model: "", provider: "" }, - )).rejects.toBe(expected); - expect(state).toEqual({ prepared: true, cloneThrew: true }); + ), state, expected); + expect(state).toEqual({ prepared: true, cloneThrew: true, boundaryFrameObserved: true }); expect(physicalAccounts).toEqual(["pool_acc_a"]); expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(429); expect(getCodexUpstreamHealth("pool-b")).toBeNull(); @@ -885,16 +911,18 @@ describe("compact alternate-account attempt (#913)", () => { await withPoolEnv("ocx-regular-retry-pre-executor-", async config => { const originalNow = Date.now; const NativeHeaders = globalThis.Headers; + const threadId = "regular-retry-pre-executor-thread"; config.accountPoolStrategy = "fill-first"; config.activeCodexAccountId = "pool-a"; config.connectTimeoutMs = 25; + const affinedAccount = resolveCodexAccountForThread(threadId, config); const probeAt = prepareHalfOpenHost(Date.now()); const hostKey = canonicalCodexUpstreamHostKey( "openai", "https://chatgpt.com/backend-api/codex/responses", )!; const expected = new Error("regular retry header clone failed before executor"); - const state = { prepared: false, cloneThrew: false }; + const state = { prepared: false, cloneThrew: false, boundaryFrameObserved: false }; const physicalAccounts: string[] = []; Date.now = () => probeAt; globalThis.Headers = throwingPreExecutorHeaders(NativeHeaders, "pool_acc_a", expected, state, 2); @@ -906,15 +934,19 @@ describe("compact alternate-account attempt (#913)", () => { }); }) as typeof fetch; try { - await expect(handleResponses( - compactionRequest(regularBody()), + const response = await handleResponses( + compactionRequest(regularBody(), undefined, { "x-codex-parent-thread-id": threadId }), config, { model: "", provider: "" }, - )).rejects.toBe(expected); - expect(state).toEqual({ prepared: true, cloneThrew: true }); + ); + expect(response.status).toBe(502); + expect(state).toEqual({ prepared: true, cloneThrew: true, boundaryFrameObserved: true }); expect(physicalAccounts).toEqual(["pool_acc_a"]); expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(503); - expect(getCodexUpstreamHostHealth(hostKey, probeAt)).toBeNull(); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(resolveCodexAccountForThread(threadId, config)).toBe(affinedAccount); + expect(config.activeCodexAccountId).toBe("pool-a"); + expect(getCodexUpstreamHostHealth(hostKey, probeAt)?.consecutiveFailures).toBe(1); expectHostAdmissionUsable(hostKey, probeAt); } finally { globalThis.Headers = NativeHeaders; @@ -927,16 +959,18 @@ describe("compact alternate-account attempt (#913)", () => { await withPoolEnv("ocx-compact-primary-pre-executor-", async config => { const originalNow = Date.now; const NativeHeaders = globalThis.Headers; + const threadId = "compact-retry-pre-executor-thread"; config.accountPoolStrategy = "fill-first"; config.activeCodexAccountId = "pool-a"; config.connectTimeoutMs = 25; + const affinedAccount = resolveCodexAccountForThread(threadId, config); const probeAt = prepareHalfOpenHost(Date.now()); const hostKey = canonicalCodexUpstreamHostKey( "openai", "https://chatgpt.com/backend-api/codex/responses", )!; const expected = new Error("compact primary header clone failed before executor"); - const state = { prepared: false, cloneThrew: false }; + const state = { prepared: false, cloneThrew: false, boundaryFrameObserved: false }; const physicalAccounts: string[] = []; Date.now = () => probeAt; globalThis.Headers = throwingPreExecutorHeaders(NativeHeaders, "pool_acc_a", expected, state); @@ -945,12 +979,12 @@ describe("compact alternate-account attempt (#913)", () => { return Response.json({ id: "must-not-run", status: "completed", output: [] }); }) as typeof fetch; try { - await expect(handleResponsesCompact( + await expectPreExecutorBoundaryRejection(() => handleResponsesCompact( compactionRequest(baseCompactionBody({})), config, { model: "", provider: "" }, - )).rejects.toBe(expected); - expect(state).toEqual({ prepared: true, cloneThrew: true }); + ), state, expected); + expect(state).toEqual({ prepared: true, cloneThrew: true, boundaryFrameObserved: true }); expect(physicalAccounts).toEqual([]); expect(getCodexUpstreamHealth("pool-a")).toBeNull(); expect(getCodexUpstreamHealth("pool-b")).toBeNull(); @@ -967,16 +1001,18 @@ describe("compact alternate-account attempt (#913)", () => { await withPoolEnv("ocx-compact-retry-pre-executor-", async config => { const originalNow = Date.now; const NativeHeaders = globalThis.Headers; + const threadId = "compact-retry-pre-executor-thread"; config.accountPoolStrategy = "fill-first"; config.activeCodexAccountId = "pool-a"; config.connectTimeoutMs = 25; + const affinedAccount = resolveCodexAccountForThread(threadId, config); const probeAt = prepareHalfOpenHost(Date.now()); const hostKey = canonicalCodexUpstreamHostKey( "openai", "https://chatgpt.com/backend-api/codex/responses", )!; const expected = new Error("compact retry header clone failed before executor"); - const state = { prepared: false, cloneThrew: false }; + const state = { prepared: false, cloneThrew: false, boundaryFrameObserved: false }; const physicalAccounts: string[] = []; Date.now = () => probeAt; globalThis.Headers = throwingPreExecutorHeaders(NativeHeaders, "pool_acc_a", expected, state, 2); @@ -988,15 +1024,19 @@ describe("compact alternate-account attempt (#913)", () => { }); }) as typeof fetch; try { - await expect(handleResponsesCompact( - compactionRequest(baseCompactionBody({})), + const response = await handleResponsesCompact( + compactionRequest(baseCompactionBody({}), undefined, { "x-codex-parent-thread-id": threadId }), config, { model: "", provider: "" }, - )).rejects.toBe(expected); - expect(state).toEqual({ prepared: true, cloneThrew: true }); + ); + expect(response.status).toBe(502); + expect(state).toEqual({ prepared: true, cloneThrew: true, boundaryFrameObserved: true }); expect(physicalAccounts).toEqual(["pool_acc_a"]); expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(503); - expect(getCodexUpstreamHostHealth(hostKey, probeAt)).toBeNull(); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(resolveCodexAccountForThread(threadId, config)).toBe(affinedAccount); + expect(config.activeCodexAccountId).toBe("pool-a"); + expect(getCodexUpstreamHostHealth(hostKey, probeAt)?.consecutiveFailures).toBe(1); expectHostAdmissionUsable(hostKey, probeAt); } finally { globalThis.Headers = NativeHeaders; @@ -1005,6 +1045,100 @@ describe("compact alternate-account attempt (#913)", () => { }); }); + test("regular preserves an actual ECONNRESET when retry setup later fails before provider execution", async () => { + await withPoolEnv("ocx-regular-reset-pre-executor-", async config => { + const originalNow = Date.now; + const NativeHeaders = globalThis.Headers; + const threadId = "regular-reset-pre-executor-thread"; + config.accountPoolStrategy = "fill-first"; + config.activeCodexAccountId = "pool-a"; + config.connectTimeoutMs = 25; + const affinedAccount = resolveCodexAccountForThread(threadId, config); + const probeAt = prepareHalfOpenHost(Date.now()); + const hostKey = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex/responses", + )!; + const expected = new Error("regular retry header clone failed after ECONNRESET"); + const state = { prepared: false, cloneThrew: false, boundaryFrameObserved: false }; + const physicalAccounts: string[] = []; + Date.now = () => probeAt; + globalThis.Headers = throwingPreExecutorHeaders(NativeHeaders, "pool_acc_a", expected, state, 2); + globalThis.fetch = (async (_url: RequestInfo | URL, init?: RequestInit) => { + physicalAccounts.push(new NativeHeaders(init?.headers).get("chatgpt-account-id") ?? ""); + const reset = new Error("upstream connection reset") as Error & { code?: string }; + reset.code = "ECONNRESET"; + throw reset; + }) as typeof fetch; + try { + const response = await handleResponses( + compactionRequest(regularBody(), undefined, { "x-codex-parent-thread-id": threadId }), + config, + { model: "", provider: "" }, + ); + expect(response.status).toBe(502); + expect(state).toEqual({ prepared: true, cloneThrew: true, boundaryFrameObserved: true }); + expect(physicalAccounts).toEqual(["pool_acc_a"]); + expect(getCodexUpstreamHealth("pool-a")).toBeNull(); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(resolveCodexAccountForThread(threadId, config)).toBe(affinedAccount); + expect(config.activeCodexAccountId).toBe("pool-a"); + expect(getCodexUpstreamHostHealth(hostKey, probeAt)?.consecutiveFailures) + .toBe(CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD + 1); + } finally { + globalThis.Headers = NativeHeaders; + Date.now = originalNow; + } + }); + }); + + test("compact preserves an actual ECONNRESET when retry setup later fails before provider execution", async () => { + await withPoolEnv("ocx-compact-reset-pre-executor-", async config => { + const originalNow = Date.now; + const NativeHeaders = globalThis.Headers; + const threadId = "compact-reset-pre-executor-thread"; + config.accountPoolStrategy = "fill-first"; + config.activeCodexAccountId = "pool-a"; + config.connectTimeoutMs = 25; + const affinedAccount = resolveCodexAccountForThread(threadId, config); + const probeAt = prepareHalfOpenHost(Date.now()); + const hostKey = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex/responses", + )!; + const expected = new Error("compact retry header clone failed after ECONNRESET"); + const state = { prepared: false, cloneThrew: false, boundaryFrameObserved: false }; + const physicalAccounts: string[] = []; + Date.now = () => probeAt; + globalThis.Headers = throwingPreExecutorHeaders(NativeHeaders, "pool_acc_a", expected, state, 2); + globalThis.fetch = (async (_url: RequestInfo | URL, init?: RequestInit) => { + physicalAccounts.push(new NativeHeaders(init?.headers).get("chatgpt-account-id") ?? ""); + const reset = new Error("upstream connection reset") as Error & { code?: string }; + reset.code = "ECONNRESET"; + throw reset; + }) as typeof fetch; + try { + const response = await handleResponsesCompact( + compactionRequest(baseCompactionBody({}), undefined, { "x-codex-parent-thread-id": threadId }), + config, + { model: "", provider: "" }, + ); + expect(response.status).toBe(502); + expect(state).toEqual({ prepared: true, cloneThrew: true, boundaryFrameObserved: true }); + expect(physicalAccounts).toEqual(["pool_acc_a"]); + expect(getCodexUpstreamHealth("pool-a")).toBeNull(); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(resolveCodexAccountForThread(threadId, config)).toBe(affinedAccount); + expect(config.activeCodexAccountId).toBe("pool-a"); + expect(getCodexUpstreamHostHealth(hostKey, probeAt)?.consecutiveFailures) + .toBe(CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD + 1); + } finally { + globalThis.Headers = NativeHeaders; + Date.now = originalNow; + } + }); + }); + test("regular settles a half-open primary response when alternate preparation throws", async () => { await withPoolEnv("ocx-regular-host-alt-throw-", async config => { const originalNow = Date.now; @@ -1139,54 +1273,60 @@ describe("compact alternate-account attempt (#913)", () => { }); test("compact settles A's half-open account probe when response header processing throws", async () => { - const originalNow = Date.now; - const now = 1_800_000_000_000; - const probeAt = now + CODEX_QUOTA_PROBE_INTERVAL_MS; - const config = nativePoolConfig(); - const expected = new Error("compact response header processing failed"); - try { - Date.now = () => now; - clearCodexUpstreamHealth(); - saveCodexAccountCredential("pool-a", { - accessToken: "pool-access-token", - refreshToken: "pool-refresh-token", - expiresAt: now + 30 * 60_000, + await withPoolEnv("ocx-compact-a-header-processing-", async config => { + const originalNow = Date.now; + const now = 1_800_000_000_000; + const probeAt = now + CODEX_QUOTA_PROBE_INTERVAL_MS; + const expected = new Error("compact response header processing failed"); + config.codexAccounts = [{ + id: "pool-a", + email: "pool@example.test", + isMain: false, chatgptAccountId: "pool_acc", - }); - recordCodexUpstreamOutcome(config, "pool-a", 429, { - now, - resetAt: Math.floor((now + 4 * 24 * 60 * 60_000) / 1_000), - modelId: "gpt-5.3-codex-spark", - }); - Date.now = () => probeAt; - globalThis.fetch = (async () => { - const response = Response.json({ error: { message: "A quota" } }, { status: 429 }); - Object.defineProperty(response, "headers", { - configurable: true, - get: () => { throw expected; }, + }] as OcxConfig["codexAccounts"]; + config.activeCodexAccountId = "pool-a"; + try { + Date.now = () => now; + saveCodexAccountCredential("pool-a", { + accessToken: "pool-access-token", + refreshToken: "pool-refresh-token", + expiresAt: now + 30 * 60_000, + chatgptAccountId: "pool_acc", }); - return response; - }) as typeof fetch; + recordCodexUpstreamOutcome(config, "pool-a", 429, { + now, + resetAt: Math.floor((now + 4 * 24 * 60 * 60_000) / 1_000), + modelId: "gpt-5.3-codex-spark", + }); + Date.now = () => probeAt; + globalThis.fetch = (async () => { + const response = Response.json({ error: { message: "A quota" } }, { status: 429 }); + Object.defineProperty(response, "headers", { + configurable: true, + get: () => { throw expected; }, + }); + return response; + }) as typeof fetch; - await expect(handleResponsesCompact( - compactionRequest(baseCompactionBody({ model: "gpt-5.3-codex-spark" })), - config, - { model: "", provider: "" }, - )).rejects.toBe(expected); + await expect(handleResponsesCompact( + compactionRequest(baseCompactionBody({ model: "gpt-5.3-codex-spark" })), + config, + { model: "", provider: "" }, + )).rejects.toBe(expected); - Date.now = () => probeAt + CODEX_QUOTA_PROBE_INTERVAL_MS; - const nextProbe = await resolveCodexAuthContext( - new Headers({ authorization: "Bearer main-token" }), - config, - "pool", - { modelId: "gpt-5.3-codex-spark" }, - ); - expect(nextProbe).toMatchObject({ accountId: "pool-a", probeQuotaScope: "spark" }); - releaseCodexAuthContextProbeLease(nextProbe); - } finally { - Date.now = originalNow; - clearCodexUpstreamHealth(); - } + Date.now = () => probeAt + CODEX_QUOTA_PROBE_INTERVAL_MS; + const nextProbe = await resolveCodexAuthContext( + new Headers({ authorization: "Bearer main-token" }), + config, + "pool", + { modelId: "gpt-5.3-codex-spark" }, + ); + expect(nextProbe).toMatchObject({ accountId: "pool-a", probeQuotaScope: "spark" }); + releaseCodexAuthContextProbeLease(nextProbe); + } finally { + Date.now = originalNow; + } + }); }); test("compact keeps B ownership when header-timeout setup fails before provider execution", async () => { @@ -1203,7 +1343,7 @@ describe("compact alternate-account attempt (#913)", () => { )!; const expected = new Error("B header clone failed before executor"); const physicalAccounts: string[] = []; - const state = { prepared: false, cloneThrew: false }; + const state = { prepared: false, cloneThrew: false, boundaryFrameObserved: false }; Date.now = () => probeAt; globalThis.Headers = throwingPreExecutorHeaders(NativeHeaders, "pool_acc_b", expected, state); globalThis.fetch = (async (_url: RequestInfo | URL, init?: RequestInit) => { @@ -1211,12 +1351,12 @@ describe("compact alternate-account attempt (#913)", () => { return Response.json({ error: { message: "A quota" } }, { status: 429 }); }) as typeof fetch; try { - await expect(handleResponsesCompact( + await expectPreExecutorBoundaryRejection(() => handleResponsesCompact( compactionRequest(baseCompactionBody({})), config, { model: "", provider: "" }, - )).rejects.toBe(expected); - expect(state).toEqual({ prepared: true, cloneThrew: true }); + ), state, expected); + expect(state).toEqual({ prepared: true, cloneThrew: true, boundaryFrameObserved: true }); expect(physicalAccounts).toEqual(["pool_acc_a"]); expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(429); expect(getCodexUpstreamHealth("pool-b")).toBeNull(); @@ -1679,7 +1819,7 @@ describe("compact alternate-account attempt (#913)", () => { sends += 1; throw new Error("opaque compact rejection"); }) as typeof fetch; - for (let attempt = 0; attempt < 3; attempt++) { + for (let attempt = 0; attempt < CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; attempt++) { expect((await handleResponsesCompact( compactionRequest(baseCompactionBody({})), config, { model: "", provider: "" }, )).status).toBe(502); @@ -1688,7 +1828,7 @@ describe("compact alternate-account attempt (#913)", () => { compactionRequest(baseCompactionBody({})), config, { model: "", provider: "" }, ); expect(blocked.status).toBe(502); - expect(sends).toBe(3); + expect(sends).toBe(CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD); expect(Number(blocked.headers.get("retry-after"))).toBeGreaterThanOrEqual(1); }); }); diff --git a/tests/server-auth.test.ts b/tests/server-auth.test.ts index ac698ad7ba..3851316b2f 100644 --- a/tests/server-auth.test.ts +++ b/tests/server-auth.test.ts @@ -2575,6 +2575,7 @@ describe("server local API auth", () => { }); test("alternate preparation failure releases A's recovery probe", async () => { + const ALTERNATE_PREPARATION_HEADER_READ = 5; const harness = await startPoolRetryHarness(() => new Response("unused"), { omitCredentialAccountIds: ["pool-b"], }); @@ -2605,7 +2606,7 @@ describe("server local API auth", () => { get(target, property) { if (property === "headers") { headerReads += 1; - if (headerReads === 5) throw new Error("synthetic alternate preparation failure"); + if (headerReads === ALTERNATE_PREPARATION_HEADER_READ) throw new Error("synthetic alternate preparation failure"); } const value = Reflect.get(target, property, target) as unknown; return typeof value === "function" ? value.bind(target) : value; @@ -2624,7 +2625,7 @@ describe("server local API auth", () => { "synthetic alternate preparation failure", ); expect(sends).toBe(1); - expect(headerReads).toBe(5); + expect(headerReads).toBe(ALTERNATE_PREPARATION_HEADER_READ); expect(observedProbeLeaseId).toEqual(expect.any(String)); expect(getCodexUpstreamHealth("pool-a")?.cooldownUntil).toEqual(expect.any(Number)); expect(getCodexUpstreamHealth("pool-a")?.probeLeaseId).toBeUndefined(); @@ -2784,6 +2785,54 @@ describe("server local API auth", () => { } }, { timeout: 30_000 }); + test("a null canonical host key does not force manual redirects across a pool retry", async () => { + const redirectModes: Array = []; + const harness = await startPoolRetryHarness(accountId => ( + accountId === "acct-pool-a" + ? new Response("rate limited", { status: 429 }) + : new Response("ok") + )); + const upstreamUrl = "https://chatgpt.com/backend-api/codex/responses"; + const redirectedFetch = globalThis.fetch; + const NativeURL = globalThis.URL; + let forcedNullHostKey = false; + + class NullHostKeyURL extends NativeURL { + constructor(input: string | URL, base?: string | URL) { + const value = typeof input === "string" ? input : input.toString(); + if (!forcedNullHostKey && base === undefined && value === upstreamUrl) { + forcedNullHostKey = true; + throw new TypeError("synthetic null canonical host key"); + } + super(input, base); + } + } + + globalThis.URL = NullHostKeyURL as typeof URL; + globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => { + const requestUrl = typeof input === "string" + ? input + : input instanceof NativeURL + ? input.toString() + : input.url; + if (requestUrl === upstreamUrl) redirectModes.push(init?.redirect); + return redirectedFetch(input, init); + }) as typeof fetch; + + try { + const response = await harness.request(); + + expect(response.status).toBe(200); + expect(forcedNullHostKey).toBe(true); + expect(harness.dispatches.map(accountId => accountId.replace("acct-", ""))).toEqual(["pool-a", "pool-b"]); + expect(redirectModes).toEqual([undefined, undefined]); + } finally { + globalThis.URL = NativeURL; + globalThis.fetch = redirectedFetch; + await stopPoolRetryHarness(harness); + } + }); + for (const path of ["/v1/responses", "/v1/responses/compact"] as const) { test(`${path} releases a pool probe after a pre-response transport failure with no host key`, async () => { const harness = await startPoolRetryHarness(() => new Response("unused"), { secondAccount: false }); From b88d9d0c6e4f1ca4aab574c5e51922d79fa69bd5 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:23:10 +0900 Subject: [PATCH 12/20] fix: admit Codex host before pool selection --- src/codex/routing.ts | 8 ++ src/server/responses/compact.ts | 44 +++++++---- src/server/responses/core.ts | 73 +++++++++++++++--- tests/responses-compaction-routing.test.ts | 88 +++++++++++++++++++++- 4 files changed, 187 insertions(+), 26 deletions(-) diff --git a/src/codex/routing.ts b/src/codex/routing.ts index b85e25e4ce..4899412f6f 100644 --- a/src/codex/routing.ts +++ b/src/codex/routing.ts @@ -240,6 +240,14 @@ export function clearThreadAccountMap(): void { threadAccountMap.clear(); } +/** Side-effect-free inspection used by routing diagnostics and contract tests. */ +export function peekCodexThreadAffinityAccountId( + threadId: string, + quotaScope?: CodexQuotaScope, +): string | null { + return getThreadAffinity(threadId, quotaScope)?.accountId ?? null; +} + export function clearThreadAccountMapForAccount(accountId: string): void { for (const [threadId, affinities] of threadAccountMap) { for (const [scope, entry] of affinities) { diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index be30e4fc00..245cb8dfd6 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -120,7 +120,12 @@ import { import { hasResponsesItemIdRepair, relaySseWithResponsesItemIdRepair } from "../responses-item-id-repair"; import type { EffectiveSubagentRoster, SpawnAgentSurface } from "../../codex/catalog"; -import { decodeRequestErrorResponse, handleResponses, usesCodexForwardPoolAuth } from "./core"; +import { + canonicalCodexForwardPoolHostKey, + decodeRequestErrorResponse, + handleResponses, + usesCodexForwardPoolAuth, +} from "./core"; import { fetchWithHeaderTimeout, providerFetch, safeHostLabel } from "./fetch-helpers"; export const COMPACT_RESPONSE_MAX_BYTES = 32 * 1024 * 1024; @@ -317,8 +322,22 @@ export async function handleResponsesCompact( // Resolve the SAME pool/thread auth context as /v1/responses — forwarding the caller's raw // headers would run compaction on the wrong account (or 401) whenever a pool account is // active for this thread while normal turns succeed. + if (req.signal.aborted) { + return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); + } + const compactHostKey = canonicalCodexForwardPoolHostKey(route); + const compactHostAdmission = compactHostKey + ? acquireCodexUpstreamHostAdmission(compactHostKey) + : null; + if (compactHostAdmission?.kind === "blocked") { + return formatErrorResponse(502, "upstream_error", "Provider host is temporarily unavailable", { + retryAfter: String(compactHostAdmission.retryAfterSeconds), + }); + } + let compactHostAdmissionLease = compactHostAdmission?.lease ?? null; let compactProvider = route.provider; let authCtx: CodexAuthContext = { kind: "main", accountId: null }; + try { const headers = new Headers({ "content-type": "application/json" }); try { if (route.codexAccountMode) { @@ -340,6 +359,7 @@ export async function handleResponsesCompact( } } } catch (err) { + releaseCodexAuthContextProbeLease(authCtx); if (err instanceof CodexAccountCooldownError) { return cooldownErrorResponse(err, Date.now(), route.codexAccountNamespace); } @@ -367,10 +387,12 @@ export async function handleResponsesCompact( const compactUrl = `${base}/responses/compact`; const compactThreadId = req.headers.get("x-codex-parent-thread-id"); const connectMs = config.connectTimeoutMs ?? 200_000; - const compactHostKey = usesCodexForwardPoolAuth(authCtx, route.provider) + const eventualCompactHostKey = usesCodexForwardPoolAuth(authCtx, route.provider) ? canonicalCodexUpstreamHostKey(route.providerName, compactUrl) : null; - let compactHostAdmissionLease: CodexUpstreamHostAdmissionLease | null = null; + if ((compactHostAdmissionLease?.key ?? null) !== eventualCompactHostKey) { + throw new Error("Codex compact upstream host changed after admission"); + } const settleObservedCompactHostResponse = (): void => { if (!compactHostAdmissionLease) return; recordCodexUpstreamHostResponse(compactHostAdmissionLease); @@ -494,16 +516,6 @@ export async function handleResponsesCompact( recordCompactPoolOutcome(authCtx, 499); return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); } - const compactHostAdmission = compactHostKey - ? acquireCodexUpstreamHostAdmission(compactHostKey) - : null; - if (compactHostAdmission?.kind === "blocked") { - releaseCodexAuthContextProbeLease(authCtx); - return formatErrorResponse(502, "upstream_error", "Provider host is temporarily unavailable", { - retryAfter: String(compactHostAdmission.retryAfterSeconds), - }); - } - compactHostAdmissionLease = compactHostAdmission?.lease ?? null; const primaryAttempts: UpstreamAttemptObservation[] = []; const primaryAttemptBoundary = { executorStarted: false }; try { @@ -685,6 +697,12 @@ export async function handleResponsesCompact( // synthetic buffer errors are not upstream bodies and stay uninspected. if (buffered.ok) inspectResponseLogJson(logCtx, await buffered.clone().text()); return buffered; + } finally { + if (compactHostAdmissionLease) { + releaseCodexUpstreamHostAdmissionLease(compactHostAdmissionLease); + releaseCodexAuthContextProbeLease(authCtx); + } + } } // ROUTED model: run the v2 synthetic-compaction turn internally (appends COMPACT_PROMPT, no diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index ed6e730b11..c7d1bcb7db 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -238,6 +238,22 @@ export function usesCodexForwardPoolAuth( && provider.authMode === "forward" && provider.adapter === "openai-responses"; } +/** + * Canonical host selected by a validated native Pool route. Host admission must + * happen before account selection, so derive it from the route-owned base URL; + * the authority-only key is checked against the eventual adapter request URL. + */ +export function canonicalCodexForwardPoolHostKey( + route: Pick, +) { + if ( + route.codexAccountMode !== "pool" + || route.provider.authMode !== "forward" + || route.provider.adapter !== "openai-responses" + ) return null; + return canonicalCodexUpstreamHostKey(route.providerName, route.provider.baseUrl ?? ""); +} + function normalizeCodexUnsupportedModelDetail(value: string): string { return value.trim().replace(/\s+/gu, " ").toLocaleLowerCase("en-US"); } @@ -804,9 +820,8 @@ async function resolveResponsesCodexAuth( route: RouteResult, options: HandleResponsesOptions, ): Promise { + let authCtx: CodexAuthContext | undefined; try { - if (route.codexAccountMode === "direct") validateForwardAdmissionCredential(req.headers, config); - let authCtx: CodexAuthContext; if (route.codexAccountMode) { authCtx = await resolveCodexAuthContext(req.headers, config, route.codexAccountMode, { accountId: route.codexAccountId, @@ -831,6 +846,7 @@ async function resolveResponsesCodexAuth( headers: headersForCodexAuthContext(req.headers, authCtx), }; } catch (err) { + releaseCodexAuthContextProbeLease(authCtx); if (err instanceof CodexAccountCooldownError) { return { ok: false, response: cooldownErrorResponse(err, Date.now(), route.codexAccountNamespace) }; } @@ -1312,6 +1328,9 @@ async function handleResponsesInner( logCtx: RequestLogContext, options: HandleResponsesOptions & { translatorBudget: TranslatorBudget }, ): Promise { + let pendingHostAdmissionLease: CodexUpstreamHostAdmissionLease | null = null; + let authCtx: CodexAuthContext = { kind: "main", accountId: null }; + try { // The Chat and Anthropic surfaces replay through here with a Responses-shaped body, // so an omitted value means a genuine Responses inbound. const inboundWire = options.inboundWire ?? "responses"; @@ -1440,7 +1459,6 @@ async function handleResponsesInner( nativeMainSelectionOnly: !nativeMainRecoveryBlocked && previewSelectionAdmission?.mainProfileDraining === true, }; - let authCtx: CodexAuthContext = { kind: "main", accountId: null }; let selectedForwardHeaders = req.headers; let subagentFallbackAccountId = config.activeCodexAccountId ?? null; let subagentQuotaFailureModel = parsed.modelId; @@ -1536,6 +1554,32 @@ async function handleResponsesInner( logCtx.provider = `${route.providerName}-${route.codexAccountNamespace}`; } + // Preserve inbound validation precedence: malformed requests, invalid routes, + // and Direct-mode admission credentials are rejected before host health can + // surface a synthetic 502. Pool credential resolution remains behind the + // atomic host admission so a blocked request cannot commit routing state. + if (route.codexAccountMode === "direct") { + try { + validateForwardAdmissionCredential(req.headers, config); + } catch (err) { + if (err instanceof ForwardAdmissionCredentialError) { + return formatErrorResponse(401, "authentication_error", err.message); + } + throw err; + } + } + if (options.abortSignal?.aborted) return clientCancelledResponse(); + const preAuthHostKey = canonicalCodexForwardPoolHostKey(route); + const preAuthHostAdmission = preAuthHostKey + ? acquireCodexUpstreamHostAdmission(preAuthHostKey) + : null; + if (preAuthHostAdmission?.kind === "blocked") { + return formatErrorResponse(502, "upstream_error", "Provider host is temporarily unavailable", { + retryAfter: String(preAuthHostAdmission.retryAfterSeconds), + }); + } + pendingHostAdmissionLease = preAuthHostAdmission?.lease ?? null; + { const finalAuth = await resolveResponsesCodexAuth(req, config, route, options); if (!finalAuth.ok) return finalAuth.response; @@ -1780,7 +1824,12 @@ async function handleResponsesInner( const hostKey = tracksCodexPoolHost ? canonicalCodexUpstreamHostKey(route.providerName, request.url) : null; - let hostAdmissionLease: CodexUpstreamHostAdmissionLease | null = null; + if ((pendingHostAdmissionLease?.key ?? null) !== hostKey) { + releaseCodexAuthContextProbeLease(authCtx); + throw new Error("Codex upstream host changed after admission"); + } + let hostAdmissionLease = pendingHostAdmissionLease; + pendingHostAdmissionLease = null; const attemptHistory: UpstreamAttemptObservation[] = []; let primaryAttemptExecutorStarted = false; const recordPoolTransportOutcome = (outcome: CodexUpstreamOutcome): void => { @@ -1834,16 +1883,10 @@ async function handleResponsesInner( try { if (options.abortSignal?.aborted) { releaseCodexAuthContextProbeLease(authCtx); + releaseCodexUpstreamHostAdmissionLease(hostAdmissionLease); + hostAdmissionLease = null; return clientCancelledResponse(); } - const hostAdmission = hostKey ? acquireCodexUpstreamHostAdmission(hostKey) : null; - if (hostAdmission?.kind === "blocked") { - releaseCodexAuthContextProbeLease(authCtx); - return formatErrorResponse(502, "upstream_error", "Provider host is temporarily unavailable", { - retryAfter: String(hostAdmission.retryAfterSeconds), - }); - } - hostAdmissionLease = hostAdmission?.lease ?? null; // Transient-5xx pre-stream retry (devlog/_plan/260716_claudecode_hardening/010): // the ChatGPT backend emits transient 502/520s that an immediate retry absorbs. // Body is a replayable string; nothing has streamed to the client yet. @@ -3324,6 +3367,12 @@ async function handleResponsesInner( } return formatErrorResponse(400, "invalid_request_error", "Non-streaming not supported by this adapter"); + } finally { + if (pendingHostAdmissionLease) { + releaseCodexUpstreamHostAdmissionLease(pendingHostAdmissionLease); + releaseCodexAuthContextProbeLease(authCtx); + } + } } diff --git a/tests/responses-compaction-routing.test.ts b/tests/responses-compaction-routing.test.ts index 39ce6ba997..4b34612d03 100644 --- a/tests/responses-compaction-routing.test.ts +++ b/tests/responses-compaction-routing.test.ts @@ -4,7 +4,7 @@ * contract; every other gateway has to be driven as a plain summarizer, or Codex * fatals on a compaction turn that came back as an ordinary message. */ -import { afterEach, describe, expect, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -12,12 +12,18 @@ import { handleResponses, handleResponsesCompact } from "../src/server/responses import { saveCodexAccountCredential } from "../src/codex/account-store"; import { CODEX_QUOTA_PROBE_INTERVAL_MS, + clearThreadAccountMap, clearCodexUpstreamHealth, + getEffectiveActiveCodexAccountId, getCodexUpstreamHealth, + peekCodexThreadAffinityAccountId, + previewCodexAccountForRequest, recordCodexUpstreamOutcome, resolveCodexAccountForThread, } from "../src/codex/routing"; import { clearAccountNeedsReauth, clearAccountQuota, updateAccountQuota } from "../src/codex/auth-api"; +import { getAccountQuota } from "../src/codex/quota"; +import { clearPoolRotationState, POOL_KEY_CODEX } from "../src/codex/pool-rotation"; import { CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD, acquireCodexUpstreamHostAdmission, @@ -35,11 +41,19 @@ import { import { supportsNativeResponsesCompactEndpoint } from "../src/providers/openai-tiers"; import type { RequestLogContext } from "../src/server/request-log"; import { acquireNativeMainProfileDrain, tryAdmitTurn } from "../src/server/lifecycle"; +import { setIcaclsRunnerForTests } from "../src/lib/windows-secret-acl"; import type { OcxConfig, OcxProviderConfig } from "../src/types"; const originalFetch = globalThis.fetch; +beforeEach(() => { + // This suite exercises response routing, not Windows ACL behavior. Match the + // account/auth suites so isolated Windows runs do not spawn icacls per fixture. + setIcaclsRunnerForTests(() => ({ success: true, exitCode: 0, timedOut: false, stdout: "" })); +}); + afterEach(() => { + setIcaclsRunnerForTests(null); globalThis.fetch = originalFetch; clearCodexUpstreamHostHealth(); }); @@ -717,6 +731,8 @@ describe("compact alternate-account attempt (#913)", () => { process.env.OPENCODEX_HOME = testDir; process.env.CODEX_HOME = testDir; clearCodexUpstreamHealth(); + clearThreadAccountMap(); + clearPoolRotationState(POOL_KEY_CODEX); clearCodexUpstreamHostHealth(); clearAccountQuota(); for (const id of ["pool-a", "pool-b"]) { @@ -732,6 +748,8 @@ describe("compact alternate-account attempt (#913)", () => { return run(twoAccountPoolConfig()).finally(() => { globalThis.fetch = originalFetch; clearCodexUpstreamHealth(); + clearThreadAccountMap(); + clearPoolRotationState(POOL_KEY_CODEX); clearCodexUpstreamHostHealth(); clearAccountQuota(); clearAccountNeedsReauth("pool-a"); @@ -769,6 +787,74 @@ describe("compact alternate-account attempt (#913)", () => { if (admission.kind === "admitted") releaseCodexUpstreamHostAdmissionLease(admission.lease, now); } + function openCanonicalHostCircuit(now = Date.now()): void { + const hostKey = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex", + )!; + for (let attempt = 0; attempt < CODEX_UPSTREAM_HOST_FAILURE_THRESHOLD; attempt++) { + const admission = acquireCodexUpstreamHostAdmission(hostKey, now + attempt); + if (admission.kind !== "admitted") throw new Error("expected host admission while opening circuit"); + recordCodexUpstreamHostFailure(admission.lease, now + attempt); + } + } + + for (const endpoint of ["regular", "compact"] as const) { + for (const strategy of ["round-robin", "fill-first"] as const) { + test(`${endpoint} open circuit is selection-neutral for an unbound ${strategy} thread`, async () => { + await withPoolEnv(`ocx-${endpoint}-${strategy}-neutral-`, async config => { + const threadId = `${endpoint}-${strategy}-blocked-thread`; + config.accountPoolStrategy = strategy; + config.accountPoolStickyLimit = 1; + config.autoSwitchThreshold = 80; + config.activeCodexAccountId = strategy === "round-robin" ? "pool-b" : "pool-a"; + if (strategy === "fill-first") updateAccountQuota("pool-a", 100); + clearCodexUpstreamHealth(); + clearThreadAccountMap(); + clearPoolRotationState(POOL_KEY_CODEX); + + const expectedNextAccount = previewCodexAccountForRequest(threadId, config); + expect(expectedNextAccount).not.toBeNull(); + const effectiveActiveBefore = getEffectiveActiveCodexAccountId(config); + const quotaBefore = new Map(["pool-a", "pool-b"].map(id => [id, structuredClone(getAccountQuota(id))])); + openCanonicalHostCircuit(); + + const physicalAccounts: string[] = []; + globalThis.fetch = (async (_url: RequestInfo | URL, init?: RequestInit) => { + physicalAccounts.push(new Headers(init?.headers).get("chatgpt-account-id") ?? ""); + return jsonResponse(completedPayload("selection-neutral recovery")); + }) as typeof fetch; + const requestFor = () => compactionRequest( + endpoint === "regular" + ? regularBody() + : baseCompactionBody({ model: "gpt-5.6-sol" }), + undefined, + { "x-codex-parent-thread-id": threadId }, + ); + const invoke = () => endpoint === "regular" + ? handleResponses(requestFor(), config, { model: "", provider: "" }) + : handleResponsesCompact(requestFor(), config, { model: "", provider: "" }); + + const blocked = await invoke(); + expect(blocked.status).toBe(502); + expect(Number(blocked.headers.get("retry-after"))).toBeGreaterThanOrEqual(1); + expect(physicalAccounts).toEqual([]); + expect(peekCodexThreadAffinityAccountId(threadId)).toBeNull(); + expect(getEffectiveActiveCodexAccountId(config)).toBe(effectiveActiveBefore); + expect(previewCodexAccountForRequest(threadId, config)).toBe(expectedNextAccount); + for (const [id, quota] of quotaBefore) expect(getAccountQuota(id)).toEqual(quota); + + clearCodexUpstreamHostHealth(); + const recovered = await invoke(); + expect(recovered.status).toBe(200); + expect(physicalAccounts).toEqual([ + expectedNextAccount === "pool-a" ? "pool_acc_a" : "pool_acc_b", + ]); + }); + }); + } + } + type PreExecutorBoundaryState = { prepared: boolean; cloneThrew: boolean; From b2cf1237ae603c61dd91e72144fa6b26548a8134 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:35:34 +0900 Subject: [PATCH 13/20] fix: preserve fixed-account auth precedence --- src/server/responses/compact.ts | 28 ++++++--- src/server/responses/core.ts | 18 +++++- tests/responses-compaction-routing.test.ts | 73 +++++++++++++++++++++- 3 files changed, 106 insertions(+), 13 deletions(-) diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index 245cb8dfd6..195f1118e7 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -325,16 +325,16 @@ export async function handleResponsesCompact( if (req.signal.aborted) { return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); } - const compactHostKey = canonicalCodexForwardPoolHostKey(route); - const compactHostAdmission = compactHostKey - ? acquireCodexUpstreamHostAdmission(compactHostKey) + const preAuthCompactHostKey = canonicalCodexForwardPoolHostKey(route); + const preAuthCompactHostAdmission = preAuthCompactHostKey + ? acquireCodexUpstreamHostAdmission(preAuthCompactHostKey) : null; - if (compactHostAdmission?.kind === "blocked") { + if (preAuthCompactHostAdmission?.kind === "blocked") { return formatErrorResponse(502, "upstream_error", "Provider host is temporarily unavailable", { - retryAfter: String(compactHostAdmission.retryAfterSeconds), + retryAfter: String(preAuthCompactHostAdmission.retryAfterSeconds), }); } - let compactHostAdmissionLease = compactHostAdmission?.lease ?? null; + let compactHostAdmissionLease = preAuthCompactHostAdmission?.lease ?? null; let compactProvider = route.provider; let authCtx: CodexAuthContext = { kind: "main", accountId: null }; try { @@ -387,10 +387,10 @@ export async function handleResponsesCompact( const compactUrl = `${base}/responses/compact`; const compactThreadId = req.headers.get("x-codex-parent-thread-id"); const connectMs = config.connectTimeoutMs ?? 200_000; - const eventualCompactHostKey = usesCodexForwardPoolAuth(authCtx, route.provider) + const compactHostKey = usesCodexForwardPoolAuth(authCtx, route.provider) ? canonicalCodexUpstreamHostKey(route.providerName, compactUrl) : null; - if ((compactHostAdmissionLease?.key ?? null) !== eventualCompactHostKey) { + if (compactHostAdmissionLease && compactHostAdmissionLease.key !== compactHostKey) { throw new Error("Codex compact upstream host changed after admission"); } const settleObservedCompactHostResponse = (): void => { @@ -516,6 +516,18 @@ export async function handleResponsesCompact( recordCompactPoolOutcome(authCtx, 499); return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); } + // Fixed namespaces resolve their credential first, then retain the original + // post-auth host admission ordering immediately before the physical send. + if (!compactHostAdmissionLease && compactHostKey) { + const lateCompactHostAdmission = acquireCodexUpstreamHostAdmission(compactHostKey); + if (lateCompactHostAdmission.kind === "blocked") { + releaseCodexAuthContextProbeLease(authCtx); + return formatErrorResponse(502, "upstream_error", "Provider host is temporarily unavailable", { + retryAfter: String(lateCompactHostAdmission.retryAfterSeconds), + }); + } + compactHostAdmissionLease = lateCompactHostAdmission.lease; + } const primaryAttempts: UpstreamAttemptObservation[] = []; const primaryAttemptBoundary = { executorStarted: false }; try { diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index c7d1bcb7db..8685459b66 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -244,10 +244,11 @@ export function usesCodexForwardPoolAuth( * the authority-only key is checked against the eventual adapter request URL. */ export function canonicalCodexForwardPoolHostKey( - route: Pick, + route: Pick, ) { if ( route.codexAccountMode !== "pool" + || route.codexAccountId !== undefined || route.provider.authMode !== "forward" || route.provider.adapter !== "openai-responses" ) return null; @@ -1824,7 +1825,7 @@ async function handleResponsesInner( const hostKey = tracksCodexPoolHost ? canonicalCodexUpstreamHostKey(route.providerName, request.url) : null; - if ((pendingHostAdmissionLease?.key ?? null) !== hostKey) { + if (pendingHostAdmissionLease && pendingHostAdmissionLease.key !== hostKey) { releaseCodexAuthContextProbeLease(authCtx); throw new Error("Codex upstream host changed after admission"); } @@ -1887,6 +1888,19 @@ async function handleResponsesInner( hostAdmissionLease = null; return clientCancelledResponse(); } + // Exact account namespaces deliberately preserve the historical ordering: + // fixed credential resolution happens before host admission, so a missing + // credential remains a 401 and never occupies the half-open host lease. + if (!hostAdmissionLease && hostKey) { + const hostAdmission = acquireCodexUpstreamHostAdmission(hostKey); + if (hostAdmission.kind === "blocked") { + releaseCodexAuthContextProbeLease(authCtx); + return formatErrorResponse(502, "upstream_error", "Provider host is temporarily unavailable", { + retryAfter: String(hostAdmission.retryAfterSeconds), + }); + } + hostAdmissionLease = hostAdmission.lease; + } // Transient-5xx pre-stream retry (devlog/_plan/260716_claudecode_hardening/010): // the ChatGPT backend emits transient 502/520s that an immediate retry absorbs. // Body is a replayable string; nothing has streamed to the client yet. diff --git a/tests/responses-compaction-routing.test.ts b/tests/responses-compaction-routing.test.ts index 4b34612d03..c1c08a478b 100644 --- a/tests/responses-compaction-routing.test.ts +++ b/tests/responses-compaction-routing.test.ts @@ -9,11 +9,12 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { handleResponses, handleResponsesCompact } from "../src/server/responses"; -import { saveCodexAccountCredential } from "../src/codex/account-store"; +import { removeCodexAccountCredential, saveCodexAccountCredential } from "../src/codex/account-store"; import { CODEX_QUOTA_PROBE_INTERVAL_MS, clearThreadAccountMap, clearCodexUpstreamHealth, + codexQuotaScopeForModel, getEffectiveActiveCodexAccountId, getCodexUpstreamHealth, peekCodexThreadAffinityAccountId, @@ -813,7 +814,14 @@ describe("compact alternate-account attempt (#913)", () => { clearThreadAccountMap(); clearPoolRotationState(POOL_KEY_CODEX); - const expectedNextAccount = previewCodexAccountForRequest(threadId, config); + const quotaScope = codexQuotaScopeForModel("gpt-5.6-sol"); + expect(quotaScope).toBe("shared"); + const expectedNextAccount = previewCodexAccountForRequest( + threadId, + config, + Date.now(), + quotaScope, + ); expect(expectedNextAccount).not.toBeNull(); const effectiveActiveBefore = getEffectiveActiveCodexAccountId(config); const quotaBefore = new Map(["pool-a", "pool-b"].map(id => [id, structuredClone(getAccountQuota(id))])); @@ -840,8 +848,9 @@ describe("compact alternate-account attempt (#913)", () => { expect(Number(blocked.headers.get("retry-after"))).toBeGreaterThanOrEqual(1); expect(physicalAccounts).toEqual([]); expect(peekCodexThreadAffinityAccountId(threadId)).toBeNull(); + expect(peekCodexThreadAffinityAccountId(threadId, quotaScope)).toBeNull(); expect(getEffectiveActiveCodexAccountId(config)).toBe(effectiveActiveBefore); - expect(previewCodexAccountForRequest(threadId, config)).toBe(expectedNextAccount); + expect(previewCodexAccountForRequest(threadId, config, Date.now(), quotaScope)).toBe(expectedNextAccount); for (const [id, quota] of quotaBefore) expect(getAccountQuota(id)).toEqual(quota); clearCodexUpstreamHostHealth(); @@ -850,11 +859,69 @@ describe("compact alternate-account attempt (#913)", () => { expect(physicalAccounts).toEqual([ expectedNextAccount === "pool-a" ? "pool_acc_a" : "pool_acc_b", ]); + expect(peekCodexThreadAffinityAccountId(threadId, quotaScope)).toBe(expectedNextAccount); }); }); } } + for (const endpoint of ["regular", "compact"] as const) { + test(`${endpoint} fixed selector keeps missing-credential 401 ahead of an open host circuit`, async () => { + await withPoolEnv(`ocx-${endpoint}-fixed-auth-precedence-`, async config => { + const threadId = `${endpoint}-fixed-auth-precedence`; + const quotaScope = codexQuotaScopeForModel("gpt-5.6-sol"); + expect(quotaScope).toBe("shared"); + config.codexAccountNamespaces = { side: "pool-a" }; + config.accountPoolStrategy = "round-robin"; + config.accountPoolStickyLimit = 1; + config.activeCodexAccountId = "pool-b"; + removeCodexAccountCredential("pool-a"); + clearCodexUpstreamHealth(); + clearThreadAccountMap(); + clearPoolRotationState(POOL_KEY_CODEX); + + const expectedNextAccount = previewCodexAccountForRequest( + threadId, + config, + Date.now(), + quotaScope, + ); + const effectiveActiveBefore = getEffectiveActiveCodexAccountId(config); + openCanonicalHostCircuit(); + const hostKey = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex", + )!; + const hostHealthBefore = getCodexUpstreamHostHealth(hostKey); + let physicalSends = 0; + globalThis.fetch = (async () => { + physicalSends += 1; + return jsonResponse(completedPayload("must not send")); + }) as typeof fetch; + + const request = compactionRequest( + endpoint === "regular" + ? { ...regularBody(), model: "side/gpt-5.6-sol" } + : baseCompactionBody({ model: "side/gpt-5.6-sol" }), + undefined, + { "x-codex-parent-thread-id": threadId }, + ); + const response = endpoint === "regular" + ? await handleResponses(request, config, { model: "", provider: "" }) + : await handleResponsesCompact(request, config, { model: "", provider: "" }); + + expect(response.status).toBe(401); + expect(physicalSends).toBe(0); + expect(getCodexUpstreamHostHealth(hostKey)).toEqual(hostHealthBefore); + expect(peekCodexThreadAffinityAccountId(threadId)).toBeNull(); + expect(peekCodexThreadAffinityAccountId(threadId, quotaScope)).toBeNull(); + expect(getEffectiveActiveCodexAccountId(config)).toBe(effectiveActiveBefore); + expect(previewCodexAccountForRequest(threadId, config, Date.now(), quotaScope)) + .toBe(expectedNextAccount); + }); + }); + } + type PreExecutorBoundaryState = { prepared: boolean; cloneThrew: boolean; From ef0437935415b4ad1ff4be729db6209bcdbfe773 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:21:34 +0900 Subject: [PATCH 14/20] fix(responses): preserve pre-executor retry evidence --- src/server/responses/compact.ts | 23 ++++-- src/server/responses/core.ts | 23 ++++-- tests/responses-compaction-routing.test.ts | 93 +++++++++++++++++----- 3 files changed, 111 insertions(+), 28 deletions(-) diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index 195f1118e7..2805635884 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -542,15 +542,28 @@ export async function handleResponsesCompact( primaryAttemptBoundary, ); } catch (err) { - if (!primaryAttemptBoundary.executorStarted && primaryAttempts.length === 0 && !req.signal.aborted) { + if (!primaryAttemptBoundary.executorStarted && !req.signal.aborted) { + const lastObservation = primaryAttempts.at(-1); const observedStatus = lastUpstreamAttemptResponseStatus(primaryAttempts); - if (observedStatus !== undefined) { - recordCompactPoolOutcome(outcomeCtx, observedStatus); - settleObservedCompactHostResponse(); - } else { + if (!lastObservation) { releaseCodexAuthContextProbeLease(outcomeCtx); releaseCodexUpstreamHostAdmissionLease(compactHostAdmissionLease); compactHostAdmissionLease = null; + } else if (lastObservation.kind === "response") { + recordCompactPoolOutcome(outcomeCtx, lastObservation.status); + settleObservedCompactHostResponse(); + } else { + if (observedStatus !== undefined) { + recordCompactPoolOutcome(outcomeCtx, observedStatus); + } else { + releaseCodexAuthContextProbeLease(outcomeCtx); + } + if (compactHostAdmissionLease) { + recordCodexUpstreamHostFailure(compactHostAdmissionLease, Date.now(), { + observedResponse: observedStatus !== undefined, + }); + } + compactHostAdmissionLease = null; } throw err; } diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 8685459b66..f48ebef40e 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -1936,15 +1936,28 @@ async function handleResponsesInner( }, ); } catch (err) { - if (!primaryAttemptExecutorStarted && attemptHistory.length === 0 && !options.abortSignal?.aborted) { + if (!primaryAttemptExecutorStarted && !options.abortSignal?.aborted) { + const lastObservation = attemptHistory.at(-1); const observedStatus = lastUpstreamAttemptResponseStatus(attemptHistory); - if (observedStatus !== undefined) { - recordPoolTransportOutcome(observedStatus); - settleObservedHostResponse(); - } else { + if (!lastObservation) { releaseCodexAuthContextProbeLease(authCtx); releaseCodexUpstreamHostAdmissionLease(hostAdmissionLease); hostAdmissionLease = null; + } else if (lastObservation.kind === "response") { + recordPoolTransportOutcome(lastObservation.status); + settleObservedHostResponse(); + } else { + if (observedStatus !== undefined) { + recordPoolTransportOutcome(observedStatus); + } else { + releaseCodexAuthContextProbeLease(authCtx); + } + if (hostAdmissionLease) { + recordCodexUpstreamHostFailure(hostAdmissionLease, Date.now(), { + observedResponse: observedStatus !== undefined, + }); + } + hostAdmissionLease = null; } throw err; } diff --git a/tests/responses-compaction-routing.test.ts b/tests/responses-compaction-routing.test.ts index c1c08a478b..d5240e2801 100644 --- a/tests/responses-compaction-routing.test.ts +++ b/tests/responses-compaction-routing.test.ts @@ -1060,7 +1060,7 @@ describe("compact alternate-account attempt (#913)", () => { }); }); - test("regular preserves a prior 503 when retry setup fails before provider execution", async () => { + test("regular rethrows the exact local error after a prior 503 and clears host health", async () => { await withPoolEnv("ocx-regular-retry-pre-executor-", async config => { const originalNow = Date.now; const NativeHeaders = globalThis.Headers; @@ -1087,19 +1087,18 @@ describe("compact alternate-account attempt (#913)", () => { }); }) as typeof fetch; try { - const response = await handleResponses( + await expectPreExecutorBoundaryRejection(() => handleResponses( compactionRequest(regularBody(), undefined, { "x-codex-parent-thread-id": threadId }), config, { model: "", provider: "" }, - ); - expect(response.status).toBe(502); + ), state, expected); expect(state).toEqual({ prepared: true, cloneThrew: true, boundaryFrameObserved: true }); expect(physicalAccounts).toEqual(["pool_acc_a"]); expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(503); expect(getCodexUpstreamHealth("pool-b")).toBeNull(); expect(resolveCodexAccountForThread(threadId, config)).toBe(affinedAccount); expect(config.activeCodexAccountId).toBe("pool-a"); - expect(getCodexUpstreamHostHealth(hostKey, probeAt)?.consecutiveFailures).toBe(1); + expect(getCodexUpstreamHostHealth(hostKey, probeAt)).toBeNull(); expectHostAdmissionUsable(hostKey, probeAt); } finally { globalThis.Headers = NativeHeaders; @@ -1150,7 +1149,7 @@ describe("compact alternate-account attempt (#913)", () => { }); }); - test("compact preserves a prior 503 when retry setup fails before provider execution", async () => { + test("compact rethrows the exact local error after a prior 503 and clears host health", async () => { await withPoolEnv("ocx-compact-retry-pre-executor-", async config => { const originalNow = Date.now; const NativeHeaders = globalThis.Headers; @@ -1177,19 +1176,18 @@ describe("compact alternate-account attempt (#913)", () => { }); }) as typeof fetch; try { - const response = await handleResponsesCompact( + await expectPreExecutorBoundaryRejection(() => handleResponsesCompact( compactionRequest(baseCompactionBody({}), undefined, { "x-codex-parent-thread-id": threadId }), config, { model: "", provider: "" }, - ); - expect(response.status).toBe(502); + ), state, expected); expect(state).toEqual({ prepared: true, cloneThrew: true, boundaryFrameObserved: true }); expect(physicalAccounts).toEqual(["pool_acc_a"]); expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(503); expect(getCodexUpstreamHealth("pool-b")).toBeNull(); expect(resolveCodexAccountForThread(threadId, config)).toBe(affinedAccount); expect(config.activeCodexAccountId).toBe("pool-a"); - expect(getCodexUpstreamHostHealth(hostKey, probeAt)?.consecutiveFailures).toBe(1); + expect(getCodexUpstreamHostHealth(hostKey, probeAt)).toBeNull(); expectHostAdmissionUsable(hostKey, probeAt); } finally { globalThis.Headers = NativeHeaders; @@ -1198,7 +1196,7 @@ describe("compact alternate-account attempt (#913)", () => { }); }); - test("regular preserves an actual ECONNRESET when retry setup later fails before provider execution", async () => { + test("regular preserves a classified/status-less rejection when retry setup later fails", async () => { await withPoolEnv("ocx-regular-reset-pre-executor-", async config => { const originalNow = Date.now; const NativeHeaders = globalThis.Headers; @@ -1224,12 +1222,11 @@ describe("compact alternate-account attempt (#913)", () => { throw reset; }) as typeof fetch; try { - const response = await handleResponses( + await expectPreExecutorBoundaryRejection(() => handleResponses( compactionRequest(regularBody(), undefined, { "x-codex-parent-thread-id": threadId }), config, { model: "", provider: "" }, - ); - expect(response.status).toBe(502); + ), state, expected); expect(state).toEqual({ prepared: true, cloneThrew: true, boundaryFrameObserved: true }); expect(physicalAccounts).toEqual(["pool_acc_a"]); expect(getCodexUpstreamHealth("pool-a")).toBeNull(); @@ -1245,7 +1242,7 @@ describe("compact alternate-account attempt (#913)", () => { }); }); - test("compact preserves an actual ECONNRESET when retry setup later fails before provider execution", async () => { + test("compact preserves a classified/status-less rejection when retry setup later fails", async () => { await withPoolEnv("ocx-compact-reset-pre-executor-", async config => { const originalNow = Date.now; const NativeHeaders = globalThis.Headers; @@ -1271,12 +1268,11 @@ describe("compact alternate-account attempt (#913)", () => { throw reset; }) as typeof fetch; try { - const response = await handleResponsesCompact( + await expectPreExecutorBoundaryRejection(() => handleResponsesCompact( compactionRequest(baseCompactionBody({}), undefined, { "x-codex-parent-thread-id": threadId }), config, { model: "", provider: "" }, - ); - expect(response.status).toBe(502); + ), state, expected); expect(state).toEqual({ prepared: true, cloneThrew: true, boundaryFrameObserved: true }); expect(physicalAccounts).toEqual(["pool_acc_a"]); expect(getCodexUpstreamHealth("pool-a")).toBeNull(); @@ -1292,6 +1288,67 @@ describe("compact alternate-account attempt (#913)", () => { }); }); + for (const endpoint of ["regular", "compact"] as const) { + test(`${endpoint} preserves response/rejection history when later retry setup fails`, async () => { + await withPoolEnv(`ocx-${endpoint}-mixed-pre-executor-`, async config => { + const originalNow = Date.now; + const NativeHeaders = globalThis.Headers; + const threadId = `${endpoint}-mixed-pre-executor-thread`; + config.accountPoolStrategy = "fill-first"; + config.activeCodexAccountId = "pool-a"; + config.connectTimeoutMs = 25; + const affinedAccount = resolveCodexAccountForThread(threadId, config); + const probeAt = prepareHalfOpenHost(Date.now()); + const hostKey = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex/responses", + )!; + const expected = new Error(`${endpoint} retry header clone failed after response and rejection`); + const state = { prepared: false, cloneThrew: false, boundaryFrameObserved: false }; + const physicalAccounts: string[] = []; + Date.now = () => probeAt; + globalThis.Headers = throwingPreExecutorHeaders(NativeHeaders, "pool_acc_a", expected, state, 3); + globalThis.fetch = (async (_url: RequestInfo | URL, init?: RequestInit) => { + physicalAccounts.push(new NativeHeaders(init?.headers).get("chatgpt-account-id") ?? ""); + if (physicalAccounts.length === 1) { + return Response.json({ error: { message: "temporarily unavailable" } }, { + status: 503, + headers: { "retry-after": "0" }, + }); + } + const reset = new Error("upstream connection reset") as Error & { code?: string }; + reset.code = "ECONNRESET"; + throw reset; + }) as typeof fetch; + try { + const run = endpoint === "regular" + ? () => handleResponses( + compactionRequest(regularBody(), undefined, { "x-codex-parent-thread-id": threadId }), + config, + { model: "", provider: "" }, + ) + : () => handleResponsesCompact( + compactionRequest(baseCompactionBody({}), undefined, { "x-codex-parent-thread-id": threadId }), + config, + { model: "", provider: "" }, + ); + await expectPreExecutorBoundaryRejection(run, state, expected); + expect(state).toEqual({ prepared: true, cloneThrew: true, boundaryFrameObserved: true }); + expect(physicalAccounts).toEqual(["pool_acc_a", "pool_acc_a"]); + expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(503); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(resolveCodexAccountForThread(threadId, config)).toBe(affinedAccount); + expect(config.activeCodexAccountId).toBe("pool-a"); + expect(getCodexUpstreamHostHealth(hostKey, probeAt)?.consecutiveFailures).toBe(1); + expectHostAdmissionUsable(hostKey, probeAt); + } finally { + globalThis.Headers = NativeHeaders; + Date.now = originalNow; + } + }); + }); + } + test("regular settles a half-open primary response when alternate preparation throws", async () => { await withPoolEnv("ocx-regular-host-alt-throw-", async config => { const originalNow = Date.now; From 5493880c94ba563846b9da6fc29717166c9522e0 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:21:34 +0900 Subject: [PATCH 15/20] fix: settle regular pool outcome before alternate failures --- src/codex/routing.ts | 10 +++- src/server/responses/core.ts | 88 ++++++++++++++++++++++++++---------- tests/server-auth.test.ts | 25 ++++++++-- 3 files changed, 92 insertions(+), 31 deletions(-) diff --git a/src/codex/routing.ts b/src/codex/routing.ts index 4899412f6f..670bb941cf 100644 --- a/src/codex/routing.ts +++ b/src/codex/routing.ts @@ -211,6 +211,8 @@ export type CodexUpstreamOutcomeMeta = { * again (which would advance a round-robin ring twice). */ promoteAccountId?: string; + /** A locally failed alternate must not advance pool rotation. */ + suppressPromotion?: boolean; /** Generation captured when this routed account was selected. */ writerGeneration?: number; }; @@ -1567,7 +1569,9 @@ export function recordCodexUpstreamOutcome( const reused = meta.promoteAccountId && meta.promoteAccountId !== accountId ? meta.promoteAccountId : null; - const fallback = reused ?? pickAlternateCodexAccount(config, accountId, now, quotaScope); + const fallback = meta.suppressPromotion + ? null + : (reused ?? pickAlternateCodexAccount(config, accountId, now, quotaScope)); if (fallback) promoteActiveCodexAccount(config, fallback); } } @@ -1613,7 +1617,9 @@ export function recordCodexUpstreamOutcome( const reused = meta.promoteAccountId && meta.promoteAccountId !== accountId ? meta.promoteAccountId : null; - const fallback = reused ?? pickAlternateCodexAccount(config, accountId, now); + const fallback = meta.suppressPromotion + ? null + : (reused ?? pickAlternateCodexAccount(config, accountId, now)); if (fallback) promoteActiveCodexAccount(config, fallback); } } diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index f48ebef40e..8ec4af1a12 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -383,6 +383,50 @@ async function retryCodexPoolOnAlternateAccount( if (firstAuthCtx.fixedAccount) return { kind: "no-alternate" }; const inboundWire = options.inboundWire ?? "responses"; let retryAuthCtx: CodexAuthContext | undefined; + let firstOutcomeSettled = false; + // A's response has already been observed when this helper starts, but B + // resolution/preparation can still throw. Keep that accounting local so A + // cannot remain eligible after a local alternate failure. + const settleFirstOutcome = async ({ + promoteAccountId, + suppressPromotion = false, + }: { + promoteAccountId?: string; + suppressPromotion?: boolean; + } = {}): Promise => { + if (firstOutcomeSettled) return; + // Claim settlement before any await. A recorder failure must not allow a + // later cleanup path to record the same upstream response again. + firstOutcomeSettled = true; + try { + const quotaMeta = codexQuotaOutcomeMeta(firstResponse); + if (outcomeStatus === 429 || outcomeStatus === 402) { + const { applyAccountQuotaFromUpstreamHeaders } = await import("../../codex/auth-api"); + applyAccountQuotaFromUpstreamHeaders( + firstAuthCtx.accountId, + firstResponse.headers, + firstAuthCtx.writerGeneration, + ); + } + if (shouldDeferCodexResetDerivedCooldown(firstResponse, options.deferCodexResetDerivedCooldown)) { + releaseCodexAuthContextProbeLease(firstAuthCtx); + return; + } + recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { + ...quotaMeta, + threadId: req.headers.get("x-codex-parent-thread-id"), + modelId: route.modelId, + probeLeaseId: codexProbeLeaseId(firstAuthCtx), + probeQuotaScope: codexProbeQuotaScope(firstAuthCtx), + writerGeneration: firstAuthCtx.writerGeneration, + ...(promoteAccountId ? { promoteAccountId } : {}), + ...(suppressPromotion ? { suppressPromotion: true } : {}), + }); + } catch (error) { + releaseCodexAuthContextProbeLease(firstAuthCtx); + throw error; + } + }; try { retryAuthCtx = await resolveCodexAuthContext( req.headers, @@ -401,6 +445,14 @@ async function retryCodexPoolOnAlternateAccount( && !(error instanceof CodexAccountCooldownError) && !(error instanceof CodexMainProfileDrainingError) ) { + try { + // B was never usable, so record A without promoting it. A settlement + // errors must not mask the original resolver error. + await settleFirstOutcome({ suppressPromotion: true }); + } catch { + // settleFirstOutcome released A's probe when its recorder failed. + } + await firstResponse.body?.cancel().catch(() => undefined); releaseCodexAuthContextProbeLease(firstAuthCtx); throw error; } @@ -411,31 +463,7 @@ async function retryCodexPoolOnAlternateAccount( const prepared = await (async () => { let request: Awaited["buildRequest"]>> | undefined; - let firstOutcomeSettled = false; try { - const quotaMeta = codexQuotaOutcomeMeta(firstResponse); - if (outcomeStatus === 429 || outcomeStatus === 402) { - const { applyAccountQuotaFromUpstreamHeaders } = await import("../../codex/auth-api"); - applyAccountQuotaFromUpstreamHeaders( - firstAuthCtx.accountId, - firstResponse.headers, - firstAuthCtx.writerGeneration, - ); - } - if (!shouldDeferCodexResetDerivedCooldown(firstResponse, options.deferCodexResetDerivedCooldown)) { - recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { - ...quotaMeta, - threadId: req.headers.get("x-codex-parent-thread-id"), - modelId: route.modelId, - probeLeaseId: codexProbeLeaseId(firstAuthCtx), - probeQuotaScope: codexProbeQuotaScope(firstAuthCtx), - writerGeneration: firstAuthCtx.writerGeneration, - // Retry already advanced the RR ring via excludeAccountId — reuse for promotion. - ...(retryAuthCtx.accountId ? { promoteAccountId: retryAuthCtx.accountId } : {}), - }); - firstOutcomeSettled = true; - } - const retryHeaders = headersForCodexAuthContext(req.headers, retryAuthCtx); const retryProvider = applyCodexAuthContextToProvider( stripCodexRuntimeProviderFields(route.provider), @@ -452,6 +480,8 @@ async function retryCodexPoolOnAlternateAccount( }); recordAdapterReasoning(logCtx, request); + // Only a fully prepared B may be reused for A's round-robin promotion. + await settleFirstOutcome({ promoteAccountId: retryAuthCtx.accountId }); await firstResponse.body?.cancel().catch(() => undefined); options.onCodexAuthContextResolved?.(retryAuthCtx); route.provider = retryProvider; @@ -464,7 +494,15 @@ async function retryCodexPoolOnAlternateAccount( return { fetcher, request, retryHeaders }; } catch (error) { request?.releaseBodyObservation?.(); - if (!firstOutcomeSettled) releaseCodexAuthContextProbeLease(firstAuthCtx); + try { + // B preparation failed, so settle A without promotion. Preserve the + // original preparation error even if the A recorder also fails. + await settleFirstOutcome({ suppressPromotion: true }); + } catch { + // settleFirstOutcome released A's probe when its recorder failed. + } + await firstResponse.body?.cancel().catch(() => undefined); + releaseCodexAuthContextProbeLease(firstAuthCtx); releaseCodexAuthContextProbeLease(retryAuthCtx); throw error; } diff --git a/tests/server-auth.test.ts b/tests/server-auth.test.ts index 3851316b2f..3f4e2b05e1 100644 --- a/tests/server-auth.test.ts +++ b/tests/server-auth.test.ts @@ -1899,7 +1899,7 @@ describe("server local API auth", () => { expect(response.headers.get("x-exact-response")).toBe("original"); expect(await response.text()).toBe(body); expect(harness.dispatches).toEqual(["acct-pool-a"]); - expect(loadConfig().activeCodexAccountId).toBe("pool-b"); + expect(loadConfig().activeCodexAccountId).toBe("pool-a"); if (status !== 400) { const cooldown = await (await harness.request({ model: `side/${POOL_RETRY_MODEL}`, callerBearer: false })).text(); expect(cooldown).toContain("selector (side)"); @@ -2635,7 +2635,7 @@ describe("server local API auth", () => { } }); - test("unexpected alternate resolver failure releases A's recovery probe and preserves the error", async () => { + test("unexpected alternate resolver failure settles a retryable A 429 once and preserves the error", async () => { const harness = await startPoolRetryHarness(() => new Response("unused"), { omitCredentialAccountIds: ["pool-b"], }); @@ -2674,7 +2674,13 @@ describe("server local API auth", () => { chatgptAccountId: "acct-pool-b", }); clearAccountNeedsReauth("pool-b"); - return rejectionResponse(unsupportedModelBody()); + return new Response("rate limited", { + status: 429, + headers: { + "retry-after": "60", + "x-codex-primary-reset-at": String(Math.floor((Date.now() + 60 * 60_000) / 1000)), + }, + }); }) as typeof fetch; try { @@ -2690,8 +2696,19 @@ describe("server local API auth", () => { expect(sends).toBe(1); expect(selectionCalls).toBe(2); expect(observedProbeLeaseId).toEqual(expect.any(String)); - expect(getCodexUpstreamHealth("pool-a")?.cooldownUntil).toEqual(expect.any(Number)); + expect(getCodexUpstreamHealth("pool-a")).toMatchObject({ + cooldownUntil: expect.any(Number), + cooldownSource: "retry-after", + cooldownGeneration: 2, + lastFailureStatus: 429, + }); expect(getCodexUpstreamHealth("pool-a")?.probeLeaseId).toBeUndefined(); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(loadConfig().activeCodexAccountId).toBe("pool-b"); + expect(getCodexUpstreamHostHealth(canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex/responses", + )!)).toBeNull(); } finally { globalThis.fetch = redirectedFetch; await stopPoolRetryHarness(harness); From 5055ada9e7a93dea5b57afcc8db4b9ce084a28c5 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 17:35:29 +0900 Subject: [PATCH 16/20] fix: harden alternate outcome settlement --- src/server/responses/core.ts | 56 +++++++++++++++++++++++++---------- tests/server-auth.test.ts | 57 +++++++++++++++++++++++++++++------- 2 files changed, 88 insertions(+), 25 deletions(-) diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 8ec4af1a12..886f50e9b0 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -383,7 +383,9 @@ async function retryCodexPoolOnAlternateAccount( if (firstAuthCtx.fixedAccount) return { kind: "no-alternate" }; const inboundWire = options.inboundWire ?? "responses"; let retryAuthCtx: CodexAuthContext | undefined; - let firstOutcomeSettled = false; + let firstOutcomeHealthSettled = false; + let firstOutcomeHealthRecordAttempted = false; + let firstOutcomeDeferred = false; // A's response has already been observed when this helper starts, but B // resolution/preparation can still throw. Keep that accounting local so A // cannot remain eligible after a local alternate failure. @@ -394,24 +396,48 @@ async function retryCodexPoolOnAlternateAccount( promoteAccountId?: string; suppressPromotion?: boolean; } = {}): Promise => { - if (firstOutcomeSettled) return; - // Claim settlement before any await. A recorder failure must not allow a - // later cleanup path to record the same upstream response again. - firstOutcomeSettled = true; + if (firstOutcomeHealthSettled || firstOutcomeHealthRecordAttempted || firstOutcomeDeferred) return; + // Header-derived quota telemetry is valuable but not authoritative health + // settlement. It must not keep A eligible if a malformed/proxied response + // header or telemetry import fails. + let quotaMeta: ReturnType = { retryAfter: null, resetAt: [] }; try { - const quotaMeta = codexQuotaOutcomeMeta(firstResponse); - if (outcomeStatus === 429 || outcomeStatus === 402) { + quotaMeta = codexQuotaOutcomeMeta(firstResponse); + } catch { + // Record A's observed outcome without optional quota-header detail. + } + if (outcomeStatus === 429 || outcomeStatus === 402) { + try { const { applyAccountQuotaFromUpstreamHeaders } = await import("../../codex/auth-api"); applyAccountQuotaFromUpstreamHeaders( firstAuthCtx.accountId, firstResponse.headers, firstAuthCtx.writerGeneration, ); + } catch { + // This cache-refresh telemetry is best-effort; the health recorder below + // remains the authoritative disposition of A's observed response. } - if (shouldDeferCodexResetDerivedCooldown(firstResponse, options.deferCodexResetDerivedCooldown)) { - releaseCodexAuthContextProbeLease(firstAuthCtx); - return; - } + } + let deferResetDerivedCooldown = false; + try { + deferResetDerivedCooldown = shouldDeferCodexResetDerivedCooldown( + firstResponse, + options.deferCodexResetDerivedCooldown, + ); + } catch { + // A header accessor failure must not bypass the account health record. + } + if (deferResetDerivedCooldown) { + firstOutcomeDeferred = true; + releaseCodexAuthContextProbeLease(firstAuthCtx); + return; + } + // This guard tracks the authoritative recorder attempt separately from a + // successful health write. A recorder exception is released, not retried by + // cleanup, so an error path cannot produce duplicate A records. + firstOutcomeHealthRecordAttempted = true; + try { recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { ...quotaMeta, threadId: req.headers.get("x-codex-parent-thread-id"), @@ -422,6 +448,7 @@ async function retryCodexPoolOnAlternateAccount( ...(promoteAccountId ? { promoteAccountId } : {}), ...(suppressPromotion ? { suppressPromotion: true } : {}), }); + firstOutcomeHealthSettled = true; } catch (error) { releaseCodexAuthContextProbeLease(firstAuthCtx); throw error; @@ -479,10 +506,6 @@ async function retryCodexPoolOnAlternateAccount( translatorBudget: options.translatorBudget, }); recordAdapterReasoning(logCtx, request); - - // Only a fully prepared B may be reused for A's round-robin promotion. - await settleFirstOutcome({ promoteAccountId: retryAuthCtx.accountId }); - await firstResponse.body?.cancel().catch(() => undefined); options.onCodexAuthContextResolved?.(retryAuthCtx); route.provider = retryProvider; logCtx.provider = formatCodexProviderForLog( @@ -491,6 +514,9 @@ async function retryCodexPoolOnAlternateAccount( config, ); const fetcher = providerFetch(route.provider); + // Only a fully prepared B may be reused for A's round-robin promotion. + await settleFirstOutcome({ promoteAccountId: retryAuthCtx.accountId }); + await firstResponse.body?.cancel().catch(() => undefined); return { fetcher, request, retryHeaders }; } catch (error) { request?.releaseBodyObservation?.(); diff --git a/tests/server-auth.test.ts b/tests/server-auth.test.ts index 3f4e2b05e1..0f15e61021 100644 --- a/tests/server-auth.test.ts +++ b/tests/server-auth.test.ts @@ -1899,7 +1899,7 @@ describe("server local API auth", () => { expect(response.headers.get("x-exact-response")).toBe("original"); expect(await response.text()).toBe(body); expect(harness.dispatches).toEqual(["acct-pool-a"]); - expect(loadConfig().activeCodexAccountId).toBe("pool-a"); + expect(loadConfig().activeCodexAccountId).toBe("pool-b"); if (status !== 400) { const cooldown = await (await harness.request({ model: `side/${POOL_RETRY_MODEL}`, callerBearer: false })).text(); expect(cooldown).toContain("selector (side)"); @@ -2574,8 +2574,8 @@ describe("server local API auth", () => { } }); - test("alternate preparation failure releases A's recovery probe", async () => { - const ALTERNATE_PREPARATION_HEADER_READ = 5; + test("A quota-header telemetry failure still records its retryable outcome once", async () => { + const QUOTA_HEADER_READ_FAILURE = 5; const harness = await startPoolRetryHarness(() => new Response("unused"), { omitCredentialAccountIds: ["pool-b"], }); @@ -2606,7 +2606,7 @@ describe("server local API auth", () => { get(target, property) { if (property === "headers") { headerReads += 1; - if (headerReads === ALTERNATE_PREPARATION_HEADER_READ) throw new Error("synthetic alternate preparation failure"); + if (headerReads === QUOTA_HEADER_READ_FAILURE) throw new Error("synthetic quota header telemetry failure"); } const value = Reflect.get(target, property, target) as unknown; return typeof value === "function" ? value.bind(target) : value; @@ -2621,13 +2621,16 @@ describe("server local API auth", () => { body: JSON.stringify({ model: POOL_RETRY_MODEL, input: "hello", stream: false }), }); const logCtx: RequestLogContext = { model: POOL_RETRY_MODEL, provider: "openai" }; - await expect(handleResponses(request, harness.config, logCtx)).rejects.toThrow( - "synthetic alternate preparation failure", - ); + const response = await handleResponses(request, harness.config, logCtx); + expect(response.status).toBe(200); expect(sends).toBe(1); - expect(headerReads).toBe(ALTERNATE_PREPARATION_HEADER_READ); + expect(headerReads).toBeGreaterThanOrEqual(QUOTA_HEADER_READ_FAILURE); expect(observedProbeLeaseId).toEqual(expect.any(String)); - expect(getCodexUpstreamHealth("pool-a")?.cooldownUntil).toEqual(expect.any(Number)); + expect(getCodexUpstreamHealth("pool-a")).toMatchObject({ + cooldownUntil: expect.any(Number), + cooldownGeneration: 2, + lastFailureStatus: 429, + }); expect(getCodexUpstreamHealth("pool-a")?.probeLeaseId).toBeUndefined(); } finally { globalThis.fetch = redirectedFetch; @@ -2704,7 +2707,7 @@ describe("server local API auth", () => { }); expect(getCodexUpstreamHealth("pool-a")?.probeLeaseId).toBeUndefined(); expect(getCodexUpstreamHealth("pool-b")).toBeNull(); - expect(loadConfig().activeCodexAccountId).toBe("pool-b"); + expect(loadConfig().activeCodexAccountId).toBe("pool-a"); expect(getCodexUpstreamHostHealth(canonicalCodexUpstreamHostKey( "openai", "https://chatgpt.com/backend-api/codex/responses", @@ -2715,6 +2718,40 @@ describe("server local API auth", () => { } }); + test("alternate hook failure settles A once without promoting an unready B", async () => { + const hookError = new Error("synthetic alternate auth hook failure"); + const harness = await startPoolRetryHarness(accountId => accountId === "acct-pool-a" + ? new Response("rate limited", { status: 429, headers: { "retry-after": "60" } }) + : new Response("unexpected alternate dispatch")); + let hookCalls = 0; + try { + const request = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json", authorization: "Bearer inbound-token" }, + body: JSON.stringify({ model: POOL_RETRY_MODEL, input: "hello", stream: false }), + }); + const logCtx: RequestLogContext = { model: POOL_RETRY_MODEL, provider: "openai" }; + await expect(handleResponses(request, harness.config, logCtx, { + onCodexAuthContextResolved: ctx => { + hookCalls += 1; + if (ctx?.accountId === "pool-b") throw hookError; + }, + })).rejects.toBe(hookError); + expect(hookCalls).toBe(2); + expect(harness.dispatches).toEqual(["acct-pool-a"]); + expect(getCodexUpstreamHealth("pool-a")).toMatchObject({ + cooldownUntil: expect.any(Number), + cooldownGeneration: 1, + lastFailureStatus: 429, + }); + expect(getCodexUpstreamHealth("pool-a")?.probeLeaseId).toBeUndefined(); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(loadConfig().activeCodexAccountId).toBe("pool-a"); + } finally { + await stopPoolRetryHarness(harness); + } + }); + test("retry-dispatch transport failure is host-only and never triple-dispatches", async () => { const harness = await startPoolRetryHarness(() => rejectionResponse(unsupportedModelBody())); const affinityThread = "retry-dispatch-host-only-affinity"; From e640e66805663be90b9760183fc27a60fed51966 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 18:02:02 +0900 Subject: [PATCH 17/20] test: harden null host key fixtures --- tests/helpers/isolated-codex-home.ts | 8 +++- tests/server-auth.test.ts | 69 +++++++++++++++++++--------- 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/tests/helpers/isolated-codex-home.ts b/tests/helpers/isolated-codex-home.ts index 975e8e81cb..23156d7fe5 100644 --- a/tests/helpers/isolated-codex-home.ts +++ b/tests/helpers/isolated-codex-home.ts @@ -18,7 +18,13 @@ export function installIsolatedCodexHome(prefix = "ocx-codex-home-"): IsolatedCo restore() { if (previousCodexHome === undefined) delete process.env.CODEX_HOME; else process.env.CODEX_HOME = previousCodexHome; - rmSync(path, { recursive: true, force: true }); + // Windows can retain short-lived file or ACL handles after a test stops. + // node:fs bounds retries for EBUSY, EPERM, and ENOTEMPTY when recursive. + if (process.platform === "win32") { + rmSync(path, { recursive: true, force: true, maxRetries: 8, retryDelay: 25 }); + } else { + rmSync(path, { recursive: true, force: true }); + } }, }; } diff --git a/tests/server-auth.test.ts b/tests/server-auth.test.ts index 0f15e61021..f9164cece6 100644 --- a/tests/server-auth.test.ts +++ b/tests/server-auth.test.ts @@ -45,6 +45,7 @@ import type { OcxConfig } from "../src/types"; import { fakeChatGptJwt } from "./helpers/fake-chatgpt-jwt"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "./helpers/isolated-codex-home"; import { configuredAdminToken } from "../src/lib/admin-secrets"; +import { setIcaclsRunnerForTests } from "../src/lib/windows-secret-acl"; const previousApiToken = process.env.OPENCODEX_API_AUTH_TOKEN; const previousOpencodexHome = process.env.OPENCODEX_HOME; @@ -125,24 +126,34 @@ function stubModelDiscoveryFor(...origins: string[]): void { } beforeEach(() => { + // This suite validates server auth and request routing, not Windows ACLs. + // Keep isolated Windows runs from spawning icacls for every config fixture. + setIcaclsRunnerForTests(() => ({ success: true, exitCode: 0, timedOut: false, stdout: "" })); isolatedCodexHome = installIsolatedCodexHome("ocx-server-auth-codex-"); }); afterEach(() => { - globalThis.fetch = originalGlobalFetch; - if (previousApiToken === undefined) delete process.env.OPENCODEX_API_AUTH_TOKEN; - else process.env.OPENCODEX_API_AUTH_TOKEN = previousApiToken; - if (previousOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; - else process.env.OPENCODEX_HOME = previousOpencodexHome; - isolatedCodexHome?.restore(); - isolatedCodexHome = null; - clearCodexUpstreamHealth(); - clearCodexUpstreamHostHealth(); - clearThreadAccountMap(); - clearAccountNeedsReauth("pool-a"); - clearAccountNeedsReauth("pool-b"); - clearAccountQuota(); - if (existsSync(TEST_DIR)) rmSync(TEST_DIR, { recursive: true }); + try { + globalThis.fetch = originalGlobalFetch; + if (previousApiToken === undefined) delete process.env.OPENCODEX_API_AUTH_TOKEN; + else process.env.OPENCODEX_API_AUTH_TOKEN = previousApiToken; + if (previousOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousOpencodexHome; + isolatedCodexHome?.restore(); + } finally { + isolatedCodexHome = null; + try { + clearCodexUpstreamHealth(); + clearCodexUpstreamHostHealth(); + clearThreadAccountMap(); + clearAccountNeedsReauth("pool-a"); + clearAccountNeedsReauth("pool-b"); + clearAccountQuota(); + if (existsSync(TEST_DIR)) rmSync(TEST_DIR, { recursive: true }); + } finally { + setIcaclsRunnerForTests(null); + } + } }); const POOL_RETRY_MODEL = "gpt-5.6-sol"; @@ -2846,16 +2857,21 @@ describe("server local API auth", () => { ? new Response("rate limited", { status: 429 }) : new Response("ok") )); + const providerBaseUrl = "https://chatgpt.com/backend-api/codex"; const upstreamUrl = "https://chatgpt.com/backend-api/codex/responses"; const redirectedFetch = globalThis.fetch; const NativeURL = globalThis.URL; - let forcedNullHostKey = false; + const uncanonicalizableInputs: string[] = []; class NullHostKeyURL extends NativeURL { constructor(input: string | URL, base?: string | URL) { const value = typeof input === "string" ? input : input.toString(); - if (!forcedNullHostKey && base === undefined && value === upstreamUrl) { - forcedNullHostKey = true; + if ( + base === undefined + && (value === providerBaseUrl || value === upstreamUrl) + && new Error().stack?.includes("canonicalCodexUpstreamHostKey") + ) { + uncanonicalizableInputs.push(value); throw new TypeError("synthetic null canonical host key"); } super(input, base); @@ -2877,7 +2893,7 @@ describe("server local API auth", () => { const response = await harness.request(); expect(response.status).toBe(200); - expect(forcedNullHostKey).toBe(true); + expect(new Set(uncanonicalizableInputs)).toEqual(new Set([providerBaseUrl, upstreamUrl])); expect(harness.dispatches.map(accountId => accountId.replace("acct-", ""))).toEqual(["pool-a", "pool-b"]); expect(redirectModes).toEqual([undefined, undefined]); } finally { @@ -2890,18 +2906,24 @@ describe("server local API auth", () => { for (const path of ["/v1/responses", "/v1/responses/compact"] as const) { test(`${path} releases a pool probe after a pre-response transport failure with no host key`, async () => { const harness = await startPoolRetryHarness(() => new Response("unused"), { secondAccount: false }); + const providerBaseUrl = "https://chatgpt.com/backend-api/codex"; const upstreamUrl = `https://chatgpt.com/backend-api/codex${path.slice(3)}`; const redirectedFetch = globalThis.fetch; const NativeURL = globalThis.URL; + const redirectModes: Array = []; const observedPoolAuth: Array<{ authorization: string | null; accountId: string | null }> = []; const observedProbeLeaseIds: Array = []; - let forcedNullHostKey = false; + const uncanonicalizableInputs: string[] = []; class NullHostKeyURL extends NativeURL { constructor(input: string | URL, base?: string | URL) { const value = typeof input === "string" ? input : input.toString(); - if (!forcedNullHostKey && base === undefined && value === upstreamUrl) { - forcedNullHostKey = true; + if ( + base === undefined + && (value === providerBaseUrl || value === upstreamUrl) + && new Error().stack?.includes("canonicalCodexUpstreamHostKey") + ) { + uncanonicalizableInputs.push(value); throw new TypeError("synthetic null canonical host key"); } super(input, base); @@ -2916,6 +2938,7 @@ describe("server local API auth", () => { ? input.toString() : input.url; if (requestUrl === upstreamUrl) { + redirectModes.push(init?.redirect); const headers = new Headers(init?.headers); observedPoolAuth.push({ authorization: headers.get("authorization"), @@ -2937,7 +2960,9 @@ describe("server local API auth", () => { const response = await harness.request({ path }); expect(response.status).toBe(502); - expect(forcedNullHostKey).toBe(true); + expect(new Set(uncanonicalizableInputs)).toEqual(new Set([providerBaseUrl, upstreamUrl])); + expect(redirectModes.length).toBeGreaterThan(0); + expect(redirectModes.every(mode => mode === undefined)).toBe(true); expect(observedPoolAuth.length).toBeGreaterThan(0); expect(observedPoolAuth).toEqual(observedPoolAuth.map(() => ({ authorization: "Bearer pool-a-token", From 79b02859cc9d5962b2a3f8820da452ea3b399ac8 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 18:24:52 +0900 Subject: [PATCH 18/20] fix(codex): preserve concurrent host leases --- src/codex/upstream-host-health.ts | 26 +++++++++++++++++------ tests/codex-upstream-host-health.test.ts | 27 ++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/src/codex/upstream-host-health.ts b/src/codex/upstream-host-health.ts index 79690207af..0a55c0c079 100644 --- a/src/codex/upstream-host-health.ts +++ b/src/codex/upstream-host-health.ts @@ -165,6 +165,14 @@ function matchingHealth(lease: CodexUpstreamHostAdmissionLease): CodexUpstreamHo return health; } +function settleLease( + health: CodexUpstreamHostHealth, + lease: CodexUpstreamHostAdmissionLease, +): void { + health.activeLeaseIds.delete(lease.leaseId); + if (health.halfOpenLeaseId === lease.leaseId) delete health.halfOpenLeaseId; +} + export function getCodexUpstreamHostHealth( key: CodexUpstreamHostKey, now = Date.now(), @@ -231,8 +239,7 @@ export function releaseCodexUpstreamHostAdmissionLease( if (!lease) return false; const health = matchingHealth(lease); if (!health) return false; - health.activeLeaseIds.delete(lease.leaseId); - if (health.halfOpenLeaseId === lease.leaseId) delete health.halfOpenLeaseId; + settleLease(health, lease); health.lastTouchedAt = now; if (health.activeLeaseIds.size === 0 && health.consecutiveFailures === 0) { upstreamHostHealth.delete(lease.key); @@ -248,8 +255,7 @@ export function recordCodexUpstreamHostFailure( ): CodexUpstreamHostHealthSnapshot | null { const current = matchingHealth(lease); if (!current) return null; - current.activeLeaseIds.delete(lease.leaseId); - if (current.halfOpenLeaseId === lease.leaseId) delete current.halfOpenLeaseId; + settleLease(current, lease); if (options.observedResponse) { current.consecutiveFailures = 1; @@ -283,8 +289,16 @@ export function recordCodexUpstreamHostResponse( lease: CodexUpstreamHostAdmissionLease, now = Date.now(), ): boolean { - if (!matchingHealth(lease)) return false; - upstreamHostHealth.delete(lease.key); + const current = matchingHealth(lease); + if (!current) return false; + settleLease(current, lease); + current.consecutiveFailures = 0; + current.lastFailureAt = 0; + current.lastTouchedAt = now; + delete current.cooldownUntil; + if (current.activeLeaseIds.size === 0) { + upstreamHostHealth.delete(lease.key); + } pruneOverflow(now); return true; } diff --git a/tests/codex-upstream-host-health.test.ts b/tests/codex-upstream-host-health.test.ts index 414d84f46f..f9d099d47c 100644 --- a/tests/codex-upstream-host-health.test.ts +++ b/tests/codex-upstream-host-health.test.ts @@ -156,6 +156,33 @@ describe("Codex upstream host health (#914)", () => { expect(afterConcurrentFailure?.cooldownUntil).toBeUndefined(); }); + test("a normal response preserves a concurrent lease and its later failure authority", () => { + const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com")!; + const now = 2_800; + expect(fail(key, now).consecutiveFailures).toBe(1); + const responseLease = admit(key, now + 1); + const failureLease = admit(key, now + 1); + + expect(recordCodexUpstreamHostResponse(responseLease, now + 2)).toBe(true); + expect(getCodexUpstreamHostHealth(key, now + 2)).toBeNull(); + + const afterConcurrentFailure = recordCodexUpstreamHostFailure(failureLease, now + 3); + expect(afterConcurrentFailure?.consecutiveFailures).toBe(1); + expect(afterConcurrentFailure?.cooldownUntil).toBeUndefined(); + }); + + test("concurrent normal responses settle independently and clean up after the last lease", () => { + const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com")!; + const now = 2_900; + const first = admit(key, now); + const second = admit(key, now); + + expect(recordCodexUpstreamHostResponse(first, now + 1)).toBe(true); + expect(recordCodexUpstreamHostResponse(second, now + 2)).toBe(true); + expect(getCodexUpstreamHostHealth(key, now + 2)).toBeNull(); + expect(releaseCodexUpstreamHostAdmissionLease(second, now + 3)).toBe(false); + }); + test("caller abort releases a half-open admission without adding evidence", () => { const key = canonicalCodexUpstreamHostKey("openai", "https://chatgpt.com")!; const trippedAt = 3_000; From b5a3e82310cf30c351dea54ab07cefa730f716f8 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 18:41:16 +0900 Subject: [PATCH 19/20] fix: suppress premature compact pool promotion --- src/server/responses/compact.ts | 67 +++++++++++++--- tests/responses-compaction-routing.test.ts | 90 +++++++++++++++++++++- 2 files changed, 143 insertions(+), 14 deletions(-) diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index 2805635884..fd9a8f1abc 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -142,6 +142,17 @@ export function compactResponseTooLargeError(): Response { +type ResolvedAlternateCompactContext = { + authCtx: CodexAuthContext; + provider: OcxProviderConfig; + headers: Headers; +}; + +type AlternateCompactResolution = + | { kind: "ready"; context: ResolvedAlternateCompactContext } + | { kind: "none" } + | { kind: "local-failure" }; + /** * Resolve one eligible pool account other than `excludeAccountId`, and build everything * the alternate send needs. Returns null when no alternate exists or construction fails, @@ -158,19 +169,21 @@ async function resolveAlternateCompactContext(args: { selectedModelId: string | undefined; excludeAccountId: string | null; turnAdmissionLease?: AdmissionLease; -}): Promise<{ authCtx: CodexAuthContext; provider: OcxProviderConfig; headers: Headers } | null> { +}): Promise { const { req, config, route, selectedModelId, excludeAccountId, turnAdmissionLease } = args; - if (!route.codexAccountMode || !excludeAccountId) return null; + if (!route.codexAccountMode || !excludeAccountId) return { kind: "none" }; let authCtx: CodexAuthContext | undefined; + let authResolved = false; try { authCtx = await resolveCodexAuthContext(req.headers, config, route.codexAccountMode, { ...(selectedModelId ? { modelId: selectedModelId } : {}), excludeAccountId, beginCodexAccountSelection: codexAccountSelectionForTurn(turnAdmissionLease), }); + authResolved = true; if (!authCtx.accountId || authCtx.accountId === excludeAccountId) { releaseCodexAuthContextProbeLease(authCtx); - return null; + return { kind: "none" }; } const provider = applyCodexAuthContextToProvider(route.provider, authCtx, route.codexAccountMode); const headers = new Headers({ "content-type": "application/json" }); @@ -185,18 +198,29 @@ async function resolveAlternateCompactContext(args: { headers.set("chatgpt-account-id", override.chatgptAccountId); } if (provider.apiKey) headers.set("authorization", `Bearer ${resolveEnvValue(provider.apiKey)}`); - return { authCtx, provider, headers }; + return { kind: "ready", context: { authCtx, provider, headers } }; } catch (err) { releaseCodexAuthContextProbeLease(authCtx); + if ( + authResolved + || !( + err instanceof CodexPoolAuthenticationError + || err instanceof CodexAuthContextError + || err instanceof CodexAccountCooldownError + || err instanceof CodexMainProfileDrainingError + ) + ) { + return { kind: "local-failure" }; + } if (err instanceof CodexMainProfileDrainingError) { // The native-main fence can start after account A has already rejected the // request. Treat the now-fenced main profile as no alternate and preserve A's // real rejection instead of replacing it with a synthetic drain response. - return null; + return { kind: "none" }; } // No eligible alternate (all cooled, affinity expired, reauth needed) — the caller // returns the first account's rejection unchanged, which is today's behavior. - return null; + return { kind: "none" }; } } @@ -408,6 +432,7 @@ export async function handleResponsesCompact( retryAfter?: string | null; resetAt?: unknown | unknown[]; promoteAccountId?: string; + suppressPromotion?: boolean; } = {}, ) => { if (!usesCodexForwardPoolAuth(ctx, route.provider)) return; @@ -586,6 +611,7 @@ export async function handleResponsesCompact( retryAfter?: string | null; resetAt?: unknown | unknown[]; promoteAccountId?: string; + suppressPromotion?: boolean; }; let pendingPrimaryPoolOutcome: PendingPrimaryPoolOutcome | null = null; let primaryPoolOutcomeSettled = false; @@ -626,7 +652,7 @@ export async function handleResponsesCompact( primaryPoolOutcome.resetAt = firstResetAt; // Build the alternate COMPLETELY before cancelling the first body: if construction // throws, the first rejection is still intact and can be returned to the client. - const alternate = await resolveAlternateCompactContext({ + const alternateResolution = await resolveAlternateCompactContext({ req, config, route, @@ -634,6 +660,17 @@ export async function handleResponsesCompact( excludeAccountId: authCtx.accountId, turnAdmissionLease, }); + const alternate = alternateResolution.kind === "ready" + ? alternateResolution.context + : null; + if (alternateResolution.kind === "local-failure") { + primaryPoolOutcome.suppressPromotion = true; + try { + settlePendingPrimaryPoolOutcome(); + } catch { + // Preserve compact's existing A-response contract for local B setup failures. + } + } pendingAlternateAuthCtx = alternate?.authCtx ?? null; // Resolution can await a credential refresh, so the client may have gone away // while we were choosing B. Re-check before spending anything: recording A, @@ -659,10 +696,6 @@ export async function handleResponsesCompact( authCtx.writerGeneration, ); } - if (alternate.authCtx.accountId) { - primaryPoolOutcome.promoteAccountId = alternate.authCtx.accountId; - } - settlePendingPrimaryPoolOutcome(); await upstream.body?.cancel().catch(() => undefined); outcomeCtx = alternate.authCtx; const alternateAttempts: UpstreamAttemptObservation[] = []; @@ -674,6 +707,10 @@ export async function handleResponsesCompact( "single", alternateAttempts, () => { + if (alternate.authCtx.accountId) { + primaryPoolOutcome.promoteAccountId = alternate.authCtx.accountId; + } + settlePendingPrimaryPoolOutcome(); alternateSendBegan = true; pendingAlternateAuthCtx = null; }, @@ -691,6 +728,10 @@ export async function handleResponsesCompact( } } } catch (error) { + if (pendingPrimaryPoolOutcome && !primaryPoolOutcomeSettled) { + delete pendingPrimaryPoolOutcome.promoteAccountId; + pendingPrimaryPoolOutcome.suppressPromotion = true; + } releaseCodexAuthContextProbeLease(pendingAlternateAuthCtx ?? undefined); try { settlePendingPrimaryPoolOutcome(); @@ -716,7 +757,9 @@ export async function handleResponsesCompact( } // Always record the real upstream status: a local buffering failure after a // 200 upstream response must not soft-avoid a healthy account or rotate a thread. - recordCompactPoolOutcome(outcomeCtx, upstream.status, { retryAfter, resetAt }); + if (outcomeCtx !== authCtx || !primaryPoolOutcomeSettled) { + recordCompactPoolOutcome(outcomeCtx, upstream.status, { retryAfter, resetAt }); + } // Lift usage and response metadata from the buffered upstream JSON into the // request log; the routed branch gets the same through handleResponses. The // synthetic buffer errors are not upstream bodies and stay uninspected. diff --git a/tests/responses-compaction-routing.test.ts b/tests/responses-compaction-routing.test.ts index d5240e2801..c7b69a102e 100644 --- a/tests/responses-compaction-routing.test.ts +++ b/tests/responses-compaction-routing.test.ts @@ -1391,11 +1391,77 @@ describe("compact alternate-account attempt (#913)", () => { }); }); + test("compact keeps A active when unexpected alternate resolution fails locally", async () => { + await withPoolEnv("ocx-compact-b-resolver-local-", async config => { + const originalNow = Date.now; + config.accountPoolStrategy = "fill-first"; + config.activeCodexAccountId = "pool-a"; + const probeAt = prepareHalfOpenHost(Date.now()); + const hostKey = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex/responses", + )!; + const accounts = config.codexAccounts; + const expectedBody = JSON.stringify({ error: { message: "A quota" } }); + let failNextAccountRead = false; + let resolverFailureObserved = false; + const physicalAccounts: string[] = []; + Object.defineProperty(config, "codexAccounts", { + configurable: true, + get: () => { + if (failNextAccountRead) { + failNextAccountRead = false; + resolverFailureObserved = true; + throw new Error("unexpected alternate resolver failure"); + } + return accounts; + }, + }); + Date.now = () => probeAt; + globalThis.fetch = (async (_url: RequestInfo | URL, init?: RequestInit) => { + physicalAccounts.push(new Headers(init?.headers).get("chatgpt-account-id") ?? ""); + failNextAccountRead = true; + return new Response(expectedBody, { + status: 429, + headers: { "content-type": "application/json", "retry-after": "31" }, + }); + }) as typeof fetch; + try { + const response = await handleResponsesCompact( + compactionRequest(baseCompactionBody({})), + config, + { model: "", provider: "" }, + ); + expect(resolverFailureObserved).toBe(true); + expect(physicalAccounts).toEqual(["pool_acc_a"]); + expect(response.status).toBe(429); + expect(response.headers.get("retry-after")).toBe("31"); + expect(await response.text()).toBe(expectedBody); + expect(getCodexUpstreamHealth("pool-a")).toMatchObject({ + lastFailureStatus: 429, + cooldownGeneration: 1, + }); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(config.activeCodexAccountId).toBe("pool-a"); + expect(getCodexUpstreamHostHealth(hostKey, probeAt)).toBeNull(); + expectHostAdmissionUsable(hostKey, probeAt); + } finally { + Date.now = originalNow; + } + }); + }); + test("compact releases B recovery ownership when alternate header construction fails", async () => { await withPoolEnv("ocx-compact-b-header-prep-", async config => { + const originalNow = Date.now; config.accountPoolStrategy = "fill-first"; config.activeCodexAccountId = "pool-a"; const NativeHeaders = globalThis.Headers; + const probeAt = prepareHalfOpenHost(Date.now()); + const hostKey = canonicalCodexUpstreamHostKey( + "openai", + "https://chatgpt.com/backend-api/codex/responses", + )!; let bHeaderAttempted = false; const physicalAccounts: string[] = []; class ThrowOnBHeaders extends NativeHeaders { @@ -1407,6 +1473,7 @@ describe("compact alternate-account attempt (#913)", () => { super.set(name, value); } } + Date.now = () => probeAt; globalThis.Headers = ThrowOnBHeaders as typeof Headers; globalThis.fetch = (async (_url: RequestInfo | URL, init?: RequestInit) => { physicalAccounts.push(new NativeHeaders(init?.headers).get("chatgpt-account-id") ?? ""); @@ -1421,13 +1488,22 @@ describe("compact alternate-account attempt (#913)", () => { expect(response.status).toBe(429); expect(bHeaderAttempted).toBe(true); expect(physicalAccounts).toEqual(["pool_acc_a"]); + expect(getCodexUpstreamHealth("pool-a")).toMatchObject({ + lastFailureStatus: 429, + cooldownGeneration: 1, + }); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(config.activeCodexAccountId).toBe("pool-a"); + expect(getCodexUpstreamHostHealth(hostKey, probeAt)).toBeNull(); + expectHostAdmissionUsable(hostKey, probeAt); } finally { globalThis.Headers = NativeHeaders; + Date.now = originalNow; } }); }); - test("compact settles half-open primary response and releases B when caller preparation throws", async () => { + test("compact suppresses promotion when quota telemetry fails after B preparation", async () => { await withPoolEnv("ocx-compact-host-alt-throw-", async config => { const originalNow = Date.now; config.accountPoolStrategy = "fill-first"; @@ -1473,6 +1549,12 @@ describe("compact alternate-account attempt (#913)", () => { )).rejects.toBe(expected); expect(bPrepared).toBe(true); expect(physicalAccounts).toEqual(["pool_acc_a"]); + expect(getCodexUpstreamHealth("pool-a")).toMatchObject({ + lastFailureStatus: 429, + cooldownGeneration: 1, + }); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(config.activeCodexAccountId).toBe("pool-a"); expect(getCodexUpstreamHostHealth(hostKey, probeAt)).toBeNull(); expectHostAdmissionUsable(hostKey, probeAt); } finally { @@ -1568,8 +1650,12 @@ describe("compact alternate-account attempt (#913)", () => { ), state, expected); expect(state).toEqual({ prepared: true, cloneThrew: true, boundaryFrameObserved: true }); expect(physicalAccounts).toEqual(["pool_acc_a"]); - expect(getCodexUpstreamHealth("pool-a")?.lastFailureStatus).toBe(429); + expect(getCodexUpstreamHealth("pool-a")).toMatchObject({ + lastFailureStatus: 429, + cooldownGeneration: 1, + }); expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + expect(config.activeCodexAccountId).toBe("pool-a"); expect(getCodexUpstreamHostHealth(hostKey, probeAt)).toBeNull(); expectHostAdmissionUsable(hostKey, probeAt); } finally { From d6c373439e3c12ad595cfb14e70e609419695980 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Tue, 4 Aug 2026 19:07:14 +0900 Subject: [PATCH 20/20] test: clarify compact alternate setup coverage --- src/server/responses/compact.ts | 4 ++-- tests/responses-compaction-routing.test.ts | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index fd9a8f1abc..2c61111f90 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -155,8 +155,8 @@ type AlternateCompactResolution = /** * Resolve one eligible pool account other than `excludeAccountId`, and build everything - * the alternate send needs. Returns null when no alternate exists or construction fails, - * in which case the caller keeps the first account's rejection intact. + * the alternate send needs. Returns a ready, none, or local-failure resolution so the + * caller can distinguish an unavailable alternate from local resolver/preparation failure. * * Mirrors the auth resolution the native compact branch already does for the first * account, so the alternate is built the same way rather than through a second, diff --git a/tests/responses-compaction-routing.test.ts b/tests/responses-compaction-routing.test.ts index c7b69a102e..4767d872d7 100644 --- a/tests/responses-compaction-routing.test.ts +++ b/tests/responses-compaction-routing.test.ts @@ -1451,7 +1451,7 @@ describe("compact alternate-account attempt (#913)", () => { }); }); - test("compact releases B recovery ownership when alternate header construction fails", async () => { + test("compact preserves A rejection when alternate B auth-header construction fails", async () => { await withPoolEnv("ocx-compact-b-header-prep-", async config => { const originalNow = Date.now; config.accountPoolStrategy = "fill-first"; @@ -1462,6 +1462,7 @@ describe("compact alternate-account attempt (#913)", () => { "openai", "https://chatgpt.com/backend-api/codex/responses", )!; + const expectedBody = JSON.stringify({ error: { message: "A quota" } }); let bHeaderAttempted = false; const physicalAccounts: string[] = []; class ThrowOnBHeaders extends NativeHeaders { @@ -1477,7 +1478,10 @@ describe("compact alternate-account attempt (#913)", () => { globalThis.Headers = ThrowOnBHeaders as typeof Headers; globalThis.fetch = (async (_url: RequestInfo | URL, init?: RequestInit) => { physicalAccounts.push(new NativeHeaders(init?.headers).get("chatgpt-account-id") ?? ""); - return Response.json({ error: { message: "A quota" } }, { status: 429 }); + return new Response(expectedBody, { + status: 429, + headers: { "content-type": "application/json", "retry-after": "37" }, + }); }) as typeof fetch; try { const response = await handleResponsesCompact( @@ -1486,6 +1490,8 @@ describe("compact alternate-account attempt (#913)", () => { { model: "", provider: "" }, ); expect(response.status).toBe(429); + expect(response.headers.get("retry-after")).toBe("37"); + expect(await response.text()).toBe(expectedBody); expect(bHeaderAttempted).toBe(true); expect(physicalAccounts).toEqual(["pool_acc_a"]); expect(getCodexUpstreamHealth("pool-a")).toMatchObject({