diff --git a/packages/harness-omp/config/models.yml b/packages/harness-omp/config/models.yml index b4b0b41..28ba53e 100644 --- a/packages/harness-omp/config/models.yml +++ b/packages/harness-omp/config/models.yml @@ -17,15 +17,15 @@ providers: models: # crusoe - id: zai/GLM-5.2 - contextWindow: 200000 + contextWindow: 1000000 reasoning: true - id: deepseek-ai/DeepSeek-V4-Pro - contextWindow: 131072 + contextWindow: 1000000 reasoning: true - id: Qwen/Qwen3-235B-A22B-Instruct-2507 contextWindow: 262144 - id: moonshotai/Kimi-K2.6 - contextWindow: 131072 + contextWindow: 262144 # openrouter (context per live catalog 2026-07-07) - id: openai/gpt-5.5 contextWindow: 1050000 @@ -73,43 +73,43 @@ providers: # mirror their base model's contextWindow; new-model windows are the # families' published values. - id: glm-5.2 - contextWindow: 200000 + contextWindow: 1000000 reasoning: true - id: glm-5.2-fast - contextWindow: 200000 + contextWindow: 1000000 reasoning: true - id: glm-5.2-flex - contextWindow: 200000 + contextWindow: 1000000 reasoning: true - id: glm-5.2-short - contextWindow: 200000 + contextWindow: 199984 reasoning: true - id: glm-5.2-short-fast - contextWindow: 200000 + contextWindow: 199984 reasoning: true - id: glm-5.2-short-fast-flex - contextWindow: 200000 + contextWindow: 199984 reasoning: true - id: glm-5.2-short-flex - contextWindow: 200000 + contextWindow: 199984 reasoning: true - id: zai/GLM-5.1 contextWindow: 200000 reasoning: true - id: kimi-k2.6 - contextWindow: 131072 + contextWindow: 262144 - id: kimi-k2.6-fast - contextWindow: 131072 + contextWindow: 262144 - id: kimi-k2.6-flex - contextWindow: 131072 + contextWindow: 262144 - id: kimi-k2.7-code contextWindow: 262144 - id: kimi-k2.7-code-flex contextWindow: 262144 - id: deepseek-ai/DeepSeek-V3-0324 - contextWindow: 131072 + contextWindow: 163840 - id: deepseek-ai/Deepseek-V4-Flash - contextWindow: 131072 + contextWindow: 1000000 reasoning: true - id: google/gemma-4-31b-it contextWindow: 131072 @@ -126,9 +126,9 @@ providers: - id: qwen3.5-397b-fast contextWindow: 262144 - id: qwen3.6-35b - contextWindow: 262144 + contextWindow: 131072 - id: qwen3.6-35b-fast - contextWindow: 262144 + contextWindow: 131072 - id: yutori/n1.5 contextWindow: 131072 ollama: diff --git a/packages/harness-pi/config/models.json b/packages/harness-pi/config/models.json index 988f77a..9d71d47 100644 --- a/packages/harness-pi/config/models.json +++ b/packages/harness-pi/config/models.json @@ -5,47 +5,191 @@ "api": "openai-completions", "apiKey": "env:UNSIGNED_LLM_API_KEY", "models": [ - { "id": "google/gemini-3.5-flash", "contextWindow": 1048576, "reasoning": true }, - { "id": "zai/GLM-5.2", "contextWindow": 200000, "reasoning": true }, - { "id": "deepseek-ai/DeepSeek-V4-Pro", "contextWindow": 131072, "reasoning": true }, - { "id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "contextWindow": 262144 }, - { "id": "moonshotai/Kimi-K2.6", "contextWindow": 131072 }, - { "id": "openai/gpt-5.5", "contextWindow": 1050000, "reasoning": true }, - { "id": "moonshotai/kimi-k2.7-code", "contextWindow": 262144 }, - { "id": "mistralai/devstral-2512", "contextWindow": 262144 }, - { "id": "claude-sonnet-5", "contextWindow": 200000, "reasoning": true }, - { "id": "claude-haiku-4-5", "contextWindow": 200000 }, - { "id": "claude-opus-4-8", "contextWindow": 200000, "reasoning": true }, - { "id": "morph-v3-fast", "contextWindow": 32768 }, - { "id": "morph-glm52-744b", "contextWindow": 1000000 }, - { "id": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", "contextWindow": 131072, "reasoning": true }, - { "id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B", "contextWindow": 131072, "reasoning": true }, - { "id": "nvidia/NVIDIA-Nemotron-3-Ultra-550B", "contextWindow": 200000, "reasoning": true }, - { "id": "nvidia/Nemotron-3-Nano-Omni-Reasoning-30B-A3B", "contextWindow": 131072, "reasoning": true }, - { "id": "glm-5.2", "contextWindow": 200000, "reasoning": true }, - { "id": "glm-5.2-fast", "contextWindow": 200000, "reasoning": true }, - { "id": "glm-5.2-flex", "contextWindow": 200000, "reasoning": true }, - { "id": "glm-5.2-short", "contextWindow": 200000, "reasoning": true }, - { "id": "glm-5.2-short-fast", "contextWindow": 200000, "reasoning": true }, - { "id": "glm-5.2-short-fast-flex", "contextWindow": 200000, "reasoning": true }, - { "id": "glm-5.2-short-flex", "contextWindow": 200000, "reasoning": true }, - { "id": "zai/GLM-5.1", "contextWindow": 200000, "reasoning": true }, - { "id": "kimi-k2.6", "contextWindow": 131072 }, - { "id": "kimi-k2.6-fast", "contextWindow": 131072 }, - { "id": "kimi-k2.6-flex", "contextWindow": 131072 }, - { "id": "kimi-k2.7-code", "contextWindow": 262144 }, - { "id": "kimi-k2.7-code-flex", "contextWindow": 262144 }, - { "id": "deepseek-ai/DeepSeek-V3-0324", "contextWindow": 131072 }, - { "id": "deepseek-ai/Deepseek-V4-Flash", "contextWindow": 131072, "reasoning": true }, - { "id": "google/gemma-4-31b-it", "contextWindow": 131072 }, - { "id": "meta-llama/Llama-3.3-70B-Instruct", "contextWindow": 131072 }, - { "id": "minimax/minimax-m2.7", "contextWindow": 200000, "reasoning": true }, - { "id": "openai/gpt-oss-120b", "contextWindow": 131072, "reasoning": true }, - { "id": "qwen3.5-397b", "contextWindow": 262144 }, - { "id": "qwen3.5-397b-fast", "contextWindow": 262144 }, - { "id": "qwen3.6-35b", "contextWindow": 262144 }, - { "id": "qwen3.6-35b-fast", "contextWindow": 262144 }, - { "id": "yutori/n1.5", "contextWindow": 131072 } + { + "id": "google/gemini-3.5-flash", + "contextWindow": 1048576, + "reasoning": true + }, + { + "id": "zai/GLM-5.2", + "contextWindow": 1000000, + "reasoning": true + }, + { + "id": "deepseek-ai/DeepSeek-V4-Pro", + "contextWindow": 1000000, + "reasoning": true + }, + { + "id": "Qwen/Qwen3-235B-A22B-Instruct-2507", + "contextWindow": 262144 + }, + { + "id": "moonshotai/Kimi-K2.6", + "contextWindow": 262144 + }, + { + "id": "openai/gpt-5.5", + "contextWindow": 1050000, + "reasoning": true + }, + { + "id": "moonshotai/kimi-k2.7-code", + "contextWindow": 262144 + }, + { + "id": "mistralai/devstral-2512", + "contextWindow": 262144 + }, + { + "id": "claude-sonnet-5", + "contextWindow": 200000, + "reasoning": true + }, + { + "id": "claude-haiku-4-5", + "contextWindow": 200000 + }, + { + "id": "claude-opus-4-8", + "contextWindow": 200000, + "reasoning": true + }, + { + "id": "morph-v3-fast", + "contextWindow": 32768 + }, + { + "id": "morph-glm52-744b", + "contextWindow": 1000000 + }, + { + "id": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B", + "contextWindow": 131072, + "reasoning": true + }, + { + "id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B", + "contextWindow": 131072, + "reasoning": true + }, + { + "id": "nvidia/NVIDIA-Nemotron-3-Ultra-550B", + "contextWindow": 200000, + "reasoning": true + }, + { + "id": "nvidia/Nemotron-3-Nano-Omni-Reasoning-30B-A3B", + "contextWindow": 131072, + "reasoning": true + }, + { + "id": "glm-5.2", + "contextWindow": 1000000, + "reasoning": true + }, + { + "id": "glm-5.2-fast", + "contextWindow": 1000000, + "reasoning": true + }, + { + "id": "glm-5.2-flex", + "contextWindow": 1000000, + "reasoning": true + }, + { + "id": "glm-5.2-short", + "contextWindow": 199984, + "reasoning": true + }, + { + "id": "glm-5.2-short-fast", + "contextWindow": 199984, + "reasoning": true + }, + { + "id": "glm-5.2-short-fast-flex", + "contextWindow": 199984, + "reasoning": true + }, + { + "id": "glm-5.2-short-flex", + "contextWindow": 199984, + "reasoning": true + }, + { + "id": "zai/GLM-5.1", + "contextWindow": 200000, + "reasoning": true + }, + { + "id": "kimi-k2.6", + "contextWindow": 262144 + }, + { + "id": "kimi-k2.6-fast", + "contextWindow": 262144 + }, + { + "id": "kimi-k2.6-flex", + "contextWindow": 262144 + }, + { + "id": "kimi-k2.7-code", + "contextWindow": 262144 + }, + { + "id": "kimi-k2.7-code-flex", + "contextWindow": 262144 + }, + { + "id": "deepseek-ai/DeepSeek-V3-0324", + "contextWindow": 163840 + }, + { + "id": "deepseek-ai/Deepseek-V4-Flash", + "contextWindow": 1000000, + "reasoning": true + }, + { + "id": "google/gemma-4-31b-it", + "contextWindow": 131072 + }, + { + "id": "meta-llama/Llama-3.3-70B-Instruct", + "contextWindow": 131072 + }, + { + "id": "minimax/minimax-m2.7", + "contextWindow": 200000, + "reasoning": true + }, + { + "id": "openai/gpt-oss-120b", + "contextWindow": 131072, + "reasoning": true + }, + { + "id": "qwen3.5-397b", + "contextWindow": 262144 + }, + { + "id": "qwen3.5-397b-fast", + "contextWindow": 262144 + }, + { + "id": "qwen3.6-35b", + "contextWindow": 131072 + }, + { + "id": "qwen3.6-35b-fast", + "contextWindow": 131072 + }, + { + "id": "yutori/n1.5", + "contextWindow": 131072 + } ] }, "ollama": { @@ -53,11 +197,30 @@ "api": "openai-completions", "apiKey": "ollama", "models": [ - { "id": "qwen3.6-27b-mtp", "contextWindow": 32768, "reasoning": true }, - { "id": "qwen3.6-35b-mtp", "contextWindow": 32768, "reasoning": true }, - { "id": "hf.co/unsloth/GLM-4.7-Flash-GGUF:UD-Q4_K_XL", "contextWindow": 32768, "reasoning": true }, - { "id": "hf.co/unsloth/Devstral-Small-2-24B-Instruct-2512-GGUF:UD-Q4_K_XL", "contextWindow": 32768 }, - { "id": "qwen3.5-9b-mtp", "contextWindow": 32768, "reasoning": true } + { + "id": "qwen3.6-27b-mtp", + "contextWindow": 32768, + "reasoning": true + }, + { + "id": "qwen3.6-35b-mtp", + "contextWindow": 32768, + "reasoning": true + }, + { + "id": "hf.co/unsloth/GLM-4.7-Flash-GGUF:UD-Q4_K_XL", + "contextWindow": 32768, + "reasoning": true + }, + { + "id": "hf.co/unsloth/Devstral-Small-2-24B-Instruct-2512-GGUF:UD-Q4_K_XL", + "contextWindow": 32768 + }, + { + "id": "qwen3.5-9b-mtp", + "contextWindow": 32768, + "reasoning": true + } ] }, "llamacpp": { @@ -65,7 +228,9 @@ "api": "openai-completions", "apiKey": "none", "models": [ - { "id": "local" } + { + "id": "local" + } ] }, "vllm": { @@ -73,7 +238,9 @@ "api": "openai-completions", "apiKey": "none", "models": [ - { "id": "served-model" } + { + "id": "served-model" + } ] } } diff --git a/packages/presets-parity/README.md b/packages/presets-parity/README.md index f3cd5bd..4e67247 100644 --- a/packages/presets-parity/README.md +++ b/packages/presets-parity/README.md @@ -26,11 +26,12 @@ on a weekly schedule (`.github/workflows/parity-live.yml`, Mon 06:17 UTC + (gateway virtual key; 1P item `unsigned-llm` — never committed). A red scheduled run means the gateway catalog drifted from the presets (OPS-729). -The live run also compares preset `contextWindow`s against the gateway's -`/v1/model/info` metadata (OPS-732): mismatches are **warned**, not failed, -while `max_input_tokens` coverage is sparse. Once the LiteLLM config carries -metadata for the full catalog (paas-side half of OPS-732), promote the -warning to an assertion. +The live run also **asserts** preset `contextWindow`s equal the gateway's +`/v1/model/info` `max_input_tokens` (OPS-732; authoritative since +unsigned-paas#1449 / chart 0.4.13 enriched all 41 catalog models), with a +coverage floor so a silent metadata collapse can't make the check vacuous. +On drift: fix the paas values first if the gateway is wrong, else align the +presets — /model/info is the source of truth. ## Verified diff --git a/packages/presets-parity/tests/parity.test.ts b/packages/presets-parity/tests/parity.test.ts index 5f839a8..486d7e2 100644 --- a/packages/presets-parity/tests/parity.test.ts +++ b/packages/presets-parity/tests/parity.test.ts @@ -91,9 +91,9 @@ describe("gateway preset parity", () => { ); // OPS-732: preset contextWindows vs the gateway's /model/info metadata. - // WARNS (does not fail) on mismatch while /model/info coverage is sparse — - // once the LiteLLM config carries max_input_tokens for every model (the - // paas-side half of OPS-732), promote the warning to an assertion. + // ASSERTS since 2026-07-18: the paas-side enrichment (unsigned-paas#1449, + // chart 0.4.13) made /model/info authoritative for all 41 catalog models — + // max_input_tokens is the source of truth and presets must match it. test.if(process.env.PARITY_LIVE === "1")( "preset contextWindows match /model/info where metadata exists", async () => { @@ -126,13 +126,14 @@ describe("gateway preset parity", () => { covered++; if (preset !== max) drifted.push(`${m.model_name}: preset=${preset} model_info=${max}`); } - if (drifted.length > 0) { - console.warn( - `[parity] contextWindow drift vs /model/info (${drifted.length}/${covered} covered ids):\n ${drifted.join("\n ")}`, - ); - } - // Hard assertion deferred until /model/info covers the full catalog. - expect(covered).toBeGreaterThanOrEqual(0); + expect( + drifted, + `contextWindow drift vs /model/info (${covered} covered ids)`, + ).toEqual([]); + // Coverage floor: the gateway must keep serving metadata for the + // catalog (a silent coverage collapse would make the drift check + // vacuous). 41 ids enriched as of chart 0.4.13. + expect(covered).toBeGreaterThanOrEqual(30); }, ); });