From 6bc61e6500b420b8f6597b31151f7222f8cc46c2 Mon Sep 17 00:00:00 2001 From: Sarthak195 Date: Wed, 15 Jul 2026 20:15:26 +0530 Subject: [PATCH] Add Gemini model fallback for overload resilience Gemini returns 503 UNAVAILABLE when a model is overloaded (observed live on gemini-3.5-flash). generateJson now tries the primary model (GEMINI_MODEL) then falls through a fallback chain (GEMINI_FALLBACK_MODELS, default gemini-3-flash-preview,gemini-flash-latest,gemini-2.0-flash) on transient codes (503/500/429); non-retryable errors (bad key, invalid request) still fail fast. Client is injectable for tests. +7 unit tests (69 total). --- CLAUDE.md | 2 +- src/lib/gemini.test.ts | 83 +++++++++++++++++++++++++++++++++++ src/lib/gemini.ts | 98 +++++++++++++++++++++++++++++++----------- 3 files changed, 157 insertions(+), 26 deletions(-) create mode 100644 src/lib/gemini.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 1ab2548..4f16f2b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,7 +18,7 @@ AI incident command copilot for stadium/tournament operations. Solo entry for ** - `src/shared/constants.ts` — demo venue (`DEMO_VENUE`, fictional "Meridian Arena, Indore"), density thresholds, sim pacing. - `src/lib/simulator/` — tick-based engine; `scenario.ts` is the scripted demo storyline (clock = minutes since gates open, kickoff at minute 60). The scenario IS the demo script — keep it aligned with `docs/DEMO_SCRIPT.md`. - `src/lib/simulator/detector.ts` — rule-based incident detection (density/queue/medical/weather thresholds). Rules open incidents; Gemini only fills `triage`. -- `src/lib/gemini.ts` — all Gemini calls go through `generateJson()` (structured output). Model from `GEMINI_MODEL` env, default `gemini-3.5-flash` (2.5-flash is retired for new API keys as of mid-2026). +- `src/lib/gemini.ts` — all Gemini calls go through `generateJson()` (structured output). Primary model from `GEMINI_MODEL` env (default `gemini-3.5-flash`; 2.5-flash is retired for new API keys as of mid-2026), then falls through the `GEMINI_FALLBACK_MODELS` chain (default `gemini-3-flash-preview,gemini-flash-latest,gemini-2.0-flash`) on a transient/overload error (503/500/429); non-retryable errors fail fast. - `src/lib/simulator/live.ts` — **THE shared match**: one server-side sim (globalThis singleton, survives HMR) broadcasting to all SSE subscribers; late joiners get a catch-up burst; ticking pauses with zero viewers and hard-stops at minute 200. Sim controls are GLOBAL via `POST /api/sim` ({type: speed|jump|restart}) — per-connection tickMs/startMinute params are gone. `api/stream` just subscribes. Client reducer dedupes by id and resets on reconnect/`{kind:"reset"}`. - `api/triage` and `api/briefing` (+`?type=handover`) are Gemini endpoints; the client posts its accumulated state. - `/tournament` — supervisor view: Meridian card uses real shared state; sister venues (`src/lib/tournament.ts`) are deterministic derived summaries with staggered kickoffs, labeled "simulated". diff --git a/src/lib/gemini.test.ts b/src/lib/gemini.test.ts new file mode 100644 index 0000000..5eed292 --- /dev/null +++ b/src/lib/gemini.test.ts @@ -0,0 +1,83 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { GoogleGenAI } from "@google/genai"; +import { generateJson, isRetryable, modelChain } from "./gemini"; + +afterEach(() => vi.unstubAllEnvs()); + +/** An SDK-style error whose message is the JSON blob Gemini returns. */ +const gErr = (code: number) => new Error(JSON.stringify({ error: { code, message: `err ${code}` } })); + +/** Fake GoogleGenAI whose per-model outcome is scripted; records call order. */ +function fakeClient(script: Record) { + const calls: string[] = []; + const client = { + models: { + generateContent: async ({ model }: { model: string }) => { + calls.push(model); + const outcome = script[model]; + if (outcome instanceof Error) throw outcome; + return outcome ?? { text: "{}" }; + }, + }, + } as unknown as GoogleGenAI; + return { client, calls }; +} + +const opts = { system: "s", prompt: "p", schema: {} }; + +describe("isRetryable", () => { + it("treats 503/500/429 as retryable", () => { + expect(isRetryable(gErr(503))).toBe(true); + expect(isRetryable(gErr(500))).toBe(true); + expect(isRetryable(gErr(429))).toBe(true); + }); + + it("treats client errors and non-JSON as non-retryable", () => { + expect(isRetryable(gErr(400))).toBe(false); + expect(isRetryable(gErr(404))).toBe(false); + expect(isRetryable(new Error("network down"))).toBe(false); + }); +}); + +describe("modelChain", () => { + it("puts the primary first, then de-duplicated fallbacks", () => { + vi.stubEnv("GEMINI_MODEL", "primary"); + vi.stubEnv("GEMINI_FALLBACK_MODELS", "primary, alt , alt, other"); + expect(modelChain()).toEqual(["primary", "alt", "other"]); + }); +}); + +describe("generateJson fallback", () => { + it("falls back to the next model when the primary is overloaded", async () => { + vi.stubEnv("GEMINI_MODEL", "a"); + vi.stubEnv("GEMINI_FALLBACK_MODELS", "b"); + const { client, calls } = fakeClient({ a: gErr(503), b: { text: '{"ok":true}' } }); + const out = await generateJson<{ ok: boolean }>(opts, client); + expect(out).toEqual({ ok: true }); + expect(calls).toEqual(["a", "b"]); + }); + + it("fails fast on a non-retryable error without trying fallbacks", async () => { + vi.stubEnv("GEMINI_MODEL", "a"); + vi.stubEnv("GEMINI_FALLBACK_MODELS", "b"); + const { client, calls } = fakeClient({ a: gErr(400), b: { text: "{}" } }); + await expect(generateJson(opts, client)).rejects.toThrow(); + expect(calls).toEqual(["a"]); // fallback never attempted + }); + + it("throws when every model in the chain is overloaded", async () => { + vi.stubEnv("GEMINI_MODEL", "a"); + vi.stubEnv("GEMINI_FALLBACK_MODELS", "b"); + const { client, calls } = fakeClient({ a: gErr(503), b: gErr(503) }); + await expect(generateJson(opts, client)).rejects.toThrow(); + expect(calls).toEqual(["a", "b"]); + }); + + it("succeeds on the primary without touching fallbacks", async () => { + vi.stubEnv("GEMINI_MODEL", "a"); + vi.stubEnv("GEMINI_FALLBACK_MODELS", "b"); + const { client, calls } = fakeClient({ a: { text: '{"n":1}' } }); + expect(await generateJson(opts, client)).toEqual({ n: 1 }); + expect(calls).toEqual(["a"]); + }); +}); diff --git a/src/lib/gemini.ts b/src/lib/gemini.ts index 8f5ff7e..05aec79 100644 --- a/src/lib/gemini.ts +++ b/src/lib/gemini.ts @@ -1,11 +1,24 @@ import { GoogleGenAI } from "@google/genai"; const DEFAULT_MODEL = "gemini-3.5-flash"; +/** Models tried, in order, when the primary is unavailable. Overridable via env. */ +const DEFAULT_FALLBACKS = "gemini-3-flash-preview,gemini-flash-latest,gemini-2.0-flash"; +/** Gemini status codes that mean "try another model": overloaded / transient / rate-limited. */ +const RETRYABLE_CODES = new Set([429, 500, 503]); export function getModel(): string { return process.env.GEMINI_MODEL || DEFAULT_MODEL; } +/** Primary model first, then de-duplicated fallbacks (env `GEMINI_FALLBACK_MODELS`). */ +export function modelChain(): string[] { + const fallbacks = (process.env.GEMINI_FALLBACK_MODELS ?? DEFAULT_FALLBACKS) + .split(",") + .map((s) => s.trim()) + .filter(Boolean); + return [...new Set([getModel(), ...fallbacks])]; +} + let client: GoogleGenAI | null = null; export function getClient(): GoogleGenAI { @@ -17,6 +30,22 @@ export function getClient(): GoogleGenAI { return client; } +/** Extract the numeric status code from an SDK error (often a JSON blob). */ +function errorCode(err: unknown): number | undefined { + const raw = err instanceof Error ? err.message : String(err); + try { + return (JSON.parse(raw) as { error?: { code?: number } }).error?.code; + } catch { + return undefined; + } +} + +/** True when the error is a model-availability/transient one worth retrying on another model. */ +export function isRetryable(err: unknown): boolean { + const code = errorCode(err); + return code !== undefined && RETRYABLE_CODES.has(code); +} + /** Turn SDK errors (often raw JSON blobs) into a line fit for the UI. */ export function geminiErrorMessage(err: unknown): string { const raw = err instanceof Error ? err.message : String(err); @@ -35,31 +64,50 @@ export function geminiErrorMessage(err: unknown): string { /** * Ask Gemini for a JSON object matching `schema` (Gemini structured-output schema, * built with the `Type` enum from @google/genai). Every AI call in the app goes - * through here so model choice and the structured-output contract live in one - * place; callers format errors for the UI via `geminiErrorMessage`. + * through here so the model chain and the structured-output contract live in one + * place. The primary model (GEMINI_MODEL) is tried first; on a transient/overload + * error (503/500/429) it falls through to the fallback models rather than failing + * the request. Non-retryable errors (bad key, invalid request) fail fast. The + * `client` param is injectable for tests; production uses the shared client. */ -export async function generateJson(opts: { - system: string; - prompt: string; - schema: object; -}): Promise { - const ai = getClient(); - const res = await ai.models.generateContent({ - model: getModel(), - contents: opts.prompt, - config: { - systemInstruction: opts.system, - responseMimeType: "application/json", - responseSchema: opts.schema, - }, - }); - const text = res.text; - if (!text) { - throw new Error("Gemini returned an empty response"); - } - try { - return JSON.parse(text) as T; - } catch { - throw new Error("Gemini returned malformed JSON"); +export async function generateJson( + opts: { + system: string; + prompt: string; + schema: object; + }, + client: GoogleGenAI = getClient(), +): Promise { + const models = modelChain(); + let lastErr: unknown = new Error("No Gemini model configured"); + + for (const model of models) { + try { + const res = await client.models.generateContent({ + model, + contents: opts.prompt, + config: { + systemInstruction: opts.system, + responseMimeType: "application/json", + responseSchema: opts.schema, + }, + }); + const text = res.text; + if (!text) { + throw new Error("Gemini returned an empty response"); + } + try { + return JSON.parse(text) as T; + } catch { + throw new Error("Gemini returned malformed JSON"); + } + } catch (err) { + lastErr = err; + // Only move to the next model on an overload/transient error; otherwise fail fast. + if (isRetryable(err)) continue; + throw err; + } } + // Every model was tried and each returned a retryable error. + throw lastErr; }