Add Gemini model fallback for overload resilience

CLAUDE.md

@@ -18,7 +18,7 @@ AI incident command copilot for stadium/tournament operations. Solo entry for **
 - `src/shared/constants.ts` — demo venue (`DEMO_VENUE`, fictional "Meridian Arena, Indore"), density thresholds, sim pacing.
 - `src/lib/simulator/` — tick-based engine; `scenario.ts` is the scripted demo storyline (clock = minutes since gates open, kickoff at minute 60). The scenario IS the demo script — keep it aligned with `docs/DEMO_SCRIPT.md`.
 - `src/lib/simulator/detector.ts` — rule-based incident detection (density/queue/medical/weather thresholds). Rules open incidents; Gemini only fills `triage`.
-- `src/lib/gemini.ts` — all Gemini calls go through `generateJson()` (structured output). Model from `GEMINI_MODEL` env, default `gemini-3.5-flash` (2.5-flash is retired for new API keys as of mid-2026).
+- `src/lib/gemini.ts` — all Gemini calls go through `generateJson()` (structured output). Primary model from `GEMINI_MODEL` env (default `gemini-3.5-flash`; 2.5-flash is retired for new API keys as of mid-2026), then falls through the `GEMINI_FALLBACK_MODELS` chain (default `gemini-3-flash-preview,gemini-flash-latest,gemini-2.0-flash`) on a transient/overload error (503/500/429); non-retryable errors fail fast.
 - `src/lib/simulator/live.ts` — **THE shared match**: one server-side sim (globalThis singleton, survives HMR) broadcasting to all SSE subscribers; late joiners get a catch-up burst; ticking pauses with zero viewers and hard-stops at minute 200. Sim controls are GLOBAL via `POST /api/sim` ({type: speed|jump|restart}) — per-connection tickMs/startMinute params are gone. `api/stream` just subscribes. Client reducer dedupes by id and resets on reconnect/`{kind:"reset"}`.
 - `api/triage` and `api/briefing` (+`?type=handover`) are Gemini endpoints; the client posts its accumulated state.
 - `/tournament` — supervisor view: Meridian card uses real shared state; sister venues (`src/lib/tournament.ts`) are deterministic derived summaries with staggered kickoffs, labeled "simulated".

src/lib/gemini.test.ts

@@ -0,0 +1,83 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import type { GoogleGenAI } from "@google/genai";
+import { generateJson, isRetryable, modelChain } from "./gemini";
+
+afterEach(() => vi.unstubAllEnvs());
+
+/** An SDK-style error whose message is the JSON blob Gemini returns. */
+const gErr = (code: number) => new Error(JSON.stringify({ error: { code, message: `err ${code}` } }));
+
+/** Fake GoogleGenAI whose per-model outcome is scripted; records call order. */
+function fakeClient(script: Record<string, { text?: string } | Error>) {
+  const calls: string[] = [];
+  const client = {
+    models: {
+      generateContent: async ({ model }: { model: string }) => {
+        calls.push(model);
+        const outcome = script[model];
+        if (outcome instanceof Error) throw outcome;
+        return outcome ?? { text: "{}" };
+      },
+    },
+  } as unknown as GoogleGenAI;
+  return { client, calls };
+}
+
+const opts = { system: "s", prompt: "p", schema: {} };
+
+describe("isRetryable", () => {
+  it("treats 503/500/429 as retryable", () => {
+    expect(isRetryable(gErr(503))).toBe(true);
+    expect(isRetryable(gErr(500))).toBe(true);
+    expect(isRetryable(gErr(429))).toBe(true);
+  });
+
+  it("treats client errors and non-JSON as non-retryable", () => {
+    expect(isRetryable(gErr(400))).toBe(false);
+    expect(isRetryable(gErr(404))).toBe(false);
+    expect(isRetryable(new Error("network down"))).toBe(false);
+  });
+});
+
+describe("modelChain", () => {
+  it("puts the primary first, then de-duplicated fallbacks", () => {
+    vi.stubEnv("GEMINI_MODEL", "primary");
+    vi.stubEnv("GEMINI_FALLBACK_MODELS", "primary, alt , alt, other");
+    expect(modelChain()).toEqual(["primary", "alt", "other"]);
+  });
+});
+
+describe("generateJson fallback", () => {
+  it("falls back to the next model when the primary is overloaded", async () => {
+    vi.stubEnv("GEMINI_MODEL", "a");
+    vi.stubEnv("GEMINI_FALLBACK_MODELS", "b");
+    const { client, calls } = fakeClient({ a: gErr(503), b: { text: '{"ok":true}' } });
+    const out = await generateJson<{ ok: boolean }>(opts, client);
+    expect(out).toEqual({ ok: true });
+    expect(calls).toEqual(["a", "b"]);
+  });
+
+  it("fails fast on a non-retryable error without trying fallbacks", async () => {
+    vi.stubEnv("GEMINI_MODEL", "a");
+    vi.stubEnv("GEMINI_FALLBACK_MODELS", "b");
+    const { client, calls } = fakeClient({ a: gErr(400), b: { text: "{}" } });
+    await expect(generateJson(opts, client)).rejects.toThrow();
+    expect(calls).toEqual(["a"]); // fallback never attempted
+  });
+
+  it("throws when every model in the chain is overloaded", async () => {
+    vi.stubEnv("GEMINI_MODEL", "a");
+    vi.stubEnv("GEMINI_FALLBACK_MODELS", "b");
+    const { client, calls } = fakeClient({ a: gErr(503), b: gErr(503) });
+    await expect(generateJson(opts, client)).rejects.toThrow();
+    expect(calls).toEqual(["a", "b"]);
+  });
+
+  it("succeeds on the primary without touching fallbacks", async () => {
+    vi.stubEnv("GEMINI_MODEL", "a");
+    vi.stubEnv("GEMINI_FALLBACK_MODELS", "b");
+    const { client, calls } = fakeClient({ a: { text: '{"n":1}' } });
+    expect(await generateJson(opts, client)).toEqual({ n: 1 });
+    expect(calls).toEqual(["a"]);
+  });
+});

src/lib/gemini.ts

@@ -1,11 +1,24 @@
 import { GoogleGenAI } from "@google/genai";
 
 const DEFAULT_MODEL = "gemini-3.5-flash";
+/** Models tried, in order, when the primary is unavailable. Overridable via env. */
+const DEFAULT_FALLBACKS = "gemini-3-flash-preview,gemini-flash-latest,gemini-2.0-flash";
+/** Gemini status codes that mean "try another model": overloaded / transient / rate-limited. */
+const RETRYABLE_CODES = new Set([429, 500, 503]);
 
 export function getModel(): string {
   return process.env.GEMINI_MODEL || DEFAULT_MODEL;
 }
 
+/** Primary model first, then de-duplicated fallbacks (env `GEMINI_FALLBACK_MODELS`). */
+export function modelChain(): string[] {
+  const fallbacks = (process.env.GEMINI_FALLBACK_MODELS ?? DEFAULT_FALLBACKS)
+    .split(",")
+    .map((s) => s.trim())
+    .filter(Boolean);
+  return [...new Set([getModel(), ...fallbacks])];
+}
+
 let client: GoogleGenAI | null = null;
 
 export function getClient(): GoogleGenAI {
@@ -17,6 +30,22 @@ export function getClient(): GoogleGenAI {
   return client;
 }
 
+/** Extract the numeric status code from an SDK error (often a JSON blob). */
+function errorCode(err: unknown): number | undefined {
+  const raw = err instanceof Error ? err.message : String(err);
+  try {
+    return (JSON.parse(raw) as { error?: { code?: number } }).error?.code;
+  } catch {
+    return undefined;
+  }
+}
+
+/** True when the error is a model-availability/transient one worth retrying on another model. */
+export function isRetryable(err: unknown): boolean {
+  const code = errorCode(err);
+  return code !== undefined && RETRYABLE_CODES.has(code);
+}
+
 /** Turn SDK errors (often raw JSON blobs) into a line fit for the UI. */
 export function geminiErrorMessage(err: unknown): string {
   const raw = err instanceof Error ? err.message : String(err);
@@ -35,31 +64,50 @@ export function geminiErrorMessage(err: unknown): string {
 /**
  * Ask Gemini for a JSON object matching `schema` (Gemini structured-output schema,
  * built with the `Type` enum from @google/genai). Every AI call in the app goes
- * through here so model choice and the structured-output contract live in one
- * place; callers format errors for the UI via `geminiErrorMessage`.
+ * through here so the model chain and the structured-output contract live in one
+ * place. The primary model (GEMINI_MODEL) is tried first; on a transient/overload
+ * error (503/500/429) it falls through to the fallback models rather than failing
+ * the request. Non-retryable errors (bad key, invalid request) fail fast. The
+ * `client` param is injectable for tests; production uses the shared client.
  */
-export async function generateJson<T>(opts: {
-  system: string;
-  prompt: string;
-  schema: object;
-}): Promise<T> {
-  const ai = getClient();
-  const res = await ai.models.generateContent({
-    model: getModel(),
-    contents: opts.prompt,
-    config: {
-      systemInstruction: opts.system,
-      responseMimeType: "application/json",
-      responseSchema: opts.schema,
-    },
-  });
-  const text = res.text;
-  if (!text) {
-    throw new Error("Gemini returned an empty response");
-  }
-  try {
-    return JSON.parse(text) as T;
-  } catch {
-    throw new Error("Gemini returned malformed JSON");
+export async function generateJson<T>(
+  opts: {
+    system: string;
+    prompt: string;
+    schema: object;
+  },
+  client: GoogleGenAI = getClient(),
+): Promise<T> {
+  const models = modelChain();
+  let lastErr: unknown = new Error("No Gemini model configured");
+
+  for (const model of models) {
+    try {
+      const res = await client.models.generateContent({
+        model,
+        contents: opts.prompt,
+        config: {
+          systemInstruction: opts.system,
+          responseMimeType: "application/json",
+          responseSchema: opts.schema,
+        },
+      });
+      const text = res.text;
+      if (!text) {
+        throw new Error("Gemini returned an empty response");
+      }
+      try {
+        return JSON.parse(text) as T;
+      } catch {
+        throw new Error("Gemini returned malformed JSON");
+      }
+    } catch (err) {
+      lastErr = err;
+      // Only move to the next model on an overload/transient error; otherwise fail fast.
+      if (isRetryable(err)) continue;
+      throw err;
+    }
   }
+  // Every model was tried and each returned a retryable error.
+  throw lastErr;
 }