diff --git a/.gitignore b/.gitignore index af6f1ac..c6cc34c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,11 @@ # ---- Dependencies ---- node_modules/ ui/node_modules/ +# This is a pnpm project — pnpm-lock.yaml is the source of truth. +# Ignore npm's lockfile so an accidental `npm install` can't reintroduce a +# second, divergent dependency graph (see CONTRIBUTING: "use pnpm, not npm"). +package-lock.json +ui/package-lock.json # ---- Build output ---- dist/ @@ -44,6 +49,9 @@ Thumbs.db .cursor/ .codeium/ .aider* +# CONTEXT.md is a local session-continuity aid (per CLAUDE.md), not part of +# the repo — keep it local-only so it never lands in a PR. +CONTEXT.md # speckit: keep specs/ + memory + templates; ignore only ephemeral state .specify/artifacts/ .specify/cache/ diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index b6b17b8..d9bc63f 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -61,7 +61,7 @@ openfusion/ │ │ ├── schema.ts # zod schemas: candidates, judge, settings │ │ ├── store.ts # read/write config.json + secrets.enc │ │ ├── crypto.ts # AES-256-GCM, machine-bound master.key -│ │ └── completeness.ts # isConfigured(): ≥2 candidates, judge set, all keys present +│ │ └── completeness.ts # isConfigured(): ≥2 candidates, judge set, key for each referenced provider that needs one (keyless providers exempt) │ ├── providers/ │ │ └── pi-ai-bridge.ts # getModel() + complete() wrapper; injects apiKey per call │ ├── store/ @@ -103,7 +103,7 @@ Progress emitted via `extra.sendNotification({ method: "notifications/progress", - **`secrets.enc`** (AES-256-GCM encrypted): `{ providers: { openai: {apiKey}, anthropic: {apiKey}, ... } }` — **one key per provider**, shared across all candidate slots + judge that use it (e.g. one OPENAI key, not one per slot). - **`master.key`** — random 256-bit key generated on first run, `chmod 600`. Machine-bound; used to encrypt/decrypt `secrets.enc`. (Simpler + sufficient for a local single-user tool; avoids native keychain deps.) -`isConfigured()` = `candidates.length ≥ 2 && judge set && every referenced provider has a key`. Minimum **2**, maximum **5** candidates (enforced in schema + UI). +`isConfigured()` = `candidates.length ≥ 2 && judge set && every referenced provider that needs a key has one` (keyless providers — e.g. the local `rapid-mlx` server — are exempt; keyed providers like `ollama-cloud` are not). Minimum **2**, maximum **5** candidates (enforced in schema + UI). ## Provider Layer (`@earendil-works/pi-ai`) @@ -132,8 +132,9 @@ All on `127.0.0.1` only (holds keys — never expose externally). No CORS (same- | GET / PUT | `/api/config` | Read/write `config.json` (model choices + settings) | | GET | `/api/secrets` | Masked key **presence** per provider (never the raw key) | | PUT | `/api/secrets` | Set a provider's key (encrypted before write) | -| GET | `/api/providers` | pi-ai `getProviders()` | -| GET | `/api/providers/:p/models` | pi-ai `getModels(p)` | +| GET | `/api/providers` | all providers (pi-ai built-ins + custom providers) with metadata | +| GET | `/api/providers/:p/models` | pi-ai `getModels(p)` for built-ins; live `/v1/models` discovery for discoverable custom providers (returns `{models, error?}` on failure) | +| GET | `/api/providers/:p/discover` | explicit retry for local discoverable providers (502 `{error}` on failure) | | POST | `/api/test` | Tiny pi-ai ping to validate a provider+model+key before save | | GET | `/api/stats` | Aggregated dashboard data (KPIs + by-model/by-day) | | GET | `/api/activity` | Paginated activity log, expandable to sub-calls | diff --git a/src/config/completeness.ts b/src/config/completeness.ts index 3fc5f7d..aa0e66d 100644 --- a/src/config/completeness.ts +++ b/src/config/completeness.ts @@ -1,9 +1,17 @@ // The configuration gate (Constitution VI). // isConfigured() = >=2 ENABLED candidates (<=5 unless benchmarkMode) + -// >=1 ENABLED judge + a key for every referenced provider. +// >=1 ENABLED judge + a key for every referenced provider that needs one. +// +// Keyless providers (e.g. the local rapid-mlx server) are EXEMPT from the +// "key for every referenced provider" clause: they run without auth, so +// requiring a stored key would block a valid local-only setup. This does not +// weaken the gate — keyed providers (e.g. ollama-cloud) are still required to +// have a stored key, and the >=2 candidates / >=1 judge rules are untouched. +// See tests/custom-providers.test.ts "completeness gate with keyless providers". import type { RawConfig } from "./schema.js"; import { referencedProviders, loadSecrets } from "./secrets.js"; import { paths } from "../util/paths.js"; +import { KEYLESS_PROVIDERS } from "../providers/custom-providers.js"; export interface CompletenessReport { configured: boolean; @@ -28,7 +36,9 @@ export function isConfigured(config: RawConfig, secretsPath = paths.secrets(), k const referenced = referencedProviders(config); if (referenced.length > 0) { const secrets = loadSecrets(secretsPath, keyPath); - const missing = referenced.filter((p) => !secrets.providers[p]?.apiKey); + // Keyless providers (e.g. rapid-mlx) don't need an API key stored in secrets. + const needsKey = referenced.filter((p) => !KEYLESS_PROVIDERS.has(p)); + const missing = needsKey.filter((p) => !secrets.providers[p]?.apiKey); if (missing.length > 0) reasons.push(`missing API key for provider(s): ${missing.join(", ")}`); } diff --git a/src/fusion/fusion.ts b/src/fusion/fusion.ts index 0aebd58..e7468aa 100644 --- a/src/fusion/fusion.ts +++ b/src/fusion/fusion.ts @@ -5,7 +5,7 @@ import { randomUUID } from "node:crypto"; import type { RawConfig } from "../config/schema.js"; import { isConfigured } from "../config/completeness.js"; import { getKey } from "../config/secrets.js"; -import { resolveModel, type AnyModel } from "../providers/pi-ai-bridge.js"; +import { resolveModel, effectiveApiKey, type AnyModel } from "../providers/pi-ai-bridge.js"; import { runParallelFanout, runSequentialFanout } from "./fanout.js"; import type { WorkerResult } from "./worker.js"; import { fusionStatusRegistry } from "./status.js"; @@ -220,7 +220,7 @@ export async function runFusion(input: FusionInput): Promise { model: safeResolve(c.provider, c.model), prompt: input.prompt, context: input.context, - apiKey: getKey(c.provider, secretsPath, keyPath) ?? "", + apiKey: effectiveApiKey(c.provider, getKey(c.provider, secretsPath, keyPath)), timeoutMs: candidateTimeoutMs, workerPrompt: personaPrompts.worker, })); @@ -288,7 +288,7 @@ export async function runFusion(input: FusionInput): Promise { // --- Judge step 1: analysis --- const judgeModel = safeResolve(judge.provider, judge.model); - const judgeApiKey = getKey(judge.provider, secretsPath, keyPath) ?? ""; + const judgeApiKey = effectiveApiKey(judge.provider, getKey(judge.provider, secretsPath, keyPath)); const candidateViews: CandidateView[] = survivors.map((w, i) => ({ index: i + 1, provider: w.provider, diff --git a/src/index.ts b/src/index.ts index c37683f..a21d265 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,8 +7,17 @@ import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js" import { createMcpServer } from "./server/mcp-server.js"; import { startUiServer } from "./server/ui-server.js"; import { printStartupBanner } from "./util/startup.js"; +import { registerConfigModels } from "./providers/pi-ai-bridge.js"; +import { loadConfig } from "./config/store.js"; async function main(): Promise { + // Register any custom provider models referenced by the saved config so + // resolveModel() works at fusion time without requiring a prior UI /models + // call. loadConfig() returns an empty config (no throw) when the file is + // absent (first run), so a genuinely corrupt config.json fails loudly here + // rather than being silently swallowed. + registerConfigModels(loadConfig()); + // First-run banner (stderr) + auto-open the dashboard on a fresh install. await printStartupBanner(); diff --git a/src/providers/custom-providers.ts b/src/providers/custom-providers.ts new file mode 100644 index 0000000..ca35de9 --- /dev/null +++ b/src/providers/custom-providers.ts @@ -0,0 +1,187 @@ +// Custom provider definitions for OpenFusion. +// +// pi-ai's static registry covers the well-known cloud providers, but two +// OpenAI-compatible endpoints aren't in it: rapid-mlx (a LOCAL MLX inference +// server on Apple Silicon) and ollama-cloud (Ollama's hosted CLOUD API). This +// module defines both so they appear in the web config dropdowns and resolve +// correctly at fusion time. Despite the branch name ("local-providers"), this +// feature intentionally covers BOTH a local server and a cloud provider. +// +// Both custom providers are discoverable — they expose a /v1/models endpoint +// so the server can fetch the actual available models at runtime. No hardcoded +// model lists: rapid-mlx's models depend on what's loaded locally, and +// ollama-cloud's catalog changes as Ollama adds new cloud models. +// +// The `local` flag distinguishes local servers (may be unreachable, so the UI +// shows a free-text input + a Discover button to retry) from cloud providers +// (always reachable, show a normal dropdown). +// +// KNOWN LIMITATION: buildModelDescriptor() bakes in default contextWindow +// (131072) and maxTokens (8192) for every discovered/typed model, because the +// OpenAI /v1/models response doesn't carry those fields. Cost is reported as 0 +// for the same reason. If a provider under-reports, the dashboard's per-model +// context badge may be inaccurate; this does not affect fusion correctness. +// +// At runtime, registerConfigModels() (called at startup + after each config +// save) registers descriptors for models referenced in the saved config so +// resolveModel() works. For discovered or user-typed models, +// registerCustomModel() is called on the fly. +import type { AnyModel } from "./pi-ai-bridge.js"; + +/** A custom provider definition. */ +export interface CustomProviderDefinition { + /** Unique provider id (used in config.json and secrets). */ + id: string; + /** Human-readable name for the UI. */ + name: string; + /** Short description shown in the UI. */ + description: string; + /** Whether this provider requires an API key. Local servers typically don't. */ + apiKeyRequired: boolean; + /** Base URL for the OpenAI-compatible API endpoint. */ + baseUrl: string; + /** pi-ai API type. All custom providers currently use openai-completions. */ + api: "openai-completions" | "openai-responses"; + /** + * Whether this provider supports /v1/models discovery. + * When true, the /models API endpoint will query the provider's /v1/models + * for a live model list and return it as a normal dropdown. + */ + discoverable: boolean; + /** + * Whether this is a local provider that may be unreachable. + * When true + discoverable, the UI shows a free-text input for model IDs + * if the server is down (no models found), plus a Discover button to retry. + * Cloud providers (local=false) always show a normal dropdown. + */ + local: boolean; + /** Compat overrides for the OpenAI completions API (auto-detected if not set). */ + compat?: Record; +} + +// ─── Provider definitions ──────────────────────────────────────────────────── + +/** rapid-mlx: local MLX inference server for Apple Silicon. No API key needed. */ +export const RAPID_MLX: CustomProviderDefinition = { + id: "rapid-mlx", + name: "Rapid-MLX (Local)", + description: + "Local MLX inference server for Apple Silicon. Runs on localhost — no API key needed. " + + "Click Discover to load available models, or type a model ID directly.", + apiKeyRequired: false, + baseUrl: "http://localhost:8000/v1", + api: "openai-completions", + discoverable: true, + local: true, + compat: { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + supportsStrictMode: false, + supportsLongCacheRetention: false, + }, +}; + +/** ollama-cloud: Ollama's cloud API at ollama.com. Requires an API key. */ +export const OLLAMA_CLOUD: CustomProviderDefinition = { + id: "ollama-cloud", + name: "Ollama Cloud", + description: + "Ollama's hosted cloud API at ollama.com. Requires an API key. " + + "Models are fetched from the cloud catalog automatically.", + apiKeyRequired: true, + baseUrl: "https://ollama.com/v1", + api: "openai-completions", + discoverable: true, + local: false, + compat: { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + supportsStrictMode: false, + supportsLongCacheRetention: false, + }, +}; + +/** All custom provider definitions, keyed by provider id. */ +export const CUSTOM_PROVIDERS: Record = { + [RAPID_MLX.id]: RAPID_MLX, + [OLLAMA_CLOUD.id]: OLLAMA_CLOUD, +}; + +/** Provider ids that don't require an API key. */ +export const KEYLESS_PROVIDERS = new Set( + Object.values(CUSTOM_PROVIDERS) + .filter((p) => !p.apiKeyRequired) + .map((p) => p.id), +); + +/** + * Build a model descriptor for a dynamically discovered or user-typed model. + * Used by registerCustomModel() and the discover endpoint. + */ +export function buildModelDescriptor( + provider: CustomProviderDefinition, + modelId: string, + overrides?: { contextWindow?: number; maxTokens?: number; reasoning?: boolean }, +): AnyModel { + return { + id: modelId, + name: modelId, + api: provider.api, + provider: provider.id, + baseUrl: provider.baseUrl, + reasoning: overrides?.reasoning ?? false, + input: ["text" as const], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: overrides?.contextWindow ?? 131072, + maxTokens: overrides?.maxTokens ?? 8192, + ...(provider.compat ? { compat: provider.compat } : {}), + }; +} + +/** Response shape from the OpenAI-compatible /v1/models endpoint. */ +export interface DiscoveryModel { + id: string; + object?: string; + created?: number; + owned_by?: string; +} + +export interface DiscoveryResponse { + object?: string; + data: DiscoveryModel[]; +} + +/** + * Discover models from a provider's /v1/models endpoint. + * Returns a list of model IDs, or throws on network/auth errors. + */ +export async function discoverModels( + provider: CustomProviderDefinition, + apiKey?: string, +): Promise { + const url = `${provider.baseUrl}/models`; + const headers: Record = { + Accept: "application/json", + }; + if (apiKey) { + headers.Authorization = `Bearer ${apiKey}`; + } + const resp = await fetch(url, { headers, signal: AbortSignal.timeout(10_000) }); + if (!resp.ok) { + const body = await resp.text().catch(() => ""); + throw new Error(`${resp.status} ${resp.statusText}${body ? `: ${body.slice(0, 200)}` : ""}`); + } + // Tolerate non-compliant /v1/models responses: a null body, a missing data + // array, or non-object elements would otherwise crash discovery. Keep only + // entries that look like { id: string }. + const json = (await resp.json()) as DiscoveryResponse | null; + const models = json && Array.isArray(json.data) ? json.data : []; + return models + .filter((m): m is DiscoveryModel => m != null && typeof m === "object" && typeof m.id === "string") + .map((m) => m.id) + .sort(); +} \ No newline at end of file diff --git a/src/providers/pi-ai-bridge.ts b/src/providers/pi-ai-bridge.ts index e1b13a2..53ae86d 100644 --- a/src/providers/pi-ai-bridge.ts +++ b/src/providers/pi-ai-bridge.ts @@ -11,6 +11,14 @@ import { type Api, } from "@earendil-works/pi-ai"; import type { Model } from "@earendil-works/pi-ai"; +import { CUSTOM_PROVIDERS, KEYLESS_PROVIDERS, buildModelDescriptor } from "./custom-providers.js"; + +/** + * Sentinel API key for providers that don't require authentication (e.g. rapid-mlx). + * pi-ai's OpenAI completions provider throws if apiKey is falsy; this sentinel + * satisfies the check while the local server ignores it. + */ +const NO_KEY_SENTINEL = "no-key"; /** The model shape used throughout OpenFusion (a pi-ai Model with a broad Api). */ export type AnyModel = Model; @@ -47,13 +55,18 @@ export function resolveModel(provider: string, model: string): AnyModel { } } -/** List provider ids (for the UI dropdowns). */ +/** List provider ids (for the UI dropdowns). Includes pi-ai's built-in + custom providers. */ export function listProviders(): string[] { - return getProviders() as string[]; + const builtIn = getProviders() as string[]; + const customIds = Object.keys(CUSTOM_PROVIDERS); + // Custom providers that aren't already in pi-ai's registry (avoid duplicates). + const added = customIds.filter((id) => !builtIn.includes(id)); + return [...builtIn, ...added]; } -/** List models for a provider (for the UI dropdowns). */ +/** List models for a provider (for the UI dropdowns). Built-in only; custom providers use discovery. */ export function listModels(provider: string) { + // Built-in pi-ai provider — delegate to the static registry. // May throw if provider unknown — let callers wrap. const models = getModels(provider as never) as Array<{ id: string; @@ -69,6 +82,37 @@ export function listModels(provider: string) { })); } +/** + * Resolve the effective API key for a provider for the COMPLETION path. pi-ai's + * openai-completions provider throws on a falsy apiKey, so keyless providers + * (e.g. rapid-mlx) that have no stored key get a module-private sentinel — the + * local server ignores it. Everyone else passes through their stored key. + * If a keyless provider has a stored key (user explicitly saved one), respect it. + * + * NOTE: this sentinel is intentionally NOT exported and is never compared + * against outside this module. Discovery (/v1/models) routes auth through + * KEYLESS_PROVIDERS directly (see server/api/providers.ts) and sends no + * Authorization header for keyless providers, so the two paths don't share the + * magic string. Callers MUST treat the returned value as opaque and must never + * compare it against literals or branch on its contents. + */ +export function effectiveApiKey(provider: string, storedKey: string | undefined): string { + if (KEYLESS_PROVIDERS.has(provider) && !storedKey) return NO_KEY_SENTINEL; + return storedKey ?? ""; +} + +/** + * Register a model for a custom provider at runtime (e.g. after discovery or + * when the user types a model ID). Also registers the descriptor with pi-ai + * so resolveModel() works at fusion time. + */ +export function registerCustomModel(provider: string, modelId: string): void { + const def = CUSTOM_PROVIDERS[provider]; + if (!def) return; // Not a custom provider — ignore (built-in providers use pi-ai's registry). + const descriptor = buildModelDescriptor(def, modelId); + registerModelDescriptor(provider, modelId, descriptor); +} + /** Run a single non-streaming completion. The single-shot worker + both judge steps use this. */ export async function runComplete( model: AnyModel, @@ -147,3 +191,27 @@ export class BridgeError extends Error { this.code = code; } } + +/** + * Register all custom provider models referenced in a config (candidates + judges). + * This must be called at startup after loading the config so that resolveModel() + * works for custom providers like rapid-mlx and ollama-cloud. Without this, a + * fusion request fails because the models were never registered — they only get + * registered when the UI calls /api/providers/:provider/models, which may not + * have happened yet. + * + * Note: listProviders() already returns custom provider ids via CUSTOM_PROVIDERS, + * so there is no separate "register providers" step — only models need registering. + */ +export function registerConfigModels(config: { candidates?: Array<{ provider: string; model: string }>; judges?: Array<{ provider: string; model: string }> }): void { + // candidates/judges are optional in the param type for defensive robustness. + // In practice loadConfig()'s zod schema (RawConfigSchema) always defaults + // these to [], so the real startup path never passes undefined — but this is + // a public export, so tolerate a partial/empty config without crashing. + const entries = [...(config?.candidates ?? []), ...(config?.judges ?? [])]; + for (const { provider, model } of entries) { + if (CUSTOM_PROVIDERS[provider] && model) { + registerCustomModel(provider, model); + } + } +} \ No newline at end of file diff --git a/src/server/api/config.ts b/src/server/api/config.ts index cccb7d3..9ab4ace 100644 --- a/src/server/api/config.ts +++ b/src/server/api/config.ts @@ -2,6 +2,7 @@ import { Router } from "express"; import { loadConfig, saveConfig, mergeAndValidate, emptyConfig } from "../../config/store.js"; import { isConfigured } from "../../config/completeness.js"; +import { registerConfigModels } from "../../providers/pi-ai-bridge.js"; export function configRouter(): Router { const r = Router(); @@ -21,6 +22,9 @@ export function configRouter(): Router { const merged = mergeAndValidate(loadConfig(), req.body); saveConfig(merged); const config = loadConfig(); + // Re-register custom provider models so resolveModel() works for the + // newly-saved providers without requiring a page reload or /models call. + registerConfigModels(config); res.json({ ...config, configured: isConfigured(config).configured }); }); diff --git a/src/server/api/providers.ts b/src/server/api/providers.ts index 6730212..80f9b34 100644 --- a/src/server/api/providers.ts +++ b/src/server/api/providers.ts @@ -1,16 +1,72 @@ -// GET /api/providers and /api/providers/:provider/models — passthrough to pi-ai. +// GET /api/providers, GET /api/providers/:provider/models, and +// GET /api/providers/:provider/discover — provider listing, model listing, +// and dynamic model discovery for custom providers (local + cloud). import { Router } from "express"; -import { listProviders, listModels } from "../../providers/pi-ai-bridge.js"; +import { listProviders, listModels, registerCustomModel } from "../../providers/pi-ai-bridge.js"; +import { CUSTOM_PROVIDERS, KEYLESS_PROVIDERS, discoverModels } from "../../providers/custom-providers.js"; +import { getKey } from "../../config/secrets.js"; + +/** + * The API key to send to a provider's /v1/models discovery endpoint, or + * undefined to send no Authorization header. Keyless providers (local servers + * like rapid-mlx) never need auth; everyone else sends their stored key (which + * may be undefined if the user hasn't saved one yet — discovery will then 401 + * and the caller surfaces the error). + */ +function discoveryApiKey(provider: string): string | undefined { + if (KEYLESS_PROVIDERS.has(provider)) return undefined; + return getKey(provider) || undefined; +} export function providersRouter(): Router { const r = Router(); + // GET /api/providers — list all providers with metadata. r.get("/", (_req, res) => { - res.json({ providers: listProviders() }); + const providers = listProviders(); + // Enrich with metadata from custom provider definitions. + const enriched = providers.map((id) => { + const custom = CUSTOM_PROVIDERS[id]; + return { + id, + name: custom?.name ?? id, + description: custom?.description ?? undefined, + keyless: KEYLESS_PROVIDERS.has(id), + discoverable: custom?.discoverable ?? false, + local: custom?.local ?? false, + }; + }); + res.json({ providers: enriched }); }); - r.get("/:provider/models", (req, res) => { + // GET /api/providers/:provider/models — list models for any provider. + // For built-in providers: returns the static registry. + // For discoverable custom providers (both local and cloud): attempts live + // discovery from the provider's /v1/models endpoint. On failure (server + // unreachable or auth rejected) returns { models: [], error } so the UI can + // tell "no models" apart from "couldn't reach the provider". + r.get("/:provider/models", async (req, res) => { const provider = req.params.provider; + const customDef = CUSTOM_PROVIDERS[provider]; + + if (customDef?.discoverable) { + const apiKey = discoveryApiKey(provider); + try { + const modelIds = await discoverModels(customDef, apiKey); + // Register discovered models so resolveModel() works at fusion time. + for (const id of modelIds) { + registerCustomModel(provider, id); + } + res.json({ models: modelIds.map((id) => ({ id })) }); + } catch (e) { + // Provider unreachable or auth failed — return empty list WITH the error + // so the dashboard can surface it (Constitution V: no silent failures). + res.json({ models: [], error: `Model discovery failed for '${provider}': ${(e as Error).message}` }); + } + return; + } + + // Built-in pi-ai provider — delegate to the static registry. try { res.json({ models: listModels(provider) }); } catch (e) { @@ -20,5 +76,29 @@ export function providersRouter(): Router { } }); + // GET /api/providers/:provider/discover — explicit discover endpoint. + // Only for local providers that may need a manual retry (e.g. after starting + // a local server). Cloud providers don't need this — they're always reachable, + // and their /models route already does discovery. + r.get("/:provider/discover", async (req, res) => { + const provider = req.params.provider; + const def = CUSTOM_PROVIDERS[provider]; + if (!def || !def.discoverable || !def.local) { + res.status(404).json({ error: `Provider '${provider}' does not support model discovery.` }); + return; + } + const apiKey = discoveryApiKey(provider); + try { + const modelIds = await discoverModels(def, apiKey); + for (const id of modelIds) { + registerCustomModel(provider, id); + } + res.json({ models: modelIds }); + } catch (e) { + const msg = (e as Error).message; + res.status(502).json({ error: `Model discovery failed for '${provider}': ${msg}` }); + } + }); + return r; -} +} \ No newline at end of file diff --git a/src/server/api/secrets.ts b/src/server/api/secrets.ts index 426d0a5..5f0069e 100644 --- a/src/server/api/secrets.ts +++ b/src/server/api/secrets.ts @@ -2,6 +2,7 @@ import { Router } from "express"; import { loadConfig } from "../../config/store.js"; import { maskedPresence, setProviderKey } from "../../config/secrets.js"; +import { KEYLESS_PROVIDERS } from "../../providers/custom-providers.js"; export function secretsRouter(): Router { const r = Router(); @@ -9,7 +10,10 @@ export function secretsRouter(): Router { // Masked presence only — NEVER the raw key. r.get("/", (_req, res) => { const config = loadConfig(); - res.json(maskedPresence(config)); + const result = maskedPresence(config); + // Annotate keyless providers so the UI can skip/hide their key input. + const keyless = [...KEYLESS_PROVIDERS]; + res.json({ ...result, keyless }); }); // Set (or clear with null/empty) one provider's key. Encrypted before write. diff --git a/src/server/api/test.ts b/src/server/api/test.ts index b473548..7b690fe 100644 --- a/src/server/api/test.ts +++ b/src/server/api/test.ts @@ -1,18 +1,26 @@ // POST /api/test — validate a provider+model+key before the user commits it (FR-013). import { Router } from "express"; -import { testPing } from "../../providers/pi-ai-bridge.js"; +import { testPing, effectiveApiKey } from "../../providers/pi-ai-bridge.js"; export function testRouter(): Router { const r = Router(); r.post("/", async (req, res) => { const { provider, model, apiKey } = req.body ?? {}; - if (!provider || !model || !apiKey) { - const e = new Error("provider, model, and apiKey are required"); + if (!provider || !model) { + const e = new Error("provider and model are required"); (e as Error & { code?: string }).code = "VALIDATION"; throw e; } - const result = await testPing(provider, model, apiKey, 10_000); + // Keyless providers (e.g. rapid-mlx) don't require an API key — supply a sentinel. + // For others, apiKey is required. + const resolvedKey = effectiveApiKey(provider, apiKey || undefined); + if (!resolvedKey) { + const e = new Error("apiKey is required for this provider"); + (e as Error & { code?: string }).code = "VALIDATION"; + throw e; + } + const result = await testPing(provider, model, resolvedKey, 10_000); res.json(result); }); diff --git a/src/ui-only.ts b/src/ui-only.ts index 7899f71..e2cc3b4 100644 --- a/src/ui-only.ts +++ b/src/ui-only.ts @@ -4,8 +4,14 @@ // Shares the same on-disk SQLite + config + secrets as the MCP server. import { startUiServer } from "./server/ui-server.js"; import { printStartupBanner } from "./util/startup.js"; +import { registerConfigModels } from "./providers/pi-ai-bridge.js"; +import { loadConfig } from "./config/store.js"; async function main(): Promise { + // Register any custom provider models referenced by the saved config so + // resolveModel() works. loadConfig() returns an empty config (no throw) when + // the file is absent (first run); a genuinely corrupt config.json fails loudly. + registerConfigModels(loadConfig()); await printStartupBanner(); await startUiServer(); console.error("OpenFusion dashboard running. Press Ctrl+C to stop."); diff --git a/tests/custom-providers.test.ts b/tests/custom-providers.test.ts new file mode 100644 index 0000000..36778c0 --- /dev/null +++ b/tests/custom-providers.test.ts @@ -0,0 +1,361 @@ +// Tests for custom provider registration, discovery, and keyless provider handling. +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { + listProviders, + listModels, + resolveModel, + clearModelDescriptors, + registerCustomModel, + registerConfigModels, + effectiveApiKey, +} from "../src/providers/pi-ai-bridge.js"; +import { + CUSTOM_PROVIDERS, + RAPID_MLX, + OLLAMA_CLOUD, + KEYLESS_PROVIDERS, + buildModelDescriptor, + discoverModels, +} from "../src/providers/custom-providers.js"; +import { isConfigured } from "../src/config/completeness.js"; +import { setProviderKey } from "../src/config/secrets.js"; +import { RawConfigSchema, type RawConfig } from "../src/config/schema.js"; +import { rmSync } from "node:fs"; + +// A secrets/key path that does not exist — loadSecrets() returns empty +// (unconfigured) without throwing, so we can assert the gate's key logic in +// isolation. Distinct per process to avoid cross-test bleed. +const NO_SECRETS = `/tmp/openfusion-test-secrets-${process.pid}.enc`; +const NO_KEY = `/tmp/openfusion-test-master-${process.pid}.key`; + +/** Build a valid RawConfig from bare candidate/judge lists (settings defaulted). */ +function cfg(candidates: Array<{ id: string; provider: string; model: string; enabled?: boolean }>, judges: Array<{ provider: string; model: string; enabled?: boolean }>): RawConfig { + return RawConfigSchema.parse({ candidates, judges }); +} + +// Clear the model override registry before each test so registrations don't leak. +beforeEach(() => { + clearModelDescriptors(); +}); + +describe("custom providers: registration", () => { + it("includes rapid-mlx and ollama-cloud in listProviders()", () => { + const providers = listProviders(); + expect(providers).toContain("rapid-mlx"); + expect(providers).toContain("ollama-cloud"); + // Built-in providers should still be present. + expect(providers).toContain("openai"); + expect(providers).toContain("anthropic"); + }); + + it("does not duplicate providers already in pi-ai's registry", () => { + const providers = listProviders(); + const openaiCount = providers.filter((p) => p === "openai").length; + expect(openaiCount).toBe(1); + }); +}); + +describe("custom providers: model listing", () => { + it("returns empty model list for discoverable custom providers", () => { + const models = listModels("rapid-mlx"); + expect(models).toEqual([]); + }); + + it("returns empty model list for ollama-cloud (discoverable)", () => { + const models = listModels("ollama-cloud"); + expect(models).toEqual([]); + }); + + it("still lists built-in provider models", () => { + const models = listModels("openai"); + expect(models.length).toBeGreaterThan(0); + }); +}); + +describe("custom providers: dynamic model registration", () => { + it("resolves a dynamically registered rapid-mlx model", () => { + registerCustomModel("rapid-mlx", "mlx-community/Qwen3.6-35B-A3B-OptiQ-4bit"); + const model = resolveModel("rapid-mlx", "mlx-community/Qwen3.6-35B-A3B-OptiQ-4bit"); + expect(model.id).toBe("mlx-community/Qwen3.6-35B-A3B-OptiQ-4bit"); + expect(model.provider).toBe("rapid-mlx"); + expect(model.api).toBe("openai-completions"); + expect(model.baseUrl).toBe("http://localhost:8000/v1"); + }); + + it("resolves a dynamically registered ollama-cloud model", () => { + registerCustomModel("ollama-cloud", "gpt-oss:120b-cloud"); + const model = resolveModel("ollama-cloud", "gpt-oss:120b-cloud"); + expect(model.id).toBe("gpt-oss:120b-cloud"); + expect(model.provider).toBe("ollama-cloud"); + expect(model.api).toBe("openai-completions"); + expect(model.baseUrl).toBe("https://ollama.com/v1"); + }); + + it("throws on unknown provider/model", () => { + expect(() => resolveModel("nonexistent-provider", "fake-model")).toThrow(); + }); +}); + +describe("custom providers: keyless handling", () => { + it("KEYLESS_PROVIDERS contains rapid-mlx but not ollama-cloud", () => { + expect(KEYLESS_PROVIDERS.has("rapid-mlx")).toBe(true); + expect(KEYLESS_PROVIDERS.has("ollama-cloud")).toBe(false); + }); + + it("effectiveApiKey returns sentinel for keyless providers with no stored key", () => { + const key = effectiveApiKey("rapid-mlx", undefined); + expect(key).toBe("no-key"); + }); + + it("effectiveApiKey returns sentinel for keyless providers with empty string", () => { + const key = effectiveApiKey("rapid-mlx", ""); + expect(key).toBe("no-key"); + }); + + it("effectiveApiKey returns stored key for keyless providers when present", () => { + const key = effectiveApiKey("rapid-mlx", "my-custom-key"); + expect(key).toBe("my-custom-key"); + }); + + it("effectiveApiKey returns stored key for normal providers", () => { + const key = effectiveApiKey("openai", "sk-abc"); + expect(key).toBe("sk-abc"); + }); + + it("effectiveApiKey returns empty string for normal providers with no key", () => { + const key = effectiveApiKey("openai", undefined); + expect(key).toBe(""); + }); +}); + +describe("custom providers: definition shape", () => { + it("RAPID_MLX has correct base URL (port 8000)", () => { + expect(RAPID_MLX.baseUrl).toBe("http://localhost:8000/v1"); + }); + + it("OLLAMA_CLOUD has correct base URL (ollama.com)", () => { + expect(OLLAMA_CLOUD.baseUrl).toBe("https://ollama.com/v1"); + }); + + it("both providers are discoverable", () => { + expect(RAPID_MLX.discoverable).toBe(true); + expect(OLLAMA_CLOUD.discoverable).toBe(true); + }); + + it("rapid-mlx is local, ollama-cloud is not", () => { + expect(RAPID_MLX.local).toBe(true); + expect(OLLAMA_CLOUD.local).toBe(false); + }); + + it("rapid-mlx is keyless", () => { + expect(RAPID_MLX.apiKeyRequired).toBe(false); + }); + + it("ollama-cloud requires a key", () => { + expect(OLLAMA_CLOUD.apiKeyRequired).toBe(true); + }); + + it("buildModelDescriptor produces correct shape for a rapid-mlx model", () => { + const descriptor = buildModelDescriptor(RAPID_MLX, "my-model"); + expect(descriptor.id).toBe("my-model"); + expect(descriptor.name).toBe("my-model"); + expect(descriptor.api).toBe("openai-completions"); + expect(descriptor.provider).toBe("rapid-mlx"); + expect(descriptor.baseUrl).toBe("http://localhost:8000/v1"); + expect(descriptor.compat).toBeDefined(); + expect((descriptor.compat as Record).maxTokensField).toBe("max_tokens"); + }); +}); + +describe("custom providers: completeness gate", () => { + it("KEYLESS_PROVIDERS set can be used to filter missing-key providers", () => { + const referenced = ["rapid-mlx", "openai"]; + const needsKey = referenced.filter((p) => !KEYLESS_PROVIDERS.has(p)); + // rapid-mlx is keyless, so only openai needs a key. + expect(needsKey).toEqual(["openai"]); + }); +}); + +describe("custom providers: registerConfigModels", () => { + it("registers custom provider models from config so resolveModel works", () => { + registerConfigModels({ + candidates: [ + { id: "c1", provider: "rapid-mlx", model: "mlx-community/Qwen3-35B-A3B-OptiQ-4bit", enabled: true }, + { id: "c2", provider: "ollama-cloud", model: "gpt-oss:120b-cloud", enabled: true }, + { id: "c3", provider: "openai", model: "gpt-4o", enabled: true }, + ], + judges: [ + { provider: "rapid-mlx", model: "mlx-community/Qwen3-35B-A3B-OptiQ-4bit", enabled: true }, + ], + }); + // rapid-mlx model should now resolve. + const rapidModel = resolveModel("rapid-mlx", "mlx-community/Qwen3-35B-A3B-OptiQ-4bit"); + expect(rapidModel.provider).toBe("rapid-mlx"); + expect(rapidModel.baseUrl).toBe("http://localhost:8000/v1"); + + // ollama-cloud model should now resolve. + const ollamaModel = resolveModel("ollama-cloud", "gpt-oss:120b-cloud"); + expect(ollamaModel.provider).toBe("ollama-cloud"); + expect(ollamaModel.baseUrl).toBe("https://ollama.com/v1"); + + // Built-in provider model should still work (not affected by registerConfigModels). + const openaiModel = resolveModel("openai", "gpt-4o"); + expect(openaiModel.provider).toBe("openai"); + }); + + it("skips entries with empty model strings", () => { + clearModelDescriptors(); + registerConfigModels({ + candidates: [ + { id: "c1", provider: "rapid-mlx", model: "", enabled: true }, + ], + judges: [], + }); + // Should not throw; empty model is simply skipped. + expect(() => resolveModel("rapid-mlx", "")).toThrow(); + }); + + it("skips providers that are not custom", () => { + clearModelDescriptors(); + // openai is a built-in provider, not a custom one — registerConfigModels should skip it. + registerConfigModels({ + candidates: [ + { id: "c1", provider: "openai", model: "gpt-4o", enabled: true }, + ], + judges: [], + }); + // gpt-4o resolves via pi-ai's built-in registry, not via modelOverrides. + const model = resolveModel("openai", "gpt-4o"); + expect(model.provider).toBe("openai"); + }); +}); + +describe("custom providers: completeness gate with keyless providers", () => { + // Constitution VI: a key is required for every referenced provider that needs + // one. Keyless providers (rapid-mlx) are exempt; keyed providers (ollama-cloud) + // are not. The >=2 candidates / >=1 judge rules are independent and untouched. + + it("is configured with only keyless providers referenced and no stored key", () => { + const report = isConfigured( + cfg( + [ + { id: "c1", provider: "rapid-mlx", model: "mlx-community/Qwen3-35B", enabled: true }, + { id: "c2", provider: "rapid-mlx", model: "mlx-community/Qwen3-8B", enabled: true }, + ], + [{ provider: "rapid-mlx", model: "mlx-community/Qwen3-35B", enabled: true }], + ), + NO_SECRETS, + NO_KEY, + ); + // No secrets file and no master key -> no stored keys, but rapid-mlx is + // keyless so the gate must NOT report a missing key. + expect(report.reasons).not.toContain("missing API key for provider(s): rapid-mlx"); + expect(report.configured).toBe(true); + }); + + it("is NOT configured when a keyed cloud provider (ollama-cloud) has no stored key", () => { + const report = isConfigured( + cfg( + [ + { id: "c1", provider: "ollama-cloud", model: "gpt-oss:120b", enabled: true }, + { id: "c2", provider: "ollama-cloud", model: "gpt-oss:20b", enabled: true }, + ], + [{ provider: "ollama-cloud", model: "gpt-oss:120b", enabled: true }], + ), + NO_SECRETS, + NO_KEY, + ); + // ollama-cloud is keyed and has no stored key -> gate must fail with a clear reason. + expect(report.configured).toBe(false); + expect(report.reasons.some((r) => r.includes("ollama-cloud"))).toBe(true); + }); + + it("is configured when a keyed cloud provider (ollama-cloud) HAS a stored key", () => { + // Real positive direction: actually persist a key for a keyed provider via + // the secrets store, then confirm the gate passes. Uses throwaway temp + // paths and cleans up so it doesn't touch the user's real secrets.enc. + const secrets = `/tmp/openfusion-test-secrets-keyed-${process.pid}.enc`; + const keyPath = `/tmp/openfusion-test-master-keyed-${process.pid}.key`; + try { + setProviderKey("ollama-cloud", "test-key-abc", secrets, keyPath); + const report = isConfigured( + cfg( + [ + { id: "c1", provider: "ollama-cloud", model: "gpt-oss:120b", enabled: true }, + { id: "c2", provider: "ollama-cloud", model: "gpt-oss:20b", enabled: true }, + ], + [{ provider: "ollama-cloud", model: "gpt-oss:120b", enabled: true }], + ), + secrets, + keyPath, + ); + expect(report.configured).toBe(true); + } finally { + rmSync(secrets, { force: true }); + rmSync(keyPath, { force: true }); + } + }); +}); + +describe("custom providers: discoverModels", () => { + // The only genuinely new runtime behavior in the feature: a live fetch to a + // provider's /v1/models, then defensive parsing. Mock globalThis.fetch so the + // suite never makes a real network call (CONTRIBUTING: no real API calls). + let fetchSpy: ReturnType; + beforeEach(() => { + fetchSpy = vi.spyOn(globalThis, "fetch"); + }); + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("(a) returns sorted model ids on a happy 200 response, with auth for a keyed provider", async () => { + fetchSpy.mockResolvedValue( + new Response(JSON.stringify({ data: [{ id: "b-model" }, { id: "a-model" }] }), { status: 200 }), + ); + const ids = await discoverModels(OLLAMA_CLOUD, "my-key"); + expect(ids).toEqual(["a-model", "b-model"]); + const [url, init] = fetchSpy.mock.calls[0] as [string, RequestInit]; + expect(url).toBe("https://ollama.com/v1/models"); + expect((init.headers as Record).Authorization).toBe("Bearer my-key"); + }); + + it("omits the Authorization header when no apiKey is supplied (keyless local provider)", async () => { + fetchSpy.mockResolvedValue(new Response(JSON.stringify({ data: [{ id: "m1" }] }), { status: 200 })); + await discoverModels(RAPID_MLX); // no apiKey + const [, init] = fetchSpy.mock.calls[0] as [string, RequestInit]; + expect((init.headers as Record).Authorization).toBeUndefined(); + }); + + it("(b) returns [] when data is null", async () => { + fetchSpy.mockResolvedValue(new Response(JSON.stringify({ data: null }), { status: 200 })); + expect(await discoverModels(OLLAMA_CLOUD)).toEqual([]); + }); + + it("(b) returns [] when data is missing entirely", async () => { + fetchSpy.mockResolvedValue(new Response(JSON.stringify({ object: "list" }), { status: 200 })); + expect(await discoverModels(OLLAMA_CLOUD)).toEqual([]); + }); + + it("filters out non-object entries and entries whose id is not a string", async () => { + fetchSpy.mockResolvedValue( + new Response( + JSON.stringify({ data: [{ id: "ok" }, null, { nope: 1 }, { id: 123 }, "raw-string" ] }), + { status: 200 }, + ), + ); + expect(await discoverModels(OLLAMA_CLOUD)).toEqual(["ok"]); + }); + + it("(c) throws with the status code on a non-200 response", async () => { + fetchSpy.mockResolvedValue(new Response("unauthorized", { status: 401, statusText: "Unauthorized" })); + await expect(discoverModels(OLLAMA_CLOUD, "bad-key")).rejects.toThrow(/401/); + }); + + it("(d) propagates a fetch rejection (e.g. an abort/timeout)", async () => { + const abort = new Error("The operation was aborted"); + (abort as Error & { name: string }).name = "AbortError"; + fetchSpy.mockRejectedValue(abort); + await expect(discoverModels(RAPID_MLX)).rejects.toThrow("The operation was aborted"); + }); +}); \ No newline at end of file diff --git a/tests/providers-api.test.ts b/tests/providers-api.test.ts new file mode 100644 index 0000000..783c50c --- /dev/null +++ b/tests/providers-api.test.ts @@ -0,0 +1,119 @@ +// /api/providers route tests — model discovery error shapes (PR #6 review). +// +// Covers the two server error paths the owner flagged: +// - GET /api/providers/:provider/models -> { models: [], error } on discovery failure +// - GET /api/providers/:provider/discover -> 502 { error } on discovery failure +// plus the happy + defensive-parse paths through the route. +// +// globalThis.fetch is spied with URL-based dispatch: calls to the in-process +// Express test server (127.0.0.1) go to the real fetch; calls to a provider's +// /v1/models endpoint return the per-test mock. The suite never makes a real +// network call (CONTRIBUTING: no real API calls). +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import express from "express"; +import { providersRouter } from "../src/server/api/providers.js"; +import { clearModelDescriptors } from "../src/providers/pi-ai-bridge.js"; + +let home: string; +let restoreHome: string | undefined; + +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), "of-providers-")); + restoreHome = process.env.OPENFUSION_HOME; + process.env.OPENFUSION_HOME = home; + clearModelDescriptors(); +}); +afterEach(() => { + if (restoreHome === undefined) delete process.env.OPENFUSION_HOME; + else process.env.OPENFUSION_HOME = restoreHome; + rmSync(home, { recursive: true, force: true }); + vi.restoreAllMocks(); +}); + +/** + * Boot the providers router on a real port and GET the given sub-path. + * `providerFetch` is called for any fetch to a provider's /v1/models URL + * (it may return a Response or throw to simulate a failure); the test's own + * call to the Express server is routed to the real fetch. + */ +async function getProvidersRoute( + subPath: string, + providerFetch: (url: string) => Response | Promise, +): Promise { + const realFetch = globalThis.fetch.bind(globalThis); + vi.spyOn(globalThis, "fetch").mockImplementation((input) => { + const url = typeof input === "string" ? input : String(input); + if (url.startsWith("http://127.0.0.1:")) return realFetch(input as URL | string) as Promise; + return Promise.resolve(providerFetch(url)); + }); + const app = express(); + app.use(express.json()); + app.use("/api/providers", providersRouter()); + const server = app.listen(0, "127.0.0.1"); + await new Promise((resolve) => server.once("listening", resolve)); + const port = (server.address() as { port: number }).port; + try { + return await fetch(`http://127.0.0.1:${port}/api/providers${subPath}`); + } finally { + server.close(); + } +} + +describe("/api/providers/:provider/models (discovery)", () => { + it("returns the discovered models on a happy 200 response", async () => { + const res = await getProvidersRoute("/ollama-cloud/models", () => + new Response(JSON.stringify({ data: [{ id: "gpt-oss:120b" }, { id: "gpt-oss:20b" }] }), { status: 200 }), + ); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.models).toEqual([{ id: "gpt-oss:120b" }, { id: "gpt-oss:20b" }]); + expect(body.error).toBeUndefined(); + }); + + it("returns { models: [], error } when discovery fails (e.g. 401 auth rejected)", async () => { + const res = await getProvidersRoute("/ollama-cloud/models", () => + new Response("unauthorized", { status: 401, statusText: "Unauthorized" }), + ); + // The /models route surfaces the error with a 200 + an `error` field (not a + // 5xx) so the UI can show "couldn't reach the provider" vs "no models". + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.models).toEqual([]); + expect(body.error).toMatch(/ollama-cloud/); + expect(body.error).toMatch(/401/); + }); + + it("returns { models: [] } (no error) when the provider responds 200 with non-array data", async () => { + const res = await getProvidersRoute("/ollama-cloud/models", () => + new Response(JSON.stringify({ data: null }), { status: 200 }), + ); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.models).toEqual([]); + expect(body.error).toBeUndefined(); + }); +}); + +describe("/api/providers/:provider/discover (local manual retry)", () => { + it("returns 502 { error } when discovery fails (e.g. local server down)", async () => { + const res = await getProvidersRoute("/rapid-mlx/discover", () => { + throw new Error("fetch failed: ECONNREFUSED"); + }); + expect(res.status).toBe(502); + const body = await res.json(); + expect(body.error).toMatch(/rapid-mlx/); + expect(body.error).toMatch(/ECONNREFUSED/); + }); + + it("returns 404 for a provider that does not support discovery (non-local)", async () => { + // ollama-cloud is discoverable but NOT local, so /discover is 404 (its + // /models route already does discovery). + const res = await getProvidersRoute("/ollama-cloud/discover", () => + new Response("{}", { status: 200 }), + ); + expect(res.status).toBe(404); + }); +}); \ No newline at end of file diff --git a/ui/src/api.ts b/ui/src/api.ts index f98c2bd..3898fe6 100644 --- a/ui/src/api.ts +++ b/ui/src/api.ts @@ -30,6 +30,8 @@ export interface Persona { export interface SecretsView { providers: Record; referenced: string[]; + /** Provider ids that don't require an API key (e.g. rapid-mlx). */ + keyless: string[]; } export interface ProviderModel { id: string; @@ -37,6 +39,16 @@ export interface ProviderModel { reasoning?: boolean | string; cost?: { input?: number; output?: number }; } +export interface ProviderInfo { + id: string; + name: string; + description?: string; + keyless: boolean; + /** Whether this provider supports /v1/models discovery. */ + discoverable: boolean; + /** Whether this is a local provider that may be unreachable. */ + local: boolean; +} export interface TestResult { ok: boolean; latencyMs: number; @@ -138,9 +150,15 @@ export const api = { getSecrets: () => getJSON("/api/secrets"), putSecret: (provider: string, apiKey: string | null) => sendJSON("PUT", "/api/secrets", { provider, apiKey }), - getProviders: () => getJSON<{ providers: string[] }>("/api/providers"), + getProviders: () => getJSON<{ providers: ProviderInfo[] }>("/api/providers"), + // `error` is present when discovery was attempted but failed (provider + // unreachable or auth rejected) — lets the UI distinguish "no models" from + // "couldn't reach the provider" (Constitution V: no silent failures). getModels: (provider: string) => - getJSON<{ models: ProviderModel[] }>(`/api/providers/${encodeURIComponent(provider)}/models`), + getJSON<{ models: ProviderModel[]; error?: string }>(`/api/providers/${encodeURIComponent(provider)}/models`), + /** Discover models from a custom provider's /v1/models endpoint. */ + discoverModels: (provider: string) => + getJSON<{ models: string[] }>(`/api/providers/${encodeURIComponent(provider)}/discover`), testProvider: (provider: string, model: string, apiKey: string) => sendJSON("POST", "/api/test", { provider, model, apiKey }), getStats: (filters?: Record) => { diff --git a/ui/src/lib/models.ts b/ui/src/lib/models.ts new file mode 100644 index 0000000..7679342 --- /dev/null +++ b/ui/src/lib/models.ts @@ -0,0 +1,26 @@ +// Shared UI helpers for provider model lists. +import type { ProviderModel } from "../api"; + +/** + * Merge two model lists, deduplicating by id. First occurrence wins (entries + * from `a` keep their richer shape — e.g. contextWindow — over a bare `{id}` + * from discovered lists). Used by Candidates + Judge to combine discovered + * models with the provider's loaded list. + */ +export function mergeModelLists(a: ProviderModel[], b: ProviderModel[]): ProviderModel[] { + const seen = new Set(); + const result: ProviderModel[] = []; + for (const m of a) { + if (!seen.has(m.id)) { + seen.add(m.id); + result.push(m); + } + } + for (const m of b) { + if (!seen.has(m.id)) { + seen.add(m.id); + result.push(m); + } + } + return result; +} \ No newline at end of file diff --git a/ui/src/pages/ApiKeys.tsx b/ui/src/pages/ApiKeys.tsx index 6634acb..7fb743e 100644 --- a/ui/src/pages/ApiKeys.tsx +++ b/ui/src/pages/ApiKeys.tsx @@ -12,6 +12,7 @@ export function ApiKeysPage({ config }: { config: AppConfig | null }) { useEffect(refresh, [config]); const referenced = secrets?.referenced ?? []; + const keyless = new Set(secrets?.keyless ?? []); const save = async (provider: string) => { setMsg(null); @@ -57,11 +58,16 @@ export function ApiKeysPage({ config }: { config: AppConfig | null }) { {referenced.map((p) => { const entry = secrets?.providers[p]; const result = results[p]; + const isKeyless = keyless.has(p); return (
{p} - {entry?.present ? ( + {isKeyless ? ( + + not required + + ) : entry?.present ? ( set · {entry.hint} @@ -69,21 +75,27 @@ export function ApiKeysPage({ config }: { config: AppConfig | null }) { missing )}
-
- setDrafts((d) => ({ ...d, [p]: e.target.value }))} - /> - - -
+ {isKeyless ? ( +

+ This provider runs locally and does not require an API key. +

+ ) : ( +
+ setDrafts((d) => ({ ...d, [p]: e.target.value }))} + /> + + +
+ )} {result && (

{result.ok diff --git a/ui/src/pages/Candidates.tsx b/ui/src/pages/Candidates.tsx index dab939c..aa460ec 100644 --- a/ui/src/pages/Candidates.tsx +++ b/ui/src/pages/Candidates.tsx @@ -1,5 +1,6 @@ import { useEffect, useState } from "react"; -import { api, type AppConfig, type CandidateSlot, type ProviderModel } from "../api"; +import { api, type AppConfig, type CandidateSlot, type ProviderInfo, type ProviderModel } from "../api"; +import { mergeModelLists } from "../lib/models.js"; /** * Serial time budget in minutes (feature 007). Mirrors the engine constants in @@ -23,11 +24,22 @@ export function CandidatesPage({ const [candidates, setCandidates] = useState([]); const [benchmark, setBenchmark] = useState(false); const [sequential, setSequential] = useState(false); - const [providers, setProviders] = useState([]); + const [providerList, setProviderList] = useState([]); const [modelsByProvider, setModelsByProvider] = useState>({}); + // Per-provider loading flags so concurrent loads (e.g. switching the provider + // select while another is still fetching) don't clobber each other's indicator, + // and to guard against duplicate in-flight requests for the same provider. + const [loadingProviders, setLoadingProviders] = useState>({}); + /** Discovered model IDs for local providers (keyed by provider). */ + const [discoveredByProvider, setDiscoveredByProvider] = useState>({}); + const [discovering, setDiscovering] = useState(null); const [saving, setSaving] = useState(false); const [msg, setMsg] = useState(null); + // Provider id list for dropdowns. + const providers = providerList.map((p) => p.id); + const providerMap = new Map(providerList.map((p) => [p.id, p])); + useEffect(() => { if (config) { setCandidates(config.candidates); @@ -37,16 +49,48 @@ export function CandidatesPage({ }, [config]); useEffect(() => { - void api.getProviders().then((r) => setProviders(r.providers)); + void api.getProviders().then((r) => setProviderList(r.providers)); }, []); + // Models are loaded lazily — on focus of the model dropdown (or when the + // provider changes). We do NOT eagerly fetch on mount: that would fire a + // live network call (with up to a 10s timeout) to every referenced custom + // provider on every page visit, hanging the UI when a local server is down. + // A saved model is always shown via the displayModels prepend below, so it + // doesn't need the full list to be loaded first. const loadModels = async (provider: string) => { - if (modelsByProvider[provider]) return; + // Skip if already loaded OR already in flight (prevents duplicate concurrent + // requests for the same provider on rapid focus/switch). + if (modelsByProvider[provider] !== undefined || loadingProviders[provider]) return; + setLoadingProviders((lp) => ({ ...lp, [provider]: true })); try { const r = await api.getModels(provider); + // On a discovery failure (auth rejected / server unreachable) the server + // returns { models: [], error }. Surface the error but DON'T cache the + // empty list — leaving modelsByProvider[provider] undefined lets a later + // focus retry, instead of permanently poisoning the cache (a [] is truthy + // and would short-circuit the guard above forever). + if (r.error) { + setMsg(r.error); + return; + } setModelsByProvider((m) => ({ ...m, [provider]: r.models })); - } catch { - /* ignore */ + } catch (e) { + setMsg(`Failed to load models for ${provider}: ${(e as Error).message}`); + } finally { + setLoadingProviders((lp) => ({ ...lp, [provider]: false })); + } + }; + + const discoverModels = async (provider: string) => { + setDiscovering(provider); + try { + const r = await api.discoverModels(provider); + setDiscoveredByProvider((d) => ({ ...d, [provider]: r.models })); + } catch (e) { + setMsg(`Discovery failed: ${(e as Error).message}`); + } finally { + setDiscovering(null); } }; @@ -161,7 +205,20 @@ export function CandidatesPage({

{candidates.map((c, i) => { + const pInfo = providerMap.get(c.provider); + const isLocal = pInfo?.local ?? false; const models = modelsByProvider[c.provider] ?? []; + const loaded = modelsByProvider[c.provider] !== undefined; + const isLoading = !!loadingProviders[c.provider]; + const discovered = discoveredByProvider[c.provider] ?? []; + // For local discoverable providers, merge discovered models into the list. + const allModels: ProviderModel[] = isLocal && discovered.length > 0 + ? mergeModelLists(discovered.map((id) => ({ id })), models) + : models; + // If the saved model isn't in the list yet, add it as an option so it's visible. + const displayModels = c.model && !allModels.some((m) => m.id === c.model) + ? [{ id: c.model }, ...allModels] + : allModels; return (
{i + 1} @@ -177,26 +234,55 @@ export function CandidatesPage({ value={c.provider} onChange={(e) => update(c.id, { provider: e.target.value, model: "" })} > - {providers.map((p) => ( - ))} - update(c.id, { model: e.target.value })} + /> + + {discovered.map((id) => ( + + +
+ ) : ( + /* Built-in and cloud providers: dropdown with saved model always visible */ + + {displayModels.map((m) => ( + + ))} + + )} @@ -206,4 +292,4 @@ export function CandidatesPage({
); -} +} \ No newline at end of file diff --git a/ui/src/pages/Judge.tsx b/ui/src/pages/Judge.tsx index 1bc9070..493532e 100644 --- a/ui/src/pages/Judge.tsx +++ b/ui/src/pages/Judge.tsx @@ -1,27 +1,68 @@ import { useEffect, useState } from "react"; -import { api, type AppConfig, type JudgeConfig, type ProviderModel } from "../api"; +import { api, type AppConfig, type JudgeConfig, type ProviderInfo, type ProviderModel } from "../api"; +import { mergeModelLists } from "../lib/models.js"; export function JudgePage({ config, onChanged }: { config: AppConfig | null; onChanged: () => void }) { const [judges, setJudges] = useState([]); - const [providers, setProviders] = useState([]); + const [providerList, setProviderList] = useState([]); const [modelsByProvider, setModelsByProvider] = useState>({}); + // Per-provider loading flags so concurrent loads don't clobber each other's + // indicator, and to guard against duplicate in-flight requests per provider. + const [loadingProviders, setLoadingProviders] = useState>({}); + /** Discovered model IDs for local providers (keyed by provider). */ + const [discoveredByProvider, setDiscoveredByProvider] = useState>({}); + const [discovering, setDiscovering] = useState(null); const [saving, setSaving] = useState(false); const [msg, setMsg] = useState(null); + // Provider id list for defaults. + const providers = providerList.map((p) => p.id); + const providerMap = new Map(providerList.map((p) => [p.id, p])); + useEffect(() => { if (config) setJudges(config.judges); }, [config]); useEffect(() => { - void api.getProviders().then((r) => setProviders(r.providers)); + void api.getProviders().then((r) => setProviderList(r.providers)); }, []); + // Models are loaded lazily — on focus of the model dropdown (or when the + // provider changes). We do NOT eagerly fetch on mount: that would fire a + // live network call (with up to a 10s timeout) to every referenced custom + // provider on every page visit, hanging the UI when a local server is down. + // A saved model is always shown via the displayModels prepend below. const loadModels = async (provider: string) => { - if (modelsByProvider[provider]) return; + // Skip if already loaded OR already in flight (prevents duplicate concurrent + // requests for the same provider on rapid focus/switch). + if (modelsByProvider[provider] !== undefined || loadingProviders[provider]) return; + setLoadingProviders((lp) => ({ ...lp, [provider]: true })); try { const r = await api.getModels(provider); + // On a discovery failure the server returns { models: [], error }. Surface + // the error but DON'T cache the empty list — leaving the cache undefined + // lets a later focus retry, instead of permanently poisoning it ([] is + // truthy and would short-circuit the guard above forever). + if (r.error) { + setMsg(r.error); + return; + } setModelsByProvider((m) => ({ ...m, [provider]: r.models })); - } catch { - /* ignore */ + } catch (e) { + setMsg(`Failed to load models for ${provider}: ${(e as Error).message}`); + } finally { + setLoadingProviders((lp) => ({ ...lp, [provider]: false })); + } + }; + + const discoverModels = async (provider: string) => { + setDiscovering(provider); + try { + const r = await api.discoverModels(provider); + setDiscoveredByProvider((d) => ({ ...d, [provider]: r.models })); + } catch (e) { + setMsg(`Discovery failed: ${(e as Error).message}`); + } finally { + setDiscovering(null); } }; @@ -88,7 +129,20 @@ export function JudgePage({ config, onChanged }: { config: AppConfig | null; onC
{judges.map((j, i) => { + const pInfo = providerMap.get(j.provider); + const isLocal = pInfo?.local ?? false; const models = modelsByProvider[j.provider] ?? []; + const loaded = modelsByProvider[j.provider] !== undefined; + const isLoading = !!loadingProviders[j.provider]; + const discovered = discoveredByProvider[j.provider] ?? []; + // For local discoverable providers, merge discovered models into the list. + const allModels: ProviderModel[] = isLocal && discovered.length > 0 + ? mergeModelLists(discovered.map((id) => ({ id })), models) + : models; + // If the saved model isn't in the list yet, add it as an option so it's visible. + const displayModels = j.model && !allModels.some((m) => m.id === j.model) + ? [{ id: j.model }, ...allModels] + : allModels; return (
update(i, { provider: e.target.value, model: "" })} > - {providers.map((p) => ( - ))} - update(i, { model: e.target.value })} + /> + + {discovered.map((id) => ( + + +
+ ) : ( + /* Built-in and cloud providers: dropdown with saved model always visible */ + + {displayModels.map((m) => ( + + ))} + + )} @@ -135,4 +218,4 @@ export function JudgePage({ config, onChanged }: { config: AppConfig | null; onC
); -} +} \ No newline at end of file