From f17ad87bc3b293d67fde56a8015c0998cac21324 Mon Sep 17 00:00:00 2001 From: Kailas Mahavarkar <66670953+KailasMahavarkar@users.noreply.github.com> Date: Tue, 28 Jul 2026 02:37:49 +0530 Subject: [PATCH] feat(market): add NVIDIA NIM as a provider Routes through litellm's native nvidia_nim provider (base https://integrate.api.nvidia.com/v1, key NVIDIA_NIM_API_KEY), following the same shape as every other provider module: fetch, build stats, is_/strip_ prefix helpers, and no rows at all without a key. Three things are specific to this catalogue. The prefix is `nim/`, not `nvidia/`. NVIDIA is an AUTHOR on OpenRouter, which serves ids like nvidia/nemotron-3-super-120b-a12b. A `nvidia/` route prefix was indistinguishable from those: the first live run returned 90 "NVIDIA" models when NIM only has 80, because ten OpenRouter rows had been absorbed into the wrong provider - and resolve_model would then have sent an OpenRouter model to NIM, where it does not exist. `nim/` collides with none of OpenRouter's 57 author segments. The catalogue is mixed modality. /v1/models lists embedders, rerankers, OCR, speech and diffusion models next to the chat ones; 22 of 102. Those cannot serve a chat completion, so they are filtered out here rather than left for the availability prober to discover one wasted call at a time. NIM's listing endpoint needs no key, unlike every other provider's. It is still gated on NVIDIA_NIM_API_KEY: a deployment that cannot call these models must not advertise them. Model ids carry their author, so rows report who BUILT the model rather than only who serves it. Context is a documented conservative floor - NIM publishes no per-model window, and claiming one we have not verified is the kind of confident-looking fiction this codebase keeps deleting. Also: the "7 providers" line in the page metadata and the chat empty state was hardcoded and had survived two provider additions. Both now count the registry and the loaded market. --- .env.local.example | 4 + DESIGN.md | 14 +- backend/main.py | 27 ++- backend/providers/authors.py | 14 ++ backend/providers/models.py | 25 ++- backend/providers/nvidia.py | 151 +++++++++++++++ backend/providers/registry.py | 2 + backend/providers/test_nvidia.py | 175 ++++++++++++++++++ backend/providers/test_registry.py | 22 +++ src/app/globals.css | 2 + src/app/layout.tsx | 4 +- src/components/ui/provider-avatar.tsx | 1 + src/config/config.test.ts | 20 ++ src/config/providers.ts | 6 + .../chat/components/chat-empty-state.tsx | 15 +- 15 files changed, 470 insertions(+), 12 deletions(-) create mode 100644 backend/providers/nvidia.py create mode 100644 backend/providers/test_nvidia.py diff --git a/.env.local.example b/.env.local.example index a0c3e74..0e5ca6a 100644 --- a/.env.local.example +++ b/.env.local.example @@ -6,6 +6,10 @@ GOOGLE_AISTUDIO_API_KEY= CEREBRAS_API_KEY= OLLAMA_API_KEY= OPENROUTER_API_KEY= +# NVIDIA NIM (build.nvidia.com). A personal account comes with free credits; the key looks like +# nvapi-... . The catalogue is public, but without a key the provider is skipped like any other - +# listing models this deployment cannot call would be advertising something that does not work. +NVIDIA_NIM_API_KEY= # Cloudflare Workers AI needs both CLOUDFLARE_API_KEY= diff --git a/DESIGN.md b/DESIGN.md index 15dc123..a47161d 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -13,8 +13,10 @@ and this file is a bug. ## 1. Identity -**Product:** real-time market intelligence for free LLM models across 7 providers, plus an -OpenAI-compatible proxy and a chat client for them. +**Product:** real-time market intelligence for free LLM models across every provider in +`src/config/providers.ts`, plus an OpenAI-compatible proxy and a chat client for them. The count is +derived from the registry wherever it is shown - it was hardcoded as "7 providers" and stayed that +way through two additions. **Personality:** warm-editorial density. Information-first, but not cold - a serif display face over a dense data table, on ivory rather than clinical grey. @@ -79,10 +81,12 @@ hardcoded hue-265 blue left behind by the palette that no longer exists. ### Provider tokens - avatars and charts only, never UI accents `openrouter` 35 · `ollama` 200 · `aistudio` 90 · `groq` 25 · `cerebras` 290 · `cloudflare` 60 · -`local` 145. +`local` 145 · `nvidia` 130. -`local` is your own machine. Its green is deliberately the quietest of the seven: it is the only -tier that is free without qualification, and it should read as calm, not as a status badge. +`local` is your own machine. Its green is deliberately the quietest of the set: it is the only tier +that is free without qualification, and it should read as calm, not as a status badge. `nvidia` sits +next to it in hue on purpose - NVIDIA's own brand green - but at much higher chroma, so the two +never read as the same provider at a glance. --- diff --git a/backend/main.py b/backend/main.py index b43f296..209fb39 100755 --- a/backend/main.py +++ b/backend/main.py @@ -38,6 +38,15 @@ strip_reasoning, strip_reasoning_chunk, ) +from providers.nvidia import ( + fetch_nvidia_models, + build_nvidia_market_stats, + is_nvidia_model, + strip_nvidia_prefix, + get_nvidia_api_key, + get_nvidia_base_url, + NVIDIA_DEFAULT_BASE_URL, +) from providers.cloudflare import ( fetch_cloudflare_models, build_cloudflare_market_stats, @@ -343,6 +352,16 @@ def resolve_model( "api_key": user_key or env_key, } + if is_nvidia_model(model_id): + # litellm has a native nvidia_nim provider, so the base URL is its concern - api_base is + # passed only when the operator has overridden it. + slug = strip_nvidia_prefix(model_id) + nvidia_extra: dict[str, Any] = {"api_key": user_key or get_nvidia_api_key()} + base = get_nvidia_base_url() + if base != NVIDIA_DEFAULT_BASE_URL: + nvidia_extra["api_base"] = base + return f"nvidia_nim/{slug}", nvidia_extra + if is_local_model(model_id): # Local server speaks the OpenAI API, so it rides the generic openai/ path with # an api_base. No BYOK: the endpoint is the user's own machine. @@ -683,6 +702,7 @@ async def _compute_rankings() -> list[dict]: cerebras_raw, cloudflare_raw, local_raw, + nvidia_raw, ) = await asyncio.gather( fetch_market_data(), fetch_unified_market_stats(), @@ -690,6 +710,7 @@ async def _compute_rankings() -> list[dict]: fetch_cerebras_models(), fetch_cloudflare_models(), fetch_local_models(), + fetch_nvidia_models(), ) ollama_stats, aistudio_stats = unified_extra @@ -697,8 +718,12 @@ async def _compute_rankings() -> list[dict]: cerebras_stats = build_cerebras_market_stats(cerebras_raw) cloudflare_stats = build_cloudflare_market_stats(cloudflare_raw) local_stats = build_local_market_stats(local_raw) + nvidia_stats = build_nvidia_market_stats(nvidia_raw) - extras = ollama_stats + aistudio_stats + groq_stats + cerebras_stats + cloudflare_stats + local_stats + extras = ( + ollama_stats + aistudio_stats + groq_stats + cerebras_stats + + cloudflare_stats + local_stats + nvidia_stats + ) all_stats = openrouter_stats + extras if not all_stats: return [] diff --git a/backend/providers/authors.py b/backend/providers/authors.py index 360cf50..ab33fbe 100644 --- a/backend/providers/authors.py +++ b/backend/providers/authors.py @@ -30,6 +30,20 @@ "moonshotai": "Moonshot AI", "zai-org": "Z.AI", "z-ai": "Z.AI", + # NVIDIA NIM ids carry an author segment too, and these are the ones whose title-cased form + # would be wrong or ugly ("Ai21Labs", "01 Ai", "Nv-Mistralai"). + "meta": "Meta", + "ai21labs": "AI21", + "01-ai": "01.AI", + "aisingapore": "AI Singapore", + "bigcode": "BigCode", + "deepseek-ai": "DeepSeek", + "minimaxai": "MiniMax", + "moonshotai": "Moonshot AI", + "nv-mistralai": "NVIDIA + Mistral", + "stepfun-ai": "StepFun", + "thinkingmachines": "Thinking Machines", + "ibm": "IBM", } diff --git a/backend/providers/models.py b/backend/providers/models.py index 15479b2..6713563 100755 --- a/backend/providers/models.py +++ b/backend/providers/models.py @@ -152,12 +152,22 @@ async def fetch_unified_models() -> dict: from .cerebras import fetch_cerebras_models from .cloudflare import fetch_cloudflare_models from .local import fetch_local_models + from .nvidia import NVIDIA_MODEL_PREFIX, fetch_nvidia_models async with httpx.AsyncClient() as client: ollama_task = _fetch_ollama_models(client) aistudio_task = _fetch_aistudio_models(client) openrouter_task = _fetch_openrouter_models(client) - ollama_raw, aistudio_raw, openrouter_raw, groq_raw, cerebras_raw, cloudflare_raw, local_raw = await asyncio.gather( + ( + ollama_raw, + aistudio_raw, + openrouter_raw, + groq_raw, + cerebras_raw, + cloudflare_raw, + local_raw, + nvidia_raw, + ) = await asyncio.gather( ollama_task, aistudio_task, openrouter_task, @@ -165,6 +175,7 @@ async def fetch_unified_models() -> dict: fetch_cerebras_models(), fetch_cloudflare_models(), fetch_local_models(), + fetch_nvidia_models(), ) data = [] @@ -190,6 +201,18 @@ async def fetch_unified_models() -> dict: if not slug: continue data.append({"id": f"local/{slug}", "object": "model", "created": m.get("created"), "owned_by": "local"}) + for m in nvidia_raw: + slug = m.get("id", "") + if not slug: + continue + data.append({ + "id": f"{NVIDIA_MODEL_PREFIX}{slug}", + "object": "model", + "created": m.get("created"), + # The author segment NIM already puts in its ids, kept rather than flattened to "nvidia": + # NVIDIA serves these, it did not build most of them. + "owned_by": m.get("owned_by") or "nvidia", + }) for m in openrouter_raw: data.append({"id": m["id"], "object": "model", "created": m.get("created"), "owned_by": "openrouter"}) diff --git a/backend/providers/nvidia.py b/backend/providers/nvidia.py new file mode 100644 index 0000000..5705be1 --- /dev/null +++ b/backend/providers/nvidia.py @@ -0,0 +1,151 @@ +"""NVIDIA NIM (build.nvidia.com) as a provider. + +NIM is OpenAI-compatible and routes through litellm's native `nvidia_nim/` provider, whose default +base is https://integrate.api.nvidia.com/v1. + +Three things make this catalogue different from the other providers': + + the prefix is `nim/`, not NVIDIA is an AUTHOR on OpenRouter, which serves ids like + `nvidia/` `nvidia/nemotron-3-super-120b-a12b`. A `nvidia/` route prefix would + have been indistinguishable from those: the market would list them + twice, the provider filter would claim OpenRouter's rows as NIM's, + and resolve_model would send an OpenRouter model to NIM, where it + does not exist. `nim/` collides with none of OpenRouter's 57 authors + and matches litellm's own provider id. + + ids carry an author segment `meta/llama-3.1-8b-instruct`, exactly like OpenRouter. Prefixed for + the market that becomes `nim/meta/llama-3.1-8b-instruct`: the first + segment is the ROUTE, the second is the AUTHOR, and they are + genuinely different facts. + + the catalogue is mixed /v1/models lists embedders, rerankers, OCR, speech and image models + alongside chat ones. Those are not chat-completable, so shipping them + would fill the market with rows that can only ever fail. They are + filtered out here rather than left for the availability prober to + discover one wasted call at a time. + +Unlike every other provider here, the catalogue needs no key - so the models are listed even on a +deployment that cannot call them. Inference still requires NVIDIA_NIM_API_KEY, and without one the +provider is skipped, matching the "providers without a key are skipped" contract. +""" + +import math +import os +import re + +import httpx + +from providers.authors import author_from_model_id +from providers.params import capability_params, parse_params + +NVIDIA_MODEL_PREFIX = "nim/" +NVIDIA_PROVIDER_NAME = "NVIDIA NIM" +NVIDIA_DEFAULT_BASE_URL = "https://integrate.api.nvidia.com/v1" +NVIDIA_TIMEOUT_S = 10.0 + +# NIM does not publish a per-model context window anywhere in /v1/models. This is a conservative +# FLOOR used only so the capability heuristic has a number to work with - deliberately not the 128k +# most of these models actually support, because claiming a window we have not verified is the kind +# of confident-looking fiction this codebase keeps having to delete. +NVIDIA_DEFAULT_CTX = 32_768 + +# Families that are not chat-completable. Matched against the full id, so `nvidia/embed-qa-4` and +# `baai/bge-m3` both go, while `meta/llama-3.1-8b-instruct` stays. +_NON_CHAT_PATTERN = re.compile( + r"embed|bge|rerank|retriever|ocr|paddle|whisper|riva|asr|tts|speech|vila|clip|molmo|esm" + r"|diffusion|stable-|sdxl|flux|video|cosmos|genmol|protein|fold|dna|audio|parakeet|canary" + r"|table-structure|graphic-elements|page-elements|chart|deplot|florence", + re.I, +) + +_REASONING_PATTERN = re.compile( + r"deepseek-r1|reasoning|thinking|qwen3|gpt-oss|nemotron-.*-reason|magistral|kimi", re.I +) + + +def get_nvidia_api_key() -> str: + """The name litellm itself reads, so a key set for one is set for both.""" + return os.getenv("NVIDIA_NIM_API_KEY", "") + + +def get_nvidia_base_url() -> str: + return os.getenv("NVIDIA_NIM_API_BASE", NVIDIA_DEFAULT_BASE_URL).rstrip("/") + + +def is_nvidia_model(model_id: str) -> bool: + return isinstance(model_id, str) and model_id.startswith(NVIDIA_MODEL_PREFIX) + + +def strip_nvidia_prefix(model_id: str) -> str: + return model_id[len(NVIDIA_MODEL_PREFIX):] if is_nvidia_model(model_id) else model_id + + +def is_chat_model(model_id: str) -> bool: + return not _NON_CHAT_PATTERN.search(model_id) + + +def _is_reasoning(model_id: str) -> bool: + return bool(_REASONING_PATTERN.search(model_id)) + + +async def fetch_nvidia_models() -> list[dict]: + """The chat-capable catalogue, or [] when no key is configured. + + The listing endpoint is public, but a deployment with no key cannot actually CALL any of these - + so it must not advertise them. Same contract as every other provider: no key, no rows. + """ + if not get_nvidia_api_key(): + return [] + try: + async with httpx.AsyncClient() as client: + r = await client.get( + f"{get_nvidia_base_url()}/models", + headers={"Authorization": f"Bearer {get_nvidia_api_key()}"}, + timeout=NVIDIA_TIMEOUT_S, + ) + r.raise_for_status() + rows = r.json().get("data", []) + except Exception as e: + print(f"NVIDIA NIM models error: {e}") + return [] + + return [m for m in rows if m.get("id") and is_chat_model(m["id"])] + + +def build_nvidia_market_stats(raw: list[dict]) -> list[dict]: + result = [] + for m in raw: + slug = m.get("id") or "" + if not slug: + continue + params = parse_params(slug) + ctx = NVIDIA_DEFAULT_CTX + capability = capability_params(params) * math.log10(ctx + 1) + result.append({ + "id": f"{NVIDIA_MODEL_PREFIX}{slug}", + "name": slug, + "params": params, + "ctx": ctx, + # Free by allocation, like Cloudflare Workers AI: a personal NVIDIA account comes with + # free credits, and the endpoints are callable until those run out. Exhausted credits + # surface as a rate-limit/quota error, which availability.py classifies as TRANSIENT - + # so running out hides the models for a while rather than deleting them permanently. + "is_free": True, + "capability": capability, + "brain": _is_reasoning(slug), + # NIM publishes no per-model tool-calling flag, and the market's other key-based + # providers (Groq, Cerebras) make the same optimistic assumption. Kept consistent rather + # than quietly different; the agentic BENCH scores are what actually drive fit_agent. + "tools": True, + "open": True, + # NVIDIA publishes no throughput figures, so this stays absent rather than estimated. + "tps": None, + "uptime": None, + "provider": NVIDIA_PROVIDER_NAME, + # The id's first segment is the author, exactly as on OpenRouter - so the row can say + # who BUILT the model instead of only who serves it. + "author": author_from_model_id(slug), + "balanced": 0.0, + "value": 0.0, + }) + return result diff --git a/backend/providers/registry.py b/backend/providers/registry.py index 70d036d..6807d4b 100644 --- a/backend/providers/registry.py +++ b/backend/providers/registry.py @@ -17,6 +17,7 @@ from .groq import GROQ_MODEL_PREFIX, GROQ_PROVIDER_NAME from .local import LOCAL_MODEL_PREFIX, LOCAL_PROVIDER_NAME from .models import AISTUDIO_PROVIDER, OLLAMA_PROVIDER, OPENROUTER_PROVIDER +from .nvidia import NVIDIA_MODEL_PREFIX, NVIDIA_PROVIDER_NAME AISTUDIO_MODEL_PREFIX = "aistudio/" OLLAMA_MODEL_PREFIX = "ollama/" @@ -28,6 +29,7 @@ CEREBRAS_MODEL_PREFIX: CEREBRAS_PROVIDER_NAME, CLOUDFLARE_MODEL_PREFIX: CLOUDFLARE_PROVIDER_NAME, LOCAL_MODEL_PREFIX: LOCAL_PROVIDER_NAME, + NVIDIA_MODEL_PREFIX: NVIDIA_PROVIDER_NAME, AISTUDIO_MODEL_PREFIX: AISTUDIO_PROVIDER, OLLAMA_MODEL_PREFIX: OLLAMA_PROVIDER, OPENROUTER_MODEL_PREFIX: OPENROUTER_PROVIDER, diff --git a/backend/providers/test_nvidia.py b/backend/providers/test_nvidia.py new file mode 100644 index 0000000..7dd4ed8 --- /dev/null +++ b/backend/providers/test_nvidia.py @@ -0,0 +1,175 @@ +"""NVIDIA NIM: an OpenAI-compatible catalogue that is public, mixed-modality, and author-prefixed. + +The mixed catalogue is the interesting part. /v1/models lists embedders, rerankers, OCR and speech +models next to the chat ones, and none of those can serve a chat completion - so shipping them whole +would fill the market with rows whose only possible outcome is a failed call. +""" + +import asyncio + +from providers.nvidia import ( + NVIDIA_DEFAULT_CTX, + NVIDIA_MODEL_PREFIX, + NVIDIA_PROVIDER_NAME, + build_nvidia_market_stats, + fetch_nvidia_models, + get_nvidia_base_url, + is_chat_model, + is_nvidia_model, + strip_nvidia_prefix, +) + + +# ---- id handling ------------------------------------------------------------------------------- + +def test_prefix_round_trip(): + assert is_nvidia_model("nim/meta/llama-3.1-8b-instruct") + assert strip_nvidia_prefix("nim/meta/llama-3.1-8b-instruct") == "meta/llama-3.1-8b-instruct" + + +def test_nvidias_own_models_keep_their_author_segment(): + """`nim/nvidia/nemotron-...` - route first, author second. Stripping removes only the ROUTE.""" + assert strip_nvidia_prefix("nim/nvidia/nemotron-4-340b-instruct") == "nvidia/nemotron-4-340b-instruct" + + +def test_an_openrouter_model_authored_by_nvidia_is_not_claimed_as_a_nim_model(): + """The reason the prefix is `nim/` and not `nvidia/`. + + NVIDIA is an author on OpenRouter, which really serves `nvidia/nemotron-3-super-120b-a12b`. With + a `nvidia/` route prefix those rows were indistinguishable from NIM's: the live catalogue came + back with 90 "NVIDIA" models when NIM only had 80, because ten OpenRouter rows had been absorbed + into the wrong provider. + """ + for openrouter_id in [ + "nvidia/nemotron-3-super-120b-a12b", + "nvidia/nemotron-3-nano-30b-a3b:free", + "nvidia/nemotron-3-ultra-550b-a55b", + ]: + assert not is_nvidia_model(openrouter_id), openrouter_id + + +def test_a_non_nvidia_id_is_left_alone(): + assert not is_nvidia_model("groq/llama-3.3-70b") + assert strip_nvidia_prefix("groq/llama-3.3-70b") == "groq/llama-3.3-70b" + + +def test_a_non_string_id_does_not_explode(): + assert is_nvidia_model(None) is False + assert is_nvidia_model(123) is False + + +# ---- modality filtering ------------------------------------------------------------------------- + +def test_chat_models_are_kept(): + for model_id in [ + "meta/llama-3.1-8b-instruct", + "deepseek-ai/deepseek-v4-pro", + "mistralai/mistral-nemotron", + "google/gemma-4-31b-it", + "openai/gpt-oss-120b", + ]: + assert is_chat_model(model_id), model_id + + +def test_models_that_cannot_serve_a_chat_completion_are_dropped(): + """Real ids from the live catalogue. Each of these would be a permanent failure on first call.""" + for model_id in [ + "baai/bge-m3", # embedding + "nvidia/embed-qa-4", # embedding + "nvidia/llama-3.2-nv-embedqa-1b-v1", # embedding + "nvidia/nemoretriever-parse", # retrieval/parse + "nvidia/nemotron-3-embed-1b", # embedding + "google/deplot", # chart-to-text + "nvidia/ai-synthetic-video-detector", # video + "google/diffusiongemma-26b-a4b-it", # diffusion + ]: + assert not is_chat_model(model_id), model_id + + +def test_the_filter_is_applied_to_the_fetched_catalogue(monkeypatch): + monkeypatch.setenv("NVIDIA_NIM_API_KEY", "nvapi-test") + + class _Response: + @staticmethod + def raise_for_status(): + return None + + @staticmethod + def json(): + return {"data": [{"id": "meta/llama-3.1-8b-instruct"}, {"id": "baai/bge-m3"}, {"id": ""}]} + + class _Client: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def get(self, *a, **kw): + return _Response() + + monkeypatch.setattr("providers.nvidia.httpx.AsyncClient", lambda *a, **kw: _Client()) + assert asyncio.run(fetch_nvidia_models()) == [{"id": "meta/llama-3.1-8b-instruct"}] + + +# ---- the no-key contract ------------------------------------------------------------------------- + +def test_no_key_means_no_models(monkeypatch): + """NIM's catalogue is public, unlike every other provider's - so this is the one place the + "no key, no rows" contract could have been broken by accident. A deployment that cannot CALL + these models must not advertise them.""" + monkeypatch.delenv("NVIDIA_NIM_API_KEY", raising=False) + assert asyncio.run(fetch_nvidia_models()) == [] + + +def test_the_base_url_is_overridable(monkeypatch): + """Self-hosted NIM containers are the whole point of the product; they are not on + integrate.api.nvidia.com.""" + monkeypatch.setenv("NVIDIA_NIM_API_BASE", "http://nim.internal:8000/v1/") + assert get_nvidia_base_url() == "http://nim.internal:8000/v1" + + +def test_the_default_base_is_nvidias_hosted_endpoint(monkeypatch): + monkeypatch.delenv("NVIDIA_NIM_API_BASE", raising=False) + assert get_nvidia_base_url() == "https://integrate.api.nvidia.com/v1" + + +# ---- market rows --------------------------------------------------------------------------------- + +def test_market_row_shape(): + [row] = build_nvidia_market_stats([{"id": "meta/llama-3.1-405b-instruct"}]) + + assert row["id"] == f"{NVIDIA_MODEL_PREFIX}meta/llama-3.1-405b-instruct" + assert row["provider"] == NVIDIA_PROVIDER_NAME + assert row["is_free"] is True + assert row["ctx"] == NVIDIA_DEFAULT_CTX + assert row["params"] == 405.0 + # Throughput is not published, so it stays absent rather than being invented. + assert row["tps"] is None + + +def test_the_author_is_read_off_the_id(): + """NIM ids name who BUILT the model. Dropping that would repeat the bug that labelled + claude-fable-5 "Google" - showing the route where the author belongs.""" + [row] = build_nvidia_market_stats([{"id": "meta/llama-3.1-8b-instruct"}]) + assert row["author"] == "Meta" + assert row["provider"] == NVIDIA_PROVIDER_NAME + + +def test_a_size_that_is_not_stated_is_none_not_one(): + [row] = build_nvidia_market_stats([{"id": "nvidia/nemotron-mini-instruct"}]) + assert row["params"] is None + + +def test_reasoning_models_are_flagged(): + [row] = build_nvidia_market_stats([{"id": "deepseek-ai/deepseek-r1"}]) + assert row["brain"] is True + + +def test_a_plain_instruct_model_is_not_flagged_as_reasoning(): + [row] = build_nvidia_market_stats([{"id": "meta/llama-3.1-8b-instruct"}]) + assert row["brain"] is False + + +def test_rows_without_a_slug_are_skipped(): + assert build_nvidia_market_stats([{"id": ""}, {}]) == [] diff --git a/backend/providers/test_registry.py b/backend/providers/test_registry.py index ab9b2f3..1a6d00c 100644 --- a/backend/providers/test_registry.py +++ b/backend/providers/test_registry.py @@ -70,6 +70,28 @@ def test_a_local_model_is_not_routed_as_openrouter(): assert "api_base" in extra +def test_an_nvidia_model_routes_to_litellms_native_nim_provider(monkeypatch): + """`nvidia_nim/` is litellm's own provider id - it owns the base URL and the NVIDIA_NIM_API_KEY + lookup, so we must not hand-roll an openai/ + api_base route for it.""" + monkeypatch.delenv("NVIDIA_NIM_API_BASE", raising=False) + routed, extra = resolve_model("nim/meta/llama-3.1-8b-instruct") + assert routed == "nvidia_nim/meta/llama-3.1-8b-instruct" + # No api_base on the default endpoint: overriding it is litellm's job unless we were told to. + assert "api_base" not in extra + + +def test_a_self_hosted_nim_endpoint_is_passed_through(monkeypatch): + monkeypatch.setenv("NVIDIA_NIM_API_BASE", "http://nim.internal:8000/v1") + _routed, extra = resolve_model("nim/meta/llama-3.1-8b-instruct") + assert extra["api_base"] == "http://nim.internal:8000/v1" + + +def test_an_nvidia_model_accepts_a_byok_key(monkeypatch): + monkeypatch.setenv("NVIDIA_NIM_API_KEY", "server-nvapi") + _routed, extra = resolve_model("nim/meta/llama-3.1-8b-instruct", user_key="user-nvapi") + assert extra["api_key"] == "user-nvapi" + + def test_a_local_model_never_carries_the_callers_provider_key(): """BYOK is meaningless for a server on your own machine, and forwarding someone's OpenRouter key to 127.0.0.1 is a credential going somewhere it has no business being.""" diff --git a/src/app/globals.css b/src/app/globals.css index 5d4b587..ee7069d 100644 --- a/src/app/globals.css +++ b/src/app/globals.css @@ -52,6 +52,7 @@ --color-provider-cerebras: oklch(0.54 0.14 290); --color-provider-cloudflare: oklch(0.64 0.15 60); --color-provider-local: oklch(0.50 0.09 145); + --color-provider-nvidia: oklch(0.58 0.16 130); /* ============ TYPOGRAPHY ============ */ --font-sans: var(--font-jakarta), ui-sans-serif, system-ui, sans-serif; @@ -207,6 +208,7 @@ --color-provider-cerebras: oklch(0.68 0.14 290); --color-provider-cloudflare: oklch(0.78 0.15 60); --color-provider-local: oklch(0.66 0.10 145); + --color-provider-nvidia: oklch(0.72 0.17 130); } /* Pre-hydration fallback only. layout.tsx stamps an explicit data-theme before paint, so this is diff --git a/src/app/layout.tsx b/src/app/layout.tsx index 9a5df40..1f4d620 100644 --- a/src/app/layout.tsx +++ b/src/app/layout.tsx @@ -5,6 +5,7 @@ import type { Metadata } from "next"; import { GeistMono } from "geist/font/mono"; import { Lora, Plus_Jakarta_Sans } from "next/font/google"; import { Providers } from "@/components/providers"; +import { PROVIDER_IDS } from "@/config/providers"; import "./globals.css"; const lora = Lora({ subsets: ["latin"], variable: "--font-lora", display: "swap" }); @@ -12,7 +13,8 @@ const jakarta = Plus_Jakarta_Sans({ subsets: ["latin"], variable: "--font-jakart export const metadata: Metadata = { title: "Gratis - Free LLM Market", - description: "Real-time market intelligence for free LLM models across 7 providers.", + // Counted from the registry, not typed out. It said "7 providers" through two provider additions. + description: `Real-time market intelligence for free LLM models across ${PROVIDER_IDS.length} providers.`, }; export default function RootLayout({ children }: { children: React.ReactNode }) { diff --git a/src/components/ui/provider-avatar.tsx b/src/components/ui/provider-avatar.tsx index e641c0d..cdaa3fc 100644 --- a/src/components/ui/provider-avatar.tsx +++ b/src/components/ui/provider-avatar.tsx @@ -11,6 +11,7 @@ const PROVIDER_VAR: Record = { "Cloudflare Workers AI": "var(--color-provider-cloudflare)", "OpenRouter": "var(--color-provider-openrouter)", "Local": "var(--color-provider-local)", + "NVIDIA NIM": "var(--color-provider-nvidia)", }; const FALLBACK_HUES = [340, 25, 60, 95, 130, 175, 210, 250, 290, 315]; diff --git a/src/config/config.test.ts b/src/config/config.test.ts index dad015c..2dead9a 100644 --- a/src/config/config.test.ts +++ b/src/config/config.test.ts @@ -19,6 +19,26 @@ describe("provider registry", () => { expect(providerForModel("ollama/qwen3")).toBe("ollama"); expect(providerForModel("openrouter/anthropic/claude-3-opus")).toBe("openrouter"); expect(providerForModel("local/Ternary-Bonsai-27B")).toBe("local"); + expect(providerForModel("nim/meta/llama-3.1-8b-instruct")).toBe("nvidia"); + }); + + it("NIM is routed as nim/, so OpenRouter's NVIDIA-authored models stay OpenRouter's", () => { + // NVIDIA is an AUTHOR on OpenRouter, which serves ids like "nvidia/nemotron-3-super-120b-a12b". + // A `nvidia/` route prefix could not be told apart from those - the market listed them twice + // and the provider filter claimed OpenRouter's rows as NIM's. + expect(providerForModel("nim/nvidia/nemotron-4-340b-instruct")).toBe("nvidia"); + expect(providerForModel("nvidia/nemotron-3-super-120b-a12b")).toBe(FALLBACK_PROVIDER); + expect(PROVIDER_BY_BACKEND_LABEL["NVIDIA NIM"]).toBe("nvidia"); + }); + + it("no route prefix may shadow an OpenRouter author segment", () => { + // The general form of the bug above. OpenRouter ids are `author/model`, so any prefix equal to + // an author name makes those models unroutable. These are real OpenRouter authors. + const openRouterAuthors = ["nvidia", "google", "meta-llama", "qwen", "anthropic", "openai", "mistralai", "deepseek"]; + const prefixes = PROVIDER_IDS.filter((id) => id !== FALLBACK_PROVIDER).map((id) => PROVIDERS[id].prefix); + for (const prefix of prefixes) { + expect(openRouterAuthors).not.toContain(prefix.replace(/\/$/, "")); + } }); it("a local model is NOT mistaken for an OpenRouter one", () => { diff --git a/src/config/providers.ts b/src/config/providers.ts index 94aee5e..da75a1a 100644 --- a/src/config/providers.ts +++ b/src/config/providers.ts @@ -13,6 +13,7 @@ export const PROVIDER_IDS = [ "ollama", "cloudflare", "local", + "nvidia", ] as const; export type ProviderId = (typeof PROVIDER_IDS)[number]; @@ -52,6 +53,11 @@ export const PROVIDERS: Readonly> = { // needsKey is false because presence on LOCAL_LLM_BASE_URL is the only auth there is - offering a // key field would ask for a credential that nothing reads. local: { id: "local", label: "Local", shortLabel: "Local", backendLabel: "Local", prefix: "local/", dotVar: "var(--color-provider-local)", keyHint: "", needsAccountId: false, needsKey: false }, + // `nim/`, NOT `nvidia/`: NVIDIA is an author on OpenRouter, which serves ids like + // "nvidia/nemotron-3-super-120b-a12b". A `nvidia/` prefix could not be told apart from those. + // NIM ids already carry their own author, so a routed id reads "nim/meta/llama-3.1-8b" - + // route first, author second. + nvidia: { id: "nvidia", label: "NVIDIA NIM", shortLabel: "NVIDIA", backendLabel: "NVIDIA NIM", prefix: "nim/", dotVar: "var(--color-provider-nvidia)", keyHint: "nvapi-...", needsAccountId: false, needsKey: true }, } as const; /** Backend provider name -> our provider id. Derived, so it cannot drift from PROVIDERS. */ diff --git a/src/features/chat/components/chat-empty-state.tsx b/src/features/chat/components/chat-empty-state.tsx index f466e8f..344912c 100644 --- a/src/features/chat/components/chat-empty-state.tsx +++ b/src/features/chat/components/chat-empty-state.tsx @@ -15,9 +15,11 @@ type PickModel = (modelId: string) => void; export function ChatEmptyState({ models }: { models: ModelStats[] }) { const startNewChat = useChatSessionStore((state) => state.startNewChat); - const suggested = useMemo( - () => models.filter((model) => model.is_free).slice(0, EMPTY_STATE_MODEL_COUNT), - [models], + const free = useMemo(() => models.filter((model) => model.is_free), [models]); + const suggested = useMemo(() => free.slice(0, EMPTY_STATE_MODEL_COUNT), [free]); + const providerCount = useMemo( + () => new Set(free.map((model) => model.provider)).size, + [free], ); const handlePick = useCallback((modelId: string) => startNewChat(modelId), [startNewChat]); @@ -28,8 +30,13 @@ export function ChatEmptyState({ models }: { models: ModelStats[] }) {

Pick a model to start

+ {/* Counted from the market that is actually loaded. It read "300+ free models across 7 + providers" as a hardcoded string, which was a claim nothing checked and which went stale + the moment a provider was added. */}

- 300+ free models across 7 providers. + {free.length > 0 + ? `${free.length} free models across ${providerCount} providers.` + : "Free models across every provider you have a key for."}

{suggested.map((model) => (