feat(chat): multi-model /tasks/chat/completions (sunflower-14b + sunflower-9b)

.env.example

@@ -79,11 +79,25 @@ SIGNED_URL_EXPIRY_MINUTES=30
 RUNPOD_API_KEY=your-runpod-api-key
 RUNPOD_ENDPOINT_ID=your-runpod-endpoint-id
 
-# Sunflower chat/inference endpoint (RunPod, OpenAI-compatible).
-# SUNFLOWER_ENDPOINT_ID is preferred; if unset, the app falls back to the
-# legacy QWEN_ENDPOINT_ID so existing deployments keep working.
-# SUNFLOWER_ENDPOINT_ID=your-sunflower-endpoint-id
-# QWEN_ENDPOINT_ID=your-legacy-endpoint-id  # deprecated alias, still honored
+# Sunflower chat/inference endpoints (RunPod, OpenAI-compatible).
+# The /tasks/chat/completions endpoint serves two models:
+#   - sunflower-14b (default) -> SUNFLOWER_14B_ENDPOINT_ID
+#   - sunflower-9b            -> SUNFLOWER_9B_ENDPOINT_ID
+#
+# For sunflower-14b the app resolves the endpoint id in this order:
+#   SUNFLOWER_14B_ENDPOINT_ID -> SUNFLOWER_ENDPOINT_ID -> QWEN_ENDPOINT_ID
+# so existing single-model deployments keep working with no config change.
+# SUNFLOWER_14B_ENDPOINT_ID=your-sunflower-14b-endpoint-id
+# SUNFLOWER_9B_ENDPOINT_ID=your-sunflower-9b-endpoint-id
+# SUNFLOWER_ENDPOINT_ID=your-sunflower-endpoint-id   # legacy 14B id, still honored
+# QWEN_ENDPOINT_ID=your-legacy-endpoint-id           # deprecated alias, still honored
+#
+# Optional: override the upstream served `model=` name per endpoint. These must
+# match each RunPod endpoint's --served-model-name. Defaults are the short names
+# below (what the deployed endpoints serve); override only if an endpoint serves
+# a different name.
+# SUNFLOWER_14B_MODEL_NAME=sunflower-14b
+# SUNFLOWER_9B_MODEL_NAME=sunflower-9b
 
 # ----------------------------------------------------------------------------
 # Rate Limiting

.github/workflows/deploy-api.yml

@@ -56,3 +56,11 @@ jobs:
           image: ${{ secrets.GCP_REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/${{ secrets.GCP_PROJECT_REPO }}/${{ secrets.APP_NAME }}:${{ github.sha }}
           region: ${{ secrets.GCP_REGION }}
           project_id: ${{ secrets.GCP_PROJECT_ID }}
+          # Sunflower chat/completions endpoints. `merge` preserves all other
+          # env vars already configured on the Cloud Run service; it only
+          # adds/updates the keys listed here. Set these two values as GitHub
+          # Actions repository secrets (RunPod serverless endpoint IDs).
+          env_vars_update_strategy: merge
+          env_vars: |-
+            SUNFLOWER_14B_ENDPOINT_ID=${{ secrets.SUNFLOWER_14B_ENDPOINT_ID }}
+            SUNFLOWER_9B_ENDPOINT_ID=${{ secrets.SUNFLOWER_9B_ENDPOINT_ID }}

.gitignore

@@ -51,3 +51,7 @@ dynamodb_exports/
 bin/runpod-modal-usage.md
 bin/show-charges-vastai.md
 bin/Runpod-State-of-AI-dark.pdf
+endpoint_usage.csv
+user_usage.csv
+users_organizations.csv
+api_user_usage.md

app/docs.py

@@ -76,9 +76,18 @@
     - `GET /tasks/modal/tts/refresh-url` → `GET /tasks/audio/speech/url`
 
 ### Inference (Sunflower Chat)
-- **`POST /tasks/chat/completions`** - OpenAI-compatible chat completions (Sunflower model).
+- **`POST /tasks/chat/completions`** - OpenAI-compatible chat completions (Sunflower models).
   Supports single instructions, multi-turn conversations, and SSE streaming (`stream: true`).
-  Use model `Sunbird/Sunflower-14B`.
+  - Select a model with the `model` field:
+    - **`sunflower-14b`** (default) — Sunbird's flagship 14B model. Covers **English
+      plus 31 Ugandan and regional languages** (32 total). Best for high-accuracy
+      translation, factual Q&A, summarization, and explanation across Ugandan languages.
+    - **`sunflower-9b`** — broader pan-African coverage: **67 African languages** across
+      good/moderate/basic tiers. Optimized for translation, instruction-following, and
+      multi-turn chat.
+  - `sunflower-14b` is the default when `model` is omitted. The old identifier
+    `Sunbird/Sunflower-14B` is **no longer accepted** — requests using it return
+    `400` with a message to use `sunflower-14b` instead.
   - **Deprecated** → superseded by the unified endpoint above:
     - `POST /tasks/sunflower_inference` → `POST /tasks/chat/completions`
     - `POST /tasks/sunflower_simple` → `POST /tasks/chat/completions` (send the instruction as a single user message)
@@ -123,7 +132,7 @@
     },
     {
         "name": "Chat",
-        "description": "OpenAI-compatible chat completions powered by the Sunflower model. Supports single instructions, multi-turn conversations, and SSE streaming.",  # noqa: E501
+        "description": "OpenAI-compatible chat completions powered by Sunbird's Sunflower models. Choose `sunflower-14b` (default; English + 31 Ugandan/regional languages) or `sunflower-9b` (67 African languages). The old `Sunbird/Sunflower-14B` identifier is no longer accepted — use `sunflower-14b`. Supports single instructions, multi-turn conversations, and SSE streaming.",  # noqa: E501
     },
     {
         "name": "Upload",

app/routers/chat.py

@@ -33,6 +33,7 @@
 )
 from app.deps import CurrentUserDep, DbDep, InferenceServiceDep, QuotaServiceDep
 from app.schemas.chat import (
+    RENAMED_MODELS,
     SUPPORTED_MODELS,
     ChatCompletionChoice,
     ChatCompletionChunk,
@@ -42,6 +43,7 @@
     ChatCompletionResponse,
     ChatCompletionResponseMessage,
     ChatCompletionUsage,
+    resolve_model,
 )
 from app.services.inference_service import (
     InferenceService,
@@ -56,10 +58,6 @@
 
 router = APIRouter()
 
-# RunPod endpoint key that serves Sunbird/Sunflower-14B (see
-# InferenceService.endpoints).
-INTERNAL_MODEL_TYPE = "qwen"
-
 
 def _completion_id() -> str:
     """Generate an OpenAI-style completion id."""
@@ -80,15 +78,30 @@ def _prepare_messages(chat_request: ChatCompletionRequest) -> List[Dict[str, str
 
 
 def _validate_model(chat_request: ChatCompletionRequest) -> None:
-    """Strict model validation: only models in SUPPORTED_MODELS are accepted."""
-    if chat_request.model not in SUPPORTED_MODELS:
+    """Reject models not in the registry.
+
+    A renamed identifier (e.g. the old ``Sunbird/Sunflower-14B``) gets a
+    targeted error pointing the client at its replacement name.
+    """
+    if resolve_model(chat_request.model) is not None:
+        return
+
+    replacement = RENAMED_MODELS.get(chat_request.model)
+    if replacement is not None:
         raise BadRequestError(
             message=(
-                f"Model '{chat_request.model}' is not supported. "
-                f"Supported models: {', '.join(SUPPORTED_MODELS)}"
+                f"Model '{chat_request.model}' is no longer accepted. "
+                f"Use '{replacement}' as the model instead."
             )
         )
 
+    raise BadRequestError(
+        message=(
+            f"Model '{chat_request.model}' is not supported. "
+            f"Supported models: {', '.join(SUPPORTED_MODELS)}"
+        )
+    )
+
 
 @router.post(
     "/chat/completions",
@@ -139,12 +152,13 @@ async def _create_chat_completion(
 ) -> ChatCompletionResponse:
     """Non-streaming path: run inference and build a chat.completion object."""
     start_time = time.time()
+    model_key = resolve_model(chat_request.model)
 
     try:
         result = await run_in_threadpool(
             lambda: service.run_inference(
                 messages=messages,
-                model_type=INTERNAL_MODEL_TYPE,
+                model_type=model_key,
                 temperature=chat_request.temperature,
                 max_tokens=chat_request.max_tokens,
                 top_p=chat_request.top_p,
@@ -377,10 +391,11 @@ async def _stream_chat_completion(
     completion_id = _completion_id()
     created = int(time.time())
     start_time = time.time()
+    model_key = resolve_model(chat_request.model)
 
     stream_gen = service.run_inference_stream(
         messages=messages,
-        model_type=INTERNAL_MODEL_TYPE,
+        model_type=model_key,
         temperature=chat_request.temperature,
         max_tokens=chat_request.max_tokens,
         top_p=chat_request.top_p,

app/schemas/chat.py

@@ -13,11 +13,23 @@
 
 from pydantic import BaseModel, Field, field_validator
 
-# The only model served today. The legacy "qwen" alias is accepted solely by
-# the deprecated /tasks/sunflower_* endpoints.
-SUPPORTED_MODELS = ("Sunbird/Sunflower-14B",)
+# Public model names accepted in the `model` field.
+SUPPORTED_MODELS = ("sunflower-14b", "sunflower-9b")
 
-DEFAULT_MODEL = SUPPORTED_MODELS[0]
+DEFAULT_MODEL = "sunflower-14b"
+
+# Old identifiers that are no longer accepted, mapped to the name that
+# replaces them. Used only to return a helpful error pointing the client at
+# the new name (these do NOT resolve to a usable model).
+RENAMED_MODELS = {"Sunbird/Sunflower-14B": "sunflower-14b"}
+
+
+def resolve_model(name: str) -> Optional[str]:
+    """Resolve a requested model name to its canonical name.
+
+    Returns None if the model is not recognised. No aliases are applied.
+    """
+    return name if name in SUPPORTED_MODELS else None
 
 
 class ChatMessage(BaseModel):
@@ -120,6 +132,8 @@ class ChatCompletionChunk(BaseModel):
 __all__ = [
     "SUPPORTED_MODELS",
     "DEFAULT_MODEL",
+    "RENAMED_MODELS",
+    "resolve_model",
     "ChatMessage",
     "ChatCompletionRequest",
     "ChatCompletionResponseMessage",

app/services/inference_service.py

@@ -494,27 +494,44 @@ def __init__(
         super().__init__()
 
         self.runpod_api_key = runpod_api_key or os.getenv("RUNPOD_API_KEY")
-        # Prefer the new SUNFLOWER_ENDPOINT_ID; fall back to the legacy
-        # QWEN_ENDPOINT_ID so existing production environments keep working.
+        # Prefer the new SUNFLOWER_14B_ENDPOINT_ID; fall back to the legacy
+        # SUNFLOWER_ENDPOINT_ID / QWEN_ENDPOINT_ID so existing production
+        # environments keep working with no config change.
         self.sunflower_endpoint_id = (
             sunflower_endpoint_id
             or qwen_endpoint_id
+            or os.getenv("SUNFLOWER_14B_ENDPOINT_ID")
             or os.getenv("SUNFLOWER_ENDPOINT_ID")
             or os.getenv("QWEN_ENDPOINT_ID")
         )
         # Backward-compatible attribute alias (some callers/tests read this).
         self.qwen_endpoint_id = self.sunflower_endpoint_id
 
-        # Single shared config; expose under both the canonical "sunflower"
-        # key and the legacy "qwen" alias so existing internal routes and
-        # OpenAI-compatible clients that send model="qwen" keep working.
-        sunflower_config = {
+        self.sunflower_9b_endpoint_id = os.getenv("SUNFLOWER_9B_ENDPOINT_ID")
+
+        # Upstream served model names (the `model=` value vLLM expects). These
+        # must match each RunPod endpoint's `--served-model-name`. The deployed
+        # endpoints serve the short names (see docs/sunflower-multi-models.md);
+        # override per endpoint via env if a deployment serves a different name.
+        sunflower_14b_model_name = os.getenv(
+            "SUNFLOWER_14B_MODEL_NAME", "sunflower-14b"
+        )
+        sunflower_9b_model_name = os.getenv("SUNFLOWER_9B_MODEL_NAME", "sunflower-9b")
+
+        sunflower_14b_config = {
             "endpoint_id": self.sunflower_endpoint_id,
-            "model_name": "Sunbird/Sunflower-14B",
+            "model_name": sunflower_14b_model_name,
         }
         self.endpoints = {
-            "sunflower": sunflower_config,
-            "qwen": sunflower_config,
+            "sunflower-14b": sunflower_14b_config,
+            "sunflower-9b": {
+                "endpoint_id": self.sunflower_9b_endpoint_id,
+                "model_name": sunflower_9b_model_name,
+            },
+            # Legacy internal aliases -> 14B (translation service, older
+            # callers, and OpenAI clients sending model="qwen").
+            "sunflower": sunflower_14b_config,
+            "qwen": sunflower_14b_config,
         }
 
         if not self.runpod_api_key: