From 8a996e133f99d068bef78a52bdc5395177e9c290 Mon Sep 17 00:00:00 2001 From: agentforce314 Date: Sat, 8 Aug 2026 09:53:11 -0700 Subject: [PATCH] feat(desktop): OpenAI-endpoint transcription + configurable STT model in Settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two follow-ups to the voice fixes: 1. Use the real OpenAI endpoint with an OpenAI key. STT now defaults to the provider's CANONICAL audio endpoint (openai -> api.openai.com/v1, groq -> api.groq.com/openai/v1) instead of the provider's chat base_url, which is often a proxy (e.g. a LiteLLM gateway) with no /audio/transcriptions route. So with an OpenAI key present, voice transcribes against api.openai.com out of the box. Fully overridable via stt..{base_url,api_key,model} or voice.stt_{base_url,api_key,model}. 2. Configure the transcription model in-app. Settings -> Voice now renders the STT provider + per-provider transcription model, backed by: - a focused /api/config/schema (stt.enabled, stt.provider select, stt.openai.model, stt.groq.model, voice.auto_tts), and - PUT /api/config, which serve was missing entirely — it deep-merges the renderer's (secret-redacted) draft over the stored config so api_keys and the env block survive, and unwraps saveClawCodexConfig's {config:{…}} envelope. This is what makes ALL in-app settings edits persist, not just voice. Model resolution: stt..model -> stt.model -> voice.stt_model -> provider default (whisper-1 / whisper-large-v3). The failure message now points at Settings -> Voice. Verified live: the Voice panel renders the STT provider + model fields; saving persists to config and survives a reload with api_keys/env intact; STT resolves to https://api.openai.com/v1 with the openai key. 65 desktop tests green (5 new: canonical endpoint, explicit stt override, model resolution ladder, PUT deep-merge preserving secrets, schema exposes the model field). Co-Authored-By: Claude Fable 5 --- src/server/desktop_audio.py | 93 ++++++++++++++++++--------- src/server/desktop_serve.py | 75 +++++++++++++++++++-- tests/server/test_desktop_audio.py | 71 +++++++++++++++----- tests/server/test_desktop_serve.py | 53 +++++++++++++++ tests/server/test_desktop_sessions.py | 4 +- 5 files changed, 246 insertions(+), 50 deletions(-) diff --git a/src/server/desktop_audio.py b/src/server/desktop_audio.py index 3fb8fd53..49b1cb69 100644 --- a/src/server/desktop_audio.py +++ b/src/server/desktop_audio.py @@ -62,53 +62,87 @@ def _ext_for(mime: str) -> str: }.get(mime.split(";")[0].strip(), "webm") -def _configured_stt_provider() -> str | None: - """An explicit ``voice.stt_provider`` from config, if set.""" +# Canonical transcription endpoints. Whisper lives here regardless of a +# provider's CHAT base URL — a user's `openai` block may point at a chat-only +# proxy (e.g. LiteLLM) that has no /audio/transcriptions route, so STT defaults +# to the provider's real audio endpoint unless the config overrides it. +_STT_BASE_URL = { + "openai": "https://api.openai.com/v1", + "groq": "https://api.groq.com/openai/v1", +} + + +def _stt_config() -> dict[str, Any]: try: from src.config import load_config - voice = (load_config() or {}).get("voice") or {} - stt = (load_config() or {}).get("stt") or {} - return voice.get("stt_provider") or stt.get("provider") + return (load_config() or {}).get("stt") or {} except Exception: # noqa: BLE001 - return None + return {} -def _stt_model_for(provider: str) -> str: - """The transcription model id: config override, else the provider default.""" +def _voice_config() -> dict[str, Any]: try: from src.config import load_config - cfg = load_config() or {} - override = (cfg.get("voice") or {}).get("stt_model") or (cfg.get("stt") or {}).get("model") - if override: - return str(override) + return (load_config() or {}).get("voice") or {} except Exception: # noqa: BLE001 - pass - return _STT_MODEL.get(provider, "whisper-1") + return {} -def _pick_provider() -> tuple[str, str, str] | None: - """(provider_id, base_url, api_key) of the STT-capable provider to use. +def _configured_stt_provider() -> str | None: + """An explicit STT provider from config (``stt.provider`` / + ``voice.stt_provider``), if set.""" + return _stt_config().get("provider") or _voice_config().get("stt_provider") - An explicit ``voice.stt_provider`` wins; otherwise the first configured - provider known to host Whisper. + +def _stt_model_for(provider: str) -> str: + """The transcription model id: ``stt..model`` → ``stt.model`` → + ``voice.stt_model`` → the provider's default (whisper-1 / whisper-large-v3).""" + stt = _stt_config() + per = stt.get(provider) if isinstance(stt.get(provider), dict) else {} + override = per.get("model") or stt.get("model") or _voice_config().get("stt_model") + return str(override) if override else _STT_MODEL.get(provider, "whisper-1") + + +def _resolve_stt() -> tuple[str, str, str] | None: + """(provider_id, base_url, api_key) for transcription, or None. + + Resolution, most specific first: + * ``stt.`` block — ``base_url`` / ``api_key`` overrides, + * ``voice.stt_base_url`` / ``voice.stt_api_key`` (flat overrides), + * the provider's own config block (its chat ``api_key``), + * the canonical audio endpoint for the provider (``_STT_BASE_URL``). + The api_key is required; the base URL always has a canonical fallback so + "use the OpenAI endpoint with an OpenAI key" works with just a key set. """ from src.config import get_provider_config configured = _configured_stt_provider() candidates = ([configured] if configured else []) + list(_STT_PROVIDERS) + stt = _stt_config() + voice = _voice_config() + for pid in candidates: if not pid: continue + per = stt.get(pid) if isinstance(stt.get(pid), dict) else {} try: - cfg = get_provider_config(pid) or {} + prov_cfg = get_provider_config(pid) or {} except Exception: # noqa: BLE001 + prov_cfg = {} + key = per.get("api_key") or voice.get("stt_api_key") or prov_cfg.get("api_key") + if not key: + continue + base = ( + per.get("base_url") + or voice.get("stt_base_url") + or _STT_BASE_URL.get(pid) + or prov_cfg.get("base_url") + ) + if not base: continue - key = cfg.get("api_key") - base = cfg.get("base_url") - if key and base: - return str(pid), str(base).rstrip("/"), str(key) + return str(pid), str(base).rstrip("/"), str(key) return None @@ -122,12 +156,13 @@ async def transcribe_data_url(data_url: str, mime_type: str | None = None) -> Tr if not audio: return TranscriptionResult(ok=False, error="empty audio clip") - picked = _pick_provider() + picked = _resolve_stt() if picked is None: return TranscriptionResult( ok=False, - error="Voice input needs a speech-to-text provider. Configure an " - "OpenAI or Groq API key (they host Whisper) in ~/.clawcodex/config.json.", + error="Voice input needs a speech-to-text provider. Add an OpenAI " + "or Groq API key (they host Whisper), then pick the transcription " + "model in Settings → Voice.", ) provider, base, key = picked filename = f"clip.{_ext_for(mime)}" @@ -155,9 +190,9 @@ async def transcribe_data_url(data_url: str, mime_type: str | None = None) -> Tr ): return TranscriptionResult( ok=False, provider=provider, - error=f"The '{provider}' endpoint doesn't offer a speech-to-text " - "model. Point an OpenAI or Groq provider at a Whisper-capable " - "base URL, or set voice.stt_model in ~/.clawcodex/config.json.", + error=f"The '{provider}' endpoint rejected the transcription " + "model. Pick a valid one in Settings → Voice (Transcription " + "model), or point the provider at a Whisper-capable base URL.", ) return TranscriptionResult( ok=False, provider=provider, diff --git a/src/server/desktop_serve.py b/src/server/desktop_serve.py index 4e382641..e2525e17 100644 --- a/src/server/desktop_serve.py +++ b/src/server/desktop_serve.py @@ -141,6 +141,19 @@ async def config(request: Request) -> Response: return JSONResponse({"error": "unauthorized"}, status_code=401) from src.config import load_config + if request.method == "PUT": + try: + body = await request.json() + except Exception: # noqa: BLE001 + body = None + if not isinstance(body, dict): + return JSONResponse({"ok": False, "error": "invalid config body"}, + status_code=400) + # The renderer wraps the payload as {"config": {...}} + # (saveClawCodexConfig); accept the bare object too. + incoming = body.get("config") if isinstance(body.get("config"), dict) else body + return JSONResponse(_save_config_merged(incoming)) + return JSONResponse(redact_secrets(load_config())) def _int_param(request: Request, name: str, default: int) -> int: @@ -220,10 +233,38 @@ async def model_auxiliary(request: Request) -> Response: async def config_schema(request: Request) -> Response: if not _token_ok(state, _rest_token(request)): return JSONResponse({"error": "unauthorized"}, status_code=401) - # No dynamic config schema in this backend — the structured settings - # panels render from their own known fields; the schema only drives the - # Advanced tab's generated rows, which degrade to none. - return JSONResponse({"fields": {}, "category_order": []}) + # There's no generated dataclass schema in this backend. Ship a focused + # schema for the voice fields the user configures — most importantly the + # transcription model, so Settings → Voice can pick it when STT fails. + # Other panels render from their own known fields. + return JSONResponse({ + "fields": { + "stt.enabled": { + "type": "boolean", + "description": "Enable voice input (speech-to-text).", + }, + "stt.provider": { + "type": "select", + "options": ["openai", "groq"], + "description": "Which provider transcribes your recordings. " + "OpenAI (whisper-1) uses api.openai.com; Groq hosts Whisper too.", + }, + "stt.openai.model": { + "type": "string", + "description": "OpenAI transcription model — whisper-1, or a " + "newer id like gpt-4o-transcribe / gpt-4o-mini-transcribe.", + }, + "stt.groq.model": { + "type": "string", + "description": "Groq transcription model — e.g. whisper-large-v3.", + }, + "voice.auto_tts": { + "type": "boolean", + "description": "Automatically speak assistant replies.", + }, + }, + "category_order": [], + }) async def cron_jobs(request: Request) -> Response: if not _token_ok(state, _rest_token(request)): @@ -443,7 +484,7 @@ async def not_found(request: Request) -> Response: routes = [ Route("/api/health", health), Route("/api/status", status), - Route("/api/config", config), + Route("/api/config", config, methods=["GET", "PUT"]), Route("/api/config/defaults", config_defaults), Route("/api/sessions", sessions_list), Route("/api/sessions/{session_id}/messages", session_messages), @@ -469,6 +510,30 @@ async def not_found(request: Request) -> Response: return Starlette(routes=routes) +def _save_config_merged(incoming: dict[str, Any]) -> dict[str, Any]: + """Deep-merge the (secret-redacted) incoming config over the stored global + config and persist it. + + GET /api/config redacts secrets, so the renderer's config draft has no + api_keys or ``env`` block. A REPLACE would wipe them — deep-merge layers + the user's edits (voice.stt.*, display, …) on top of the stored config so + credentials survive. Mirrors ``saveClawCodexConfig``'s documented merge + semantics (not the REPLACE variant). + """ + from src.config import _deep_merge, _get_default_manager + + try: + manager = _get_default_manager() + current = manager.load_global() + merged = _deep_merge(current, incoming) + manager.save_global(merged) + manager.invalidate() + return {"ok": True} + except Exception as exc: # noqa: BLE001 + logger.warning("desktop: config save failed", exc_info=True) + return {"ok": False, "error": str(exc)} + + _SECRET_KEY_MARKERS = ("api_key", "apikey", "token", "secret", "password") diff --git a/tests/server/test_desktop_audio.py b/tests/server/test_desktop_audio.py index 4927b177..27ae9797 100644 --- a/tests/server/test_desktop_audio.py +++ b/tests/server/test_desktop_audio.py @@ -30,7 +30,8 @@ def test_decode_rejects_non_data_url() -> None: @pytest.mark.asyncio async def test_transcribe_no_provider_is_actionable(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(desktop_audio, "_configured_stt_provider", lambda: None) + monkeypatch.setattr(desktop_audio, "_stt_config", lambda: {}) + monkeypatch.setattr(desktop_audio, "_voice_config", lambda: {}) monkeypatch.setattr("src.config.get_provider_config", lambda pid: {}) result = await transcribe_data_url(_wav_data_url()) assert result.ok is False @@ -43,8 +44,8 @@ async def test_transcribe_success(monkeypatch: pytest.MonkeyPatch) -> None: "src.config.get_provider_config", lambda pid: {"api_key": "k", "base_url": "https://stt.example/v1"} if pid == "openai" else {}, ) - monkeypatch.setattr(desktop_audio, "_configured_stt_provider", lambda: None) - monkeypatch.setattr(desktop_audio, "_stt_model_for", lambda p: "whisper-1") + monkeypatch.setattr(desktop_audio, "_stt_config", lambda: {}) + monkeypatch.setattr(desktop_audio, "_voice_config", lambda: {}) captured = {} @@ -56,29 +57,59 @@ def handler(request: httpx.Request) -> httpx.Response: transport = httpx.MockTransport(handler) orig_client = httpx.AsyncClient - - def client_factory(*a, **k): - k["transport"] = transport - return orig_client(*a, **k) - - monkeypatch.setattr(httpx, "AsyncClient", client_factory) + monkeypatch.setattr(httpx, "AsyncClient", + lambda *a, **k: orig_client(*a, **{**k, "transport": transport})) result = await transcribe_data_url(_wav_data_url(), "audio/wav") assert result.ok is True assert result.transcript == "hello world" assert result.provider == "openai" - assert captured["url"] == "https://stt.example/v1/audio/transcriptions" + # openai STT defaults to the CANONICAL OpenAI endpoint (not the provider's + # chat base_url, which may be a proxy without /audio/transcriptions). + assert captured["url"] == "https://api.openai.com/v1/audio/transcriptions" assert captured["auth"] == "Bearer k" assert captured["has_multipart"] is True +@pytest.mark.asyncio +async def test_transcribe_honors_explicit_stt_config(monkeypatch: pytest.MonkeyPatch) -> None: + """stt.openai.{base_url,api_key,model} override the provider + canonical + defaults — this is how a user points STT at their own endpoint/model.""" + monkeypatch.setattr("src.config.get_provider_config", lambda pid: {}) + monkeypatch.setattr(desktop_audio, "_stt_config", lambda: { + "provider": "openai", + "openai": {"base_url": "https://my.stt/v1", "api_key": "sk-real", "model": "gpt-4o-transcribe"}, + }) + monkeypatch.setattr(desktop_audio, "_voice_config", lambda: {}) + + captured = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["url"] = str(request.url) + captured["auth"] = request.headers.get("authorization") + captured["model_ok"] = b"gpt-4o-transcribe" in request.content + return httpx.Response(200, json={"text": "hi"}) + + transport = httpx.MockTransport(handler) + orig_client = httpx.AsyncClient + monkeypatch.setattr(httpx, "AsyncClient", + lambda *a, **k: orig_client(*a, **{**k, "transport": transport})) + + result = await transcribe_data_url(_wav_data_url(), "audio/wav") + assert result.ok is True + assert captured["url"] == "https://my.stt/v1/audio/transcriptions" + assert captured["auth"] == "Bearer sk-real" + assert captured["model_ok"] is True + + @pytest.mark.asyncio async def test_transcribe_model_rejection_is_actionable(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( "src.config.get_provider_config", - lambda pid: {"api_key": "k", "base_url": "https://chat.example/v1"} if pid == "openai" else {}, + lambda pid: {"api_key": "k"} if pid == "openai" else {}, ) - monkeypatch.setattr(desktop_audio, "_configured_stt_provider", lambda: None) + monkeypatch.setattr(desktop_audio, "_stt_config", lambda: {}) + monkeypatch.setattr(desktop_audio, "_voice_config", lambda: {}) def handler(request: httpx.Request) -> httpx.Response: return httpx.Response(400, json={"error": "Invalid model name passed in model=whisper-1"}) @@ -89,7 +120,7 @@ def handler(request: httpx.Request) -> httpx.Response: result = await transcribe_data_url(_wav_data_url(), "audio/wav") assert result.ok is False - assert "doesn't offer a speech-to-text model" in result.error + assert "Settings → Voice" in result.error @pytest.mark.asyncio @@ -99,6 +130,16 @@ async def test_transcribe_empty_clip() -> None: assert "empty" in result.error.lower() -def test_stt_model_config_override(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr("src.config.load_config", lambda: {"voice": {"stt_model": "whisper-large-v3"}}) +def test_stt_model_resolution(monkeypatch: pytest.MonkeyPatch) -> None: + # Per-provider stt.

.model wins. + monkeypatch.setattr("src.config.load_config", + lambda: {"stt": {"openai": {"model": "gpt-4o-transcribe"}}}) + assert desktop_audio._stt_model_for("openai") == "gpt-4o-transcribe" + # Legacy voice.stt_model still honored. + monkeypatch.setattr("src.config.load_config", + lambda: {"voice": {"stt_model": "whisper-large-v3"}}) assert desktop_audio._stt_model_for("openai") == "whisper-large-v3" + # Default when nothing set. + monkeypatch.setattr("src.config.load_config", lambda: {}) + assert desktop_audio._stt_model_for("openai") == "whisper-1" + assert desktop_audio._stt_model_for("groq") == "whisper-large-v3" diff --git a/tests/server/test_desktop_serve.py b/tests/server/test_desktop_serve.py index 1f67470c..c1f066bc 100644 --- a/tests/server/test_desktop_serve.py +++ b/tests/server/test_desktop_serve.py @@ -74,6 +74,59 @@ def test_index_serves_adoptable_token(client: TestClient) -> None: assert json.dumps(TOKEN) in res.text +def test_config_put_deep_merges_preserving_secrets( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """PUT /api/config layers the (secret-redacted) edit over the stored config + so api_keys/env survive — the Voice panel writes stt.* this way.""" + stored = { + "display": {"skin": "dark"}, + "providers": {"openai": {"api_key": "sk-keep", "base_url": "https://x"}}, + "env": {"TAVILY_API_KEY": "tvly-keep"}, + } + saved: dict = {} + + class _Manager: + def load_global(self): + return dict(stored) + + def save_global(self, data): + saved.update(data) + + def invalidate(self): + pass + + monkeypatch.setattr("src.config._get_default_manager", lambda: _Manager()) + + # The renderer sends a redacted draft (no api_key/env) plus its edit, + # wrapped as {"config": {...}} by saveClawCodexConfig. + res = client.put( + "/api/config", + headers={"X-ClawCodex-Session-Token": TOKEN}, + json={"config": {"display": {"skin": "dark"}, "stt": {"provider": "openai", + "openai": {"model": "whisper-1"}}}}, + ) + assert res.status_code == 200 and res.json() == {"ok": True} + # The edit applied… + assert saved["stt"] == {"provider": "openai", "openai": {"model": "whisper-1"}} + # …and the secrets the draft omitted were preserved by the deep-merge. + assert saved["providers"]["openai"]["api_key"] == "sk-keep" + assert saved["env"]["TAVILY_API_KEY"] == "tvly-keep" + + +def test_config_put_requires_token(client: TestClient) -> None: + assert client.put("/api/config", json={"x": 1}).status_code == 401 + + +def test_config_schema_exposes_transcription_model(client: TestClient) -> None: + schema = client.get("/api/config/schema", + headers={"X-ClawCodex-Session-Token": TOKEN}).json() + fields = schema["fields"] + assert "stt.openai.model" in fields + assert fields["stt.provider"]["type"] == "select" + assert "openai" in fields["stt.provider"]["options"] + + def test_config_requires_token_and_redacts_secrets( client: TestClient, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/tests/server/test_desktop_sessions.py b/tests/server/test_desktop_sessions.py index b683d21f..6744bb30 100644 --- a/tests/server/test_desktop_sessions.py +++ b/tests/server/test_desktop_sessions.py @@ -165,7 +165,9 @@ def test_settings_panel_rest_routes( assert aux["tasks"] == [] schema = rest.get("/api/config/schema", headers=AUTH).json() - assert schema == {"fields": {}, "category_order": []} + assert "fields" in schema and isinstance(schema["fields"], dict) + # Ships a focused voice schema (transcription model, provider, …). + assert "stt.openai.model" in schema["fields"] assert rest.get("/api/cron/jobs", headers=AUTH).json() == {"jobs": []} assert rest.get("/api/audio/elevenlabs/voices", headers=AUTH).json() == {"voices": []}