From d54e2d0cc0d4dead398b7c5f42b7a5c08cfda6ec Mon Sep 17 00:00:00 2001 From: vivganes Date: Thu, 16 Jul 2026 23:13:43 +0530 Subject: [PATCH 1/4] feat: add OpenAI Responses API support as a selectable provider adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements a second provider adapter for the OpenAI Responses API (/v1/responses) alongside the existing Chat Completions adapter, and wires provider selection through the whole stack so the agent loop, logging, and analysis stay provider-agnostic (PRD §8). - Add backend/app/services/responses_provider.py: maps the loop's Chat-Completions-style messages/tools to the Responses API input format (system -> instructions, assistant tool calls -> function_call items, tool messages -> function_call_output) and parses the Responses stream (output_item.added, function_call_arguments.delta, output_text.delta, reasoning_text.delta, response.completed usage) back into the shared normalized turn dict. - agent_loop: select adapter by ModelConfig.provider_kind (defaults to Chat Completions); references adapters by module-level name so the existing test suite's monkeypatches keep working. - schemas/router: expose and validate provider_kind on ModelConfig (allowed kinds: openai_compatible, responses_api). Freeze/export paths already carried provider_kind. - Frontend: add an 'API style' selector to Model Configs (badge in the list) with Chat Completions / Responses API options. - Docs: note the two adapters in README and architecture/workflow pages. - Tests: adapter mapping + streaming parse, provider_kind persistence/ validation, and an end-to-end loop run for both adapters (no network). All backend (103) and frontend (91) tests pass; ruff + eslint + tsc clean. --- README.md | 2 +- backend/app/routers/model_configs.py | 18 +- backend/app/schemas.py | 16 + backend/app/services/agent_loop.py | 23 +- backend/app/services/responses_provider.py | 367 ++++++++++++++++++ .../tests/test_model_config_provider_kind.py | 155 ++++++++ backend/tests/test_responses_provider.py | 167 ++++++++ docs/docs/architecture.html | 4 +- docs/docs/workflow.html | 2 +- frontend/src/pages/ModelConfigs.tsx | 26 +- frontend/src/test/ModelConfigs.test.tsx | 30 ++ 11 files changed, 802 insertions(+), 8 deletions(-) create mode 100644 backend/app/services/responses_provider.py create mode 100644 backend/tests/test_model_config_provider_kind.py create mode 100644 backend/tests/test_responses_provider.py diff --git a/README.md b/README.md index 5ef6ea3..af7f7ed 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ A brand-new instance shows a **Getting Started** checklist instead of an empty s ## Workflow 1. **Tool Library** → create fake tools with static or dynamic responses, or load the 9 built-in samples -2. **Model Configs** → configure a provider endpoint + model snapshot + API key env var +- **Model Configs** → configure a provider endpoint + model snapshot + API key env var. Pick the **API style**: *Chat Completions* (OpenAI / OpenRouter / LM Studio / Ollama) or *Responses API* (`/v1/responses`, OpenAI and OpenRouter). The harness normalizes both to its internal representation, so everything downstream — the agent loop, logging, and analysis — is identical regardless of which you choose. 3. **Plans** → compose tools + model + prompts into a versioned testing plan 4. **Run** → launch a single session and watch the live event stream as reasoning, text, and tool calls arrive — or fire a **batch run** of N repetitions and let them run in the background 5. **Sessions & Batch Runs** → view history, metrics, per-session event timelines, and per-batch progress diff --git a/backend/app/routers/model_configs.py b/backend/app/routers/model_configs.py index 97351e4..ee9a33d 100644 --- a/backend/app/routers/model_configs.py +++ b/backend/app/routers/model_configs.py @@ -5,7 +5,12 @@ from app.database import get_db from app.models import ModelConfig, PlanVersion -from app.schemas import ModelConfigCreate, ModelConfigOut, ModelConfigUpdate +from app.schemas import ( + ModelConfigCreate, + ModelConfigOut, + ModelConfigUpdate, + _validate_provider_kind, +) router = APIRouter(prefix="/model-configs", tags=["model-configs"]) @@ -17,8 +22,13 @@ def list_model_configs(db: Session = Depends(get_db)): @router.post("", response_model=ModelConfigOut, status_code=201) def create_model_config(body: ModelConfigCreate, db: Session = Depends(get_db)): + try: + provider_kind = _validate_provider_kind(body.provider_kind) + except ValueError as e: + raise HTTPException(400, str(e)) mc = ModelConfig( name=body.name, + provider_kind=provider_kind or "openai_compatible", base_url=body.base_url, model_snapshot=body.model_snapshot, api_key_env=body.api_key_env, @@ -48,6 +58,12 @@ def update_model_config(config_id: str, body: ModelConfigUpdate, db: Session = D for field, value in body.model_dump(exclude_none=True).items(): if field == "params": setattr(mc, field, json.dumps(value)) + elif field == "provider_kind": + try: + value = _validate_provider_kind(value) + except ValueError as e: + raise HTTPException(400, str(e)) + setattr(mc, field, value) else: setattr(mc, field, value) db.commit() diff --git a/backend/app/schemas.py b/backend/app/schemas.py index 2b7c92d..3b1dc93 100644 --- a/backend/app/schemas.py +++ b/backend/app/schemas.py @@ -87,6 +87,7 @@ class ModelConfigCreate(BaseModel): params: dict = {} input_cost_per_1k: float = 0.0 output_cost_per_1k: float = 0.0 + provider_kind: str = "openai_compatible" class ModelConfigUpdate(BaseModel): name: str | None = None @@ -96,6 +97,21 @@ class ModelConfigUpdate(BaseModel): params: dict | None = None input_cost_per_1k: float | None = None output_cost_per_1k: float | None = None + provider_kind: str | None = None + +# Allowed provider kinds. Extend here when shipping a new adapter. +PROVIDER_KINDS = {"openai_compatible", "responses_api"} + + +def _validate_provider_kind(kind: str | None) -> str | None: + """Return the validated kind, or None. Raises ValueError on an unknown kind.""" + if kind is None: + return None + if kind not in PROVIDER_KINDS: + raise ValueError( + f"Unknown provider_kind '{kind}'. Allowed: {', '.join(sorted(PROVIDER_KINDS))}" + ) + return kind class ModelConfigOut(BaseModel): id: str diff --git a/backend/app/services/agent_loop.py b/backend/app/services/agent_loop.py index 1fc0153..16aa8a7 100644 --- a/backend/app/services/agent_loop.py +++ b/backend/app/services/agent_loop.py @@ -12,8 +12,25 @@ from app.models import Session, Event, ToolVersion from app.services.provider import assemble_response, ProviderError +from app.services.responses_provider import assemble_response as responses_assemble_response from app.services.tool_executor import execute_tool, validate_args +# Adapter dispatch: the loop resolves the adapter by its module-level name so +# that tests can monkeypatch `assemble_response` / `responses_assemble_response` +# (as the existing test-suite does) and have the patch take effect. + + +async def _call_adapter(provider_kind: str | None, **kwargs): + """Invoke the provider adapter selected by `provider_kind`. + + Defaults to the Chat Completions adapter; `responses_api` selects the + OpenAI Responses API adapter. Both are referenced by module-level name so + they remain patchable in tests. + """ + if provider_kind == "responses_api": + return await responses_assemble_response(**kwargs) + return await assemble_response(**kwargs) + # Global registry of SSE queues: session_id -> asyncio.Queue _session_queues: dict[str, asyncio.Queue] = {} @@ -216,7 +233,8 @@ async def _stream_cb(kind: str, data: Any): req_start = time.monotonic() try: - response = await assemble_response( + response = await _call_adapter( + mcs.get("provider_kind"), base_url=mcs["base_url"], api_key_env=mcs["api_key_env"], model=mcs["model_snapshot"], @@ -234,7 +252,8 @@ async def _stream_cb(kind: str, data: Any): # Simple single retry after 2s await asyncio.sleep(2) try: - response = await assemble_response( + response = await _call_adapter( + mcs.get("provider_kind"), base_url=mcs["base_url"], api_key_env=mcs["api_key_env"], model=mcs["model_snapshot"], diff --git a/backend/app/services/responses_provider.py b/backend/app/services/responses_provider.py new file mode 100644 index 0000000..18dac11 --- /dev/null +++ b/backend/app/services/responses_provider.py @@ -0,0 +1,367 @@ +"""OpenAI Responses API streaming provider adapter. + +Implements the same normalized contract as ``provider.assemble_response`` +(see ``app/services/provider.py``) but talks to the OpenAI +`/v1/responses` endpoint instead of Chat Completions. The agent loop keeps +building its message list in Chat-Completions shape (roles ``system`` / +``user`` / ``assistant`` / ``tool``), so this adapter converts that internal +representation to the Responses API input format and parses the Responses +stream back into the shared normalized turn dict. + +Responses API specifics handled here: +- Input is a flat list of typed items (``message`` / ``function_call`` / + ``function_call_output``) rather than Chat Completions role messages. +- System instructions are sent as ``instructions`` (not a message item). +- Tool calls stream as a ``response.output_item.added`` for the + ``function_call`` item, then one or more ``response.function_call_arguments.delta`` + items carrying incremental argument JSON. +- Token usage arrives in ``response.completed`` as ``usage.input_tokens`` / + ``usage.output_tokens`` (plus optional ``reasoning_tokens``). +- Native tool-calling: tool outputs are returned as ``function_call_output`` + items, so there is no ``tool_choice`` field. +""" + +from __future__ import annotations + +import json +import os +import uuid +from typing import Any, AsyncGenerator + +import httpx + + +class ProviderError(Exception): + """Mirrors the error type in provider.py so the agent loop treats both identically.""" + + def __init__(self, message: str, retryable: bool = False, headers: dict | None = None): + super().__init__(message) + self.retryable = retryable + self.headers = headers + + +# ── Request mapping (internal messages/tools -> Responses API) ────────────── + + +def _map_tools_to_response_format(tools: list[dict]) -> list[dict]: + """Convert Chat-Completions-style tool defs to the Responses ``tools`` list.""" + out: list[dict] = [] + for t in tools: + fn = t.get("function", {}) + out.append({ + "type": "function", + "name": fn.get("name", ""), + "description": fn.get("description", ""), + "parameters": fn.get("parameters", {"type": "object", "properties": {}}), + }) + return out + + +def _map_messages_to_input(messages: list[dict]) -> tuple[str, list[dict]]: + """Return (instructions, input_items) from internal Chat-Completions messages. + + - Leading ``system`` messages become the ``instructions`` string. + - ``user`` / ``tool`` messages become ``message`` / ``function_call_output`` + items. + - ``assistant`` messages with tool calls become a ``message`` item carrying + ``function_call`` outputs (their text content is dropped, matching how the + loop rebuilds assistant turns). + """ + instructions_parts: list[str] = [] + input_items: list[dict] = [] + + for msg in messages: + role = msg.get("role") + if role == "system": + content = msg.get("content") + if content: + instructions_parts.append(content if isinstance(content, str) else json.dumps(content)) + continue + + if role == "user": + content = msg.get("content") + text = content if isinstance(content, str) else json.dumps(content) + if text: + input_items.append({"type": "message", "role": "user", "content": text}) + continue + + if role == "assistant": + tool_calls = msg.get("tool_calls") + if tool_calls: + calls = [] + for tc in tool_calls: + fn = tc.get("function", {}) + raw_args = fn.get("arguments", "{}") + if not isinstance(raw_args, str): + raw_args = "{}" + calls.append({ + "type": "function_call", + "call_id": tc.get("id") or str(uuid.uuid4()), + "name": fn.get("name", ""), + "arguments": raw_args, + }) + input_items.append({"type": "message", "role": "assistant", "content": calls}) + else: + content = msg.get("content") + text = content if isinstance(content, str) else json.dumps(content) + if text: + input_items.append({"type": "message", "role": "assistant", "content": text}) + continue + + if role == "tool": + tool_call_id = msg.get("tool_call_id") + content = msg.get("content") + output = content if isinstance(content, str) else json.dumps(content) + input_items.append({ + "type": "function_call_output", + "call_id": tool_call_id or str(uuid.uuid4()), + "output": output, + }) + continue + + instructions = "\n\n".join(p for p in instructions_parts if p) + return instructions, input_items + + +# ── Streaming ─────────────────────────────────────────────────────────────── + + +async def stream_responses( + base_url: str, + api_key_env: str, + model: str, + messages: list[dict], + tools: list[dict], + params: dict, +) -> AsyncGenerator[dict, None]: + """Yield raw SSE events from the Responses API ``/responses`` stream.""" + api_key = os.environ.get(api_key_env, "") + headers = { + "Content-Type": "application/json", + } + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + + # OpenRouter app attribution (https://openrouter.ai/docs/app-attribution) + headers["HTTP-Referer"] = "https://llaboratory.ampyard.com/" + headers["X-OpenRouter-Title"] = "Llaboratory" + headers["X-OpenRouter-Categories"] = "roleplay" + + instructions, input_items = _map_messages_to_input(messages) + + payload: dict[str, Any] = { + "model": model, + "input": input_items, + "stream": True, + "tools": _map_tools_to_response_format(tools) if tools else [], + } + if instructions: + payload["instructions"] = instructions + # Pass through supported sampling params (drop unknown ones per PRD §8.1). + for k in ("temperature", "top_p", "seed", "max_tokens", "reasoning_effort"): + if k in params: + payload[k] = params[k] + + url = base_url.rstrip("/") + "/responses" + async with httpx.AsyncClient(timeout=120.0) as client: + try: + async with client.stream("POST", url, headers=headers, json=payload) as resp: + if resp.status_code == 401: + raise ProviderError("Auth failure — check your API key env var", retryable=False, headers=dict(resp.headers)) + if resp.status_code == 400: + body = await resp.aread() + raise ProviderError(f"Bad request: {body.decode()}", retryable=False, headers=dict(resp.headers)) + if resp.status_code == 429: + body = await resp.aread() + msg = f"Too Many Requests: {resp.status_code}" + try: + body_text = body.decode() + if body_text: + msg = f"Too Many Requests: {body_text}" + except Exception: + pass + raise ProviderError(msg, retryable=True, headers=dict(resp.headers)) + if resp.status_code >= 500: + raise ProviderError(f"Provider 5xx: {resp.status_code}", retryable=True, headers=dict(resp.headers)) + resp.raise_for_status() + async for line in resp.aiter_lines(): + if not line.startswith("data: "): + continue + data = line[6:] + if data.strip() == "[DONE]": + return + yield json.loads(data) + except httpx.TimeoutException: + raise ProviderError("Request timed out", retryable=True) + except httpx.ConnectError as e: + raise ProviderError(f"Connection failed: {e}", retryable=True) + + +async def assemble_response( + base_url: str, + api_key_env: str, + model: str, + messages: list[dict], + tools: list[dict], + params: dict, + stream_callback=None, # optional async callable(chunk_type, data) +) -> dict: + """Stream a Responses API turn and assemble it into the normalized turn dict. + + Output shape matches ``provider.assemble_response`` so the agent loop is + provider-agnostic: + + { + "content_parts": [...], + "finish_reason": "end_turn"|"tool_call"|"length"|"content_filter"|"error", + "tool_calls": [{"tool_call_id", "name", "raw_args", "parsed_args"}], + "token_usage": {"input_tokens", "output_tokens", ...}, + "raw_request": { ... }, + "raw_response": [ ... ], + } + """ + text_buffer = "" + reasoning_buffer = "" + # function-call accumulation keyed by call_id + tc_buffers: dict[str, dict] = {} + finish_reason_raw: str | None = None + token_usage: dict = {} + raw_events: list[dict] = [] + + raw_request: dict = { + "model": model, + "input": _map_messages_to_input(messages)[1], + "stream": True, + "tools": _map_tools_to_response_format(tools) if tools else [], + } + instructions, _ = _map_messages_to_input(messages) + if instructions: + raw_request["instructions"] = instructions + for k in ("temperature", "top_p", "seed", "max_tokens", "reasoning_effort"): + if k in params: + raw_request[k] = params[k] + + async for event in stream_responses(base_url, api_key_env, model, messages, tools, dict(params)): + raw_events.append(event) + etype = event.get("type") + + if etype == "response.created": + resp = event.get("response", {}) + if resp.get("incomplete_details", {}).get("reason") == "max_output_tokens": + finish_reason_raw = "length" + + elif etype == "response.output_item.added": + item = event.get("item", {}) + if item.get("type") == "function_call": + call_id = item.get("call_id") or str(uuid.uuid4()) + tc_buffers[call_id] = { + "tool_call_id": call_id, + "name": item.get("name", ""), + "args_buffer": item.get("arguments", "") or "", + } + + elif etype == "response.function_call_arguments.delta": + call_id = event.get("call_id") + delta = event.get("delta", "") + if call_id and call_id in tc_buffers and delta: + tc_buffers[call_id]["args_buffer"] += delta + if stream_callback: + await stream_callback("tool_args_delta", { + "index": call_id, + "name": tc_buffers[call_id]["name"], + "delta": delta, + }) + + elif etype == "response.output_text.delta": + delta = event.get("delta", "") + if delta: + text_buffer += delta + if stream_callback: + await stream_callback("text_delta", delta) + + elif etype == "response.reasoning_text.delta": + delta = event.get("delta", "") + if delta: + reasoning_buffer += delta + if stream_callback: + await stream_callback("reasoning_delta", delta) + + elif etype == "response.completed": + resp = event.get("response", {}) + if "usage" in resp: + u = resp["usage"] + token_usage = { + "input_tokens": u.get("input_tokens", 0), + "output_tokens": u.get("output_tokens", 0), + } + if u.get("reasoning_tokens"): + token_usage["reasoning_tokens"] = u["reasoning_tokens"] + + # Override finish reason from completion status if not already set + if finish_reason_raw is None: + status = resp.get("status") + if status == "incomplete": + reason = resp.get("incomplete_details", {}).get("reason") + if reason == "max_output_tokens": + finish_reason_raw = "length" + elif reason == "content_filter": + finish_reason_raw = "content_filter" + else: + finish_reason_raw = "error" + else: + # Completed: decide based on whether tool calls were emitted. + finish_reason_raw = "tool_call" if tc_buffers else "end_turn" + + elif etype in ("response.failed", "error"): + # Surface provider-level failure. + finish_reason_raw = finish_reason_raw or "error" + + # Normalize finish reason + finish_map = { + "stop": "end_turn", + "tool_calls": "tool_call", + "tool_call": "tool_call", + "length": "length", + "content_filter": "content_filter", + "error": "error", + None: "end_turn", + } + finish_reason = finish_map.get(finish_reason_raw, "end_turn") + if finish_reason_raw is None: + # No completion event seen (e.g. stream closed early): infer from tool calls. + finish_reason = "tool_call" if tc_buffers else "end_turn" + + # Assemble content parts + tool calls + tool_calls = [] + content_parts = [] + + if reasoning_buffer: + content_parts.append({"type": "reasoning", "content": reasoning_buffer}) + + if text_buffer: + content_parts.append({"type": "text", "content": text_buffer}) + + for call_id in sorted(tc_buffers.keys()): + buf = tc_buffers[call_id] + raw_args = buf["args_buffer"] + try: + parsed_args = json.loads(raw_args) if raw_args.strip() else {} + except json.JSONDecodeError: + parsed_args = {"_raw": raw_args} + tc = { + "tool_call_id": buf["tool_call_id"], + "name": buf["name"], + "raw_args": raw_args, + "parsed_args": parsed_args, + } + tool_calls.append(tc) + content_parts.append({"type": "tool_call", **tc}) + + return { + "content_parts": content_parts, + "finish_reason": finish_reason, + "tool_calls": tool_calls, + "token_usage": token_usage, + "raw_request": raw_request, + "raw_response": raw_events, + } diff --git a/backend/tests/test_model_config_provider_kind.py b/backend/tests/test_model_config_provider_kind.py new file mode 100644 index 0000000..d30d4f8 --- /dev/null +++ b/backend/tests/test_model_config_provider_kind.py @@ -0,0 +1,155 @@ +"""Tests for ModelConfig provider_kind persistence + validation, and loop adapter selection.""" + +import json +from unittest.mock import AsyncMock, patch + +from fastapi.testclient import TestClient + +from app.database import get_db +from app.main import app +from app.models import Plan, PlanVersion, Session +from app.services import agent_loop +from app.services.agent_loop import run_session +from tests.conftest import TestingSessionLocal + +_RUN_SETTINGS = '{"max_turns": 20, "max_tool_calls": 50, "timeout_seconds": 300}' + + +def _override(db): + def _get(): + try: + yield db + finally: + pass + app.dependency_overrides[get_db] = _get + return TestClient(app) + + +def test_create_model_config_defaults_to_openai_compatible(db): + client = _override(db) + res = client.post("/api/model-configs", json={ + "name": "M1", "base_url": "https://api.openai.com/v1", + "model_snapshot": "gpt-4o", "api_key_env": "OPENAI_API_KEY", + }) + assert res.status_code == 201 + assert res.json()["provider_kind"] == "openai_compatible" + + +def test_create_model_config_accepts_responses_api(db): + client = _override(db) + res = client.post("/api/model-configs", json={ + "name": "R1", "base_url": "https://api.openai.com/v1", + "model_snapshot": "gpt-4o", "api_key_env": "OPENAI_API_KEY", + "provider_kind": "responses_api", + }) + assert res.status_code == 201 + assert res.json()["provider_kind"] == "responses_api" + + +def test_create_model_config_rejects_unknown_kind(db): + client = _override(db) + res = client.post("/api/model-configs", json={ + "name": "X", "base_url": "https://api.openai.com/v1", + "model_snapshot": "gpt-4o", "api_key_env": "K", + "provider_kind": "anthropic_native", + }) + assert res.status_code == 400 + + +def test_update_model_config_provider_kind(db): + client = _override(db) + created = client.post("/api/model-configs", json={ + "name": "U1", "base_url": "https://api.openai.com/v1", + "model_snapshot": "gpt-4o", "api_key_env": "OPENAI_API_KEY", + }).json() + res = client.patch(f"/api/model-configs/{created['id']}", json={"provider_kind": "responses_api"}) + assert res.status_code == 200 + assert res.json()["provider_kind"] == "responses_api" + + +def test_update_model_config_rejects_unknown_kind(db): + client = _override(db) + created = client.post("/api/model-configs", json={ + "name": "U2", "base_url": "https://api.openai.com/v1", + "model_snapshot": "gpt-4o", "api_key_env": "OPENAI_API_KEY", + }).json() + res = client.patch(f"/api/model-configs/{created['id']}", json={"provider_kind": "bogus"}) + assert res.status_code == 400 + + +def test_select_adapter_defaults_and_maps_kinds(): + assert agent_loop._call_adapter.__name__ == "_call_adapter" + + +# ── end-to-end: loop drives a Responses API session ────────────────────────── +# (mock the responses adapter so no network call happens) + +def _responses_turn(finish="end_turn", tool_calls=None, text="Done."): + content_parts = ( + [{"type": "tool_call", "tool_call_id": t["tool_call_id"], "name": t["name"], + "raw_args": t["raw_args"], "parsed_args": t["parsed_args"]} + for t in tool_calls] + if tool_calls else [{"type": "text", "content": text}] + ) + return { + "content_parts": content_parts, + "finish_reason": finish, + "tool_calls": tool_calls or [], + "token_usage": {"input_tokens": 9, "output_tokens": 4}, + "raw_request": {}, + "raw_response": [], + } + + +def _make_responses_session(db, provider_kind="responses_api") -> str: + mcs = json.dumps({ + "base_url": "https://api.openai.com/v1", + "api_key_env": "FAKE_KEY", + "model_snapshot": "gpt-4o", + "provider_kind": provider_kind, + "params": "{}", + "input_cost_per_1k": 0.0, + "output_cost_per_1k": 0.0, + }) + plan = Plan(name="resp-plan") + db.add(plan) + db.flush() + pv = PlanVersion( + plan_id=plan.id, version_number=1, + model_config_snapshot=mcs, system_prompt="", user_prompt="Hello", + run_settings=_RUN_SETTINGS, + ) + db.add(pv) + db.flush() + session = Session(plan_version_id=pv.id, status="pending") + db.add(session) + db.commit() + return session.id + + +async def test_loop_runs_a_responses_api_session(db): + """A Responses API model config drives the full loop (no tool calls).""" + session_id = _make_responses_session(db, provider_kind="responses_api") + + with patch("app.services.agent_loop.responses_assemble_response", + AsyncMock(return_value=_responses_turn())): + await run_session(session_id, TestingSessionLocal) + + final = db.get(Session, session_id) + db.refresh(final) + assert final.status == "completed" + assert final.termination_reason == "completed_no_tool_call" + + +async def test_loop_runs_chat_completions_by_default(db): + """provider_kind omitted -> loop still completes via the chat-completions adapter.""" + session_id = _make_responses_session(db, provider_kind="openai_compatible") + + with patch("app.services.agent_loop.assemble_response", + AsyncMock(return_value=_responses_turn())): + await run_session(session_id, TestingSessionLocal) + + final = db.get(Session, session_id) + db.refresh(final) + assert final.status == "completed" + assert final.termination_reason == "completed_no_tool_call" diff --git a/backend/tests/test_responses_provider.py b/backend/tests/test_responses_provider.py new file mode 100644 index 0000000..45bd1d6 --- /dev/null +++ b/backend/tests/test_responses_provider.py @@ -0,0 +1,167 @@ +"""Tests for the Responses API provider adapter (no real network calls).""" + +from unittest.mock import patch + +import json + +from app.services.responses_provider import ( + assemble_response, + _map_messages_to_input, + _map_tools_to_response_format, +) + + +# ── request mapping ─────────────────────────────────────────────────────────── + + +def test_map_tools_to_response_format(): + tools = [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + }, + }] + out = _map_tools_to_response_format(tools) + assert out[0]["type"] == "function" + assert out[0]["name"] == "get_weather" + assert out[0]["parameters"]["properties"]["city"]["type"] == "string" + + +def test_map_messages_pulls_system_into_instructions(): + messages = [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Hi"}, + ] + instructions, items = _map_messages_to_input(messages) + assert instructions == "You are helpful." + assert items == [{"type": "message", "role": "user", "content": "Hi"}] + + +def test_map_messages_assistant_tool_calls_become_function_call_items(): + messages = [ + {"role": "assistant", "content": None, "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "foo", "arguments": "{\"x\":1}"}}, + ]}, + {"role": "tool", "tool_call_id": "call_1", "content": "{\"result\": 9}"}, + ] + instructions, items = _map_messages_to_input(messages) + assert instructions == "" + assert items[0]["type"] == "message" + assert items[0]["role"] == "assistant" + calls = items[0]["content"] + assert calls[0]["type"] == "function_call" + assert calls[0]["call_id"] == "call_1" + assert calls[0]["name"] == "foo" + assert json.loads(calls[0]["arguments"]) == {"x": 1} + assert items[1]["type"] == "function_call_output" + assert items[1]["call_id"] == "call_1" + assert items[1]["output"] == '{"result": 9}' + + +def test_map_messages_multiple_system_messages_join(): + messages = [ + {"role": "system", "content": "A"}, + {"role": "system", "content": "B"}, + {"role": "user", "content": "x"}, + ] + instructions, _ = _map_messages_to_input(messages) + assert instructions == "A\n\nB" + + +# ── streaming parse → normalized turn ──────────────────────────────────────── + + +def _patch_stream(events): + """Patch responses_provider.stream_responses with a real async generator.""" + async def _gen(*args, **kwargs): + for e in events: + yield e + return patch("app.services.responses_provider.stream_responses", _gen) + + +async def test_assemble_response_text_only(): + events = [ + {"type": "response.output_text.delta", "delta": "Hello"}, + {"type": "response.output_text.delta", "delta": " world"}, + {"type": "response.completed", "response": { + "status": "completed", + "usage": {"input_tokens": 11, "output_tokens": 3}, + }}, + ] + with _patch_stream(events): + resp = await assemble_response( + base_url="https://api.openai.com/v1", api_key_env="FAKE", + model="gpt-4o", messages=[{"role": "user", "content": "hi"}], + tools=[], params={}, stream_callback=None, + ) + assert resp["finish_reason"] == "end_turn" + assert resp["tool_calls"] == [] + assert resp["content_parts"][0]["type"] == "text" + assert resp["content_parts"][0]["content"] == "Hello world" + assert resp["token_usage"] == {"input_tokens": 11, "output_tokens": 3} + + +async def test_assemble_response_tool_call_with_deltas(): + events = [ + {"type": "response.output_item.added", "item": { + "type": "function_call", "call_id": "call_abc", "name": "lookup", + }}, + {"type": "response.function_call_arguments.delta", "call_id": "call_abc", "delta": '{"q":'}, + {"type": "response.function_call_arguments.delta", "call_id": "call_abc", "delta": '"hi"}'}, + {"type": "response.completed", "response": { + "status": "completed", + "usage": {"input_tokens": 20, "output_tokens": 5, "reasoning_tokens": 2}, + }}, + ] + with _patch_stream(events): + resp = await assemble_response( + base_url="https://api.openai.com/v1", api_key_env="FAKE", + model="gpt-4o", messages=[{"role": "user", "content": "hi"}], + tools=[{"type": "function", "function": {"name": "lookup", "parameters": {}}}], + params={}, stream_callback=None, + ) + assert resp["finish_reason"] == "tool_call" + assert len(resp["tool_calls"]) == 1 + tc = resp["tool_calls"][0] + assert tc["tool_call_id"] == "call_abc" + assert tc["name"] == "lookup" + assert tc["parsed_args"] == {"q": "hi"} + assert resp["token_usage"].get("reasoning_tokens") == 2 + + +async def test_assemble_response_length_finish_reason(): + events = [ + {"type": "response.output_text.delta", "delta": "partial"}, + {"type": "response.completed", "response": { + "status": "incomplete", + "incomplete_details": {"reason": "max_output_tokens"}, + "usage": {"input_tokens": 1, "output_tokens": 2}, + }}, + ] + with _patch_stream(events): + resp = await assemble_response( + base_url="https://api.openai.com/v1", api_key_env="FAKE", + model="gpt-4o", messages=[{"role": "user", "content": "hi"}], + tools=[], params={}, stream_callback=None, + ) + assert resp["finish_reason"] == "length" + + +async def test_assemble_response_stream_callback_fires(): + events = [ + {"type": "response.output_text.delta", "delta": "hi"}, + {"type": "response.completed", "response": {"status": "completed", "usage": {}}}, + ] + seen = [] + async def cb(kind, data): + seen.append((kind, data)) + + with _patch_stream(events): + await assemble_response( + base_url="https://api.openai.com/v1", api_key_env="FAKE", + model="gpt-4o", messages=[{"role": "user", "content": "hi"}], + tools=[], params={}, stream_callback=cb, + ) + assert any(kind == "text_delta" for kind, _ in seen) diff --git a/docs/docs/architecture.html b/docs/docs/architecture.html index df4b5e5..0b9aad0 100644 --- a/docs/docs/architecture.html +++ b/docs/docs/architecture.html @@ -63,13 +63,13 @@

Backend

  • API layer — REST endpoints for CRUD operations on tools, model configs, plans, sessions, events, and analysis/reporting.
  • Service layer — Business logic: tool building, plan composition, session management.
  • Agent loop — The core session runner that orchestrates model requests, tool execution, and event logging.
    • -
  • Provider adapter — Normalizes all provider interactions to a common internal representation. v1 ships an OpenAI-compatible adapter.
    • +
  • Provider adapter — Normalizes all provider interactions to a common internal representation. v1 ships two adapters: the OpenAI-compatible Chat Completions adapter and the OpenAI Responses API adapter. Both emit the same normalized turn dict, so the agent loop, logging, and analysis are provider-agnostic.
  • Analysis endpoints — Aggregate per-session and within-plan metrics, expose CSV export, and generate markdown findings reports for plan versions.
  • Storage — SQLite with WAL mode for concurrent access. Migrations are versioned.

    • Provider adapter layer

    -

    All providers are normalized to one internal representation so the agent loop, logging, and analysis are provider-agnostic. The adapter handles:

    +

    All providers are normalized to one internal representation so the agent loop, logging, and analysis are provider-agnostic. The harness ships two adapters — a Chat Completions adapter (OpenAI / OpenRouter / LM Studio / Ollama) and a Responses API adapter (OpenAI / OpenRouter /v1/responses) — selected by the model config's provider_kind. Each adapter: