Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ A brand-new instance shows a **Getting Started** checklist instead of an empty s
## Workflow

1. **Tool Library** → create fake tools with static or dynamic responses, or load the 9 built-in samples
2. **Model Configs** → configure a provider endpoint + model snapshot + API key env var
- **Model Configs** → configure a provider endpoint + model snapshot + API key env var. Pick the **API style**: *Chat Completions* (OpenAI / OpenRouter / LM Studio / Ollama) or *Responses API* (`/v1/responses`, OpenAI and OpenRouter). The harness normalizes both to its internal representation, so everything downstream — the agent loop, logging, and analysis — is identical regardless of which you choose.
3. **Plans** → compose tools + model + prompts into a versioned testing plan
4. **Run** → launch a single session and watch the live event stream as reasoning, text, and tool calls arrive — or fire a **batch run** of N repetitions and let them run in the background
5. **Sessions & Batch Runs** → view history, metrics, per-session event timelines, and per-batch progress
Expand Down
18 changes: 17 additions & 1 deletion backend/app/routers/model_configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,12 @@

from app.database import get_db
from app.models import ModelConfig, PlanVersion
from app.schemas import ModelConfigCreate, ModelConfigOut, ModelConfigUpdate
from app.schemas import (
ModelConfigCreate,
ModelConfigOut,
ModelConfigUpdate,
_validate_provider_kind,
)

router = APIRouter(prefix="/model-configs", tags=["model-configs"])

Expand All @@ -17,8 +22,13 @@ def list_model_configs(db: Session = Depends(get_db)):

@router.post("", response_model=ModelConfigOut, status_code=201)
def create_model_config(body: ModelConfigCreate, db: Session = Depends(get_db)):
try:
provider_kind = _validate_provider_kind(body.provider_kind)
except ValueError as e:
raise HTTPException(400, str(e))
mc = ModelConfig(
name=body.name,
provider_kind=provider_kind or "openai_compatible",
base_url=body.base_url,
model_snapshot=body.model_snapshot,
api_key_env=body.api_key_env,
Expand Down Expand Up @@ -48,6 +58,12 @@ def update_model_config(config_id: str, body: ModelConfigUpdate, db: Session = D
for field, value in body.model_dump(exclude_none=True).items():
if field == "params":
setattr(mc, field, json.dumps(value))
elif field == "provider_kind":
try:
value = _validate_provider_kind(value)
except ValueError as e:
raise HTTPException(400, str(e))
setattr(mc, field, value)
else:
setattr(mc, field, value)
db.commit()
Expand Down
16 changes: 16 additions & 0 deletions backend/app/schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ class ModelConfigCreate(BaseModel):
params: dict = {}
input_cost_per_1k: float = 0.0
output_cost_per_1k: float = 0.0
provider_kind: str = "openai_compatible"

class ModelConfigUpdate(BaseModel):
name: str | None = None
Expand All @@ -96,6 +97,21 @@ class ModelConfigUpdate(BaseModel):
params: dict | None = None
input_cost_per_1k: float | None = None
output_cost_per_1k: float | None = None
provider_kind: str | None = None

# Allowed provider kinds. Extend here when shipping a new adapter.
PROVIDER_KINDS = {"openai_compatible", "responses_api"}


def _validate_provider_kind(kind: str | None) -> str | None:
"""Return the validated kind, or None. Raises ValueError on an unknown kind."""
if kind is None:
return None
if kind not in PROVIDER_KINDS:
raise ValueError(
f"Unknown provider_kind '{kind}'. Allowed: {', '.join(sorted(PROVIDER_KINDS))}"
)
return kind

class ModelConfigOut(BaseModel):
id: str
Expand Down
23 changes: 21 additions & 2 deletions backend/app/services/agent_loop.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,25 @@

from app.models import Session, Event, ToolVersion
from app.services.provider import assemble_response, ProviderError
from app.services.responses_provider import assemble_response as responses_assemble_response
from app.services.tool_executor import execute_tool, validate_args

# Adapter dispatch: the loop resolves the adapter by its module-level name so
# that tests can monkeypatch `assemble_response` / `responses_assemble_response`
# (as the existing test-suite does) and have the patch take effect.


async def _call_adapter(provider_kind: str | None, **kwargs):
"""Invoke the provider adapter selected by `provider_kind`.

Defaults to the Chat Completions adapter; `responses_api` selects the
OpenAI Responses API adapter. Both are referenced by module-level name so
they remain patchable in tests.
"""
if provider_kind == "responses_api":
return await responses_assemble_response(**kwargs)
return await assemble_response(**kwargs)

# Global registry of SSE queues: session_id -> asyncio.Queue
_session_queues: dict[str, asyncio.Queue] = {}

Expand Down Expand Up @@ -216,7 +233,8 @@ async def _stream_cb(kind: str, data: Any):

req_start = time.monotonic()
try:
response = await assemble_response(
response = await _call_adapter(
mcs.get("provider_kind"),
base_url=mcs["base_url"],
api_key_env=mcs["api_key_env"],
model=mcs["model_snapshot"],
Expand All @@ -234,7 +252,8 @@ async def _stream_cb(kind: str, data: Any):
# Simple single retry after 2s
await asyncio.sleep(2)
try:
response = await assemble_response(
response = await _call_adapter(
mcs.get("provider_kind"),
base_url=mcs["base_url"],
api_key_env=mcs["api_key_env"],
model=mcs["model_snapshot"],
Expand Down
Loading
Loading