diff --git a/src/agentpool_server/acp_server/acp_agent.py b/src/agentpool_server/acp_server/acp_agent.py index aa0f48c43..1bd800b78 100644 --- a/src/agentpool_server/acp_server/acp_agent.py +++ b/src/agentpool_server/acp_server/acp_agent.py @@ -61,13 +61,15 @@ async def get_session_model_state(agent: BaseAgent) -> SessionModelState | None: - """Get SessionModelState from an agent using its get_available_models() method. + """Get SessionModelState from an agent, including configured variants. - Converts tokonomics ModelInfo to ACP ModelInfo format. + Converts tokonomics ModelInfo to ACP ModelInfo format, then merges + with configured model variants from the manifest. + + Configured variants take precedence over discovered models with the same ID. Args: agent: Any agent with get_available_models() method - current_model: Currently active model ID (defaults to first available) Returns: SessionModelState with all available models, None if no models available @@ -78,8 +80,22 @@ async def get_session_model_state(agent: BaseAgent) -> SessionModelState | None: logger.exception("Failed to get available models from agent") return None + # Get configured variants from manifest + configured = [] + agent_pool = getattr(agent, "agent_pool", None) + manifest = getattr(agent_pool, "manifest", None) if agent_pool else None + + if manifest and manifest.model_variants: + configured = [ + ACPModelInfo( + model_id=name, + name=name, + ) + for name in manifest.model_variants + ] + # Convert tokonomics ModelInfo to ACP ModelInfo - acp_models = [ + acp_models_from_tokonomics = [ ACPModelInfo( model_id=toko.id_override if toko.id_override else toko.id, name=toko.name, @@ -87,21 +103,36 @@ async def get_session_model_state(agent: BaseAgent) -> SessionModelState | None: ) for toko in toko_models or [] ] - if not acp_models: + + # Merge lists (configured variants take precedence over discovered) + all_models: dict[str, ACPModelInfo] = {} + + # Add tokonomics models first + for model in acp_models_from_tokonomics: + all_models[model.model_id] = model + + # Override/add configured variants (they take precedence) + for model in configured: + all_models[model.model_id] = model + + if not all_models: return None + # Ensure current model is in the list - all_ids = [model.model_id for model in acp_models] + acp_models_list = list(all_models.values()) + all_ids = [model.model_id for model in acp_models_list] current_model = agent.model_name + if current_model and current_model not in all_ids: # Add current model to the list so the UI shows it desc = "Currently configured model" model_info = ACPModelInfo(model_id=current_model, name=current_model, description=desc) - acp_models.insert(0, model_info) + acp_models_list.insert(0, model_info) current_model_id = current_model else: current_model_id = current_model if current_model in all_ids else all_ids[0] - return SessionModelState(available_models=acp_models, current_model_id=current_model_id) + return SessionModelState(available_models=acp_models_list, current_model_id=current_model_id) async def get_session_mode_state(agent: BaseAgent) -> SessionModeState | None: @@ -595,7 +626,8 @@ async def set_session_model( """Set the session model. Changes the model for the active agent in the session. - Validates that the requested model is available via agent.get_available_models(). + Validates that the requested model is available via agent.get_available_models() + OR is a configured variant in the manifest. """ session = self.session_manager.get_session(params.session_id) if not session: @@ -603,13 +635,29 @@ async def set_session_model( logger.warning(msg, session_id=params.session_id) return None try: - # Get available models from agent and validate + # Build list of valid model IDs from both tokonomics and configured variants + valid_model_ids: set[str] = set() + + # Get tokonomics models if toko_models := await session.agent.get_available_models(): - # Build list of valid model IDs (using id_override if set) - ids = [m.id_override if m.id_override else m.id for m in toko_models] - if (id_ := params.model_id) not in ids: - logger.warning("Model not in available models", model_id=id_, available=ids) - return None + valid_model_ids.update( + m.id_override if m.id_override else m.id for m in toko_models + ) + + # Get configured variants from manifest + agent_pool = getattr(session.agent, "agent_pool", None) + manifest = getattr(agent_pool, "manifest", None) if agent_pool else None + if manifest and manifest.model_variants: + valid_model_ids.update(manifest.model_variants.keys()) + + # Validate the requested model + if params.model_id not in valid_model_ids: + logger.warning( + "Model not in available models", + model_id=params.model_id, + available=list(valid_model_ids), + ) + return None # Set the model on the agent (all agents now have async set_model) await session.agent.set_model(params.model_id) diff --git a/src/agentpool_server/opencode_server/routes/config_routes.py b/src/agentpool_server/opencode_server/routes/config_routes.py index 248bf86f0..c786e8b40 100644 --- a/src/agentpool_server/opencode_server/routes/config_routes.py +++ b/src/agentpool_server/opencode_server/routes/config_routes.py @@ -4,11 +4,13 @@ from collections import defaultdict from datetime import timedelta +import logging import os -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from fastapi import APIRouter +from agentpool.models.manifest import AgentsManifest from agentpool_server.opencode_server.dependencies import StateDep from agentpool_server.opencode_server.models import ( Config, @@ -20,6 +22,19 @@ ProviderListResponse, ProvidersResponse, ) +from agentpool_server.shared.constants import ( + DEFAULT_MODEL_CONTEXT_LIMIT, + DEFAULT_MODEL_INPUT_COST, + DEFAULT_MODEL_OUTPUT_COST, + DEFAULT_MODEL_OUTPUT_LIMIT, +) +from agentpool_server.shared.model_utils import ( + _build_providers_from_tokonomics, + _extract_provider, +) + + +logger = logging.getLogger(__name__) if TYPE_CHECKING: @@ -85,6 +100,155 @@ async def _get_available_models() -> list[TokoModelInfo]: return await get_all_models(max_age=max_age) +async def _get_configured_variants( + manifest: AgentsManifest | None, +) -> dict[str, dict[str, Any]]: + """Get model variants from manifest configuration. + + Returns empty dict if manifest or model_variants is None/empty. + + Args: + manifest: The agents manifest containing model_variants configuration. + + Returns: + Dictionary mapping variant names to their config dicts with provider info. + """ + variants: dict[str, dict[str, Any]] = {} + + # Check manifest model_variants + if manifest and manifest.model_variants: + for name, config in manifest.model_variants.items(): + variants[name] = { + "provider": _extract_provider(config), + # Note: variant-specific settings (temp, thinking) are not exposed to clients; + # they are applied internally by the agent + } + + return variants + + +def _build_providers_from_configured( + configured: dict[str, dict[str, Any]], +) -> list[Provider]: + """Build providers list from configured variants. + + Args: + configured: Dictionary mapping variant names to their config dicts. + + Returns: + List of Provider objects with models grouped by provider. + """ + providers_by_name: dict[str, Provider] = {} + + for variant_name, variant_config in configured.items(): + provider_name = variant_config.get("provider", "unknown") + + if provider_name not in providers_by_name: + providers_by_name[provider_name] = Provider( + id=provider_name.lower(), + name=provider_name.title(), + models={}, + ) + + providers_by_name[provider_name].models[variant_name] = Model( + id=variant_name, + name=variant_name, + cost=ModelCost( + input=DEFAULT_MODEL_INPUT_COST, + output=DEFAULT_MODEL_OUTPUT_COST, + ), + limit=ModelLimit( + context=DEFAULT_MODEL_CONTEXT_LIMIT, + output=DEFAULT_MODEL_OUTPUT_LIMIT, + ), + ) + + return list(providers_by_name.values()) + + +def _build_providers_from_variants( + variants: dict[str, dict[str, object]], +) -> list[Provider]: + """Build providers list from agent variant modes. + + For agents with thought_level modes (Codex, Claude Code), creates + a single provider with all variants as models. + + Args: + variants: Dictionary mapping variant names to their config dicts. + + Returns: + List of Provider objects containing the variant models. + """ + # For agent-specific modes, create a single provider with all variants + return [ + Provider( + id="agent", + name="Agent Modes", + models={ + name: Model( + id=name, + name=name, + cost=ModelCost( + input=DEFAULT_MODEL_INPUT_COST, + output=DEFAULT_MODEL_OUTPUT_COST, + ), + limit=ModelLimit( + context=DEFAULT_MODEL_CONTEXT_LIMIT, + output=DEFAULT_MODEL_OUTPUT_LIMIT, + ), + ) + for name in variants + }, + ) + ] + + +async def _build_providers_with_fallback( + manifest: AgentsManifest | None, + agent: object | None = None, +) -> list[Provider]: + """Build providers list with fallback hierarchy. + + 1. Primary: Use configured variants from manifest + 2. Secondary: Dynamically discover via tokonomics + 3. Tertiary: Get agent modes (Codex/Claude thought levels) + 4. Last resort: Return empty list with warning + + Args: + manifest: The agents manifest containing model_variants configuration. + agent: Optional agent instance to get agent-specific modes from. + + Returns: + List of Provider objects following the fallback hierarchy. + """ + # Primary: Configured variants + configured = await _get_configured_variants(manifest) + if configured: + logger.debug(f"Using {len(configured)} configured variants from manifest") + return _build_providers_from_configured(configured) + + # Secondary: Tokonomics discovery + try: + toko_models = await _get_available_models() + if toko_models: + logger.debug(f"Using {len(toko_models)} models from tokonomics discovery") + return _build_providers_from_tokonomics(toko_models) + except Exception as e: # noqa: BLE001 + logger.warning(f"Tokonomics discovery failed: {e}") + + # Tertiary: Agent-specific modes + if agent: + agent_variants = await _get_variants_from_agent(agent) + if agent_variants: + logger.debug(f"Using {len(agent_variants)} variants from agent modes") + return _build_providers_from_variants(agent_variants) + + # Last resort: Empty with warning + logger.warning("No model variants configured and no models available from discovery") + return [] + + @router.get("/config") async def get_config(state: StateDep) -> Config: """Get server configuration.""" @@ -171,69 +335,18 @@ async def _get_variants_from_agent(agent: object) -> dict[str, dict[str, object] return {} -def _apply_variants_to_providers( - providers: list[Provider], variants: dict[str, dict[str, object]] -) -> list[Provider]: - """Apply variants to all models in all providers. - - For agents with known thought_level modes (Codex, Claude Code), - the same variants apply to all models. - """ - if not variants: - return providers - - updated_providers = [] - for provider in providers: - updated_models = { - model_id: model.model_copy(update={"variants": variants}) - for model_id, model in provider.models.items() - } - updated_providers.append(provider.model_copy(update={"models": updated_models})) - return updated_providers - - -def _get_dummy_providers() -> list[Provider]: - """Return a single dummy provider for testing.""" - dummy_model = Model( - id="gpt-4o", - name="GPT-4o", - attachment=True, - cost=ModelCost(input=5.0, output=15.0), - limit=ModelLimit(context=128000.0, output=4096.0), - reasoning=False, - release_date="2024-05-13", - temperature=True, - tool_call=True, - ) - dummy_provider = Provider( - id="openai", - name="OpenAI", - env=["OPENAI_API_KEY"], - models={"gpt-4o": dummy_model}, - ) - return [dummy_provider] - - @router.get("/config/providers") async def get_providers(state: StateDep) -> ProvidersResponse: """Get available providers and models from agent.""" - providers: list[Provider] = [] - # Try to get models from the agent + # Get manifest from agent pool (may be None if not loaded) + manifest: AgentsManifest | None = None try: - toko_models = await state.agent.get_available_models() - if toko_models: - providers = _build_providers(toko_models) - except Exception: # noqa: BLE001 - pass # Fall through to dummy providers - - # Fall back to dummy providers if no models available - if not providers: - providers = _get_dummy_providers() + manifest = state.pool.manifest + except (AttributeError, RuntimeError): + pass # No manifest available - # Get variants from agent's thought_level modes (for Codex, Claude Code, etc.) - variants = await _get_variants_from_agent(state.agent) - if variants: - providers = _apply_variants_to_providers(providers, variants) + # Build providers using fallback hierarchy + providers = await _build_providers_with_fallback(manifest, state.agent) # Build default models map: use first model for each connected provider default_models: dict[str, str] = {} @@ -252,26 +365,15 @@ async def get_providers(state: StateDep) -> ProvidersResponse: @router.get("/provider") async def list_providers(state: StateDep) -> ProviderListResponse: """List all providers.""" - import os - - providers: list[Provider] = [] - - # Try to get models from the agent + # Get manifest from agent pool (may be None if not loaded) + manifest: AgentsManifest | None = None try: - toko_models = await state.agent.get_available_models() - if toko_models: - providers = _build_providers(toko_models) - except Exception: # noqa: BLE001 - pass # Fall through to dummy providers - - # Fall back to dummy providers if no models available - if not providers: - providers = _get_dummy_providers() + manifest = state.pool.manifest + except (AttributeError, RuntimeError): + pass # No manifest available - # Get variants from agent's thought_level modes (for Codex, Claude Code, etc.) - variants = await _get_variants_from_agent(state.agent) - if variants: - providers = _apply_variants_to_providers(providers, variants) + # Build providers using fallback hierarchy + providers = await _build_providers_with_fallback(manifest, state.agent) # Determine which providers are "connected" based on env vars connected = [ diff --git a/src/agentpool_server/shared/__init__.py b/src/agentpool_server/shared/__init__.py new file mode 100644 index 000000000..0cd98ad33 --- /dev/null +++ b/src/agentpool_server/shared/__init__.py @@ -0,0 +1,17 @@ +"""Shared utilities for AgentPool servers.""" + +from __future__ import annotations + +from agentpool_server.shared.constants import ( + DEFAULT_MODEL_CONTEXT_LIMIT, + DEFAULT_MODEL_INPUT_COST, + DEFAULT_MODEL_OUTPUT_COST, + DEFAULT_MODEL_OUTPUT_LIMIT, +) + +__all__ = [ + "DEFAULT_MODEL_CONTEXT_LIMIT", + "DEFAULT_MODEL_INPUT_COST", + "DEFAULT_MODEL_OUTPUT_COST", + "DEFAULT_MODEL_OUTPUT_LIMIT", +] diff --git a/src/agentpool_server/shared/constants.py b/src/agentpool_server/shared/constants.py new file mode 100644 index 000000000..541c97cf1 --- /dev/null +++ b/src/agentpool_server/shared/constants.py @@ -0,0 +1,11 @@ +"""Shared constants for AgentPool servers.""" + +from __future__ import annotations + +# Default model limits used when creating placeholder models +DEFAULT_MODEL_CONTEXT_LIMIT: float = 128000.0 +DEFAULT_MODEL_OUTPUT_LIMIT: float = 4096.0 + +# Default model costs used when creating placeholder models +DEFAULT_MODEL_INPUT_COST: float = 0.0 +DEFAULT_MODEL_OUTPUT_COST: float = 0.0 diff --git a/src/agentpool_server/shared/model_utils.py b/src/agentpool_server/shared/model_utils.py new file mode 100644 index 000000000..dbe3fa7d6 --- /dev/null +++ b/src/agentpool_server/shared/model_utils.py @@ -0,0 +1,194 @@ +"""Shared model utilities for AgentPool servers. + +This module provides helper functions for extracting provider information, +building provider lists from tokonomics discovery, and merging configured +variants across ACP and OpenCode servers. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from llmling_models_config import ( + AnthropicModelConfig, + AnyModelConfig, + FallbackModelConfig, + GeminiModelConfig, + OpenAIModelConfig, + StringModelConfig, +) + +from agentpool_server.shared.constants import ( + DEFAULT_MODEL_CONTEXT_LIMIT, + DEFAULT_MODEL_INPUT_COST, + DEFAULT_MODEL_OUTPUT_COST, + DEFAULT_MODEL_OUTPUT_LIMIT, +) + + +if TYPE_CHECKING: + from tokonomics.model_discovery.model_info import ModelInfo as TokoModelInfo + + from agentpool_server.opencode_server.models import Provider + + +def _extract_provider_from_identifier(identifier: str) -> str: + """Extract provider name from a model identifier string. + + Args: + identifier: Model identifier string (e.g., "openai:gpt-4o") + + Returns: + Provider name extracted from identifier (e.g., "openai"), or "unknown" + if no provider prefix found. + """ + if ":" in identifier: + return identifier.split(":", 1)[0] + return "unknown" + + +def _extract_provider(config: AnyModelConfig) -> str: + """Extract provider name from AnyModelConfig. + + Handles: + - StringModelConfig: Extract provider from identifier (e.g., "openai:gpt-4o" -> "openai") + - AnthropicModelConfig: Returns "anthropic" + - OpenAIModelConfig: Returns "openai" + - GeminiModelConfig: Returns "google" + - FallbackModelConfig: Returns provider of first model in chain + + Args: + config: Model configuration to extract provider from. + + Returns: + Provider name as a string. + """ + match config: + case StringModelConfig(identifier=identifier): + return _extract_provider_from_identifier(str(identifier)) + + case AnthropicModelConfig(): + return "anthropic" + + case OpenAIModelConfig(): + return "openai" + + case GeminiModelConfig(): + return "google" + + case FallbackModelConfig(models=models) if models: + first = models[0] + match first: + case StringModelConfig(identifier=identifier): + return _extract_provider_from_identifier(str(identifier)) + case AnthropicModelConfig(): + return "anthropic" + case OpenAIModelConfig(): + return "openai" + case GeminiModelConfig(): + return "google" + case FallbackModelConfig(): + return _extract_provider(first) + case _: + return "unknown" + + case _: + return "unknown" + + +def _build_providers_from_tokonomics(toko_models: list[TokoModelInfo]) -> list[Provider]: + """Build providers list from tokonomics discovery results. + + Groups models by (provider, provider_display_name) and creates Provider + objects with their associated models. + + Args: + toko_models: List of tokonomics ModelInfo objects from discovery. + + Returns: + List of Provider objects with models converted using Model.from_tokonomics(). + """ + from agentpool_server.opencode_server.models import Model, Provider + + providers_by_name: dict[str, Provider] = {} + + for info in toko_models: + # Skip embedding models + if info.is_embedding: + continue + + provider_id = info.provider + + if provider_id not in providers_by_name: + providers_by_name[provider_id] = Provider( + id=provider_id, + name=provider_id.title(), + models={}, + ) + + model_id = info.id_override or info.id + providers_by_name[provider_id].models[model_id] = Model.from_tokonomics(info) + + return list(providers_by_name.values()) + + +def _apply_configured_variants( + providers: list[Provider], + configured_variants: dict[str, dict[str, Any]], +) -> None: + """Merge configured variants into providers list. + + Configured variants with matching IDs override discovered models. + New configured variants are added to their respective providers. + + Args: + providers: List of Provider objects to modify in place. + configured_variants: Dictionary mapping variant names to their + configuration dictionaries. Each config dict should have a + "provider" key indicating which provider the variant belongs to. + + Note: + This function modifies the providers list in place. New providers + are created if a configured variant references a non-existent provider. + """ + from agentpool_server.opencode_server.models import Model, ModelCost, ModelLimit, Provider + + # Build lookup for provider name -> Provider object + provider_lookup: dict[str, Provider] = {} + for provider in providers: + provider_lookup[provider.id.lower()] = provider + + for variant_name, variant_config in configured_variants.items(): + provider_name = variant_config.get("provider", "unknown").lower() + + if provider_name not in provider_lookup: + # Create new provider entry for this variant + provider_lookup[provider_name] = Provider( + id=provider_name, + name=provider_name.title(), + models={}, + ) + providers.append(provider_lookup[provider_name]) + + provider = provider_lookup[provider_name] + + # Check if model with this ID already exists + if variant_name in provider.models: + # Override existing (configured takes precedence) + existing = provider.models[variant_name] + existing.name = variant_name + # Note: variant-specific settings (temp, thinking) not exposed to client + else: + # Add new model - use a minimal Model creation + provider.models[variant_name] = Model( + id=variant_name, + name=variant_name, + cost=ModelCost( + input=DEFAULT_MODEL_INPUT_COST, + output=DEFAULT_MODEL_OUTPUT_COST, + ), + limit=ModelLimit( + context=DEFAULT_MODEL_CONTEXT_LIMIT, + output=DEFAULT_MODEL_OUTPUT_LIMIT, + ), + ) diff --git a/tests/agentpool_server/shared/test_model_utils.py b/tests/agentpool_server/shared/test_model_utils.py new file mode 100644 index 000000000..24f6e621d --- /dev/null +++ b/tests/agentpool_server/shared/test_model_utils.py @@ -0,0 +1,378 @@ +"""Tests for model_utils module.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any + +import pytest +from tokonomics.model_discovery.model_info import ModelInfo, ModelPricing + +from agentpool_server.opencode_server.models import ModelCost, ModelLimit +from agentpool_server.shared.model_utils import ( + _apply_configured_variants, + _build_providers_from_tokonomics, + _extract_provider, + _extract_provider_from_identifier, +) + + +def create_toko_model_info( + model_id: str, + name: str, + provider: str, + is_embedding: bool = False, + **kwargs: Any, +) -> ModelInfo: + """Create a ModelInfo instance for testing.""" + return ModelInfo( + id=model_id, + name=name, + provider=provider, + is_embedding=is_embedding, + **kwargs, + ) + + +class TestExtractProviderFromIdentifier: + """Tests for _extract_provider_from_identifier function.""" + + def test_extract_provider_with_colon(self) -> None: + """Extract provider from identifier with colon separator.""" + result = _extract_provider_from_identifier("openai:gpt-4o") + assert result == "openai" + + def test_extract_provider_anthropic(self) -> None: + """Extract provider from anthropic identifier.""" + result = _extract_provider_from_identifier("anthropic:claude-3-opus") + assert result == "anthropic" + + def test_extract_provider_without_colon(self) -> None: + """Return unknown for identifier without colon.""" + result = _extract_provider_from_identifier("gpt-4o") + assert result == "unknown" + + def test_extract_provider_empty_string(self) -> None: + """Return unknown for empty string.""" + result = _extract_provider_from_identifier("") + assert result == "unknown" + + def test_extract_provider_multiple_colons(self) -> None: + """Extract only first part when multiple colons present.""" + result = _extract_provider_from_identifier("provider:ns:model") + assert result == "provider" + + +class TestExtractProvider: + """Tests for _extract_provider function with AnyModelConfig.""" + + def test_string_config_openai(self) -> None: + """Extract provider from StringModelConfig with openai identifier.""" + from llmling_models_config import StringModelConfig + + config = StringModelConfig(identifier="openai:gpt-4o") + result = _extract_provider(config) + assert result == "openai" + + def test_string_config_anthropic(self) -> None: + """Extract provider from StringModelConfig with anthropic identifier.""" + from llmling_models_config import StringModelConfig + + config = StringModelConfig(identifier="anthropic:claude-sonnet-4") + result = _extract_provider(config) + assert result == "anthropic" + + def test_string_config_no_provider(self) -> None: + """Return unknown for StringModelConfig without provider prefix.""" + from llmling_models_config import StringModelConfig + + config = StringModelConfig(identifier="gpt-4o") + result = _extract_provider(config) + assert result == "unknown" + + def test_anthropic_config(self) -> None: + """Return anthropic for AnthropicModelConfig.""" + from llmling_models_config import AnthropicModelConfig + + config = AnthropicModelConfig(identifier="claude-opus-4-5") + result = _extract_provider(config) + assert result == "anthropic" + + def test_openai_config(self) -> None: + """Return openai for OpenAIModelConfig.""" + from llmling_models_config import OpenAIModelConfig + + config = OpenAIModelConfig(identifier="gpt-5") + result = _extract_provider(config) + assert result == "openai" + + def test_gemini_config(self) -> None: + """Return google for GeminiModelConfig.""" + from llmling_models_config import GeminiModelConfig + + config = GeminiModelConfig(identifier="gemini-2.0-flash") + result = _extract_provider(config) + assert result == "google" + + def test_fallback_config_first_string(self) -> None: + """Extract provider from first model in FallbackModelConfig.""" + from llmling_models_config import FallbackModelConfig, StringModelConfig + + config = FallbackModelConfig(models=[StringModelConfig(identifier="openai:gpt-4o")]) + result = _extract_provider(config) + assert result == "openai" + + def test_fallback_config_first_anthropic(self) -> None: + """Extract anthropic when first model is AnthropicModelConfig.""" + from llmling_models_config import AnthropicModelConfig, FallbackModelConfig + + config = FallbackModelConfig(models=[AnthropicModelConfig(identifier="claude-opus-4-5")]) + result = _extract_provider(config) + assert result == "anthropic" + + def test_fallback_config_empty_models(self) -> None: + """Return unknown for FallbackModelConfig with empty models list - minimum 1 required.""" + # Note: FallbackModelConfig requires at least 1 model, so we test with String instead + from llmling_models_config import FallbackModelConfig + + # Single model fallback with unknown provider string + config = FallbackModelConfig(models=["unknown-model-name"]) + result = _extract_provider(config) + assert result == "unknown" + + def test_fallback_config_nested_fallback(self) -> None: + """Handle nested FallbackModelConfig.""" + from llmling_models_config import ( + AnthropicModelConfig, + FallbackModelConfig, + ) + + inner = FallbackModelConfig(models=[AnthropicModelConfig(identifier="claude-opus-4-5")]) + outer = FallbackModelConfig(models=[inner]) + result = _extract_provider(outer) + assert result == "anthropic" + + +class TestBuildProvidersFromTokonomics: + """Tests for _build_providers_from_tokonomics function.""" + + def test_empty_list(self) -> None: + """Return empty list for empty input.""" + result = _build_providers_from_tokonomics([]) + assert result == [] + + def test_single_model(self) -> None: + """Build provider with single model.""" + model = create_toko_model_info( + model_id="gpt-4o", + name="GPT-4o", + provider="openai", + ) + result = _build_providers_from_tokonomics([model]) + + assert len(result) == 1 + assert result[0].id == "openai" + assert result[0].name == "Openai" + assert "gpt-4o" in result[0].models + + def test_multiple_models_same_provider(self) -> None: + """Group multiple models from same provider.""" + models = [ + create_toko_model_info(model_id="gpt-4o", name="GPT-4o", provider="openai"), + create_toko_model_info(model_id="gpt-4o-mini", name="GPT-4o Mini", provider="openai"), + ] + result = _build_providers_from_tokonomics(models) + + assert len(result) == 1 + assert result[0].id == "openai" + assert len(result[0].models) == 2 + assert "gpt-4o" in result[0].models + assert "gpt-4o-mini" in result[0].models + + def test_multiple_providers(self) -> None: + """Create separate providers for different providers.""" + models = [ + create_toko_model_info(model_id="gpt-4o", name="GPT-4o", provider="openai"), + create_toko_model_info( + model_id="claude-3-opus", name="Claude 3 Opus", provider="anthropic" + ), + ] + result = _build_providers_from_tokonomics(models) + + assert len(result) == 2 + provider_ids = {p.id for p in result} + assert provider_ids == {"openai", "anthropic"} + + def test_skip_embedding_models(self) -> None: + """Skip models marked as embeddings.""" + models = [ + create_toko_model_info( + model_id="text-embedding-3-small", + name="Embedding Small", + provider="openai", + is_embedding=True, + ), + create_toko_model_info( + model_id="gpt-4o", name="GPT-4o", provider="openai", is_embedding=False + ), + ] + result = _build_providers_from_tokonomics(models) + + assert len(result) == 1 + assert len(result[0].models) == 1 + assert "gpt-4o" in result[0].models + assert "text-embedding-3-small" not in result[0].models + + def test_id_override(self) -> None: + """Use id_override when available.""" + model = ModelInfo( + id="claude-opus-4-20250514", + name="Claude Opus 4", + provider="anthropic", + id_override="opus", + ) + result = _build_providers_from_tokonomics([model]) + + assert "opus" in result[0].models + assert "claude-opus-4-20250514" not in result[0].models + + +class TestApplyConfiguredVariants: + """Tests for _apply_configured_variants function.""" + + @pytest.fixture + def sample_provider(self) -> Any: + """Create a sample Provider for testing.""" + from agentpool_server.opencode_server.models import Model, Provider + + model = Model( + id="gpt-4o", + name="GPT-4o", + cost=ModelCost(input=5.0, output=15.0), + limit=ModelLimit(context=128000.0, output=4096.0), + ) + return Provider( + id="openai", + name="OpenAI", + models={"gpt-4o": model}, + ) + + def test_empty_variants(self, sample_provider: Any) -> None: + """Handle empty configured variants dict.""" + providers = [sample_provider] + _apply_configured_variants(providers, {}) + + assert len(providers) == 1 + assert len(providers[0].models) == 1 + + def test_new_provider_creation(self, sample_provider: Any) -> None: + """Create new provider when variant references unknown provider.""" + providers = [sample_provider] + variants = {"custom-model": {"provider": "customai"}} + + _apply_configured_variants(providers, variants) + + assert len(providers) == 2 + custom_provider = next(p for p in providers if p.id == "customai") + assert "custom-model" in custom_provider.models + + def test_model_override(self, sample_provider: Any) -> None: + """Override existing model when variant ID matches.""" + providers = [sample_provider] + variants = {"gpt-4o": {"provider": "openai"}} + + _apply_configured_variants(providers, variants) + + assert len(providers) == 1 + assert len(providers[0].models) == 1 + assert providers[0].models["gpt-4o"].name == "gpt-4o" + + def test_add_model_to_existing_provider(self, sample_provider: Any) -> None: + """Add new model to existing provider.""" + providers = [sample_provider] + variants = {"gpt-5": {"provider": "openai"}} + + _apply_configured_variants(providers, variants) + + assert len(providers[0].models) == 2 + assert "gpt-4o" in providers[0].models + assert "gpt-5" in providers[0].models + + def test_provider_name_case_insensitive(self, sample_provider: Any) -> None: + """Treat provider names case-insensitively.""" + providers = [sample_provider] + variants = {"new-model": {"provider": "OPENAI"}} + + _apply_configured_variants(providers, variants) + + assert len(providers) == 1 + assert "new-model" in providers[0].models + + def test_multiple_variants_same_provider(self, sample_provider: Any) -> None: + """Handle multiple variants for the same provider.""" + providers = [sample_provider] + variants = { + "fast": {"provider": "openai"}, + "smart": {"provider": "openai"}, + } + + _apply_configured_variants(providers, variants) + + assert len(providers[0].models) == 3 + assert "fast" in providers[0].models + assert "smart" in providers[0].models + assert "gpt-4o" in providers[0].models + + def test_default_provider_unknown(self, sample_provider: Any) -> None: + """Use unknown provider when not specified.""" + providers = [sample_provider] + variants: dict[str, dict[str, Any]] = {"orphan-model": {}} + + _apply_configured_variants(providers, variants) + + unknown_provider = next(p for p in providers if p.id == "unknown") + assert "orphan-model" in unknown_provider.models + + +class TestIntegration: + """Integration tests combining multiple functions.""" + + def test_end_to_end_workflow(self) -> None: + """Test complete workflow from tokonomics to merged providers.""" + # Create tokonomics models + toko_models = [ + create_toko_model_info( + model_id="gpt-4o", + name="GPT-4o", + provider="openai", + pricing=ModelPricing(prompt=0.00001, completion=0.00003), + context_window=128000, + max_output_tokens=4096, + created_at=datetime(2024, 5, 13), + ), + create_toko_model_info( + model_id="claude-3-opus", + name="Claude 3 Opus", + provider="anthropic", + ), + ] + + # Build providers from tokonomics + providers = _build_providers_from_tokonomics(toko_models) + assert len(providers) == 2 + + # Apply configured variants + configured_variants = { + "fast": {"provider": "openai"}, + "smart": {"provider": "anthropic"}, + } + _apply_configured_variants(providers, configured_variants) + + # Verify merged results + openai_provider = next(p for p in providers if p.id == "openai") + anthropic_provider = next(p for p in providers if p.id == "anthropic") + + assert "gpt-4o" in openai_provider.models + assert "fast" in openai_provider.models + assert "claude-3-opus" in anthropic_provider.models + assert "smart" in anthropic_provider.models