diff --git a/skills/afk-coder/SKILL.md b/skills/afk-coder/SKILL.md index 26249f9..bc6eb85 100644 --- a/skills/afk-coder/SKILL.md +++ b/skills/afk-coder/SKILL.md @@ -33,7 +33,10 @@ Read these files for detailed API references. Recommended order: | 6 | [security-and-policies.md](./references/security-and-policies.md) | PolicyEngine, SandboxProfile, SkillToolPolicy, fail-safe | | 7 | [multi-agent-and-delegation.md](./references/multi-agent-and-delegation.md) | Subagents, DelegationPlan, A2A protocol, MCP | | 8 | [evals-and-testing.md](./references/evals-and-testing.md) | EvalCase, EvalSuite, assertions, budgets, testing patterns | -| 9 | [cookbook-examples.md](./references/cookbook-examples.md) | 11 complete runnable examples | +| 9 | [debugger.md](./references/debugger.md) | Debugger, DebuggerConfig, debug instrumentation | +| 10 | [queues.md](./references/queues.md) | TaskQueue, TaskWorker, retry policies | +| 11 | [environment-variables.md](./references/environment-variables.md) | All AFK_* env vars, loading from .env | +| 12 | [cookbook-examples.md](./references/cookbook-examples.md) | 11 complete runnable examples | ## Architecture @@ -180,11 +183,14 @@ Key source files for implementation details: | Memory store | `src/afk/memory/store.py` | | Memory factory | `src/afk/memory/factory.py` | | Memory backends | `src/afk/memory/adapters/` | +| Memory lifecycle | `src/afk/memory/lifecycle.py` | | Streaming | `src/afk/core/streaming.py` | | Interaction | `src/afk/core/interaction.py` | | Policy engine | `src/afk/agents/policy/engine.py` | | Delegation | `src/afk/core/runtime/dispatcher.py` | | Evals | `src/afk/evals/` | +| Debugger | `src/afk/debugger/` | +| Queues | `src/afk/queues/` | | MCP server | `src/afk/mcp/server/` | | A2A protocol | `src/afk/agents/a2a/` | diff --git a/skills/afk-coder/references/agents-and-runner.md b/skills/afk-coder/references/agents-and-runner.md index 712b797..25c94f5 100644 --- a/skills/afk-coder/references/agents-and-runner.md +++ b/skills/afk-coder/references/agents-and-runner.md @@ -177,11 +177,28 @@ defaults. Pass it to `Runner(config=RunnerConfig(...))`. | `sanitize_tool_output` | `bool` | `True` | Sanitize tool output before forwarding to the model | | `untrusted_tool_preamble` | `bool` | `True` | Inject untrusted-data warning preamble into tool results | | `tool_output_max_chars` | `int` | `12_000` | Max characters of tool output forwarded to model | +| `default_sandbox_profile` | `SandboxProfile \| None` | `None` | Default sandbox profile for tool execution | +| `sandbox_profile_provider` | `SandboxProfileProvider \| None` | `None` | Runtime sandbox profile resolver | +| `secret_scope_provider` | `SecretScopeProvider \| None` | `None` | Secret injector per tool call | +| `default_allowlisted_commands` | `tuple[str, ...]` | `(ls, cat, head, tail, rg, pwd, echo)` | Allowed shell commands | +| `max_parallel_subagents_global` | `int` | `64` | Global cap for concurrently executing subagent tasks | +| `max_parallel_subagents_per_parent` | `int` | `8` | Per-parent-run cap for concurrent subagent fanout | +| `max_parallel_subagents_per_target_agent` | `int` | `4` | Per-target-agent cap under broad fanout | +| `subagent_queue_backpressure_limit` | `int` | `512` | Max pending subagent nodes before backpressure | | `checkpoint_async_writes` | `bool` | `True` | Enable asynchronous checkpoint/state writes | +| `checkpoint_queue_maxsize` | `int` | `1024` | Maximum queued checkpoint writes | +| `checkpoint_flush_timeout_s` | `float` | `10.0` | Timeout for terminal checkpoint flush | +| `checkpoint_coalesce_runtime_state` | `bool` | `True` | Coalesce runtime-state writes by key | | `debug` | `bool` | `False` | Enable debug instrumentation for run events | +| `debug_config` | `RunnerDebugConfig \| None` | `None` | Advanced debug settings | | `background_tools_enabled` | `bool` | `True` | Allow tools to be deferred into background execution | -| `max_parallel_subagents_global` | `int` | `64` | Global cap for concurrently executing subagent tasks | -| `max_parallel_subagents_per_parent` | `int` | `8` | Per-parent-run cap for concurrent subagent fanout | +| `background_tool_default_grace_s` | `float` | `0.0` | Grace window before backgrounding | +| `background_tool_max_pending` | `int` | `256` | Max unresolved background tools per run | +| `background_tool_poll_interval_s` | `float` | `0.5` | Poll interval for persisted tool state | +| `background_tool_result_ttl_s` | `float` | `3600.0` | TTL for pending background tool tickets | +| `background_tool_interrupt_on_resolve` | `bool` | `True` | Hint loop wake-up on resolution | + +`RunnerConfig` also has a `from_env()` class method that loads configuration from environment variables. --- diff --git a/skills/afk-coder/references/cookbook-examples.md b/skills/afk-coder/references/cookbook-examples.md index b35565d..650cbe3 100644 --- a/skills/afk-coder/references/cookbook-examples.md +++ b/skills/afk-coder/references/cookbook-examples.md @@ -197,9 +197,8 @@ print(result2.final_text) # "Your name is Alice and you love Python." Sandbox profiles, tool output limits, and fail-safe settings. ```python -from afk.agents import Agent, Runner -from afk.agents.types import FailSafeConfig -from afk.core.runner import RunnerConfig +from afk.agents import Agent, Runner, FailSafeConfig +from afk.core import RunnerConfig from afk.tools.security import SandboxProfile sandbox = SandboxProfile( @@ -230,9 +229,9 @@ agent = Agent( tools=[...], max_steps=10, fail_safe=FailSafeConfig( - max_consecutive_errors=3, - max_total_errors=5, - max_empty_responses=2, + max_steps=10, + max_wall_time_s=60.0, + max_total_cost_usd=0.50, ), ) @@ -287,11 +286,9 @@ See [llm-configuration.md](./llm-configuration.md) for profiles, routing, and en Interactive approval flow for sensitive operations. ```python -from afk.agents import Agent, Runner -from afk.agents.policy import PolicyRule, PolicyRuleCondition, PolicyEngine +from afk.agents import Agent, Runner, PolicyRule, PolicyRuleCondition, PolicyEngine from afk.agents.types import ApprovalDecision -from afk.core.interaction import InteractionProvider -from afk.core.runner import RunnerConfig +from afk.core import InteractionProvider, RunnerConfig class TerminalApprovalProvider: """Simple terminal-based approval provider.""" @@ -345,43 +342,42 @@ result = runner.run_sync(agent, user_message="Clean up old records") Automated evaluation of agent behavior. ```python -from afk.evals import EvalCase, EvalSuite, BudgetConfig +from afk.agents import Agent +from afk.evals import ( + EvalCase, + EvalSuiteConfig, + EvalBudget, + StateCompletedAssertion, + FinalTextContainsAssertion, + arun_suite, +) cases = [ EvalCase( - case_id="greeting", - input="Hello!", - expected_output="friendly greeting", - assertions=[ - {"type": "contains", "value": "hello", "case_insensitive": True}, - ], - tags=["basic"], + name="greeting", + agent=Agent(model="gpt-4.1-mini", instructions="Greet the user."), + user_message="Say hello", ), EvalCase( - case_id="math", - input="What is 15 * 7?", - expected_output="105", - assertions=[ - {"type": "contains", "value": "105"}, - ], - tags=["tools"], + name="math", + agent=Agent(model="gpt-4.1-mini", instructions="Answer math questions."), + user_message="What is 15 * 7?", ), ] -suite = EvalSuite( - suite_id="agent-v1", +result = await arun_suite( + runner_factory=None, # or pass Runner class cases=cases, - budget=BudgetConfig( - max_steps=5, - timeout_s=30.0, - max_cost_usd=0.10, + config=EvalSuiteConfig( + assertions=( + StateCompletedAssertion(), + FinalTextContainsAssertion("hello", name="contains_greeting"), + ), ), ) - -# Run evaluation (pseudo code -- see evals reference for full API) -results = await suite.run(agent=agent, runner=runner) -for r in results: - print(f"{r.case_id}: {'PASS' if r.passed else 'FAIL'}") +print(f"Passed: {result.passed}/{result.total}") +for row in result.results: + print(f" {row.case}: {'PASS' if row.passed else 'FAIL'}") ``` See [evals-and-testing.md](./evals-and-testing.md) for assertions, scorers, and datasets. @@ -395,8 +391,8 @@ Long-running tools that execute in the background. ```python import uuid from pydantic import BaseModel -from afk.agents import Agent, Runner -from afk.core.runner import RunnerConfig +from afk.agents import Agent +from afk.core import Runner, RunnerConfig from afk.tools import tool, ToolResult, ToolDeferredHandle class ReportArgs(BaseModel): diff --git a/skills/afk-coder/references/debugger.md b/skills/afk-coder/references/debugger.md new file mode 100644 index 0000000..ef4011b --- /dev/null +++ b/skills/afk-coder/references/debugger.md @@ -0,0 +1,135 @@ +# Debugger + +AFK debugger module: Debug instrumentation, payload redaction, and run event enrichment. + +- Doc page: https://afk.arpan.sh/library/debugger +- Source: `src/afk/debugger/` +- Cross-refs: `agents-and-runner.md` + +--- + +## Overview + +The `Debugger` module provides debug instrumentation for agent runs, including: +- Debug metadata enrichment and formatting hooks +- Payload redaction for sensitive data +- Verbosity levels for different detail amounts + +Key public imports: + +```python +from afk.debugger import Debugger, DebuggerConfig +``` + +--- + +## DebuggerConfig + +Configuration for debugger formatting and payload redaction. + +```python +from afk.debugger import DebuggerConfig + +config = DebuggerConfig( + enabled=True, + verbosity="detailed", + include_content=True, + redact_secrets=True, + max_payload_chars=4000, + emit_timestamps=True, + emit_step_snapshots=True, +) +``` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `enabled` | `bool` | `True` | Enable debug metadata enrichment | +| `verbosity` | `str` | `"detailed"` | Detail level: `"basic"`, `"detailed"`, or `"trace"` | +| `include_content` | `bool` | `True` | Include event/tool/message content in debug payloads | +| `redact_secrets` | `bool` | `True` | Redact sensitive keys from debug payload | +| `max_payload_chars` | `int` | `4000` | Truncate debug payload fields to this length | +| `emit_timestamps` | `bool` | `True` | Attach timestamp metadata to debug payloads | +| `emit_step_snapshots` | `bool` | `True` | Emit summarized per-step snapshot metadata | + +--- + +## Debugger + +The `Debugger` class provides a facade for creating configured `Runner` instances with debug instrumentation. + +```python +from afk.debugger import Debugger, DebuggerConfig + +# Create debugger with custom config +debugger = Debugger( + DebuggerConfig( + enabled=True, + verbosity="detailed", + redact_secrets=True, + ) +) + +# Get a debug-configured runner +runner = debugger.runner() + +# Or access the RunnerConfig directly +config = debugger.config() +``` + +### Debugger Constructor + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `config` | `DebuggerConfig \| None` | `None` | Debug configuration | + +### Debugger Methods + +| Method | Returns | Description | +|--------|---------|-------------| +| `runner(config=None)` | `Runner` | Create a runner with debug instrumentation | +| `config()` | `RunnerConfig` | Get the debug-configured RunnerConfig | + +--- + +## Usage with Runner + +The debugger can be used to wrap runner creation with debug settings: + +```python +from afk.debugger import Debugger, DebuggerConfig +from afk.agents import Agent + +debugger = Debugger( + DebuggerConfig( + enabled=True, + verbosity="detailed", + redact_secrets=True, + ) +) + +runner = debugger.runner() +agent = Agent(model="gpt-4.1-mini", instructions="You are helpful.") + +result = runner.run_sync( + agent, + user_message="Hello", + thread_id="debug-thread", +) +``` + +--- + +## Source Files + +| File | Purpose | +|------|---------| +| `src/afk/debugger/__init__.py` | Public API exports | +| `src/afk/debugger/core.py` | `Debugger` implementation | +| `src/afk/debugger/types.py` | `DebuggerConfig` dataclass | + +--- + +## Cross-References + +- **Runner API**: See [agents-and-runner.md](./agents-and-runner.md) for `RunnerConfig.debug` and `RunnerConfig.debug_config` +- **Tools**: See [tools-system.md](./tools-system.md) for debugging tools \ No newline at end of file diff --git a/skills/afk-coder/references/environment-variables.md b/skills/afk-coder/references/environment-variables.md new file mode 100644 index 0000000..2a1430f --- /dev/null +++ b/skills/afk-coder/references/environment-variables.md @@ -0,0 +1,375 @@ +# Environment Variable Reference + +> Complete reference for all AFK environment variables. + +--- + +## LLM Settings + +| Variable | Default | Purpose | +|----------|---------|---------| +| `AFK_LLM_PROVIDER` | `litellm` | Primary provider | +| `AFK_LLM_PROVIDER_ORDER` | -- | Fallback order (comma-separated): `openai,anthropic,litellm` | +| `AFK_MCP_PORT` | `8000` | Bind port | +| `AFK_MCP_INSTRUCTIONS` | -- | Server instructions | +| `AFK_MCP_PATH` | `/mcp` | JSON-RPC endpoint | +| `AFK_MCP_SSE_PATH` | `/mcp/sse` | SSE endpoint | +| `AFK_MCP_HEALTH_PATH` | `/health` | Health check endpoint | +| `AFK_MCP_ENABLE_SSE` | `true` | Enable SSE transport | +| `AFK_MCP_ENABLE_HEALTH` | `true` | Enable health endpoint | +| `AFK_MCP_ALLOW_BATCH` | `true` | Allow batch requests | + +--- + +## Runner / Safety + +| Variable | Default | Purpose | +|----------|---------|---------| +| `AFK_ALLOWED_COMMANDS` | `ls,cat,head,tail,rg,pwd,echo` | Allowed shell commands (comma-separated) | + +--- + +## Observability + +| Variable | Default | Purpose | +|----------|---------|---------| +| `AFK_OTEL_ENDPOINT` | -- | OpenTelemetry collector endpoint | +| `AFK_OTEL_INSECURE` | `false` | Use insecure OTEL connection | + +--- + +## Hedging Policy (Tail Latency) + +Enable hedging to fire duplicate requests after a delay for latency-sensitive applications: + +```python +from afk.llms import create_llm_client, HedgingPolicy + +# Enable hedging - fires backup request after delay_s +client = create_llm_client( + provider="openai", + hedging_policy=HedgingPolicy(enabled=True, delay_s=0.1), +) +``` + +Or via LLMBuilder: + +```python +from afk.llms import LLMBuilder + +client = ( + LLMBuilder() + .provider("openai") + .profile("low_latency") # includes hedging enabled + .build() +) +``` + +Environment variable support: + +```bash +# Not automatically wired - use code for now +export AFK_LLM_HEDGING_DELAY_S="0.1" +``` + +--- + +## Per-Tool Circuit Breaker (Advanced) + +You can configure per-tool circuit breakers for fault isolation. This allows failing tools to be isolated without affecting other tools: + +```python +from afk.llms import LLMClient, CircuitBreakerPolicy + +# Default: one breaker for all calls +client = LLMClient( + provider="openai", + circuit_breaker_policy=CircuitBreakerPolicy(failure_threshold=5), +) + +# Per-tool isolation requires custom implementation: +# 1. Create separate clients per tool +# 2. Or use tool routing with different clients + +# Example: isolate external API calls +api_client = LLMClient( + provider="openai", + circuit_breaker_policy=CircuitBreakerPolicy(failure_threshold=3), # strict +) +llm_client = LLMClient( + provider="openai", + circuit_breaker_policy=CircuitBreakerPolicy(failure_threshold=10), # lenient +) +``` + +### Circuit Breaker States + +| State | Description | +|------|-------------| +| **Closed** | Normal operation, requests flow through | +| **Open** | Too many failures, requests fail fast | +| **Half-Open** | Probe after cooldown to test recovery | + +| Failure Threshold | Cooldown | Behavior | +|-------------------|----------|-----------| +| 3 | 30s | Strict (for unreliable tools/APIs) | +| 5 | 30s | Normal default | +| 10 | 60s | Lenient (for primary providers) | + +--- + +## Multi-Provider Routing + +AFK supports multiple providers with automatic fallback via `RoutePolicy`: + +```python +from afk.llms import LLMClient, RoutePolicy + +# Client with fallback chain +client = LLMClient( + provider="openai", # primary + settings=LLMSettings( + default_model="gpt-4.1", + ), + route_policy=RoutePolicy( + provider_order=("openai", "anthropic", "litellm"), + ), +) +``` + +The router tries providers in order until one succeeds: + +```python +# Use LLMBuilder for cleaner API +from afk.llms import LLMBuilder, RoutePolicy + +client = ( + LLMBuilder() + .provider("openai") + .model("gpt-4.1") + .with_router(RoutePolicy(provider_order=("openai", "anthropic"))) + .build() +) +``` + +Environment variable for default provider order: + +```bash +# Primary +export AFK_LLM_PROVIDER="openai" # primary + +# Router tries in order: openai → anthropic → litellm +``` + +--- + +## Per-Provider Settings + +Configure each provider with its own API key and base URL: + +```python +from afk.llms import LLMClient, LLMSettings, RoutePolicy + +# Per-provider settings (keyed by provider name) +provider_settings = { + "openai": { + "api_key": "sk-openai-...", + "api_base_url": "https://api.openai.com/v1", + }, + "anthropic": { + "api_key": "sk-ant-...", + }, + "litellm": { + "api_key": "sk-litellm-...", + "api_base_url": "https://my-proxy.com/v1", + }, +} + +client = LLMClient( + provider="openai", + settings=LLMSettings(default_model="gpt-4.1"), + provider_settings=provider_settings, + route_policy=RoutePolicy( + provider_order=("openai", "anthropic", "litellm"), + ), +) +``` + +Or using LLMBuilder: + +```python +from afk.llms import LLMBuilder, RoutePolicy + +client = ( + LLMBuilder() + .provider("openai") + .model("gpt-4.1") + .with_provider_settings("openai", { + "api_key": "sk-openai-...", + }) + .with_provider_settings("anthropic", { + "api_key": "sk-ant-...", + }) + .with_provider_settings("litellm", { + "api_key": "sk-litellm-...", + "api_base_url": "https://my-proxy.com/v1", + }) + .with_router(RoutePolicy( + provider_order=("openai", "anthropic", "litellm"), + )) + .build() +) +``` + +### Environment Variable Pattern + +For env vars, prefix with provider name: + +```bash +# Primary +export AFK_LLM_PROVIDER="openai" +export AFK_LLM_MODEL="gpt-4.1" + +# OpenAI +export AFK_OPENAI_API_KEY="sk-openai-..." + +# Anthropic +export AFK_ANTHROPIC_API_KEY="sk-ant-..." + +# LiteLLM (custom proxy) +export AFK_LITELLM_API_KEY="sk-litellm-..." +export AFK_LITELLM_API_BASE_URL="https://my-proxy.com/v1" + +# Fallback order +export AFK_LLM_PROVIDER_ORDER="openai,anthropic,litellm" +``` + +Common provider env var prefixes: + +| Provider | API Key Var | API Base URL Var | +|----------|------------|----------------| +| `openai` | `AFK_OPENAI_API_KEY` | `AFK_OPENAI_API_BASE_URL` | +| `anthropic` | `AFK_ANTHROPIC_API_KEY` | -- | +| `litellm` | `AFK_LITELLM_API_KEY` | `AFK_LITELLM_API_BASE_URL` | + +--- + +## Multiple API Keys for Same Provider + +You can create separate clients for different API keys: + +```python +from afk.llms import LLMBuilder + +# Create separate clients per API key +client_a = ( + LLMBuilder() + .provider("openai") + .with_provider_settings("openai", {"api_key": "sk-proj-a-..."}) + .build() +) + +client_b = ( + LLMBuilder() + .provider("openai") + .with_provider_settings("openai", {"api_key": "sk-proj-b-"}) + .build() +) + +# Use whichever you need +result = await client_a.chat.completions.create( + model="gpt-4.1", + messages=[{"role": "user", "content": "Hello"}], +) +``` + +Or with Agents - pass the client explicitly: + +```python +from afk.agents import Agent +from afk.core import Runner +from afk.llms import LLMBuilder + +# Different agents with different API keys +agent_a = Agent( + model="gpt-4.1", + instructions="You are helpful.", + llm=LLMBuilder() + .provider("openai") + .with_provider_settings("openai", {"api_key": "sk-proj-a-..."}) + .build(), +) + +agent_b = Agent( + model="gpt-4.1", + instructions="You are helpful.", + llm=LLMBuilder() + .provider("openai") + .with_provider_settings("openai", {"api_key": "sk-proj-b-..."}) + .build(), +) +``` + +Or use LiteLLM as a proxy with multiple keys: + +```bash +# LiteLLM can route to different keys via its config +export AFK_LITELLM_API_KEY="sk-litellm-master" +export AFK_LITELLM_API_BASE_URL="https://your-proxy.com/v1" + +# In LiteLLM config.yaml: +model_list: + - model_name: gpt-4.1 + litellm_params: + api_key: os.environ/OPENAI_PROJECT_A_KEY + - model_name: gpt-4.1-a + litellm_params: + api_key: os.environ/OPENAI_PROJECT_B_KEY +``` + +--- + +## Quick Setup + +```bash +# LLM Configuration +export AFK_LLM_API_KEY="sk-your-key-here" +export AFK_LLM_PROVIDER="openai" # or litellm, anthropic +export AFK_LLM_MODEL="gpt-4.1" + +# Memory (optional: defaults to SQLite) +export AFK_MEMORY_BACKEND="sqlite" +# or Redis: +export AFK_MEMORY_BACKEND="redis" +export AFK_REDIS_URL="redis://localhost:6379/0" + +# MCP Server (optional) +export AFK_MCP_PORT=8080 +``` + +--- + +## Loading from Code + +Most modules automatically load settings from environment: + +```python +# Automatic - reads AFK_* env vars +from afk.llms import LLMBuilder +client = LLMBuilder().build() + +# Manual override +from afk.llms import LLMSettings +settings = LLMSettings.from_env() +custom = settings.model_dump(update={"timeout_s": 60.0}) +``` + +For direct env access: + +```python +from afk.config import LLMEnv, MemoryEnv, MCPEnv + +llm_cfg = LLMEnv.from_env() +memory_cfg = MemoryEnv.from_env() +mcp_cfg = MCPEnv.from_env() +``` \ No newline at end of file diff --git a/skills/afk-coder/references/memory-and-state.md b/skills/afk-coder/references/memory-and-state.md index 8426247..fa7b6c8 100644 --- a/skills/afk-coder/references/memory-and-state.md +++ b/skills/afk-coder/references/memory-and-state.md @@ -273,9 +273,14 @@ Standalone cosine utility: `cosine_similarity([1.0, 0.0], [0.0, 1.0]) # 0.0` ```python from afk.memory import ( MemoryStore, InMemoryMemoryStore, # ABC + default backend + SQLiteMemoryStore, # SQLite backend + RedisMemoryStore, # Redis backend (optional) + PostgresMemoryStore, # Postgres backend (optional) MemoryEvent, LongTermMemory, # Data models - RetentionPolicy, compact_thread_memory, # Lifecycle + RetentionPolicy, StateRetentionPolicy, # Retention policies + MemoryCompactionResult, # Compaction result + compact_thread_memory, apply_event_retention, apply_state_retention, # Lifecycle create_memory_store_from_env, # Factory - cosine_similarity, # Vector utility + cosine_similarity, # Vector utility ) ``` diff --git a/skills/afk-coder/references/multi-agent-and-delegation.md b/skills/afk-coder/references/multi-agent-and-delegation.md index 62a3ccc..9adc5c7 100644 --- a/skills/afk-coder/references/multi-agent-and-delegation.md +++ b/skills/afk-coder/references/multi-agent-and-delegation.md @@ -410,6 +410,65 @@ agent = Agent(model="gpt-4.1-mini", mcp_servers=[ref]) | Dict | `{"name": "x", "url": "..."}` | Mapped to `MCPServerRef` fields | | `MCPServerRef` | `MCPServerRef(name="x", url="...")` | Used directly | +### MCP Server Hosting + +Expose tools as an MCP server using `MCPServer`: + +```python +from afk.tools import ToolRegistry, tool +from afk.mcp.server import MCPServer, MCPServerConfig + +registry = ToolRegistry() + +@tool(name="greet", description="Greet someone") +def greet(name: str) -> str: + return f"Hello, {name}!" + +registry.register(greet) + +server = MCPServer( + registry, + config=MCPServerConfig( + host="0.0.0.0", + port=8080, + ), +) + +server.run() +``` + +### MCPServerConfig + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `host` | `str` | `"0.0.0.0"` | Server host | +| `port` | `int` | `8000` | Server port | +| `title` | `str` | `"AFK MCP Server"` | Server title | +| `version` | `str` | `"1.0.0"` | Server version | + +### create_mcp_server + +Create an MCP server from a pre-built registry: + +```python +from afk.tools import ToolRegistry, tool +from afk.mcp.server import create_mcp_server + +registry = ToolRegistry() + +@tool(name="greet", description="Greet someone") +def greet(name: str) -> str: + return f"Hello, {name}!" + +registry.register(greet) + +server = create_mcp_server( + registry, + config=MCPServerConfig(host="0.0.0.0", port=8080), +) +server.run() +``` + --- ## 7. Agent Skills diff --git a/skills/afk-coder/references/queues.md b/skills/afk-coder/references/queues.md new file mode 100644 index 0000000..8446847 --- /dev/null +++ b/skills/afk-coder/references/queues.md @@ -0,0 +1,287 @@ +# Queues + +AFK task queue subsystem: `TaskQueue` for distributed agent execution with retry policies, +`TaskWorker` consumer loop, and pluggable backends. + +- Doc page: https://afk.arpan.sh/library/queues +- Source: `src/afk/queues/` +- Cross-refs: `agents-and-runner.md`, `evals-and-testing.md` + +--- + +## Overview + +AFK provides a persistent task queue for enqueuing, dequeuing, and tracking agent tasks with: +- Automatic retry with configurable backoff +- Pluggable backends (InMemory, Redis) +- Worker consumer loop with metrics +- Execution contracts for agent invocation + +Key public imports: + +```python +from afk.queues import ( + TaskQueue, + TaskItem, + TaskStatus, + InMemoryTaskQueue, + TaskWorker, + TaskWorkerConfig, + RetryPolicy, + RunnerChatExecutionContract, + JobDispatchExecutionContract, + create_task_queue_from_env, +) +``` + +--- + +## TaskItem + +One queued task item. + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `task_id` | `str` | -- | Unique task identifier | +| `contract_key` | `str` | -- | Execution contract key | +| `payload` | `dict[str, JSONValue]` | -- | Task payload | +| `status` | `TaskStatus` | `"pending"` | Current task status | +| `attempt` | `int` | `0` | Current attempt number | +| `created_at` | `int` | -- | Epoch milliseconds | +| `next_attempt_at` | `int \| None` | `None` | Next attempt scheduled time | +| `dead_letter_reason` | `str \| None` | `None` | Reason if moved to dead letter | + +--- + +## TaskStatus + +| Value | Description | +|-------|-------------| +| `"pending"` | Task waiting to be processed | +| `"running"` | Task currently being executed | +| `"completed"` | Task completed successfully | +| `"failed"` | Task failed after retries exhausted | +| `"dead_letter"` | Task moved to dead letter queue | + +--- + +## TaskQueue + +Abstract queue interface. + +| Method | Signature | Description | +|--------|-----------|-------------| +| `enqueue_contract` | `(contract, payload, *, agent_name, thread_id=None) -> str` | Enqueue a task using contract | +| `dequeue` | `(timeout=None) -> TaskItem \| None` | Dequeue the next pending task | +| `ack` | `(task_id, success) -> None` | Acknowledge task completion | +| `get` | `(task_id) -> TaskItem \| None` | Get task by ID | +| `list` | `(status, limit=100) -> list[TaskItem]` | List tasks by status | +| `update_status` | `(task_id, status) -> None` | Update task status | + +--- + +## RetryPolicy + +Retry configuration for failed tasks. + +```python +RetryPolicy( + max_attempts=3, + backoff_base_s=1.0, + backoff_max_s=60.0, + backoff_jitter_s=0.5, +) +``` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `max_attempts` | `int` | `3` | Maximum retry attempts | +| `backoff_base_s` | `float` | `1.0` | Base backoff duration in seconds | +| `backoff_max_s` | `float` | `60.0` | Maximum backoff cap | +| `backoff_jitter_s` | `float` | `0.5` | Random jitter added to backoff | + +--- + +## Execution Contracts + +### RunnerChatExecutionContract + +Execute an agent via Runner chat contract. + +```python +from afk.queues import RunnerChatExecutionContract + +contract = RunnerChatExecutionContract( + agent=my_agent, + max_steps=20, + fail_safe=FailSafeConfig(max_steps=20), +) +``` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `agent` | `BaseAgent` | -- | Agent to execute | +| `payload_key` | `"user_message"` | User message key in payload | +| `context_key` | `"context"` | Context key | +| `thread_id_key` | `"thread_id"` | Thread ID key | +| `max_steps` | `int` | `None` | Override agent max_steps | +| `fail_safe` | `FailSafeConfig \| None` | `None` | Fail-safe config | + +### JobDispatchExecutionContract + +Execute a job handler function. + +```python +from afk.queues import JobDispatchExecutionContract + +contract = JobDispatchExecutionContract( + handler=my_handler_function, +) +``` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `handler` | `Callable` | -- | Job handler function | + +--- + +## TaskWorker + +Consumer loop that processes tasks from a queue. + +```python +from afk.queues import TaskWorker, TaskWorkerConfig + +worker = TaskWorker( + queue, + agents={"greeter": my_agent}, + config=TaskWorkerConfig( + poll_interval_s=1.0, + max_concurrency=4, + ), +) + +await worker.start() +``` + +### TaskWorkerConfig + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `poll_interval_s` | `float` | `1.0` | Queue poll interval | +| `max_concurrency` | `int` | `4` | Max concurrent task processing | +| `shutdown_timeout_s` | `float` | `30.0` | Graceful shutdown timeout | + +### TaskWorker Methods + +| Method | Returns | Description | +|--------|---------|-------------| +| `start()` | `None` | Start the worker loop | +| `stop()` | `None` | Stop the worker loop | +| `running()` | `bool` | Check if worker is running | + +--- + +## In-Memory Queue + +Simple in-memory task queue for development and testing. + +```python +from afk.queues import InMemoryTaskQueue + +queue = InMemoryTaskQueue() +await queue.enqueue_contract( + RunnerChatExecutionContract(agent=my_agent), + payload={"user_message": "Hello!"}, + agent_name="greeter", +) +``` + +--- + +## Redis Queue + +Production queue with Redis backend (optional extra). + +```python +from afk.queues import RedisTaskQueue + +queue = RedisTaskQueue( + url="redis://localhost:6379/0", + key_prefix="afk:tasks", +) +``` + +Requires `redis` package: `pip install afk[redis]` + +--- + +## Factory + +Create queue from environment variables. + +```python +from afk.queues import create_task_queue_from_env + +# Respects AFK_QUEUE_BACKEND env var +queue = create_task_queue_from_env() +``` + +| Variable | Default | Purpose | +|----------|---------|---------| +| `AFK_QUEUE_BACKEND` | `memory` | `memory` or `redis` | +| `AFK_REDIS_URL` | -- | Redis connection URL | + +--- + +## Basic Example + +```python +from afk.agents import Agent +from afk.queues import ( + InMemoryTaskQueue, + RunnerChatExecutionContract, + TaskWorker, +) + +agent = Agent(model="gpt-4.1-mini", instructions="You are a greeter.") +queue = InMemoryTaskQueue() + +# Enqueue a task +task_id = await queue.enqueue_contract( + RunnerChatExecutionContract(agent=agent), + payload={"user_message": "Hello!"}, + agent_name="greeter", +) + +# Start worker to process +worker = TaskWorker(queue, agents={"greeter": agent}) +await worker.start() + +# Or manually process +item = await queue.dequeue() +# ... process item ... +await queue.ack(item.task_id, success=True) +``` + +--- + +## Source Files + +| File | Purpose | +|------|---------| +| `src/afk/queues/__init__.py` | Public API exports | +| `src/afk/queues/base.py` | `BaseTaskQueue` abstract class | +| `src/afk/queues/memory.py` | `InMemoryTaskQueue` implementation | +| `src/afk/queues/redis_queue.py` | `RedisTaskQueue` implementation | +| `src/afk/queues/factory.py` | `create_task_queue_from_env` | +| `src/afk/queues/worker.py` | `TaskWorker` implementation | +| `src/afk/queues/types.py` | `TaskItem`, `TaskStatus`, `RetryPolicy` | +| `src/afk/queues/contracts.py` | Execution contracts | + +--- + +## Cross-References + +- **Agents**: See [agents-and-runner.md](./agents-and-runner.md) for `Agent` configuration +- **Evals**: See [evals-and-testing.md](./evals-and-testing.md) for testing queued agents \ No newline at end of file diff --git a/skills/afk-coder/references/security-and-policies.md b/skills/afk-coder/references/security-and-policies.md index 98361ae..2f4a2ec 100644 --- a/skills/afk-coder/references/security-and-policies.md +++ b/skills/afk-coder/references/security-and-policies.md @@ -217,6 +217,9 @@ profile = SandboxProfile( | `deny_shell_operators` | `bool` | `True` | Block `&&`, `\|\|`, `;`, `\|`, backticks, `$(`, redirects | | `allowed_paths` | `list[str]` | `[]` | Allowed filesystem paths (empty = no restriction) | | `denied_paths` | `list[str]` | `[]` | Denied filesystem paths (checked before allowlist) | +| `file_allowed_extensions` | `list[str]` | `[]` | Allowed file extensions (empty = no restriction) | +| `file_max_size_mb` | `int \| None` | `None` | Maximum file size in MB for file operations | +| `network_allowed_domains` | `list[str]` | `[]` | Allowed domains for network access (empty = no restriction) | | `command_timeout_s` | `float \| None` | `None` | Command execution timeout | | `max_output_chars` | `int` | `20_000` | Maximum output characters (truncated beyond this) | diff --git a/src/afk/config.py b/src/afk/config.py new file mode 100644 index 0000000..8bf003e --- /dev/null +++ b/src/afk/config.py @@ -0,0 +1,247 @@ +""" +MIT License +Copyright (c) 2026 arpan404 +See LICENSE file for full license text. + +Centralized, type-safe environment variable resolution. + +``EnvVarField`` descriptors declare env vars; ``Settings.from_env()`` resolves +them and returns an immutable typed object. +""" + +from __future__ import annotations + +import os +from typing import Any, Callable, TypeVar + +T = TypeVar("T") + + +# --------------------------------------------------------------------------- +# Parsers +# --------------------------------------------------------------------------- + + +def _csv_list(raw: str) -> list[str]: + """Parse a comma-separated string into a list of stripped, non-empty tokens.""" + return [s.strip() for s in raw.split(",") if s.strip()] + + +def _bool(raw: str) -> bool: + """Parse a boolean from a raw string (raises on invalid input).""" + if raw.strip().lower() in ("1", "true", "yes", "y", "on"): + return True + if raw.strip().lower() in ("0", "false", "no", "n", "off"): + return False + raise ValueError(f"Invalid boolean value: {raw!r}") + + +def _env_bool(name: str, default: bool = False) -> bool: + """Parse a boolean from an environment variable (returns default if unset).""" + raw = os.getenv(name) + if raw is None: + return default + return _bool(raw) + + +def _float(raw: str) -> float: + return float(raw) + + +def _int(raw: str) -> int: + return int(raw) + + +# --------------------------------------------------------------------------- +# EnvVarField descriptor +# --------------------------------------------------------------------------- + + +class EnvVarField: + """ + A class-level descriptor that binds a settings field to an environment variable. + + Usage:: + + class MySettings: + host: str = EnvVarField("MY_HOST", default="localhost") + port: int = EnvVarField("MY_PORT", default=8000, parser=_int) + debug: bool = EnvVarField("DEBUG", default=False, parser=_bool) + """ + + __slots__ = ("env_name", "default", "parser") + + def __init__( + self, + env_name: str, + default: Any, + parser: Callable[[str], Any] = lambda s: s, + ) -> None: + self.env_name = env_name + self.default = default + self.parser = parser + + def get(self) -> Any: + raw = os.getenv(self.env_name) + if raw is None: + return self.default + if raw == "": + return self.default + return self.parser(raw) + + +# --------------------------------------------------------------------------- +# Settings base +# --------------------------------------------------------------------------- + + +class Settings: + """ + Base class for settings that load from environment variables. + + Subclasses declare ``EnvVarField`` class variables. ``from_env()`` resolves + each from the environment and returns an instance of the subclass. + """ + + __slots__: ClassVar[list[str]] = [] + + @classmethod + def from_env(cls: type[T]) -> T: + kwargs = {} + for name in dir(cls): + if name.startswith("_"): + continue + val = getattr(cls, name, None) + if not isinstance(val, EnvVarField): + continue + kwargs[name] = val.get() + instance = object.__new__(cls) + for k, v in kwargs.items(): + object.__setattr__(instance, k, v) + return instance # type: ignore[return-value] + + def __setattr__(self, name: str, value: Any) -> None: + raise AttributeError(f"{type(self).__name__} is immutable") + + +# --------------------------------------------------------------------------- +# Concrete env bindings +# --------------------------------------------------------------------------- + + +class MCPServerEnv(Settings): + """Environment variable bindings for the MCP server.""" + + AFK_CORS_ORIGINS: list[str] = EnvVarField( + "AFK_CORS_ORIGINS", default=[], parser=_csv_list + ) + AFK_MCP_NAME: str = EnvVarField("AFK_MCP_NAME", default="afk-mcp-server") + AFK_MCP_VERSION: str = EnvVarField("AFK_MCP_VERSION", default="1.0.0") + AFK_MCP_HOST: str = EnvVarField("AFK_MCP_HOST", default="0.0.0.0") + AFK_MCP_PORT: int = EnvVarField("AFK_MCP_PORT", default=8000, parser=_int) + AFK_MCP_INSTRUCTIONS: str | None = EnvVarField( + "AFK_MCP_INSTRUCTIONS", default=None + ) + AFK_MCP_PATH: str = EnvVarField("AFK_MCP_PATH", default="/mcp") + AFK_MCP_SSE_PATH: str = EnvVarField("AFK_MCP_SSE_PATH", default="/mcp/sse") + AFK_MCP_HEALTH_PATH: str = EnvVarField("AFK_MCP_HEALTH_PATH", default="/health") + AFK_MCP_ENABLE_SSE: bool = EnvVarField( + "AFK_MCP_ENABLE_SSE", default=True, parser=_bool + ) + AFK_MCP_ENABLE_HEALTH: bool = EnvVarField( + "AFK_MCP_ENABLE_HEALTH", default=True, parser=_bool + ) + AFK_MCP_ALLOW_BATCH: bool = EnvVarField( + "AFK_MCP_ALLOW_BATCH", default=True, parser=_bool + ) + + +class RunnerEnv(Settings): + """Environment variable bindings for runner behavior.""" + + AFK_ALLOWED_COMMANDS: list[str] = EnvVarField( + "AFK_ALLOWED_COMMANDS", default=[], parser=_csv_list + ) + + +class MemoryEnv(Settings): + """Environment variable bindings for memory stores and queues.""" + + AFK_MEMORY_BACKEND: str = EnvVarField("AFK_MEMORY_BACKEND", default="sqlite") + AFK_SQLITE_PATH: str = EnvVarField("AFK_SQLITE_PATH", default="afk_memory.sqlite3") + AFK_REDIS_URL: str | None = EnvVarField("AFK_REDIS_URL", default=None) + AFK_REDIS_HOST: str = EnvVarField("AFK_REDIS_HOST", default="localhost") + AFK_REDIS_PORT: int = EnvVarField("AFK_REDIS_PORT", default=6379, parser=_int) + AFK_REDIS_DB: int = EnvVarField("AFK_REDIS_DB", default=0, parser=_int) + AFK_REDIS_PASSWORD: str = EnvVarField("AFK_REDIS_PASSWORD", default="") + AFK_REDIS_EVENTS_MAX: int = EnvVarField( + "AFK_REDIS_EVENTS_MAX", default=2000, parser=_int + ) + AFK_PG_DSN: str | None = EnvVarField("AFK_PG_DSN", default=None) + AFK_PG_HOST: str = EnvVarField("AFK_PG_HOST", default="localhost") + AFK_PG_PORT: int = EnvVarField("AFK_PG_PORT", default=5432, parser=_int) + AFK_PG_USER: str = EnvVarField("AFK_PG_USER", default="postgres") + AFK_PG_PASSWORD: str = EnvVarField("AFK_PG_PASSWORD", default="") + AFK_PG_DB: str = EnvVarField("AFK_PG_DB", default="afk") + AFK_PG_SSL: bool = EnvVarField("AFK_PG_SSL", default=False, parser=_bool) + AFK_PG_POOL_MIN: int = EnvVarField("AFK_PG_POOL_MIN", default=1, parser=_int) + AFK_PG_POOL_MAX: int = EnvVarField("AFK_PG_POOL_MAX", default=10, parser=_int) + AFK_VECTOR_DIM: int | None = EnvVarField("AFK_VECTOR_DIM", default=None, parser=_int) + AFK_QUEUE_BACKEND: str = EnvVarField("AFK_QUEUE_BACKEND", default="inmemory") + AFK_QUEUE_RETRY_BACKOFF_BASE_S: float = EnvVarField( + "AFK_QUEUE_RETRY_BACKOFF_BASE_S", default=0.5, parser=_float + ) + AFK_QUEUE_RETRY_BACKOFF_MAX_S: float = EnvVarField( + "AFK_QUEUE_RETRY_BACKOFF_MAX_S", default=30.0, parser=_float + ) + AFK_QUEUE_RETRY_BACKOFF_JITTER_S: float = EnvVarField( + "AFK_QUEUE_RETRY_BACKOFF_JITTER_S", default=0.2, parser=_float + ) + AFK_QUEUE_REDIS_PREFIX: str = EnvVarField( + "AFK_QUEUE_REDIS_PREFIX", default="afk:queue" + ) + AFK_QUEUE_REDIS_URL: str | None = EnvVarField("AFK_QUEUE_REDIS_URL", default=None) + AFK_QUEUE_REDIS_HOST: str = EnvVarField("AFK_QUEUE_REDIS_HOST", default="localhost") + AFK_QUEUE_REDIS_PORT: int = EnvVarField( + "AFK_QUEUE_REDIS_PORT", default=6379, parser=_int + ) + AFK_QUEUE_REDIS_DB: int = EnvVarField("AFK_QUEUE_REDIS_DB", default=0, parser=_int) + AFK_QUEUE_REDIS_PASSWORD: str = EnvVarField( + "AFK_QUEUE_REDIS_PASSWORD", default="" + ) + + +class LLMEnv(Settings): + """Environment variable bindings for LLM providers.""" + + AFK_LLM_PROVIDER: str = EnvVarField("AFK_LLM_PROVIDER", default="litellm") + AFK_LLM_PROVIDER_ORDER: list[str] = EnvVarField( + "AFK_LLM_PROVIDER_ORDER", default=[], parser=_csv_list + ) + AFK_LLM_MODEL: str = EnvVarField("AFK_LLM_MODEL", default="gpt-4.1-mini") + AFK_EMBED_MODEL: str | None = EnvVarField("AFK_EMBED_MODEL", default=None) + AFK_LLM_API_BASE_URL: str | None = EnvVarField( + "AFK_LLM_API_BASE_URL", default=None + ) + AFK_LLM_API_KEY: str | None = EnvVarField("AFK_LLM_API_KEY", default=None) + AFK_LLM_TIMEOUT_S: float = EnvVarField( + "AFK_LLM_TIMEOUT_S", default=30.0, parser=_float + ) + AFK_LLM_MAX_RETRIES: int = EnvVarField( + "AFK_LLM_MAX_RETRIES", default=3, parser=_int + ) + AFK_LLM_BACKOFF_BASE_S: float = EnvVarField( + "AFK_LLM_BACKOFF_BASE_S", default=0.5, parser=_float + ) + AFK_LLM_BACKOFF_JITTER_S: float = EnvVarField( + "AFK_LLM_BACKOFF_JITTER_S", default=0.15, parser=_float + ) + AFK_LLM_JSON_MAX_RETRIES: int = EnvVarField( + "AFK_LLM_JSON_MAX_RETRIES", default=2, parser=_int + ) + AFK_LLM_MAX_INPUT_CHARS: int = EnvVarField( + "AFK_LLM_MAX_INPUT_CHARS", default=200000, parser=_int + ) + AFK_LLM_STREAM_IDLE_TIMEOUT_S: float = EnvVarField( + "AFK_LLM_STREAM_IDLE_TIMEOUT_S", default=45.0, parser=_float + ) diff --git a/src/afk/debugger/core.py b/src/afk/debugger/core.py index 635caaf..589e1ae 100644 --- a/src/afk/debugger/core.py +++ b/src/afk/debugger/core.py @@ -8,9 +8,12 @@ from __future__ import annotations +import asyncio +import fnmatch import json +import time from collections.abc import Callable -from dataclasses import asdict +from dataclasses import asdict, dataclass, field from typing import Any from ..agents.types import AgentRunEvent, AgentRunHandle @@ -113,3 +116,313 @@ def _redact_value(self, value: Any) -> Any: if isinstance(value, list): return [self._redact_value(item) for item in value] return value + + +# === Interactive Step Debugger === + + +@dataclass +class BreakpointConfig: + """ + Configuration for interactive debugging breakpoints. + + Attributes: + pause_on_tool: Glob patterns for tool names to pause on. + pause_on_llm_error: Pause when LLM returns an error. + pause_on_tool_error: Pause when tool execution fails. + pause_on_state: Pause on specific agent states. + max_steps_before_auto_resume: Auto-resume after N steps. + auto_resume_timeout_s: Auto-resume after timeout. + emit_step_snapshots: Emit state snapshots at each step. + """ + + pause_on_tool: list[str] = field(default_factory=list) + pause_on_llm_error: bool = False + pause_on_tool_error: bool = True + pause_on_state: list[str] = field(default_factory=list) + max_steps_before_auto_resume: int = 0 + auto_resume_timeout_s: float = 0.0 + emit_step_snapshots: bool = True + + +@dataclass +class StepSnapshot: + """ + Snapshot of agent state at a specific step. + + Attributes: + step: Step number. + state: Agent state. + tool_name: Tool being executed (if any). + tool_args: Tool arguments (may be redacted). + llm_response: LLM response text (truncated). + message_count: Number of messages in context. + timestamp_ms: Snapshot timestamp. + """ + + step: int + state: str + tool_name: str | None = None + tool_args: dict[str, Any] | None = None + llm_response: str | None = None + message_count: int = 0 + timestamp_ms: int = field(default_factory=lambda: int(time.time() * 1000)) + + +@dataclass +class DebugSession: + """ + Active debug session for an agent run. + + Tracks breakpoints, snapshots, and user interactions. + """ + + run_id: str + thread_id: str + config: BreakpointConfig + breakpoints_enabled: bool = True + paused: bool = False + pause_reason: str | None = None + step_snapshots: list[StepSnapshot] = field(default_factory=list) + breakpoints_hit: list[str] = field(default_factory=list) + resume_callback: Callable[[], None] | None = None + + async def pause(self, reason: str) -> None: + """Pause the debug session.""" + self.paused = True + self.pause_reason = reason + + async def resume(self) -> None: + """Resume the debug session.""" + self.paused = False + self.pause_reason = None + + def add_snapshot(self, snapshot: StepSnapshot) -> None: + """Add a step snapshot to the session.""" + self.step_snapshots.append(snapshot) + + def record_breakpoint_hit(self, pattern: str) -> None: + """Record that a breakpoint was hit.""" + self.breakpoints_hit.append(pattern) + + +class InteractiveDebugger: + """ + Interactive debugger for agent execution. + + Provides breakpoint support, step-through capability, + and state inspection during agent runs. + """ + + def __init__(self) -> None: + self._sessions: dict[str, DebugSession] = {} + self._global_breakpoints: list[str] = [] + self._default_config = BreakpointConfig() + + def enable_debugger( + self, + run_id: str, + thread_id: str, + config: BreakpointConfig | None = None, + ) -> DebugSession: + """ + Enable debugging for a specific run. + + Args: + run_id: Run identifier. + thread_id: Thread identifier. + config: Optional breakpoint configuration. + + Returns: + DebugSession for this run. + """ + session = DebugSession( + run_id=run_id, + thread_id=thread_id, + config=config or self._default_config, + ) + self._sessions[run_id] = session + return session + + def disable_debugger(self, run_id: str) -> None: + """Disable debugging for a run.""" + self._sessions.pop(run_id, None) + + def get_session(self, run_id: str) -> DebugSession | None: + """Get active debug session for a run.""" + return self._sessions.get(run_id) + + def add_global_breakpoint(self, pattern: str) -> None: + """Add a global breakpoint pattern.""" + self._global_breakpoints.append(pattern) + + def remove_global_breakpoint(self, pattern: str) -> None: + """Remove a global breakpoint pattern.""" + self._global_breakpoints.remove(pattern) + + def should_break( + self, + run_id: str, + tool_name: str | None = None, + event_type: str | None = None, + state: str | None = None, + has_error: bool = False, + ) -> tuple[bool, str | None]: + """ + Check if execution should break for debugging. + + Args: + run_id: Run identifier. + tool_name: Current tool name (if any). + event_type: Current event type. + state: Current agent state. + has_error: Whether an error occurred. + + Returns: + Tuple of (should_break, reason). + """ + session = self._sessions.get(run_id) + if not session or not session.breakpoints_enabled: + return False, None + + config = session.config + + # Check tool breakpoints + if tool_name: + for pattern in config.pause_on_tool + self._global_breakpoints: + if fnmatch.fnmatch(tool_name, pattern): + session.record_breakpoint_hit(pattern) + return True, f"tool breakpoint: {pattern}" + # Check negated patterns (starting with !) + if pattern.startswith("!"): + if fnmatch.fnmatch(tool_name, pattern[1:]): + return False, None + + # Check error breakpoints + if has_error: + if config.pause_on_tool_error: + return True, "tool error" + if config.pause_on_llm_error: + return True, "LLM error" + + # Check state breakpoints + if state and state in config.pause_on_state: + return True, f"state: {state}" + + return False, None + + async def on_step_start( + self, + run_id: str, + step: int, + state: str, + tool_name: str | None = None, + tool_args: dict[str, Any] | None = None, + llm_response: str | None = None, + message_count: int = 0, + ) -> StepSnapshot | None: + """ + Called at the start of each step to check breakpoints. + + Args: + run_id: Run identifier. + step: Step number. + state: Current agent state. + tool_name: Tool being executed. + tool_args: Tool arguments. + llm_response: LLM response text. + message_count: Number of messages in context. + + Returns: + StepSnapshot if breakpoint triggered, None otherwise. + """ + session = self._sessions.get(run_id) + if not session: + return None + + should_break, reason = self.should_break( + run_id=run_id, + tool_name=tool_name, + state=state, + ) + + snapshot = StepSnapshot( + step=step, + state=state, + tool_name=tool_name, + tool_args=tool_args, + llm_response=llm_response[:200] if llm_response else None, + message_count=message_count, + ) + + if session.config.emit_step_snapshots: + session.add_snapshot(snapshot) + + if should_break: + await session.pause(reason or "breakpoint") + return snapshot + + return None + + async def wait_for_resume(self, run_id: str) -> None: + """ + Wait for user resume signal. + + Args: + run_id: Run identifier. + """ + session = self._sessions.get(run_id) + if not session: + return + + # If auto-resume is configured, set up timeout + if session.config.auto_resume_timeout_s > 0: + + async def auto_resume(): + await asyncio.sleep(session.config.auto_resume_timeout_s) + if session.paused: + await session.resume() + + asyncio.create_task(auto_resume()) + + # Wait until not paused + while session.paused: + await asyncio.sleep(0.1) + + def get_snapshots(self, run_id: str) -> list[StepSnapshot]: + """Get all step snapshots for a run.""" + session = self._sessions.get(run_id) + return list(session.step_snapshots) if session else [] + + def get_breakpoints_hit(self, run_id: str) -> list[str]: + """Get list of breakpoints hit during a run.""" + session = self._sessions.get(run_id) + return list(session.breakpoints_hit) if session else [] + + def set_resume_callback( + self, run_id: str, callback: Callable[[], None] + ) -> None: + """Set callback to be called when resumed.""" + session = self._sessions.get(run_id) + if session: + session.resume_callback = callback + + async def resume(self, run_id: str) -> None: + """Resume a paused debug session.""" + session = self._sessions.get(run_id) + if session: + await session.resume() + if session.resume_callback: + session.resume_callback() + + +# Global debugger instance +_global_debugger: InteractiveDebugger | None = None + + +def get_debugger() -> InteractiveDebugger: + """Get the global debugger instance.""" + global _global_debugger + if _global_debugger is None: + _global_debugger = InteractiveDebugger() + return _global_debugger \ No newline at end of file diff --git a/src/afk/llms/__init__.py b/src/afk/llms/__init__.py index 33a289f..d30f0bc 100644 --- a/src/afk/llms/__init__.py +++ b/src/afk/llms/__init__.py @@ -30,7 +30,7 @@ from .llm import LLM from .middleware import MiddlewareStack from .observability import LLMLifecycleEvent, LLMObserver -from .profiles import PROFILES +from .profiles import LLMProfile, PROFILES from .providers import ( AnthropicAgentProvider, LiteLLMProvider, @@ -98,14 +98,53 @@ def create_llm_client( provider_settings: dict[str, dict] | None = None, middlewares: MiddlewareStack | None = None, observers: list[LLMObserver] | None = None, + router=None, + retry_policy: "RetryPolicy | None" = None, + timeout_policy: "TimeoutPolicy | None" = None, + rate_limit_policy: "RateLimitPolicy | None" = None, + circuit_breaker_policy: "CircuitBreakerPolicy | None" = None, + hedging_policy: "HedgingPolicy | None" = None, + cache_policy: "CachePolicy | None" = None, + coalescing_policy: "CoalescingPolicy | None" = None, ) -> LLMClient: - """Create enterprise runtime client with explicit provider selection.""" + """Create enterprise runtime client with explicit provider selection. + + Args: + provider: Provider id (e.g., "openai", "anthropic_agent", "litellm") + settings: LLMSettings instance (defaults to env) + provider_settings: Per-provider API keys/URLs + middlewares: MiddlewareStack for transport paths + observers: LLM lifecycle observers + router: LLMRouter for multi-provider fallback + retry_policy: Retry policy for transient failures + timeout_policy: Request/stream timeout policy + rate_limit_policy: Rate limiting policy + circuit_breaker_policy: Circuit breaker for fault isolation + hedging_policy: Tail latency hedging (fires duplicate requests) + cache_policy: Response caching policy + coalescing_policy: Request coalescing for identical payloads + + Example: + # Use hedging for latency-sensitive applications + client = create_llm_client( + provider="openai", + hedging_policy=HedgingPolicy(enabled=True, delay_s=0.1), + ) + """ return LLMClient( provider=provider, settings=settings or LLMSettings.from_env(), provider_settings=provider_settings, middlewares=middlewares, observers=observers, + router=router, + retry_policy=retry_policy, + timeout_policy=timeout_policy, + rate_limit_policy=rate_limit_policy, + circuit_breaker_policy=circuit_breaker_policy, + hedging_policy=hedging_policy, + cache_policy=cache_policy, + coalescing_policy=coalescing_policy, ) @@ -113,6 +152,7 @@ def create_llm_client( "LLM", "LLMClient", "LLMBuilder", + "LLMProfile", "LLMSettings", "LLMProvider", "LLMTransport", diff --git a/src/afk/llms/builder.py b/src/afk/llms/builder.py index efa834b..97ffee5 100644 --- a/src/afk/llms/builder.py +++ b/src/afk/llms/builder.py @@ -66,16 +66,16 @@ def settings(self, settings: LLMSettings) -> LLMBuilder: def profile(self, name: str) -> LLMBuilder: """Apply one named runtime profile from `afk.llms.profiles.PROFILES`.""" key = name.strip().lower() - row = PROFILES.get(key) - if row is None: + profile = PROFILES.get(key) + if profile is None: raise ValueError(f"Unknown llm profile '{name}'") - self._retry_policy = row["retry"] - self._timeout_policy = row["timeout"] - self._rate_limit_policy = row["rate_limit"] - self._breaker_policy = row["breaker"] - self._hedging_policy = row["hedging"] - self._cache_policy = row["cache"] - self._coalescing_policy = row["coalescing"] + self._retry_policy = profile.retry + self._timeout_policy = profile.timeout + self._rate_limit_policy = profile.rate_limit + self._breaker_policy = profile.breaker + self._hedging_policy = profile.hedging + self._cache_policy = profile.cache + self._coalescing_policy = profile.coalescing return self def for_agent_runtime(self) -> LLMBuilder: diff --git a/src/afk/llms/profiles.py b/src/afk/llms/profiles.py index ce5d314..33502b0 100644 --- a/src/afk/llms/profiles.py +++ b/src/afk/llms/profiles.py @@ -8,6 +8,8 @@ from __future__ import annotations +from dataclasses import dataclass + from .runtime.contracts import ( CachePolicy, CircuitBreakerPolicy, @@ -18,49 +20,80 @@ TimeoutPolicy, ) -PROFILES = { - "development": { - "retry": RetryPolicy(max_retries=1, backoff_base_s=0.2, backoff_jitter_s=0.05), - "timeout": TimeoutPolicy(request_timeout_s=30.0, stream_idle_timeout_s=90.0), - "rate_limit": RateLimitPolicy(requests_per_second=50.0, burst=100), - "breaker": CircuitBreakerPolicy( + +@dataclass(frozen=True, slots=True) +class LLMProfile: + """ + Named profile containing all runtime policies for an LLM client. + + Attributes: + name: Profile identifier. + retry: Retry policy for transient failures. + timeout: Request/stream timeout policy. + rate_limit: Rate limiting policy. + breaker: Circuit breaker policy. + hedging: Hedging policy for tail latency reduction. + cache: Caching policy for responses. + coalescing: Request coalescing policy for identical payloads. + """ + + name: str + retry: RetryPolicy + timeout: TimeoutPolicy + rate_limit: RateLimitPolicy + breaker: CircuitBreakerPolicy + hedging: HedgingPolicy + cache: CachePolicy + coalescing: CoalescingPolicy + + +PROFILES: dict[str, LLMProfile] = { + "development": LLMProfile( + name="development", + retry=RetryPolicy(max_retries=1, backoff_base_s=0.2, backoff_jitter_s=0.05), + timeout=TimeoutPolicy(request_timeout_s=30.0, stream_idle_timeout_s=90.0), + rate_limit=RateLimitPolicy(requests_per_second=50.0, burst=100), + breaker=CircuitBreakerPolicy( failure_threshold=8, cooldown_s=10.0, half_open_max_calls=2 ), - "hedging": HedgingPolicy(enabled=False, delay_s=0.2), - "cache": CachePolicy(enabled=False, ttl_s=15.0), - "coalescing": CoalescingPolicy(enabled=True), - }, - "production": { - "retry": RetryPolicy(max_retries=3, backoff_base_s=0.5, backoff_jitter_s=0.15), - "timeout": TimeoutPolicy(request_timeout_s=30.0, stream_idle_timeout_s=45.0), - "rate_limit": RateLimitPolicy(requests_per_second=20.0, burst=40), - "breaker": CircuitBreakerPolicy( + hedging=HedgingPolicy(enabled=False, delay_s=0.2), + cache=CachePolicy(enabled=False, ttl_s=15.0), + coalescing=CoalescingPolicy(enabled=True), + ), + "production": LLMProfile( + name="production", + retry=RetryPolicy(max_retries=3, backoff_base_s=0.5, backoff_jitter_s=0.15), + timeout=TimeoutPolicy(request_timeout_s=30.0, stream_idle_timeout_s=45.0), + rate_limit=RateLimitPolicy(requests_per_second=20.0, burst=40), + breaker=CircuitBreakerPolicy( failure_threshold=5, cooldown_s=30.0, half_open_max_calls=1 ), - "hedging": HedgingPolicy(enabled=False, delay_s=0.2), - "cache": CachePolicy(enabled=False, ttl_s=30.0), - "coalescing": CoalescingPolicy(enabled=True), - }, - "high_throughput": { - "retry": RetryPolicy(max_retries=2, backoff_base_s=0.3, backoff_jitter_s=0.1), - "timeout": TimeoutPolicy(request_timeout_s=20.0, stream_idle_timeout_s=40.0), - "rate_limit": RateLimitPolicy(requests_per_second=120.0, burst=200), - "breaker": CircuitBreakerPolicy( + hedging=HedgingPolicy(enabled=False, delay_s=0.2), + cache=CachePolicy(enabled=False, ttl_s=30.0), + coalescing=CoalescingPolicy(enabled=True), + ), + "high_throughput": LLMProfile( + name="high_throughput", + retry=RetryPolicy(max_retries=2, backoff_base_s=0.3, backoff_jitter_s=0.1), + timeout=TimeoutPolicy(request_timeout_s=20.0, stream_idle_timeout_s=40.0), + rate_limit=RateLimitPolicy(requests_per_second=120.0, burst=200), + breaker=CircuitBreakerPolicy( failure_threshold=10, cooldown_s=20.0, half_open_max_calls=3 ), - "hedging": HedgingPolicy(enabled=False, delay_s=0.2), - "cache": CachePolicy(enabled=True, ttl_s=20.0), - "coalescing": CoalescingPolicy(enabled=True), - }, - "low_latency": { - "retry": RetryPolicy(max_retries=1, backoff_base_s=0.2, backoff_jitter_s=0.05), - "timeout": TimeoutPolicy(request_timeout_s=10.0, stream_idle_timeout_s=20.0), - "rate_limit": RateLimitPolicy(requests_per_second=30.0, burst=60), - "breaker": CircuitBreakerPolicy( + hedging=HedgingPolicy(enabled=False, delay_s=0.2), + cache=CachePolicy(enabled=True, ttl_s=20.0), + coalescing=CoalescingPolicy(enabled=True), + ), + "low_latency": LLMProfile( + name="low_latency", + retry=RetryPolicy(max_retries=1, backoff_base_s=0.2, backoff_jitter_s=0.05), + timeout=TimeoutPolicy(request_timeout_s=10.0, stream_idle_timeout_s=20.0), + rate_limit=RateLimitPolicy(requests_per_second=30.0, burst=60), + breaker=CircuitBreakerPolicy( failure_threshold=3, cooldown_s=15.0, half_open_max_calls=1 ), - "hedging": HedgingPolicy(enabled=True, delay_s=0.08), - "cache": CachePolicy(enabled=True, ttl_s=10.0), - "coalescing": CoalescingPolicy(enabled=True), - }, + hedging=HedgingPolicy(enabled=True, delay_s=0.08), + cache=CachePolicy(enabled=True, ttl_s=10.0), + coalescing=CoalescingPolicy(enabled=True), + ), } diff --git a/tests/core/test_debugger.py b/tests/core/test_debugger.py index ae6f4dc..33a7e8c 100644 --- a/tests/core/test_debugger.py +++ b/tests/core/test_debugger.py @@ -2,10 +2,11 @@ import asyncio +import pytest from pydantic import BaseModel from afk.agents import Agent -from afk.core import RunnerConfig +from afk.core import Runner, RunnerConfig from afk.debugger import Debugger, DebuggerConfig from afk.tools import tool @@ -19,6 +20,74 @@ def emit_secret(args: _SecretArgs) -> dict[str, str]: return {"token": args.value} +# --------------------------------------------------------------------------- +# DebuggerConfig tests +# --------------------------------------------------------------------------- + + +class TestDebuggerConfigDefaults: + def test_default_values(self): + config = DebuggerConfig() + assert config.enabled is True + assert config.verbosity == "detailed" + assert config.include_content is True + assert config.redact_secrets is True + assert config.max_payload_chars == 4000 + assert config.emit_timestamps is True + assert config.emit_step_snapshots is True + + def test_custom_values(self): + config = DebuggerConfig( + enabled=False, + verbosity="basic", + include_content=False, + redact_secrets=False, + max_payload_chars=100, + emit_timestamps=False, + emit_step_snapshots=False, + ) + assert config.enabled is False + assert config.verbosity == "basic" + assert config.include_content is False + assert config.redact_secrets is False + assert config.max_payload_chars == 100 + assert config.emit_timestamps is False + assert config.emit_step_snapshots is False + + def test_frozen(self): + config = DebuggerConfig() + with pytest.raises(AttributeError): + config.enabled = False + + @pytest.mark.parametrize("verbosity", ["basic", "detailed", "trace"]) + def test_verbosity_values(self, verbosity): + config = DebuggerConfig(verbosity=verbosity) + assert config.verbosity == verbosity + + +# --------------------------------------------------------------------------- +# Debugger tests +# --------------------------------------------------------------------------- + + +class TestDebugger: + def test_debugger_creates_runner_with_debug_enabled(self): + debugger = Debugger() + runner = debugger.runner() + assert runner.config.debug is True + + def test_debugger_with_custom_config(self): + config = DebuggerConfig(enabled=True, redact_secrets=True, verbosity="trace") + debugger = Debugger(config) + runner = debugger.runner() + assert runner.config.debug is True + + +# --------------------------------------------------------------------------- +# Integration tests +# --------------------------------------------------------------------------- + + def test_debugger_runner_enables_debug_and_redacts_secret_payloads(): from afk.llms import LLM from afk.llms.types import ( @@ -127,8 +196,6 @@ async def _embed_core(self, req: EmbeddingRequest) -> EmbeddingResponse: raise NotImplementedError async def _scenario(): - from afk.core import Runner - runner = Runner(config=RunnerConfig(debug=True)) handle = await runner.run_handle(Agent(model=_LLM(), instructions="x")) first = None @@ -141,3 +208,51 @@ async def _scenario(): event = asyncio.run(_scenario()) assert event is not None assert "debug" in event.data + + +def test_debugger_disabled_emits_no_debug_payload(): + from afk.llms import LLM + from afk.llms.types import ( + EmbeddingRequest, + EmbeddingResponse, + LLMCapabilities, + LLMRequest, + LLMResponse, + ) + + class _LLM(LLM): + @property + def provider_id(self) -> str: + return "dbg-disabled" + + @property + def capabilities(self) -> LLMCapabilities: + return LLMCapabilities(chat=True, streaming=False, tool_calling=True) + + async def _chat_core(self, req: LLMRequest, *, response_model=None) -> LLMResponse: + _ = req + _ = response_model + return LLMResponse(text="ok") + + async def _chat_stream_core(self, req: LLMRequest, *, response_model=None): + _ = req + _ = response_model + raise NotImplementedError + + async def _embed_core(self, req: EmbeddingRequest) -> EmbeddingResponse: + _ = req + raise NotImplementedError + + async def _scenario(): + debugger = Debugger(DebuggerConfig(enabled=False)) + runner = debugger.runner() + handle = await runner.run_handle(Agent(model=_LLM(), instructions="x")) + events = [] + async for event in handle.events: + events.append(event) + _ = await handle.await_result() + return events + + events = asyncio.run(_scenario()) + first = events[0] + assert "debug" not in first.data diff --git a/tests/llms/test_llm_builder_structured.py b/tests/llms/test_llm_builder_structured.py index f18dd00..b75e7da 100644 --- a/tests/llms/test_llm_builder_structured.py +++ b/tests/llms/test_llm_builder_structured.py @@ -503,9 +503,9 @@ def test_all_expected_keys(self): class TestProfilesStructure: - """Every profile must have all 7 required policy keys with correct types.""" + """Every LLMProfile must have all 7 required policy attributes with correct types.""" - _REQUIRED_KEYS = ("retry", "timeout", "rate_limit", "breaker", "hedging", "cache", "coalescing") + _REQUIRED_ATTRS = ("retry", "timeout", "rate_limit", "breaker", "hedging", "cache", "coalescing") _EXPECTED_TYPES = { "retry": RetryPolicy, "timeout": TimeoutPolicy, @@ -517,139 +517,139 @@ class TestProfilesStructure: } @pytest.mark.parametrize("profile_name", ["development", "production", "high_throughput", "low_latency"]) - def test_has_all_required_keys(self, profile_name): + def test_has_all_required_attrs(self, profile_name): profile = PROFILES[profile_name] - for key in self._REQUIRED_KEYS: - assert key in profile, f"Profile '{profile_name}' missing key '{key}'" + for attr in self._REQUIRED_ATTRS: + assert hasattr(profile, attr), f"Profile '{profile_name}' missing attr '{attr}'" @pytest.mark.parametrize("profile_name", ["development", "production", "high_throughput", "low_latency"]) def test_correct_policy_types(self, profile_name): profile = PROFILES[profile_name] - for key, expected_type in self._EXPECTED_TYPES.items(): - assert isinstance(profile[key], expected_type), ( - f"Profile '{profile_name}' key '{key}' expected {expected_type.__name__}, " - f"got {type(profile[key]).__name__}" + for attr, expected_type in self._EXPECTED_TYPES.items(): + assert isinstance(getattr(profile, attr), expected_type), ( + f"Profile '{profile_name}' attr '{attr}' expected {expected_type.__name__}, " + f"got {type(getattr(profile, attr)).__name__}" ) @pytest.mark.parametrize("profile_name", ["development", "production", "high_throughput", "low_latency"]) - def test_no_extra_keys(self, profile_name): + def test_has_name(self, profile_name): profile = PROFILES[profile_name] - assert set(profile.keys()) == set(self._REQUIRED_KEYS) + assert profile.name == profile_name class TestProductionProfileValues: """Verify specific values in the production profile.""" def test_retry_max_retries(self): - assert PROFILES["production"]["retry"].max_retries == 3 + assert PROFILES["production"].retry.max_retries == 3 def test_retry_backoff_base(self): - assert PROFILES["production"]["retry"].backoff_base_s == 0.5 + assert PROFILES["production"].retry.backoff_base_s == 0.5 def test_retry_backoff_jitter(self): - assert PROFILES["production"]["retry"].backoff_jitter_s == 0.15 + assert PROFILES["production"].retry.backoff_jitter_s == 0.15 def test_timeout_request(self): - assert PROFILES["production"]["timeout"].request_timeout_s == 30.0 + assert PROFILES["production"].timeout.request_timeout_s == 30.0 def test_timeout_stream_idle(self): - assert PROFILES["production"]["timeout"].stream_idle_timeout_s == 45.0 + assert PROFILES["production"].timeout.stream_idle_timeout_s == 45.0 def test_rate_limit_rps(self): - assert PROFILES["production"]["rate_limit"].requests_per_second == 20.0 + assert PROFILES["production"].rate_limit.requests_per_second == 20.0 def test_rate_limit_burst(self): - assert PROFILES["production"]["rate_limit"].burst == 40 + assert PROFILES["production"].rate_limit.burst == 40 def test_breaker_failure_threshold(self): - assert PROFILES["production"]["breaker"].failure_threshold == 5 + assert PROFILES["production"].breaker.failure_threshold == 5 def test_breaker_cooldown(self): - assert PROFILES["production"]["breaker"].cooldown_s == 30.0 + assert PROFILES["production"].breaker.cooldown_s == 30.0 def test_breaker_half_open_max(self): - assert PROFILES["production"]["breaker"].half_open_max_calls == 1 + assert PROFILES["production"].breaker.half_open_max_calls == 1 def test_hedging_disabled(self): - assert PROFILES["production"]["hedging"].enabled is False + assert PROFILES["production"].hedging.enabled is False def test_cache_disabled(self): - assert PROFILES["production"]["cache"].enabled is False + assert PROFILES["production"].cache.enabled is False def test_cache_ttl(self): - assert PROFILES["production"]["cache"].ttl_s == 30.0 + assert PROFILES["production"].cache.ttl_s == 30.0 def test_coalescing_enabled(self): - assert PROFILES["production"]["coalescing"].enabled is True + assert PROFILES["production"].coalescing.enabled is True class TestLowLatencyProfileValues: """Low latency profile has hedging enabled and aggressive timeouts.""" def test_hedging_enabled(self): - assert PROFILES["low_latency"]["hedging"].enabled is True + assert PROFILES["low_latency"].hedging.enabled is True def test_hedging_delay(self): - assert PROFILES["low_latency"]["hedging"].delay_s == 0.08 + assert PROFILES["low_latency"].hedging.delay_s == 0.08 def test_request_timeout(self): - assert PROFILES["low_latency"]["timeout"].request_timeout_s == 10.0 + assert PROFILES["low_latency"].timeout.request_timeout_s == 10.0 def test_stream_idle_timeout(self): - assert PROFILES["low_latency"]["timeout"].stream_idle_timeout_s == 20.0 + assert PROFILES["low_latency"].timeout.stream_idle_timeout_s == 20.0 def test_cache_enabled(self): - assert PROFILES["low_latency"]["cache"].enabled is True + assert PROFILES["low_latency"].cache.enabled is True def test_breaker_failure_threshold(self): - assert PROFILES["low_latency"]["breaker"].failure_threshold == 3 + assert PROFILES["low_latency"].breaker.failure_threshold == 3 def test_retry_max_retries(self): - assert PROFILES["low_latency"]["retry"].max_retries == 1 + assert PROFILES["low_latency"].retry.max_retries == 1 class TestHighThroughputProfileValues: """High throughput profile has high RPS and cache enabled.""" def test_rate_limit_rps(self): - assert PROFILES["high_throughput"]["rate_limit"].requests_per_second == 120.0 + assert PROFILES["high_throughput"].rate_limit.requests_per_second == 120.0 def test_rate_limit_burst(self): - assert PROFILES["high_throughput"]["rate_limit"].burst == 200 + assert PROFILES["high_throughput"].rate_limit.burst == 200 def test_cache_enabled(self): - assert PROFILES["high_throughput"]["cache"].enabled is True + assert PROFILES["high_throughput"].cache.enabled is True def test_cache_ttl(self): - assert PROFILES["high_throughput"]["cache"].ttl_s == 20.0 + assert PROFILES["high_throughput"].cache.ttl_s == 20.0 def test_breaker_failure_threshold(self): - assert PROFILES["high_throughput"]["breaker"].failure_threshold == 10 + assert PROFILES["high_throughput"].breaker.failure_threshold == 10 class TestDevelopmentProfileValues: """Development profile has relaxed settings.""" def test_retry_max_retries(self): - assert PROFILES["development"]["retry"].max_retries == 1 + assert PROFILES["development"].retry.max_retries == 1 def test_rate_limit_rps(self): - assert PROFILES["development"]["rate_limit"].requests_per_second == 50.0 + assert PROFILES["development"].rate_limit.requests_per_second == 50.0 def test_rate_limit_burst(self): - assert PROFILES["development"]["rate_limit"].burst == 100 + assert PROFILES["development"].rate_limit.burst == 100 def test_breaker_failure_threshold(self): - assert PROFILES["development"]["breaker"].failure_threshold == 8 + assert PROFILES["development"].breaker.failure_threshold == 8 def test_hedging_disabled(self): - assert PROFILES["development"]["hedging"].enabled is False + assert PROFILES["development"].hedging.enabled is False def test_cache_disabled(self): - assert PROFILES["development"]["cache"].enabled is False + assert PROFILES["development"].cache.enabled is False def test_coalescing_enabled(self): - assert PROFILES["development"]["coalescing"].enabled is True + assert PROFILES["development"].coalescing.enabled is True # ============================== Structured Output ============================== diff --git a/tests/llms/test_llm_settings.py b/tests/llms/test_llm_settings.py index 2785607..5c3df41 100644 --- a/tests/llms/test_llm_settings.py +++ b/tests/llms/test_llm_settings.py @@ -230,12 +230,12 @@ def test_override_all_vars_at_once(self): class TestLLMSettingsFromEnvInvalidValues: - """from_env() should fall back to defaults when env vars are invalid.""" + """from_env() falls back to defaults on invalid env var values (forgiving DX).""" def test_invalid_float_timeout_s(self): os.environ["AFK_LLM_TIMEOUT_S"] = "not_a_float" s = LLMSettings.from_env() - assert s.timeout_s == LLMSettings().timeout_s + assert s.timeout_s == LLMSettings().timeout_s # falls back to default def test_invalid_float_backoff_base_s(self): os.environ["AFK_LLM_BACKOFF_BASE_S"] = "abc"