diff --git a/AGENTS.md b/AGENTS.md index f4f6668..4cd3ce0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,11 +40,13 @@ objects (`ListToolsResult`, `CallToolResult`, `ListResourcesResult`). | `src/deep_agentic_core_mcp/config.py` | Server identity and shared constants | | `src/deep_agentic_core_mcp/tools/` | Tool implementations (`core.py`) and registry (`registry.py`) | | `src/deep_agentic_core_mcp/resources/` | MCP resource definitions and catalog | -| `src/deep_agentic_core_mcp/prompts/` | Reusable prompt templates (planned) | +| `src/deep_agentic_core_mcp/prompts/` | Reusable prompt templates, wired into `prompts/list`/`prompts/get` | | `src/deep_agentic_core_mcp/schemas/` | Request/response contracts | | `src/deep_agentic_core_mcp/services/` | Shared orchestration logic | | `src/deep_agentic_core_mcp/adapters/` | Integration boundaries to agenticlens and agentic-chaos | | `tests/` | Pytest tests (asyncio_mode=auto) | +| `scripts/generate_tools_doc.py` | Generates `docs/tools.md` from `tools/registry.py` | +| `docs/tools.md` | **Generated** — never hand-edit, run `make docs` | | `server.json` | MCP Registry metadata | | `Makefile` | Local dev automation | @@ -56,9 +58,11 @@ objects (`ListToolsResult`, `CallToolResult`, `ListResourcesResult`). ## Adding a New Tool 1. Add implementation in `tools/` (return a dict) -2. Register in `tools/registry.py` (name + description) +2. Register in `tools/registry.py` (name, description, and the metadata + fields: `category`, `prerequisites`, `expected_duration`, `mutates_session`) 3. Add handler entry in `server.py` `_TOOL_DISPATCH` 4. Add test in `tests/test_server.py` +5. Run `make docs` to regenerate `docs/tools.md` ## Package Boundaries @@ -70,6 +74,9 @@ objects (`ListToolsResult`, `CallToolResult`, `ListResourcesResult`). ## Pre-push Checklist Run `make check` before every push. It runs: lint → format-check → typecheck → test. +If `tools/registry.py` changed, also run `make docs-check` (regenerates +`docs/tools.md` and fails if that changed anything you didn't commit) — +not part of `check` itself so the default gate stays fast. ## Release diff --git a/CHANGELOG.md b/CHANGELOG.md index 6080c9c..3f561d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,61 @@ All notable changes to this project will be documented here. This project follows [Semantic Versioning](https://semver.org/). +## 0.2.0 - 2026-08-08 + +### Added + +- `core.verify` tool checking agenticlens, agentic-chaos, and ai-operations-spec connectivity. +- `core.session_state` tool exposing the in-memory session's stored artifacts and call history. +- In-memory session store (`services/session.py`) so `lens.analyze_workflow` -> `lens.report_summary` + -> `lens.compare_runs` -> `chaos.run_experiment` can share artifacts without the client resending + them; tools accept an optional `session_id` argument. +- `lens.report_summary` tool rendering a Markdown workflow report via AgenticLens's `MarkdownExporter`. +- `lens.compare_runs` tool wrapping AgenticLens's baseline/candidate trace comparison and regression + detection. +- `lens.slo_summary` tool applying release-gate style SLO thresholds to an evaluation report. +- `lens.audit_report` tool returning case-by-case evaluation detail, optionally with an HTML report. +- `chaos.run_experiment` tool running a workspace-sandboxed target script inside a chaos session and + reporting the resulting fault events (mirrors the agentic-chaos CLI's `chaos run`). +- `examples/chaos_target.py`, a minimal `chaos_call()`-instrumented script for `chaos.run_experiment`. +- Real `prompts/list` and `prompts/get` handlers, with prompt arguments and rendered templates + (previously the prompt registry existed but was never wired into the server). +- Tool metadata (`category`, `prerequisites`, `expected_duration`, `mutates_session`) on every tool, + surfaced to MCP hosts via `Tool.annotations`/`Tool._meta`. +- `core.health` now returns adapter availability/version, loaded tool/resource/prompt counts, the + resolved workspace root, and recent successful-call timestamps, instead of just `{"status": "ok"}`. +- `docs/tools.md`, a generated tool reference (name, description, category, prerequisites, expected + duration, mutation/side-effect flags, and full input schema per tool) produced by + `scripts/generate_tools_doc.py` from `tools/registry.py`, so it can't drift out of sync with what + `tools/list` actually returns. `make docs` regenerates it; `make docs-check` fails if it's stale. + +### Changed + +- Adapters (`adapters/agenticlens.py`, `adapters/agentic_chaos.py`, `adapters/ai_operations_spec.py`) + now import their sibling repo defensively: a missing/broken sibling no longer crashes server boot, + it surfaces as `"available": false` through `core.verify`/`core.health` and a structured tool error. + +### Fixed + +- `chaos.run_experiment`'s `timeout_seconds` now actually bounds wall-clock time. It previously ran + the worker thread inside a `with ThreadPoolExecutor(...)` block, whose `__exit__` calls + `shutdown(wait=True)` and blocked for the thread to finish regardless of the timeout having already + fired. +- `server.py` no longer indexes `SCHEMA_DOCUMENTS` directly when building the schemas resources + (`SCHEMA_DOCUMENTS["workflow.schema.json"]`, etc.); a missing/broken `ai-operations-spec` sibling + used to raise `KeyError` at import time, crashing server boot before `core.verify` could report it + as unavailable. `ai_operations_spec.py` now exposes `schema_resource_content()`/ + `list_schema_resources()` that both derive from what actually loaded, so `resources/list` and + `resources/read` degrade consistently with everything else. +- `examples/sample_workflow.json` previously failed `Workflow` validation outright (missing + `start_time`) and was too thin to exercise the recommendation engine even if fixed. It's now a + valid, richer workflow (6 steps, real metrics) that produces real `lens.analyze_workflow`/ + `lens.report_summary` recommendations (excessive retrieved chunks, a duplicate tool call, long + conversation history) instead of an empty or erroring result. +- `handle_call_tool` now catches any exception a tool handler raises (e.g. a pydantic + `ValidationError` from malformed workflow/run input) and returns a structured + `{"ok": false, "error": ...}` payload instead of letting it propagate past the MCP dispatch boundary. + ## 0.1.3 - 2026-08-07 ### Added diff --git a/Makefile b/Makefile index f2db7e0..b84e647 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL := help -.PHONY: help install lint format format-check typecheck test test-cov clean build check +.PHONY: help install lint format format-check typecheck test test-cov clean build check docs docs-check help: ## Show this help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | \ @@ -33,4 +33,10 @@ clean: ## Remove build artifacts build: ## Build package distributions uv run python -m build +docs: ## Regenerate generated docs (docs/tools.md) from tools/registry.py + uv run python scripts/generate_tools_doc.py + +docs-check: docs ## Fail if docs/tools.md is out of date (regenerates, then diffs) + git diff --exit-code docs/tools.md + check: lint format-check typecheck test ## Run all quality gates diff --git a/README.md b/README.md index 12fdb7e..6f69d5c 100644 --- a/README.md +++ b/README.md @@ -45,35 +45,41 @@ Planned capability areas: - Python-first: package and publish through PyPI - Thin orchestration layer: reuse `agenticlens` and `agentic-chaos` instead of re-implementing their logic -- Local-first: work well as a stdio MCP server for developer workflows +- Local-first: work well as a stdio MCP server for developer workflows — + this matters because `chaos.run_experiment` executes real code (see + [SECURITY.md](SECURITY.md)), so this server is meant for trusted, + local/stdio use, not exposure to untrusted clients - Expandable: leave room for a later remote deployment mode if needed -## Initial Scope - -The first milestone is foundation only: - -- repository structure -- packaging metadata -- MCP registry metadata -- roadmap and product framing -- minimal server entrypoint and tool layout - -The first working implementation can stay intentionally small while the shape of -the tool surface stabilizes. - -## Proposed MCP Surface - -Possible first tool groups: - -- `lens.profile_workflow` -- `lens.analyze_workflow` -- `chaos.run_experiment` -- `chaos.list_faults` -- `core.health` -- `core.version` - -These names are placeholders, but the structure matters: one server can expose -multiple tools without needing multiple MCP packages or registry entries. +## MCP Surface (current, `0.2.0`) + +- `core.health` — rich diagnostics: adapter availability/version, loaded + tool/resource/prompt counts, workspace root, recent successful calls +- `core.version` — server package version +- `core.verify` — checks agenticlens/agentic-chaos/ai-operations-spec + connectivity and reports readiness +- `core.session_state` — inspect what the active session has accumulated +- `lens.analyze_workflow` — run AgenticLens recommendations against a + workflow artifact +- `lens.report_summary` — render a Markdown workflow report +- `lens.compare_runs` — compare baseline/candidate trace runs for + regressions +- `lens.slo_summary` — apply release-gate style SLO thresholds to an + evaluation report +- `lens.audit_report` — case-by-case evaluation detail, optionally with HTML +- `chaos.list_faults` — list the supported fault types +- `chaos.run_experiment` — run a workspace-sandboxed target script under + selected faults ([executes real code — see `SECURITY.md`](SECURITY.md)) +- `spec.validate_artifact` — validate a workflow/run artifact against the AI + Operations v0.4 draft + +Sequential tool calls can share context via an optional `session_id` +argument, backed by an in-memory session store — see `ROADMAP.md` Phase 2. + +See [ROADMAP.md](ROADMAP.md) for what's shipped per phase and what's still +open, and [docs/tools.md](docs/tools.md) for full input schemas and +per-tool metadata (generated from `tools/registry.py`, run `make docs` to +refresh it after changing that file). ## Repository Layout @@ -85,9 +91,13 @@ mcp-server/ ├── server.json ├── .gitignore ├── docs/ -│ └── architecture.md +│ ├── architecture.md +│ └── tools.md # generated - see scripts/generate_tools_doc.py ├── examples/ -│ └── sample_workflow.json +│ ├── sample_workflow.json +│ └── chaos_target.py +├── scripts/ +│ └── generate_tools_doc.py ├── src/ │ └── deep_agentic_core_mcp/ │ ├── __init__.py @@ -104,20 +114,26 @@ mcp-server/ │ │ └── tooling.py │ ├── services/ │ │ ├── __init__.py -│ │ └── registry.py +│ │ ├── registry.py +│ │ └── session.py │ ├── adapters/ │ │ ├── __init__.py │ │ ├── agentic_chaos.py -│ │ └── agenticlens.py +│ │ ├── agenticlens.py +│ │ └── ai_operations_spec.py │ └── tools/ │ ├── __init__.py │ ├── registry.py │ ├── chaos.py │ ├── core.py -│ └── lens.py +│ ├── lens.py +│ └── spec.py └── tests/ + ├── test_degraded_boot.py ├── test_imports.py - └── test_registry.py + ├── test_registry.py + ├── test_server.py + └── test_session.py ``` ## MCP-Oriented Structure @@ -130,12 +146,11 @@ MCP server: workflow examples - `prompts/` for reusable prompt templates exposed through the server - `schemas/` for typed request and response contracts -- `services/` for shared orchestration logic that keeps tool modules thin -- `adapters/` for integration boundaries to `agenticlens` and - `agentic-chaos` - -The implementation is still early, but the file structure now reflects that -shape so we can add functionality without reshuffling the repo later. +- `services/` for shared orchestration logic that keeps tool modules thin, + including the in-memory session store (`services/session.py`) +- `adapters/` for integration boundaries to `agenticlens`, `agentic-chaos`, + and `ai-operations-spec` — each degrades to `"available": false` rather + than crashing server boot if its sibling repo is missing ## Packaging and Publishing Model @@ -147,28 +162,21 @@ shape so we can add functionality without reshuffling the repo later. For PyPI-based verification, the `mcp-name` marker above must match the `name` field in `server.json`. -## Near-Term Build Order - -1. Lock the canonical namespace and package metadata. -2. Implement the stdio MCP server entrypoint. -3. Add a minimal `core.health` tool. -4. Add the first `agenticlens` and `agentic-chaos` adapter-backed tools. -5. Add examples and publishable packaging checks. - ## What's Next -Upcoming capabilities (see [ROADMAP.md](ROADMAP.md) for full details): +Phase 2 (session management, rich diagnostics, tool annotations, prompt +registry, `core.verify`) and Phase 3b (Agentic Chaos) are complete as of +`0.2.0`. What's still open (see [ROADMAP.md](ROADMAP.md) for full detail): -- **Session management** — sequential tool calls share context without - resending artifacts -- **Rich diagnostics** — `core.health` returns adapter availability, dependency - versions, and config validation -- **Tool annotations** — category, prerequisites, duration, and mutation - metadata on every tool -- **Prompt registry** — reusable prompt templates for analysis, comparison, and - experiment workflows -- **Integration verification** — `core.verify` checks agenticlens and - agentic-chaos connectivity +- **Phase 3a (AgenticLens)** — provenance verification on + `lens.analyze_workflow`'s response shape +- **Phase 3c (AI Operations Specification)** — multi-version schema support + and conformance-style reporting, both blocked on upstream `ai-operations-spec` + work landing first +- **Phase 4 (Unified Workflows)** — joined observability + chaos workflows, + incident/readiness reporting, a higher-level control surface +- **Phase 5/6** — PyPI + MCP Registry publishing, operational intelligence + features ## Development @@ -178,6 +186,8 @@ A `Makefile` provides shorthand for common tasks: make install # install dev dependencies make check # run all quality gates (lint + format + typecheck + test) make test-cov # tests with coverage report +make docs # regenerate docs/tools.md from tools/registry.py +make docs-check # fail if docs/tools.md is out of date make help # list all available targets ``` diff --git a/ROADMAP.md b/ROADMAP.md index 5d4b4af..a0b1e48 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,5 +1,28 @@ # Roadmap +## Release Status + +Current shipped version: `0.2.0` (2026-08-08) — see [CHANGELOG.md](CHANGELOG.md). + +- **Phase 0: Foundation** ✅ Complete +- **Phase 1: Minimal MCP Server** ✅ Complete — `core.health`/`core.version` + shipped in `0.1.2` +- **Phase 2: Session Management & Diagnostics** ✅ Complete — session state, + rich `core.health` diagnostics, tool metadata/annotations, `core.verify`, + and real `prompts/list`/`prompts/get` support all shipped in `0.2.0` +- **Phase 3a: AgenticLens Integration** 🏗️ In progress — `lens.analyze_workflow` + shipped in `0.1.3`; `lens.report_summary`, `lens.compare_runs`, + `lens.slo_summary`, and `lens.audit_report` shipped in `0.2.0` +- **Phase 3b: Agentic Chaos Integration** ✅ Complete — `chaos.list_faults` + shipped in `0.1.3`; `chaos.run_experiment` shipped in `0.2.0` +- **Phase 3c: AI Operations Specification Conformance** 🏗️ In progress — + `spec.validate_artifact` and schema resources shipped in `0.1.3`, ahead of + where this roadmap originally planned them; remaining work still blocked on + upstream (see below) +- **Phase 4: Unified Workflows** 🚧 Planned +- **Phase 5: Publishing and Adoption** 🚧 Planned +- **Phase 6: Operational Intelligence** 🚧 Planned + ## Vision Build one public MCP server for the DeepAgentLabs ecosystem that unifies: @@ -42,7 +65,7 @@ That means: ## Phase 0: Foundation -Status: current +Status: complete Goals: @@ -61,6 +84,8 @@ Deliverables: ## Phase 1: Minimal MCP Server +Status: complete — `core.health` and `core.version` shipped in 0.1.2. + Goals: - create a runnable stdio MCP server @@ -76,27 +101,48 @@ Success criteria: ## Phase 2: Session Management & Diagnostics +Status: complete, shipped in 0.2.0. In-memory session state +(`services/session.py`, `core.session_state`), rich `core.health` +diagnostics, tool metadata/annotations, `core.verify`, and real +`prompts/list`/`prompts/get` support (the 0.1.3 prompt registry was data +only and was never wired into the server) all landed together. + Goals: -- add lightweight in-memory session state so sequential tool calls share context -- expand `core.health` into rich diagnostics (adapter availability, dependency - versions, loaded tools/resources, config validation, last successful runs) -- add tool metadata and annotations (category, prerequisites, expected duration, +- ✅ add lightweight in-memory session state so sequential tool calls share context +- ✅ expand `core.health` into rich diagnostics (adapter availability, dependency + versions, loaded tools/resources/prompts, workspace root, last successful calls) +- ✅ add tool metadata and annotations (category, prerequisites, expected duration, whether the tool mutates session state) -- implement prompt registry support — expose reusable prompt templates as MCP - prompts/resources for analysis, comparison, and experiment workflows -- add integration verification flow — a `core.verify` tool that checks - agenticlens and agentic-chaos connectivity and reports readiness +- ✅ implement prompt registry support — `prompts/list`/`prompts/get` are wired + into the server, with real arguments and rendered templates +- ✅ add integration verification flow — a `core.verify` tool that checks + agenticlens, agentic-chaos, and ai-operations-spec connectivity and reports + readiness Success criteria: -- `lens.profile` → `chaos.run` → `lens.compare` can share a session without - the client resending artifacts -- `core.health` returns structured diagnostics, not just `{"status": "ok"}` -- MCP clients can discover tool categories, prerequisites, and output types -- a new contributor can run `core.verify` and see what's connected - -## Phase 3: AgenticLens Integration +- ✅ `lens.analyze_workflow` → `lens.compare_runs` → `chaos.run_experiment` can + share a session (via an optional `session_id` argument) without the client + resending artifacts — the roadmap's original `lens.profile`/`chaos.run` + names don't exist as MCP tools, so this is verified against the tool names + that actually ship +- ✅ `core.health` returns structured diagnostics, not just `{"status": "ok"}` +- ✅ MCP clients can discover tool categories, prerequisites, and output types + (via `Tool._meta`/`Tool.annotations`) +- ✅ a new contributor can run `core.verify` and see what's connected +- as a prerequisite for the above, adapters now degrade instead of crashing + server boot when a sibling repo is missing/broken (`AdapterUnavailableError`, + per-adapter `probe()`) — `core.verify`/`core.health` couldn't report + "not connected" otherwise + +## Phase 3a: AgenticLens Integration + +Status: in progress — `lens.analyze_workflow` and the `agenticlens` adapter +layer shipped in 0.1.3; `lens.report_summary`, `lens.compare_runs`, +`lens.slo_summary`, and `lens.audit_report` shipped in 0.2.0, each backed by +real `agenticlens` capability (`MarkdownExporter`, `comparison.runner`, +`evaluation.gate`, `evaluation.html_report`) rather than reimplemented logic. - wire `agenticlens` into the MCP server through adapter functions - expose a first analysis-oriented tool surface @@ -106,39 +152,82 @@ Success criteria: Possible tools: -- `lens.analyze_workflow` -- `lens.report_summary` -- `lens.compare_runs` -- `lens.slo_summary` -- `lens.audit_report` +- [x] `lens.analyze_workflow` — shipped in 0.1.3 +- [x] `lens.report_summary` — shipped in 0.2.0 +- [x] `lens.compare_runs` — shipped in 0.2.0 +- [x] `lens.slo_summary` — shipped in 0.2.0 +- [x] `lens.audit_report` — shipped in 0.2.0 Success criteria: - a saved workflow artifact can be analyzed through MCP - recommendations are returned in a host-friendly schema -- every finding includes source provenance (step, span, artifact) +- every finding includes source provenance (step, span, artifact) — not yet + verified against `lens.analyze_workflow`'s current response shape - the analyzed artifact remains traceable to the AI Operations Specification contract -## Phase 3: Agentic Chaos Integration +Remaining work: this phase is functionally complete against agenticlens's +current API surface; provenance verification above is still open. + +## Phase 3b: Agentic Chaos Integration + +Status: complete, shipped in 0.2.0. `chaos.list_faults` and the +`agentic-chaos` adapter layer shipped in 0.1.3; `chaos.run_experiment` +shipped in 0.2.0, running a workspace-sandboxed target script inside a real +`chaos_session()` (mirroring the agentic-chaos CLI's `chaos run`), with a +`timeout_seconds` guard and a documented limitation that Python cannot force +-kill the worker thread on timeout. Goals: -- wire `agentic-chaos` into the MCP server through adapter functions -- expose fault listing and experiment execution -- return structured experiment results +- ✅ wire `agentic-chaos` into the MCP server through adapter functions +- ✅ expose fault listing and experiment execution +- ✅ return structured experiment results Possible tools: -- `chaos.list_faults` -- `chaos.run_experiment` +- [x] `chaos.list_faults` — shipped in 0.1.3 +- [x] `chaos.run_experiment` — shipped in 0.2.0 Success criteria: -- a target script or workflow can be exercised with selected faults -- results can be summarized alongside normal run output +- ✅ a target script or workflow can be exercised with selected faults +- ✅ results can be summarized alongside normal run output - chaos results are readable as or convertible to AI Operations Specification - artifacts + artifacts — still open; `chaos.run_experiment`'s output is `ChaosReport`-shaped + but not yet run through `spec.validate_artifact` + +## Phase 3c: AI Operations Specification Conformance + +Status: in progress — delivered ahead of where this roadmap had it planned. + +Goals: + +- wire `ai-operations-spec` into the MCP server through adapter functions +- expose structural and semantic artifact validation +- expose the specification's schemas as MCP resources so hosts can fetch the + contract directly instead of vendoring copies + +Delivered in 0.1.3: + +- `spec.validate_artifact` tool, validating workflow/run artifacts against + the AI Operations v0.4 draft +- MCP resource endpoints for the v0.4 workflow, run, and common schemas +- a `resources/read` handler returning resource contents +- an `ai-operations-spec` adapter layer + +Remaining work (re-checked during the 0.2.0 pass, still blocked upstream): + +- validation coverage beyond the v0.4 draft (versioned/multi-version support) + — `ai-operations-spec`'s `v0.1`–`v0.3` directories don't have populated + `schemas/` yet, only `v0.4` does, so there's nothing to switch between +- conformance-style reporting that distinguishes spec-defined pass/fail rules + from server-specific presentation, mirroring `agenticlens`'s own + conformance direction — no defined conformance-rule format exists upstream + to wire against yet +- resource coverage for additional artifact and schema types as the + specification grows ## Phase 4: Unified Workflows @@ -185,6 +274,41 @@ Goals: - Should `deep-agentic-core-mcp` be a thin wrapper package or eventually own workflow orchestration logic directly? - Is stdio-only enough for v0, or do we want a remote deployment path early? + If yes, see the known limitation directly below — it needs to be fixed + first, not concurrently. + +## Known Limitations + +- **Tool handlers are synchronous and block the event loop.** `handle_call_tool` + in `server.py` calls each tool handler directly (not via `asyncio.to_thread()` + or similar), so a slow call — most notably `chaos.run_experiment`, which can + run for up to `timeout_seconds` (default 30s) — blocks the server from + processing anything else for its duration, including cancellation/other + requests from the same client. Harmless for today's single-client stdio + transport, but this must be fixed (wrap dispatch in `asyncio.to_thread()`, + or make handlers genuinely async) before any remote/multi-session/SSE + transport (Phase 4+) is added — it would otherwise let one slow call stall + every other client. + +## Documentation Backlog + +Identified while writing `docs/tools.md` in `0.2.0` and re-flagged by a +subsequent review; deliberately deferred rather than missed. All three are +about *using* the server (a new integrator's first fifteen minutes), not +about the tool surface itself, which `docs/tools.md` already covers: + +- **`docs/getting-started.md`** — install + MCP client config (e.g. Claude + Desktop) + a first `core.health`/`lens.analyze_workflow` call, walked + through end to end. +- **Session workflow walkthrough** — `lens.analyze_workflow` -> + `lens.compare_runs` -> `chaos.run_experiment` sharing a `session_id`. + Phase 2's headline feature (see above); currently only demonstrated in + test code (`tests/test_server.py`), not in any doc a new integrator would + read. +- **Prompts overview** — what the 3 shipped prompts (`lens.workflow_summary`, + `lens.compare_summary`, `chaos.experiment_brief`) actually render, given an + example set of arguments. Currently only discoverable by reading + `prompts/registry.py` directly. ## Capability North Star diff --git a/SECURITY.md b/SECURITY.md index e8de69c..b364d3b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -27,3 +27,35 @@ Please do not open a public issue for suspected vulnerabilities until the issue This MCP server orchestrates tool calls across AgenticLens and Agentic Chaos. It does not store API keys or manage cloud credentials directly. If a vulnerability depends on a specific runtime environment or MCP client, include those environment details in the report. + +## `chaos.run_experiment` executes real code + +Unlike every other tool in this server, `chaos.run_experiment` is a genuine +code-execution primitive: it runs a target Python script (via `runpy`) inside +an `agentic-chaos` `chaos_session()`. That's intentional — it's what makes +fault injection real instead of simulated — but it means any MCP client that +can call this tool can run arbitrary code that already exists somewhere in +the workspace. + +Mitigations currently in place: + +- **Sandboxed to the workspace** — `script` is resolved against the + workspace root (the directory containing `mcp-server` and its sibling + repos) and rejected if it resolves outside it or doesn't exist. It cannot + be pointed at arbitrary paths on the host machine. +- **`timeout_seconds` guard** — the script runs on a worker thread with a + configurable timeout. Note this is *not* a hard kill: Python cannot + forcibly terminate a thread, so on timeout the script's thread may still + be running in the background after the tool call returns a `timed_out` + result. + +What this does **not** do: it does not sandbox the script's actual +capabilities (filesystem, network, subprocess access are all whatever the +server process itself has), and it does not authenticate or authorize the +MCP client making the call — that's the host/transport's job. + +**Only expose this server to trusted MCP clients and keep it stdio/local.** +If a remote deployment mode is ever added (see `ROADMAP.md`), this tool +needs a real sandbox (container, restricted user, etc.) before it can be +exposed to untrusted callers — workspace-path confinement alone is not +sufficient at that point. diff --git a/docs/tools.md b/docs/tools.md new file mode 100644 index 0000000..9503f36 --- /dev/null +++ b/docs/tools.md @@ -0,0 +1,370 @@ + + + +# Tool Reference + +12 tools, generated from `tools/registry.py` - the same data MCP +clients see via `tools/list`. See [README.md](../README.md) for a one-line-per-tool +overview and [ROADMAP.md](../ROADMAP.md) for what's shipped per phase. + +## Core + +### `core.health` — Health Check + +Return rich server diagnostics: adapter availability, loaded tools/resources/prompts, and recent successful calls. + +| | | +| --- | --- | +| Category | `core` | +| Prerequisites | none | +| Expected duration | instant | +| Mutates session | no | + +Input schema: + +```json +{ + "type": "object", + "properties": {}, + "additionalProperties": false +} +``` + +### `core.session_state` — Session State + +Inspect the artifacts and call history accumulated in a session. + +| | | +| --- | --- | +| Category | `core` | +| Prerequisites | none | +| Expected duration | instant | +| Mutates session | no | + +Input schema: + +```json +{ + "type": "object", + "properties": { + "session_id": { + "type": "string", + "description": "Session to read/write shared state under. Defaults to 'default'." + } + }, + "additionalProperties": false +} +``` + +### `core.verify` — Verify Integrations + +Check connectivity to agenticlens, agentic-chaos, and ai-operations-spec, and report readiness. + +| | | +| --- | --- | +| Category | `core` | +| Prerequisites | none | +| Expected duration | instant | +| Mutates session | no | + +Input schema: + +```json +{ + "type": "object", + "properties": {}, + "additionalProperties": false +} +``` + +### `core.version` — Server Version + +Return the current server package version. + +| | | +| --- | --- | +| Category | `core` | +| Prerequisites | none | +| Expected duration | instant | +| Mutates session | no | + +Input schema: + +```json +{ + "type": "object", + "properties": {}, + "additionalProperties": false +} +``` + +## AgenticLens (`lens.*`) + +### `lens.analyze_workflow` — Analyze Workflow + +Analyze an AgenticLens-compatible workflow artifact. + +| | | +| --- | --- | +| Category | `lens` | +| Prerequisites | `agenticlens` | +| Expected duration | fast | +| Mutates session | yes | + +Input schema: + +```json +{ + "type": "object", + "properties": { + "artifact": { + "type": "object" + }, + "session_id": { + "type": "string", + "description": "Session to read/write shared state under. Defaults to 'default'." + } + }, + "required": [ + "artifact" + ], + "additionalProperties": false +} +``` + +### `lens.audit_report` — Audit Report + +Return case-by-case evaluation detail for an audit trail. + +| | | +| --- | --- | +| Category | `lens` | +| Prerequisites | `agenticlens` | +| Expected duration | fast | +| Mutates session | no | + +Input schema: + +```json +{ + "type": "object", + "properties": { + "report": { + "type": "object" + }, + "include_html": { + "type": "boolean" + }, + "session_id": { + "type": "string", + "description": "Session to read/write shared state under. Defaults to 'default'." + } + }, + "required": [ + "report" + ], + "additionalProperties": false +} +``` + +### `lens.compare_runs` — Compare Runs + +Compare baseline and candidate trace runs for regressions. 'baseline'/'candidate' may be omitted to reuse the session's stored runs. + +| | | +| --- | --- | +| Category | `lens` | +| Prerequisites | `agenticlens` | +| Expected duration | fast | +| Mutates session | yes | + +Input schema: + +```json +{ + "type": "object", + "properties": { + "baseline": { + "type": "array", + "items": { + "type": "object" + } + }, + "candidate": { + "type": "array", + "items": { + "type": "object" + } + }, + "regression_threshold": { + "type": "number" + }, + "session_id": { + "type": "string", + "description": "Session to read/write shared state under. Defaults to 'default'." + } + }, + "additionalProperties": false +} +``` + +### `lens.report_summary` — Workflow Report Summary + +Render a Markdown workflow report and recommendation summary. 'artifact' may be omitted to reuse the session's stored workflow. + +| | | +| --- | --- | +| Category | `lens` | +| Prerequisites | `agenticlens` | +| Expected duration | fast | +| Mutates session | yes | + +Input schema: + +```json +{ + "type": "object", + "properties": { + "artifact": { + "type": "object" + }, + "session_id": { + "type": "string", + "description": "Session to read/write shared state under. Defaults to 'default'." + } + }, + "additionalProperties": false +} +``` + +### `lens.slo_summary` — SLO Summary + +Apply release-gate style SLO thresholds to an evaluation report. + +| | | +| --- | --- | +| Category | `lens` | +| Prerequisites | `agenticlens` | +| Expected duration | fast | +| Mutates session | no | + +Input schema: + +```json +{ + "type": "object", + "properties": { + "report": { + "type": "object" + }, + "thresholds": { + "type": "object" + }, + "session_id": { + "type": "string", + "description": "Session to read/write shared state under. Defaults to 'default'." + } + }, + "required": [ + "report" + ], + "additionalProperties": false +} +``` + +## Agentic Chaos (`chaos.*`) + +### `chaos.list_faults` — List Chaos Faults + +List the supported fault types for chaos experiments. + +| | | +| --- | --- | +| Category | `chaos` | +| Prerequisites | `agentic_chaos` | +| Expected duration | instant | +| Mutates session | no | + +Input schema: + +```json +{ + "type": "object", + "properties": {}, + "additionalProperties": false +} +``` + +### `chaos.run_experiment` — Run Chaos Experiment + +Run a workspace-sandboxed target script under selected chaos faults and report the resulting events. + +| | | +| --- | --- | +| Category | `chaos` | +| Prerequisites | `agentic_chaos` | +| Expected duration | slow | +| Mutates session | yes | +| Executes external code | **yes** — see [SECURITY.md](../SECURITY.md) | + +Input schema: + +```json +{ + "type": "object", + "properties": { + "script": { + "type": "string", + "description": "Script path, resolved inside the workspace root." + }, + "faults": { + "type": "array", + "items": { + "type": "string" + } + }, + "timeout_seconds": { + "type": "number" + }, + "session_id": { + "type": "string", + "description": "Session to read/write shared state under. Defaults to 'default'." + } + }, + "required": [ + "script", + "faults" + ], + "additionalProperties": false +} +``` + +## AI Operations Specification (`spec.*`) + +### `spec.validate_artifact` — Validate AI Operations Artifact + +Validate a workflow or run artifact against the AI Operations v0.4 draft. + +| | | +| --- | --- | +| Category | `spec` | +| Prerequisites | `ai_operations_spec` | +| Expected duration | fast | +| Mutates session | no | + +Input schema: + +```json +{ + "type": "object", + "properties": { + "artifact": { + "type": "object" + } + }, + "required": [ + "artifact" + ], + "additionalProperties": false +} +``` diff --git a/examples/chaos_target.py b/examples/chaos_target.py new file mode 100644 index 0000000..8666b9e --- /dev/null +++ b/examples/chaos_target.py @@ -0,0 +1,22 @@ +"""Minimal target script for the `chaos.run_experiment` MCP tool. + +Standalone: `python examples/chaos_target.py` +Under chaos (CLI): `agentic-chaos chaos run examples/chaos_target.py --inject silent_degradation` +Under chaos (MCP): call `chaos.run_experiment` with +`{"script": "mcp-server/examples/chaos_target.py", "faults": ["silent_degradation"]}` + +Outside a chaos session `chaos_call()` is transparent, so running this script +directly behaves exactly like calling `answer_question()` itself. +""" + +from agentic_chaos import chaos_call + + +def answer_question(prompt: str) -> str: + """Stand-in for a real LLM call - deterministic so faults are easy to see.""" + return f"answer to: {prompt}" + + +if __name__ == "__main__": + result = chaos_call(answer_question, "What is the capital of France?", step_name="answer") + print(result) diff --git a/examples/sample_workflow.json b/examples/sample_workflow.json index 5e8d438..08f2872 100644 --- a/examples/sample_workflow.json +++ b/examples/sample_workflow.json @@ -1,18 +1,97 @@ { - "name": "sample-agentic-workflow", - "description": "Placeholder workflow artifact for early MCP resource examples.", + "name": "Support workflow: refund policy question", + "start_time": "2026-08-08T09:00:00Z", + "end_time": "2026-08-08T09:00:12Z", "steps": [ { - "name": "Planner", - "type": "planner" + "name": "Plan support response", + "type": "planner", + "provider": "anthropic", + "model": "claude-sonnet-5", + "metrics": { + "prompt_tokens": 320, + "completion_tokens": 80, + "total_tokens": 400, + "latency": 0.9, + "cost": 0.0021 + } }, { - "name": "Retriever", - "type": "retriever" + "name": "Search knowledge base", + "type": "retriever", + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "latency": 0.4 + }, + "metadata": { + "chunk_count": 14, + "avg_tokens_per_chunk": 180, + "retrieved_chunks": [ + "Refunds are issued within 5-7 business days of approval.", + "Store credit is available as an alternative to a cash refund.", + "Digital goods are non-refundable once downloaded." + ] + } }, { - "name": "Final Answer", - "type": "final_response" + "name": "Look up order status", + "type": "tool_call", + "metrics": { + "prompt_tokens": 40, + "completion_tokens": 15, + "total_tokens": 55, + "latency": 0.3, + "cost": 0.0003 + }, + "metadata": { + "tool_name": "search_docs", + "tool_args": { "query": "refund policy" } + } + }, + { + "name": "Re-check order status", + "type": "tool_call", + "metrics": { + "prompt_tokens": 40, + "completion_tokens": 15, + "total_tokens": 55, + "latency": 0.3, + "cost": 0.0003 + }, + "metadata": { + "tool_name": "search_docs", + "tool_args": { "query": "refund policy" } + } + }, + { + "name": "Draft response", + "type": "llm_call", + "provider": "anthropic", + "model": "claude-sonnet-5", + "metrics": { + "prompt_tokens": 5400, + "completion_tokens": 210, + "total_tokens": 5610, + "latency": 2.1, + "cost": 0.0298 + }, + "metadata": { + "history_tokens": 5000 + } + }, + { + "name": "Final answer", + "type": "final_response", + "metrics": { + "prompt_tokens": 0, + "completion_tokens": 120, + "total_tokens": 120, + "latency": 0.5, + "cost": 0.0006 + } } - ] + ], + "chaos_events": [] } diff --git a/pyproject.toml b/pyproject.toml index 12b5aad..bf84fc8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "deep-agentic-core-mcp" -version = "0.1.3" +version = "0.2.0" description = "Unified MCP server for AgenticLens and Agentic Chaos workflows." readme = "README.md" requires-python = ">=3.10" diff --git a/scripts/generate_tools_doc.py b/scripts/generate_tools_doc.py new file mode 100644 index 0000000..840543a --- /dev/null +++ b/scripts/generate_tools_doc.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Generate docs/tools.md from the canonical tool registry. + +Run `make docs` (or `python scripts/generate_tools_doc.py`) after changing +`tools/registry.py`. The output is derived entirely from `list_tools()` - +the same data MCP clients see via `tools/list` - plus the open-world/ +destructive annotation set `server.py` computes tool annotations from, so +this doc cannot drift out of sync with either. Never hand-edit +`docs/tools.md`; it will be silently overwritten on the next run. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from deep_agentic_core_mcp.server import _OPEN_WORLD_TOOLS +from deep_agentic_core_mcp.tools.registry import list_tools + +REPO_ROOT = Path(__file__).resolve().parents[1] +OUTPUT_PATH = REPO_ROOT / "docs" / "tools.md" + +# Known categories get a friendly heading and a fixed order; anything else +# (a category added to registry.py without updating this map) still renders, +# alphabetically, at the end - so a new category can't silently vanish from +# the doc, it just looks less polished until this map is updated too. +_CATEGORY_ORDER = ["core", "lens", "chaos", "spec"] +_CATEGORY_TITLES = { + "core": "Core", + "lens": "AgenticLens (`lens.*`)", + "chaos": "Agentic Chaos (`chaos.*`)", + "spec": "AI Operations Specification (`spec.*`)", +} + + +def _render_tool(tool: dict[str, Any]) -> str: + lines = [f"### `{tool['name']}` — {tool['title']}", "", tool["description"], ""] + lines.append("| | |") + lines.append("| --- | --- |") + lines.append(f"| Category | `{tool['category']}` |") + prereqs = ", ".join(f"`{p}`" for p in tool["prerequisites"]) or "none" + lines.append(f"| Prerequisites | {prereqs} |") + lines.append(f"| Expected duration | {tool['expected_duration']} |") + lines.append(f"| Mutates session | {'yes' if tool['mutates_session'] else 'no'} |") + if tool["name"] in _OPEN_WORLD_TOOLS: + lines.append("| Executes external code | **yes** — see [SECURITY.md](../SECURITY.md) |") + lines.append("") + lines.append("Input schema:") + lines.append("") + lines.append("```json") + lines.append(json.dumps(tool["input_schema"], indent=2)) + lines.append("```") + lines.append("") + return "\n".join(lines) + + +def generate() -> str: + tools = list_tools() + by_category: dict[str, list[dict[str, Any]]] = {} + for tool in tools: + by_category.setdefault(tool["category"], []).append(tool) + + lines = [ + "", + "", + "", + "# Tool Reference", + "", + f"{len(tools)} tools, generated from `tools/registry.py` - the same data MCP", + "clients see via `tools/list`. See [README.md](../README.md) for a one-line-per-tool", + "overview and [ROADMAP.md](../ROADMAP.md) for what's shipped per phase.", + "", + ] + for category in _CATEGORY_ORDER: + category_tools = by_category.pop(category, []) + if not category_tools: + continue + lines.append(f"## {_CATEGORY_TITLES.get(category, category)}") + lines.append("") + for tool in sorted(category_tools, key=lambda t: str(t["name"])): + lines.append(_render_tool(tool)) + for category, category_tools in sorted(by_category.items()): + lines.append(f"## {category}") + lines.append("") + for tool in sorted(category_tools, key=lambda t: str(t["name"])): + lines.append(_render_tool(tool)) + return "\n".join(lines).rstrip() + "\n" + + +def main() -> None: + OUTPUT_PATH.write_text(generate(), encoding="utf-8") + print(f"Wrote {OUTPUT_PATH.relative_to(REPO_ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/server.json b/server.json index 397992e..055e7bc 100644 --- a/server.json +++ b/server.json @@ -3,7 +3,7 @@ "name": "io.github.DeepAgentLabs/deep-agentic-core-mcp", "title": "Deep Agentic Core MCP", "description": "Unified MCP server for AgenticLens and Agentic Chaos capabilities.", - "version": "0.1.3", + "version": "0.2.0", "repository": { "url": "https://github.com/DeepAgentLabs/mcp-server", "source": "github" @@ -12,7 +12,7 @@ { "registryType": "pypi", "identifier": "deep-agentic-core-mcp", - "version": "0.1.3", + "version": "0.2.0", "transport": { "type": "stdio" } diff --git a/src/deep_agentic_core_mcp/__init__.py b/src/deep_agentic_core_mcp/__init__.py index 4383e0b..977a2bf 100644 --- a/src/deep_agentic_core_mcp/__init__.py +++ b/src/deep_agentic_core_mcp/__init__.py @@ -2,4 +2,4 @@ __all__ = ["__version__"] -__version__ = "0.1.3" +__version__ = "0.2.0" diff --git a/src/deep_agentic_core_mcp/adapters/__init__.py b/src/deep_agentic_core_mcp/adapters/__init__.py index 15e0b7a..b2d80b5 100644 --- a/src/deep_agentic_core_mcp/adapters/__init__.py +++ b/src/deep_agentic_core_mcp/adapters/__init__.py @@ -6,6 +6,29 @@ from pathlib import Path +class AdapterUnavailableError(RuntimeError): + """Raised when a sibling repo (agenticlens, agentic-chaos, ...) can't be reached. + + Carries the adapter name so callers (tool wrappers, `core.verify`) can + report which integration is down without parsing the message text. + """ + + def __init__(self, adapter_name: str, cause: BaseException) -> None: + self.adapter_name = adapter_name + self.cause = cause + super().__init__(f"{adapter_name} adapter is unavailable: {cause}") + + +def workspace_root() -> Path: + """Return the workspace directory containing this repo and its siblings. + + The MCP server is developed alongside the reference projects in the same + parent directory, so tools that need to resolve a sibling repo or a + sandboxed script path share this single definition of "the workspace". + """ + return Path(__file__).resolve().parents[4] + + def ensure_repo_on_path(repo_name: str, *, src: bool = True) -> Path: """Make a sibling repository importable from this workspace. @@ -13,8 +36,7 @@ def ensure_repo_on_path(repo_name: str, *, src: bool = True) -> Path: parent directory, so we can resolve them directly during local use and in tests without requiring wheel installation first. """ - workspace_root = Path(__file__).resolve().parents[4] - repo_root = workspace_root / repo_name + repo_root = workspace_root() / repo_name import_root = repo_root / "src" if src else repo_root import_root_str = str(import_root) if import_root_str not in sys.path: diff --git a/src/deep_agentic_core_mcp/adapters/agentic_chaos.py b/src/deep_agentic_core_mcp/adapters/agentic_chaos.py index 7017b20..67b80fc 100644 --- a/src/deep_agentic_core_mcp/adapters/agentic_chaos.py +++ b/src/deep_agentic_core_mcp/adapters/agentic_chaos.py @@ -2,13 +2,47 @@ from __future__ import annotations +import runpy +import uuid +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import TimeoutError as FutureTimeoutError +from datetime import datetime, timezone +from pathlib import Path from typing import Any -from deep_agentic_core_mcp.adapters import ensure_repo_on_path +from deep_agentic_core_mcp.adapters import ( + AdapterUnavailableError, + ensure_repo_on_path, + workspace_root, +) -ensure_repo_on_path("agentic-chaos") +_IMPORT_ERROR: Exception | None = None +_version: str | None = None -from agentic_chaos.chaos.faults import FAULT_REGISTRY # noqa: E402 +try: + ensure_repo_on_path("agentic-chaos") + + import agentic_chaos as _agentic_chaos_pkg + from agentic_chaos.chaos.faults import FAULT_REGISTRY, resolve_faults + from agentic_chaos.chaos.session import chaos_session + + _version = _agentic_chaos_pkg.__version__ +except Exception as exc: # noqa: BLE001 - captured for core.verify/core.health reporting + _IMPORT_ERROR = exc + + +def _require_available() -> None: + if _IMPORT_ERROR is not None: + raise AdapterUnavailableError("agentic_chaos", _IMPORT_ERROR) + + +def probe() -> dict[str, Any]: + """Report whether the agentic-chaos integration is reachable.""" + return { + "available": _IMPORT_ERROR is None, + "version": _version, + "error": None if _IMPORT_ERROR is None else str(_IMPORT_ERROR), + } def describe_capabilities() -> list[str]: @@ -18,6 +52,7 @@ def describe_capabilities() -> list[str]: def list_faults() -> dict[str, list[dict[str, Any]]]: """Return the registered chaos fault inventory.""" + _require_available() return { "faults": [ { @@ -28,3 +63,88 @@ def list_faults() -> dict[str, list[dict[str, Any]]]: for name, fault_cls in sorted(FAULT_REGISTRY.items()) ] } + + +def _resolve_sandboxed_script(script: str) -> Path: + """Resolve `script` and reject anything outside the workspace. + + Mirrors the confinement `chaos.run_experiment` promises MCP clients: the + tool can execute code, but only code already living somewhere in this + workspace (mcp-server, its sibling repos, or the caller's own checkout + alongside them) - not an arbitrary path on the host machine. + """ + root = workspace_root().resolve() + raw_path = Path(script) + candidate = raw_path.resolve() if raw_path.is_absolute() else (root / raw_path).resolve() + try: + candidate.relative_to(root) + except ValueError as exc: + raise ValueError(f"script must resolve inside the workspace ({root}): {script}") from exc + if not candidate.is_file(): + raise ValueError(f"script not found: {candidate}") + return candidate + + +def run_experiment( + script: str, + faults: list[str], + *, + timeout_seconds: float = 30.0, +) -> dict[str, Any]: + """Run a sandboxed target script inside a chaos_session and report what happened. + + Mirrors what the agentic-chaos CLI's `chaos run` command does internally + (runpy.run_path inside chaos_session) against the library's public API - + that command's own helper is private, so it isn't imported directly. + + Runs on a worker thread so `timeout_seconds` can be enforced even when a + fault (e.g. TokenTimeoutFault) sleeps past it. Python cannot forcibly + kill a thread, so on timeout the script's thread may keep running in the + background after this function returns a "timed_out" result - this is a + real limitation of in-process sandboxing, not a hard kill. + """ + _require_available() + script_path = _resolve_sandboxed_script(script) + resolved_faults = resolve_faults(faults) # raises ValueError on unknown fault names + + def _run() -> tuple[Any, Exception | None]: + with chaos_session(resolved_faults) as session: + crashed: Exception | None = None + try: + runpy.run_path(str(script_path), run_name="__main__") + except Exception as exc: # noqa: BLE001 - reported back, not swallowed silently + crashed = exc + return session, crashed + + started_at = datetime.now(timezone.utc) + timed_out = False + session = None + crashed: Exception | None = None + # Deliberately not a `with` block: ThreadPoolExecutor.__exit__ calls + # shutdown(wait=True), which would block for the worker thread to finish + # regardless of the timeout below - defeating the whole point of it. + # shutdown(wait=False) lets this function return promptly on timeout; the + # thread is intentionally left to finish (or not) in the background, per + # this function's documented can't-force-kill-a-thread limitation. + executor = ThreadPoolExecutor(max_workers=1) + try: + future = executor.submit(_run) + try: + session, crashed = future.result(timeout=timeout_seconds) + except FutureTimeoutError: + timed_out = True + finally: + executor.shutdown(wait=False) + ended_at = datetime.now(timezone.utc) + + events = session.events_as_json() if session is not None else [] + return { + "ok": not timed_out and crashed is None, + "id": str(uuid.uuid4()), + "name": script_path.stem, + "start_time": started_at.isoformat(), + "end_time": ended_at.isoformat(), + "chaos_events": events, + "crashed": repr(crashed) if crashed is not None else None, + "timed_out": timed_out, + } diff --git a/src/deep_agentic_core_mcp/adapters/agenticlens.py b/src/deep_agentic_core_mcp/adapters/agenticlens.py index 6f9cad7..f8e16e7 100644 --- a/src/deep_agentic_core_mcp/adapters/agenticlens.py +++ b/src/deep_agentic_core_mcp/adapters/agenticlens.py @@ -2,36 +2,79 @@ from __future__ import annotations +import tempfile +from pathlib import Path from typing import Any -from deep_agentic_core_mcp.adapters import ensure_repo_on_path +from deep_agentic_core_mcp.adapters import AdapterUnavailableError, ensure_repo_on_path -ensure_repo_on_path("agenticlens") +_IMPORT_ERROR: Exception | None = None +_version: str | None = None -from agenticlens.models.workflow import Workflow # noqa: E402 -from agenticlens.recommenders.engine import RecommendationEngine # noqa: E402 +try: + ensure_repo_on_path("agenticlens") + + import agenticlens as _agenticlens_pkg + from agenticlens.comparison.runner import compare_runs as _compare_runs + from agenticlens.evaluation.gate import GateConfig, evaluate_gate + from agenticlens.evaluation.html_report import render_html_report + from agenticlens.evaluation.models import EvaluationReport + from agenticlens.exporters.markdown_exporter import MarkdownExporter + from agenticlens.models.trace import Run + from agenticlens.models.workflow import Workflow + from agenticlens.recommenders.engine import RecommendationEngine + + _version = _agenticlens_pkg.__version__ +except Exception as exc: # noqa: BLE001 - captured for core.verify/core.health reporting + _IMPORT_ERROR = exc + + +def _require_available() -> None: + if _IMPORT_ERROR is not None: + raise AdapterUnavailableError("agenticlens", _IMPORT_ERROR) + + +def probe() -> dict[str, Any]: + """Report whether the agenticlens integration is reachable.""" + return { + "available": _IMPORT_ERROR is None, + "version": _version, + "error": None if _IMPORT_ERROR is None else str(_IMPORT_ERROR), + } def describe_capabilities() -> list[str]: """Return the supported AgenticLens-backed capabilities.""" - return ["analyze_workflow", "profile_workflow"] + return [ + "analyze_workflow", + "profile_workflow", + "report_summary", + "compare_runs", + "slo_summary", + "audit_report", + ] + + +def _workflow_summary(workflow: Workflow) -> dict[str, Any]: + return { + "id": workflow.id, + "name": workflow.name, + "step_count": len(workflow.steps), + "total_tokens": workflow.total_tokens, + "total_cost": workflow.total_cost, + "latency_seconds": workflow.latency, + "chaos_event_count": len(workflow.chaos_events), + } def analyze_workflow(artifact: dict[str, Any]) -> dict[str, Any]: """Run AgenticLens recommendations against a workflow-shaped artifact.""" + _require_available() workflow = Workflow.model_validate(artifact) engine = RecommendationEngine() recommendations = engine.run(workflow) return { - "workflow": { - "id": workflow.id, - "name": workflow.name, - "step_count": len(workflow.steps), - "total_tokens": workflow.total_tokens, - "total_cost": workflow.total_cost, - "latency_seconds": workflow.latency, - "chaos_event_count": len(workflow.chaos_events), - }, + "workflow": _workflow_summary(workflow), "recommendation_count": len(recommendations), "estimated_savings_pct": RecommendationEngine.estimated_savings_pct( workflow, recommendations @@ -41,3 +84,86 @@ def analyze_workflow(artifact: dict[str, Any]) -> dict[str, Any]: recommendation.model_dump(mode="json") for recommendation in recommendations ], } + + +def report_summary(artifact: dict[str, Any]) -> dict[str, Any]: + """Analyze a workflow and render it through AgenticLens's own Markdown exporter. + + Reuses `MarkdownExporter` rather than reimplementing report formatting - + the exporter only writes to a path, so a temp file bridges the gap. + """ + _require_available() + workflow = Workflow.model_validate(artifact) + engine = RecommendationEngine() + recommendations = engine.run(workflow) + + with tempfile.TemporaryDirectory() as tmp_dir: + report_path = Path(tmp_dir) / "report.md" + MarkdownExporter().export(workflow, report_path, recommendations) + markdown_report = report_path.read_text(encoding="utf-8") + + return { + "workflow": _workflow_summary(workflow), + "recommendation_count": len(recommendations), + "estimated_savings_pct": RecommendationEngine.estimated_savings_pct( + workflow, recommendations + ), + "estimated_cost_savings": RecommendationEngine.estimated_cost_savings(recommendations), + "markdown_report": markdown_report, + } + + +def compare_runs( + baseline: list[dict[str, Any]], + candidate: list[dict[str, Any]], + *, + regression_threshold: float = 0.05, +) -> dict[str, Any]: + """Compare repeated baseline and candidate trace runs for regressions.""" + _require_available() + baseline_runs = [Run.model_validate(item) for item in baseline] + candidate_runs = [Run.model_validate(item) for item in candidate] + report = _compare_runs( + baseline_runs, + candidate_runs, + regression_threshold=regression_threshold, + ) + dumped: dict[str, Any] = report.model_dump(mode="json") + return dumped + + +def slo_summary( + report: dict[str, Any], + thresholds: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Apply release-gate style SLO thresholds to an evaluation report.""" + _require_available() + evaluation_report = EvaluationReport.model_validate(report) + gate_config = GateConfig.model_validate(thresholds or {}) + decision = evaluate_gate(evaluation_report, gate_config) + return { + "suite_name": evaluation_report.suite_name, + "suite_version": evaluation_report.suite_version, + "summary": evaluation_report.summary.model_dump(mode="json"), + "thresholds": gate_config.model_dump(mode="json"), + "decision": decision.model_dump(mode="json"), + } + + +def audit_report( + report: dict[str, Any], + *, + include_html: bool = False, +) -> dict[str, Any]: + """Return the full case-by-case evaluation detail, for audit trails.""" + _require_available() + evaluation_report = EvaluationReport.model_validate(report) + result: dict[str, Any] = { + "suite_name": evaluation_report.suite_name, + "suite_version": evaluation_report.suite_version, + "summary": evaluation_report.summary.model_dump(mode="json"), + "cases": [case.model_dump(mode="json") for case in evaluation_report.cases], + } + if include_html: + result["html_report"] = render_html_report(evaluation_report) + return result diff --git a/src/deep_agentic_core_mcp/adapters/ai_operations_spec.py b/src/deep_agentic_core_mcp/adapters/ai_operations_spec.py index 345099f..5c54874 100644 --- a/src/deep_agentic_core_mcp/adapters/ai_operations_spec.py +++ b/src/deep_agentic_core_mcp/adapters/ai_operations_spec.py @@ -10,22 +10,45 @@ from jsonschema import Draft202012Validator, FormatChecker from referencing import Registry, Resource -from deep_agentic_core_mcp.adapters import ensure_repo_on_path +from deep_agentic_core_mcp.adapters import AdapterUnavailableError, ensure_repo_on_path -SPEC_REPO = ensure_repo_on_path("ai-operations-spec", src=False) -SPEC_V04_DIR = SPEC_REPO / "specification" / "v0.4" -SCHEMA_DIR = SPEC_V04_DIR / "schemas" CLAIM = "Aligned with AI Operations Specification v0.4-draft as observed on 2026-08-07." +_IMPORT_ERROR: Exception | None = None +SCHEMA_DOCUMENTS: dict[str, dict[str, Any]] = {} +REGISTRY: Registry = Registry() + def _load_json(path: Path) -> dict[str, Any]: return json.loads(path.read_text(encoding="utf-8")) # type: ignore[no-any-return] -SCHEMA_DOCUMENTS = {path.name: _load_json(path) for path in SCHEMA_DIR.glob("*.schema.json")} -REGISTRY = Registry().with_resources( - (schema["$id"], Resource.from_contents(schema)) for schema in SCHEMA_DOCUMENTS.values() -) +try: + SPEC_REPO = ensure_repo_on_path("ai-operations-spec", src=False) + SPEC_V04_DIR = SPEC_REPO / "specification" / "v0.4" + SCHEMA_DIR = SPEC_V04_DIR / "schemas" + SCHEMA_DOCUMENTS = {path.name: _load_json(path) for path in SCHEMA_DIR.glob("*.schema.json")} + if not SCHEMA_DOCUMENTS: + raise FileNotFoundError(f"no v0.4 schema documents found under {SCHEMA_DIR}") + REGISTRY = Registry().with_resources( + (schema["$id"], Resource.from_contents(schema)) for schema in SCHEMA_DOCUMENTS.values() + ) +except Exception as exc: # noqa: BLE001 - captured for core.verify/core.health reporting + _IMPORT_ERROR = exc + + +def _require_available() -> None: + if _IMPORT_ERROR is not None: + raise AdapterUnavailableError("ai_operations_spec", _IMPORT_ERROR) + + +def probe() -> dict[str, Any]: + """Report whether the ai-operations-spec schemas are reachable.""" + return { + "available": _IMPORT_ERROR is None, + "version": "v0.4" if _IMPORT_ERROR is None else None, + "error": None if _IMPORT_ERROR is None else str(_IMPORT_ERROR), + } def describe_capabilities() -> list[str]: @@ -33,29 +56,49 @@ def describe_capabilities() -> list[str]: return ["validate_artifact", "semantic_validate_run"] +# Single source of truth for schema-file -> MCP resource mapping, so the +# advertised resource list (`list_schema_resources`) and its actual content +# (`schema_resource_content`) can never drift out of sync with each other or +# with what's really in `SCHEMA_DOCUMENTS`. Both derive from this and skip +# any file that isn't loaded (e.g. because the sibling repo is unavailable), +# rather than assuming all three are always present. +_SCHEMA_RESOURCES: dict[str, tuple[str, str]] = { + "workflow.schema.json": ( + "resource://schemas/aiops/v0.4/workflow", + "AI Operations v0.4 workflow schema", + ), + "run.schema.json": ( + "resource://schemas/aiops/v0.4/run", + "AI Operations v0.4 run schema", + ), + "common.schema.json": ( + "resource://schemas/aiops/v0.4/common", + "AI Operations v0.4 common schema", + ), +} + + def list_schema_resources() -> list[dict[str, str]]: - """Expose the draft schema assets as MCP resources.""" + """Expose the draft schema assets as MCP resources - only those actually loaded.""" return [ - { - "uri": "resource://schemas/aiops/v0.4/workflow", - "name": "AI Operations v0.4 workflow schema", - "kind": "json-schema", - }, - { - "uri": "resource://schemas/aiops/v0.4/run", - "name": "AI Operations v0.4 run schema", - "kind": "json-schema", - }, - { - "uri": "resource://schemas/aiops/v0.4/common", - "name": "AI Operations v0.4 common schema", - "kind": "json-schema", - }, + {"uri": uri, "name": name, "kind": "json-schema"} + for filename, (uri, name) in _SCHEMA_RESOURCES.items() + if filename in SCHEMA_DOCUMENTS ] +def schema_resource_content() -> dict[str, dict[str, Any]]: + """Map each available schema resource's URI to its document, for resources/read.""" + return { + uri: SCHEMA_DOCUMENTS[filename] + for filename, (uri, _name) in _SCHEMA_RESOURCES.items() + if filename in SCHEMA_DOCUMENTS + } + + def validate_artifact(artifact: dict[str, Any]) -> dict[str, Any]: """Validate a draft workflow or run artifact structurally and semantically.""" + _require_available() artifact_type = artifact.get("artifact_type") if artifact_type not in {"workflow", "run"}: return { diff --git a/src/deep_agentic_core_mcp/prompts/registry.py b/src/deep_agentic_core_mcp/prompts/registry.py index 901d038..625df89 100644 --- a/src/deep_agentic_core_mcp/prompts/registry.py +++ b/src/deep_agentic_core_mcp/prompts/registry.py @@ -1,15 +1,93 @@ -"""Reusable prompt catalog placeholders.""" +"""Reusable prompt catalog, wired into the server's `prompts/list` and `prompts/get`.""" +from typing import Any -def list_prompts() -> list[dict[str, str]]: - """Return the initial prompt registry.""" + +def list_prompts() -> list[dict[str, Any]]: + """Return the current prompt registry, with MCP-shaped `arguments`.""" return [ { "name": "lens.workflow_summary", "description": "Summarize an analyzed workflow for a human operator.", + "arguments": [ + { + "name": "workflow_name", + "description": "Name of the analyzed workflow.", + "required": True, + }, + { + "name": "recommendation_count", + "description": "Number of recommendations produced.", + "required": False, + }, + { + "name": "estimated_savings_pct", + "description": "Estimated token savings percentage.", + "required": False, + }, + ], + }, + { + "name": "lens.compare_summary", + "description": "Summarize a baseline-vs-candidate run comparison for a release " + "decision.", + "arguments": [ + { + "name": "baseline_label", + "description": "Baseline group label.", + "required": False, + }, + { + "name": "candidate_label", + "description": "Candidate group label.", + "required": False, + }, + ], }, { "name": "chaos.experiment_brief", "description": "Explain the intent and expected impact of a chaos run.", + "arguments": [ + { + "name": "faults", + "description": "Comma-separated fault names being injected.", + "required": True, + }, + { + "name": "script", + "description": "Target script being exercised.", + "required": False, + }, + ], }, ] + + +def render_prompt(name: str, arguments: dict[str, str] | None = None) -> str: + """Render a prompt template's user-message text for `prompts/get`.""" + arguments = arguments or {} + if name == "lens.workflow_summary": + savings = arguments.get("estimated_savings_pct") + savings_clause = f" with an estimated {savings}% token savings" if savings else "" + return ( + "Summarize the AgenticLens analysis of workflow " + f"'{arguments.get('workflow_name', 'the workflow')}' for a human operator. " + f"It produced {arguments.get('recommendation_count', 'an unknown number of')} " + f"recommendation(s){savings_clause}. Call out the highest-severity recommendation " + "first, in plain language." + ) + if name == "lens.compare_summary": + return ( + "Summarize the run comparison between " + f"'{arguments.get('baseline_label', 'baseline')}' and " + f"'{arguments.get('candidate_label', 'candidate')}' for a human operator, " + "highlighting any regressions and whether the candidate is safe to ship." + ) + if name == "chaos.experiment_brief": + return ( + "Explain the intent and expected impact of injecting the fault(s) " + f"'{arguments.get('faults', 'the configured faults')}' into " + f"'{arguments.get('script', 'the target script')}', for someone who has not " + "read the agentic-chaos documentation." + ) + raise KeyError(f"Unknown prompt: {name}") diff --git a/src/deep_agentic_core_mcp/schemas/tooling.py b/src/deep_agentic_core_mcp/schemas/tooling.py index b45ed7d..68fedde 100644 --- a/src/deep_agentic_core_mcp/schemas/tooling.py +++ b/src/deep_agentic_core_mcp/schemas/tooling.py @@ -13,6 +13,17 @@ class ToolDescriptor(BaseModel): default_factory=dict, description="JSON Schema describing tool arguments.", ) + category: str = Field(default="core", description="Rough grouping (core, lens, chaos, spec).") + prerequisites: list[str] = Field( + default_factory=list, + description="Adapter names that must be available for the tool to succeed.", + ) + expected_duration: str = Field( + default="fast", description="Rough duration hint: instant, fast, or slow." + ) + mutates_session: bool = Field( + default=False, description="Whether the call writes to the in-memory session store." + ) class ResourceDescriptor(BaseModel): @@ -28,3 +39,7 @@ class PromptDescriptor(BaseModel): name: str description: str + arguments: list[dict[str, object]] = Field( + default_factory=list, + description="MCP PromptArgument-shaped entries (name, description, required).", + ) diff --git a/src/deep_agentic_core_mcp/server.py b/src/deep_agentic_core_mcp/server.py index f890084..3f71d49 100644 --- a/src/deep_agentic_core_mcp/server.py +++ b/src/deep_agentic_core_mcp/server.py @@ -10,23 +10,38 @@ from mcp.types import ( CallToolRequestParams, CallToolResult, + GetPromptRequestParams, + GetPromptResult, + ListPromptsResult, ListResourcesResult, ListToolsResult, PaginatedRequestParams, + Prompt, + PromptArgument, + PromptMessage, ReadResourceRequestParams, ReadResourceResult, Resource, TextContent, TextResourceContents, Tool, + ToolAnnotations, ) -from deep_agentic_core_mcp.adapters.ai_operations_spec import SCHEMA_DOCUMENTS +from deep_agentic_core_mcp.adapters.ai_operations_spec import schema_resource_content from deep_agentic_core_mcp.config import SERVER_NAME +from deep_agentic_core_mcp.prompts.registry import list_prompts as _registry_prompts +from deep_agentic_core_mcp.prompts.registry import render_prompt from deep_agentic_core_mcp.resources.catalog import list_resources as _catalog_resources -from deep_agentic_core_mcp.tools.chaos import list_faults -from deep_agentic_core_mcp.tools.core import health, version -from deep_agentic_core_mcp.tools.lens import analyze_workflow +from deep_agentic_core_mcp.tools.chaos import list_faults, run_experiment +from deep_agentic_core_mcp.tools.core import health, session_state, verify, version +from deep_agentic_core_mcp.tools.lens import ( + analyze_workflow, + audit_report, + compare_runs, + report_summary, + slo_summary, +) from deep_agentic_core_mcp.tools.registry import list_tools as _registry_tools from deep_agentic_core_mcp.tools.spec import validate_artifact @@ -43,30 +58,59 @@ _TOOL_DISPATCH: dict[str, Callable[[dict[str, Any] | None], dict[str, Any]]] = { "core.health": health, "core.version": version, + "core.verify": verify, + "core.session_state": session_state, "lens.analyze_workflow": analyze_workflow, + "lens.report_summary": report_summary, + "lens.compare_runs": compare_runs, + "lens.slo_summary": slo_summary, + "lens.audit_report": audit_report, "chaos.list_faults": list_faults, + "chaos.run_experiment": run_experiment, "spec.validate_artifact": validate_artifact, } +# Tools whose execution has real-world side effects (they run external code), +# rather than just reading/deriving from arguments already in hand. +_OPEN_WORLD_TOOLS = {"chaos.run_experiment"} + # --------------------------------------------------------------------------- -# Tool definitions (derived from the canonical registry) +# Tool and prompt definitions (derived from the canonical registries) # --------------------------------------------------------------------------- def _build_tools() -> list[Tool]: """Build Tool objects from the central tool registry. - Only tools with a registered handler are advertised. + Only tools with a registered handler are advertised. Registry metadata + beyond the MCP-standard fields (category, prerequisites, expected + duration) rides in `_meta`; `mutates_session`/side effects map onto the + standard MCP tool annotations so hosts get them without custom parsing. """ - return [ - Tool( - name=entry["name"], - description=str(entry["description"]), - input_schema=entry["input_schema"], + tools = [] + for entry in _registry_tools(): + if entry["name"] not in _TOOL_DISPATCH: + continue + open_world = entry["name"] in _OPEN_WORLD_TOOLS + tools.append( + Tool( + name=entry["name"], + title=str(entry["title"]), + description=str(entry["description"]), + input_schema=entry["input_schema"], + annotations=ToolAnnotations( + read_only_hint=not entry.get("mutates_session", False), + destructive_hint=open_world or None, + open_world_hint=open_world or None, + ), + _meta={ + "category": entry.get("category"), + "prerequisites": entry.get("prerequisites", []), + "expected_duration": entry.get("expected_duration"), + }, + ) ) - for entry in _registry_tools() - if entry["name"] in _TOOL_DISPATCH - ] + return tools def _build_resources() -> list[Resource]: @@ -81,8 +125,21 @@ def _build_resources() -> list[Resource]: ] +def _build_prompts() -> list[Prompt]: + """Build Prompt objects from the central prompt registry.""" + return [ + Prompt( + name=entry["name"], + description=entry["description"], + arguments=[PromptArgument(**argument) for argument in entry.get("arguments", [])], + ) + for entry in _registry_prompts() + ] + + TOOLS: list[Tool] = _build_tools() RESOURCES: list[Resource] = _build_resources() +PROMPTS: list[Prompt] = _build_prompts() RESOURCE_CONTENT: dict[str, dict[str, Any]] = { "resource://examples/sample_workflow": { "name": "Customer support answer", @@ -92,9 +149,7 @@ def _build_resources() -> list[Resource]: "chaos_events": [], }, "resource://catalogs/chaos_faults": list_faults(), - "resource://schemas/aiops/v0.4/workflow": SCHEMA_DOCUMENTS["workflow.schema.json"], - "resource://schemas/aiops/v0.4/run": SCHEMA_DOCUMENTS["run.schema.json"], - "resource://schemas/aiops/v0.4/common": SCHEMA_DOCUMENTS["common.schema.json"], + **schema_resource_content(), } # --------------------------------------------------------------------------- @@ -108,11 +163,24 @@ async def handle_list_tools() -> list[Tool]: async def handle_call_tool(name: str, arguments: dict[str, Any] | None) -> list[TextContent]: - """Dispatch tool calls and return JSON results.""" + """Dispatch tool calls and return JSON results. + + Tool handlers are expected to catch their own known failure modes (e.g. + `AdapterUnavailableError`) and return a structured `{"ok": False, ...}` + payload. This is the safety net for whatever they don't: malformed + client input can still surface as a raw exception from deep inside a + handler (a pydantic `ValidationError` from `Workflow.model_validate`, a + `KeyError` from an artifact missing an expected field, ...), and that + must become a structured MCP tool error here rather than propagate. + """ handler = _TOOL_DISPATCH.get(name) if handler is None: return [TextContent(type="text", text=json.dumps({"error": f"Unknown tool: {name}"}))] - result = handler(arguments) + try: + result = handler(arguments) + except Exception as exc: # noqa: BLE001 - last-resort boundary, see docstring + error = {"ok": False, "error": f"{type(exc).__name__}: {exc}"} + return [TextContent(type="text", text=json.dumps(error))] return [TextContent(type="text", text=json.dumps(result))] @@ -141,6 +209,24 @@ async def handle_read_resource(uri: str) -> list[TextResourceContents]: ] +async def handle_list_prompts() -> list[Prompt]: + """Advertise available prompts.""" + return PROMPTS + + +async def handle_get_prompt(name: str, arguments: dict[str, str] | None) -> GetPromptResult: + """Render a prompt template into a ready-to-send message.""" + try: + text = render_prompt(name, arguments) + except KeyError: + return GetPromptResult(description=f"Unknown prompt: {name}", messages=[]) + description = next((p.description for p in PROMPTS if p.name == name), None) + return GetPromptResult( + description=description, + messages=[PromptMessage(role="user", content=TextContent(type="text", text=text))], + ) + + async def _on_list_tools(_: Any, params: PaginatedRequestParams) -> ListToolsResult: del params return ListToolsResult(tools=await handle_list_tools()) @@ -167,10 +253,21 @@ async def _on_read_resource(_: Any, params: ReadResourceRequestParams) -> ReadRe return ReadResourceResult(contents=list(await handle_read_resource(str(params.uri)))) +async def _on_list_prompts(_: Any, params: PaginatedRequestParams) -> ListPromptsResult: + del params + return ListPromptsResult(prompts=await handle_list_prompts()) + + +async def _on_get_prompt(_: Any, params: GetPromptRequestParams) -> GetPromptResult: + return await handle_get_prompt(params.name, params.arguments) + + server.add_request_handler("tools/list", PaginatedRequestParams, _on_list_tools) server.add_request_handler("tools/call", CallToolRequestParams, _on_call_tool) server.add_request_handler("resources/list", PaginatedRequestParams, _on_list_resources) server.add_request_handler("resources/read", ReadResourceRequestParams, _on_read_resource) +server.add_request_handler("prompts/list", PaginatedRequestParams, _on_list_prompts) +server.add_request_handler("prompts/get", GetPromptRequestParams, _on_get_prompt) # --------------------------------------------------------------------------- diff --git a/src/deep_agentic_core_mcp/services/session.py b/src/deep_agentic_core_mcp/services/session.py new file mode 100644 index 0000000..3666afd --- /dev/null +++ b/src/deep_agentic_core_mcp/services/session.py @@ -0,0 +1,85 @@ +"""Lightweight in-memory session state shared across sequential tool calls. + +The MCP server runs as a single stdio process per client, so a simple +module-level store keyed by an optional `session_id` (defaulting to +`"default"`) is enough to let tools such as `lens.analyze_workflow` -> +`lens.compare_runs` -> `chaos.run_experiment` share artifacts without the +client resending them on every call. This intentionally does not persist +across process restarts. +""" + +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any + +DEFAULT_SESSION_ID = "default" +_HISTORY_LIMIT = 50 + + +@dataclass +class SessionState: + """Artifacts and recent activity for one session.""" + + workflow: dict[str, Any] | None = None + last_analysis: dict[str, Any] | None = None + baseline_runs: list[dict[str, Any]] | None = None + candidate_runs: list[dict[str, Any]] | None = None + last_comparison: dict[str, Any] | None = None + last_chaos_report: dict[str, Any] | None = None + history: deque[dict[str, Any]] = field(default_factory=lambda: deque(maxlen=_HISTORY_LIMIT)) + + def summary(self) -> dict[str, Any]: + """A compact, JSON-friendly view of what this session currently holds.""" + return { + "has_workflow": self.workflow is not None, + "has_analysis": self.last_analysis is not None, + "baseline_run_count": len(self.baseline_runs) if self.baseline_runs else 0, + "candidate_run_count": len(self.candidate_runs) if self.candidate_runs else 0, + "has_comparison": self.last_comparison is not None, + "has_chaos_report": self.last_chaos_report is not None, + "history": list(self.history), + } + + +_SESSIONS: dict[str, SessionState] = {} + + +def get_session(session_id: str = DEFAULT_SESSION_ID) -> SessionState: + """Return the session state for `session_id`, creating it if needed.""" + return _SESSIONS.setdefault(session_id, SessionState()) + + +def record_call(session_id: str, tool: str, ok: bool, note: str = "") -> None: + """Append a call outcome to the session's history.""" + session = get_session(session_id) + entry: dict[str, Any] = { + "tool": tool, + "ok": ok, + "at": datetime.now(timezone.utc).isoformat(), + } + if note: + entry["note"] = note + session.history.append(entry) + + +def last_successful_calls(session_id: str = DEFAULT_SESSION_ID) -> dict[str, str]: + """Return the most recent successful-call timestamp per tool name.""" + session = get_session(session_id) + latest: dict[str, str] = {} + for entry in session.history: + if entry["ok"]: + latest[entry["tool"]] = entry["at"] + return latest + + +def reset_session(session_id: str = DEFAULT_SESSION_ID) -> None: + """Discard a session's stored state.""" + _SESSIONS.pop(session_id, None) + + +def all_sessions() -> dict[str, SessionState]: + """Return every tracked session, for diagnostics.""" + return dict(_SESSIONS) diff --git a/src/deep_agentic_core_mcp/tools/chaos.py b/src/deep_agentic_core_mcp/tools/chaos.py index 32208f2..d468025 100644 --- a/src/deep_agentic_core_mcp/tools/chaos.py +++ b/src/deep_agentic_core_mcp/tools/chaos.py @@ -2,8 +2,11 @@ from typing import Any +from deep_agentic_core_mcp.adapters import AdapterUnavailableError from deep_agentic_core_mcp.adapters.agentic_chaos import describe_capabilities from deep_agentic_core_mcp.adapters.agentic_chaos import list_faults as adapter_list_faults +from deep_agentic_core_mcp.adapters.agentic_chaos import run_experiment as adapter_run_experiment +from deep_agentic_core_mcp.services import session def capabilities() -> dict[str, list[str]]: @@ -11,6 +14,30 @@ def capabilities() -> dict[str, list[str]]: return {"chaos": describe_capabilities()} -def list_faults(_: dict[str, Any] | None = None) -> dict[str, list[dict[str, Any]]]: +def list_faults(_: dict[str, Any] | None = None) -> dict[str, Any]: """Return the supported chaos faults.""" - return adapter_list_faults() + try: + return adapter_list_faults() + except AdapterUnavailableError as exc: + return {"ok": False, "error": str(exc)} + + +def run_experiment(arguments: dict[str, Any] | None) -> dict[str, Any]: + """Run a sandboxed target script under selected chaos faults.""" + if not arguments or "script" not in arguments or "faults" not in arguments: + return {"ok": False, "error": "Missing required 'script' and/or 'faults' argument"} + session_id = arguments.get("session_id", session.DEFAULT_SESSION_ID) + timeout_seconds = arguments.get("timeout_seconds", 30.0) + try: + result = adapter_run_experiment( + arguments["script"], + arguments["faults"], + timeout_seconds=timeout_seconds, + ) + except (AdapterUnavailableError, ValueError) as exc: + session.record_call(session_id, "chaos.run_experiment", ok=False, note=str(exc)) + return {"ok": False, "error": str(exc)} + state = session.get_session(session_id) + state.last_chaos_report = result + session.record_call(session_id, "chaos.run_experiment", ok=result["ok"]) + return result diff --git a/src/deep_agentic_core_mcp/tools/core.py b/src/deep_agentic_core_mcp/tools/core.py index 9666bad..cc04ac7 100644 --- a/src/deep_agentic_core_mcp/tools/core.py +++ b/src/deep_agentic_core_mcp/tools/core.py @@ -2,14 +2,59 @@ from typing import Any +from deep_agentic_core_mcp.adapters import agentic_chaos as agentic_chaos_adapter +from deep_agentic_core_mcp.adapters import agenticlens as agenticlens_adapter +from deep_agentic_core_mcp.adapters import ai_operations_spec as ai_operations_spec_adapter +from deep_agentic_core_mcp.adapters import workspace_root from deep_agentic_core_mcp.config import SERVER_NAME, VERSION +from deep_agentic_core_mcp.prompts.registry import list_prompts +from deep_agentic_core_mcp.resources.catalog import list_resources +from deep_agentic_core_mcp.services import session +from deep_agentic_core_mcp.tools.registry import list_tools +_ADAPTER_PROBES = { + "agenticlens": agenticlens_adapter.probe, + "agentic_chaos": agentic_chaos_adapter.probe, + "ai_operations_spec": ai_operations_spec_adapter.probe, +} -def health(_: dict[str, Any] | None = None) -> dict[str, str]: - """Return a basic health payload.""" - return {"status": "ok", "server": SERVER_NAME} + +def _probe_adapters() -> dict[str, dict[str, Any]]: + return {name: probe() for name, probe in _ADAPTER_PROBES.items()} + + +def health(_: dict[str, Any] | None = None) -> dict[str, Any]: + """Return rich diagnostics: adapter availability, loaded surface, recent activity.""" + adapters = _probe_adapters() + status = "ok" if all(info["available"] for info in adapters.values()) else "degraded" + return { + "status": status, + "server": SERVER_NAME, + "version": VERSION, + "adapters": adapters, + "tools_loaded": len(list_tools()), + "resources_loaded": len(list_resources()), + "prompts_loaded": len(list_prompts()), + "workspace_root": str(workspace_root()), + "last_successful_calls": session.last_successful_calls(), + } def version(_: dict[str, Any] | None = None) -> dict[str, str]: """Return the current package version.""" return {"version": VERSION} + + +def verify(_: dict[str, Any] | None = None) -> dict[str, Any]: + """Check connectivity to each integrated sibling project and report readiness.""" + adapters = _probe_adapters() + return { + "ok": all(info["available"] for info in adapters.values()), + "adapters": adapters, + } + + +def session_state(arguments: dict[str, Any] | None = None) -> dict[str, Any]: + """Return what the active session currently holds, so shared context is inspectable.""" + session_id = (arguments or {}).get("session_id", session.DEFAULT_SESSION_ID) + return {"session_id": session_id, **session.get_session(session_id).summary()} diff --git a/src/deep_agentic_core_mcp/tools/lens.py b/src/deep_agentic_core_mcp/tools/lens.py index 8f701bd..fd2cef8 100644 --- a/src/deep_agentic_core_mcp/tools/lens.py +++ b/src/deep_agentic_core_mcp/tools/lens.py @@ -2,8 +2,14 @@ from typing import Any +from deep_agentic_core_mcp.adapters import AdapterUnavailableError from deep_agentic_core_mcp.adapters.agenticlens import analyze_workflow as adapter_analyze_workflow +from deep_agentic_core_mcp.adapters.agenticlens import audit_report as adapter_audit_report +from deep_agentic_core_mcp.adapters.agenticlens import compare_runs as adapter_compare_runs from deep_agentic_core_mcp.adapters.agenticlens import describe_capabilities +from deep_agentic_core_mcp.adapters.agenticlens import report_summary as adapter_report_summary +from deep_agentic_core_mcp.adapters.agenticlens import slo_summary as adapter_slo_summary +from deep_agentic_core_mcp.services import session def capabilities() -> dict[str, list[str]]: @@ -15,4 +21,105 @@ def analyze_workflow(arguments: dict[str, Any] | None) -> dict[str, Any]: """Analyze an AgenticLens-compatible workflow artifact.""" if not arguments or "artifact" not in arguments: return {"ok": False, "error": "Missing required 'artifact' argument"} - return {"ok": True, **adapter_analyze_workflow(arguments["artifact"])} + session_id = arguments.get("session_id", session.DEFAULT_SESSION_ID) + try: + result = {"ok": True, **adapter_analyze_workflow(arguments["artifact"])} + except AdapterUnavailableError as exc: + session.record_call(session_id, "lens.analyze_workflow", ok=False, note=str(exc)) + return {"ok": False, "error": str(exc)} + state = session.get_session(session_id) + state.workflow = arguments["artifact"] + state.last_analysis = result + session.record_call(session_id, "lens.analyze_workflow", ok=True) + return result + + +def report_summary(arguments: dict[str, Any] | None) -> dict[str, Any]: + """Render a Markdown workflow report alongside the usual analysis metrics. + + `artifact` may be omitted if a workflow was already analyzed in this + session (via `lens.analyze_workflow` or a previous `report_summary` + call) - the stored artifact is reused. + """ + arguments = arguments or {} + session_id = arguments.get("session_id", session.DEFAULT_SESSION_ID) + state = session.get_session(session_id) + artifact = arguments.get("artifact", state.workflow) + if artifact is None: + return {"ok": False, "error": "Missing 'artifact' and no workflow stored in session"} + try: + result = {"ok": True, **adapter_report_summary(artifact)} + except AdapterUnavailableError as exc: + session.record_call(session_id, "lens.report_summary", ok=False, note=str(exc)) + return {"ok": False, "error": str(exc)} + state.workflow = artifact + session.record_call(session_id, "lens.report_summary", ok=True) + return result + + +def compare_runs(arguments: dict[str, Any] | None) -> dict[str, Any]: + """Compare baseline and candidate trace runs for regressions. + + `baseline`/`candidate` may be omitted if a comparison already stored + runs in this session under the same slots. + """ + arguments = arguments or {} + session_id = arguments.get("session_id", session.DEFAULT_SESSION_ID) + state = session.get_session(session_id) + baseline = arguments.get("baseline", state.baseline_runs) + candidate = arguments.get("candidate", state.candidate_runs) + if not baseline or not candidate: + return { + "ok": False, + "error": "Missing 'baseline'/'candidate' and none stored in session", + } + regression_threshold = arguments.get("regression_threshold", 0.05) + try: + result = { + "ok": True, + **adapter_compare_runs(baseline, candidate, regression_threshold=regression_threshold), + } + except AdapterUnavailableError as exc: + session.record_call(session_id, "lens.compare_runs", ok=False, note=str(exc)) + return {"ok": False, "error": str(exc)} + state.baseline_runs = baseline + state.candidate_runs = candidate + state.last_comparison = result + session.record_call(session_id, "lens.compare_runs", ok=True) + return result + + +def slo_summary(arguments: dict[str, Any] | None) -> dict[str, Any]: + """Apply release-gate style SLO thresholds to an evaluation report.""" + if not arguments or "report" not in arguments: + return {"ok": False, "error": "Missing required 'report' argument"} + session_id = arguments.get("session_id", session.DEFAULT_SESSION_ID) + try: + result = { + "ok": True, + **adapter_slo_summary(arguments["report"], arguments.get("thresholds")), + } + except AdapterUnavailableError as exc: + session.record_call(session_id, "lens.slo_summary", ok=False, note=str(exc)) + return {"ok": False, "error": str(exc)} + session.record_call(session_id, "lens.slo_summary", ok=True) + return result + + +def audit_report(arguments: dict[str, Any] | None) -> dict[str, Any]: + """Return case-by-case evaluation detail for an audit trail.""" + if not arguments or "report" not in arguments: + return {"ok": False, "error": "Missing required 'report' argument"} + session_id = arguments.get("session_id", session.DEFAULT_SESSION_ID) + try: + result = { + "ok": True, + **adapter_audit_report( + arguments["report"], include_html=bool(arguments.get("include_html", False)) + ), + } + except AdapterUnavailableError as exc: + session.record_call(session_id, "lens.audit_report", ok=False, note=str(exc)) + return {"ok": False, "error": str(exc)} + session.record_call(session_id, "lens.audit_report", ok=True) + return result diff --git a/src/deep_agentic_core_mcp/tools/registry.py b/src/deep_agentic_core_mcp/tools/registry.py index 65e3983..86deba5 100644 --- a/src/deep_agentic_core_mcp/tools/registry.py +++ b/src/deep_agentic_core_mcp/tools/registry.py @@ -2,21 +2,75 @@ from typing import Any +_SESSION_ID_PROPERTY = { + "session_id": { + "type": "string", + "description": "Session to read/write shared state under. Defaults to 'default'.", + } +} + def list_tools() -> list[dict[str, Any]]: - """Return the current tool inventory for the server.""" + """Return the current tool inventory for the server. + + Beyond the MCP-standard `name`/`title`/`description`/`input_schema`, + each entry carries server-specific metadata surfaced to hosts via + `Tool._meta`/`Tool.annotations` in `server.py`: + + - `category`: rough grouping (`core`, `lens`, `chaos`, `spec`) + - `prerequisites`: adapter names (see `adapters/`) that must be + available for the tool to succeed + - `expected_duration`: `"instant" | "fast" | "slow"`, a rough hint for + hosts deciding whether to show a spinner + - `mutates_session`: whether the call writes to the in-memory session + store (`services/session.py`) + """ return [ { "name": "core.health", "title": "Health Check", - "description": "Return the basic health status of the MCP server.", + "description": "Return rich server diagnostics: adapter availability, loaded " + "tools/resources/prompts, and recent successful calls.", "input_schema": {"type": "object", "properties": {}, "additionalProperties": False}, + "category": "core", + "prerequisites": [], + "expected_duration": "instant", + "mutates_session": False, }, { "name": "core.version", "title": "Server Version", "description": "Return the current server package version.", "input_schema": {"type": "object", "properties": {}, "additionalProperties": False}, + "category": "core", + "prerequisites": [], + "expected_duration": "instant", + "mutates_session": False, + }, + { + "name": "core.verify", + "title": "Verify Integrations", + "description": "Check connectivity to agenticlens, agentic-chaos, and " + "ai-operations-spec, and report readiness.", + "input_schema": {"type": "object", "properties": {}, "additionalProperties": False}, + "category": "core", + "prerequisites": [], + "expected_duration": "instant", + "mutates_session": False, + }, + { + "name": "core.session_state", + "title": "Session State", + "description": "Inspect the artifacts and call history accumulated in a session.", + "input_schema": { + "type": "object", + "properties": {**_SESSION_ID_PROPERTY}, + "additionalProperties": False, + }, + "category": "core", + "prerequisites": [], + "expected_duration": "instant", + "mutates_session": False, }, { "name": "lens.analyze_workflow", @@ -24,16 +78,121 @@ def list_tools() -> list[dict[str, Any]]: "description": "Analyze an AgenticLens-compatible workflow artifact.", "input_schema": { "type": "object", - "properties": {"artifact": {"type": "object"}}, + "properties": {"artifact": {"type": "object"}, **_SESSION_ID_PROPERTY}, "required": ["artifact"], "additionalProperties": False, }, + "category": "lens", + "prerequisites": ["agenticlens"], + "expected_duration": "fast", + "mutates_session": True, + }, + { + "name": "lens.report_summary", + "title": "Workflow Report Summary", + "description": "Render a Markdown workflow report and recommendation summary. " + "'artifact' may be omitted to reuse the session's stored workflow.", + "input_schema": { + "type": "object", + "properties": {"artifact": {"type": "object"}, **_SESSION_ID_PROPERTY}, + "additionalProperties": False, + }, + "category": "lens", + "prerequisites": ["agenticlens"], + "expected_duration": "fast", + "mutates_session": True, + }, + { + "name": "lens.compare_runs", + "title": "Compare Runs", + "description": "Compare baseline and candidate trace runs for regressions. " + "'baseline'/'candidate' may be omitted to reuse the session's stored runs.", + "input_schema": { + "type": "object", + "properties": { + "baseline": {"type": "array", "items": {"type": "object"}}, + "candidate": {"type": "array", "items": {"type": "object"}}, + "regression_threshold": {"type": "number"}, + **_SESSION_ID_PROPERTY, + }, + "additionalProperties": False, + }, + "category": "lens", + "prerequisites": ["agenticlens"], + "expected_duration": "fast", + "mutates_session": True, + }, + { + "name": "lens.slo_summary", + "title": "SLO Summary", + "description": "Apply release-gate style SLO thresholds to an evaluation report.", + "input_schema": { + "type": "object", + "properties": { + "report": {"type": "object"}, + "thresholds": {"type": "object"}, + **_SESSION_ID_PROPERTY, + }, + "required": ["report"], + "additionalProperties": False, + }, + "category": "lens", + "prerequisites": ["agenticlens"], + "expected_duration": "fast", + "mutates_session": False, + }, + { + "name": "lens.audit_report", + "title": "Audit Report", + "description": "Return case-by-case evaluation detail for an audit trail.", + "input_schema": { + "type": "object", + "properties": { + "report": {"type": "object"}, + "include_html": {"type": "boolean"}, + **_SESSION_ID_PROPERTY, + }, + "required": ["report"], + "additionalProperties": False, + }, + "category": "lens", + "prerequisites": ["agenticlens"], + "expected_duration": "fast", + "mutates_session": False, }, { "name": "chaos.list_faults", "title": "List Chaos Faults", "description": "List the supported fault types for chaos experiments.", "input_schema": {"type": "object", "properties": {}, "additionalProperties": False}, + "category": "chaos", + "prerequisites": ["agentic_chaos"], + "expected_duration": "instant", + "mutates_session": False, + }, + { + "name": "chaos.run_experiment", + "title": "Run Chaos Experiment", + "description": "Run a workspace-sandboxed target script under selected chaos " + "faults and report the resulting events.", + "input_schema": { + "type": "object", + "properties": { + "script": { + "type": "string", + "description": "Script path, resolved inside the workspace root.", + }, + "faults": {"type": "array", "items": {"type": "string"}}, + "timeout_seconds": {"type": "number"}, + **_SESSION_ID_PROPERTY, + }, + "required": ["script", "faults"], + "additionalProperties": False, + }, + "category": "chaos", + "prerequisites": ["agentic_chaos"], + "expected_duration": "slow", + "mutates_session": True, }, { "name": "spec.validate_artifact", @@ -47,5 +206,9 @@ def list_tools() -> list[dict[str, Any]]: "required": ["artifact"], "additionalProperties": False, }, + "category": "spec", + "prerequisites": ["ai_operations_spec"], + "expected_duration": "fast", + "mutates_session": False, }, ] diff --git a/src/deep_agentic_core_mcp/tools/spec.py b/src/deep_agentic_core_mcp/tools/spec.py index e49dd9d..c9a7e63 100644 --- a/src/deep_agentic_core_mcp/tools/spec.py +++ b/src/deep_agentic_core_mcp/tools/spec.py @@ -4,6 +4,7 @@ from typing import Any +from deep_agentic_core_mcp.adapters import AdapterUnavailableError from deep_agentic_core_mcp.adapters.ai_operations_spec import describe_capabilities from deep_agentic_core_mcp.adapters.ai_operations_spec import ( validate_artifact as adapter_validate_artifact, @@ -19,4 +20,7 @@ def validate_artifact(arguments: dict[str, Any] | None) -> dict[str, Any]: """Validate a workflow or run artifact against the v0.4 draft.""" if not arguments or "artifact" not in arguments: return {"ok": False, "error": "Missing required 'artifact' argument"} - return adapter_validate_artifact(arguments["artifact"]) + try: + return adapter_validate_artifact(arguments["artifact"]) + except AdapterUnavailableError as exc: + return {"ok": False, "error": str(exc)} diff --git a/tests/test_degraded_boot.py b/tests/test_degraded_boot.py new file mode 100644 index 0000000..2dfd3f8 --- /dev/null +++ b/tests/test_degraded_boot.py @@ -0,0 +1,83 @@ +"""Verify the server boots and stays inspectable when a sibling repo is unavailable. + +Runs in a fresh subprocess rather than monkeypatching the current process: +`ai_operations_spec.py` (like the other adapters) resolves its sibling repo +and loads schema documents at *import* time, so the only faithful way to +exercise "the sibling repo is missing" is to make that true before +`deep_agentic_core_mcp.server` is first imported. +""" + +import json +import subprocess +import sys +import textwrap +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] + +# Points `adapters.workspace_root()` at an empty temp directory before any +# adapter module is imported, so every sibling-repo lookup (agenticlens, +# agentic-chaos, ai-operations-spec) fails to find its repo - without +# touching the real sibling directories on disk. +_SCRIPT = textwrap.dedent( + """ + import asyncio + import json + import sys + import tempfile + from pathlib import Path + + sys.path.insert(0, "src") + import deep_agentic_core_mcp.adapters as adapters_pkg + + adapters_pkg.workspace_root = lambda: Path(tempfile.mkdtemp()) + + import deep_agentic_core_mcp.server as server_module # must not raise + + async def main() -> None: + verify = json.loads((await server_module.handle_call_tool("core.verify", {}))[0].text) + health = json.loads((await server_module.handle_call_tool("core.health", {}))[0].text) + validate = json.loads( + ( + await server_module.handle_call_tool( + "spec.validate_artifact", {"artifact": {"artifact_type": "workflow"}} + ) + )[0].text + ) + schema_resource_uris = [r.uri for r in server_module.RESOURCES if "schema" in r.uri] + print(json.dumps({ + "verify": verify, + "health_status": health["status"], + "validate": validate, + "schema_resource_uris": schema_resource_uris, + })) + + asyncio.run(main()) + """ +) + + +def test_server_boots_and_degrades_when_sibling_repos_are_missing() -> None: + result = subprocess.run( + [sys.executable, "-c", _SCRIPT], + cwd=REPO_ROOT, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + + output = json.loads(result.stdout.strip().splitlines()[-1]) + assert output["verify"]["ok"] is False + assert output["verify"]["adapters"]["ai_operations_spec"]["available"] is False + assert output["health_status"] == "degraded" + + # The crash this test guards against: server.py used to index + # SCHEMA_DOCUMENTS["workflow.schema.json"] directly at import time, which + # raised KeyError (not a caught AdapterUnavailableError) as soon as the + # sibling repo's schemas weren't loaded - before core.verify could ever + # run. Reaching this point at all is most of what this test proves; + # these assertions confirm the *degraded* behavior is also correct. + assert output["validate"]["ok"] is False + assert "unavailable" in output["validate"]["error"] + assert output["schema_resource_uris"] == [] diff --git a/tests/test_imports.py b/tests/test_imports.py index 54a9202..314ef46 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -7,6 +7,8 @@ def test_health_payload() -> None: payload = health() assert payload["status"] == "ok" + assert set(payload["adapters"]) == {"agenticlens", "agentic_chaos", "ai_operations_spec"} + assert all(info["available"] for info in payload["adapters"].values()) def test_version_payload() -> None: diff --git a/tests/test_registry.py b/tests/test_registry.py index 57459a6..f5c59de 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -16,6 +16,35 @@ def test_tool_registry_has_core_health() -> None: assert "spec.validate_artifact" in names +def test_tool_registry_has_phase_2_and_3_additions() -> None: + names = {tool["name"] for tool in list_tools()} + assert { + "core.verify", + "core.session_state", + "lens.report_summary", + "lens.compare_runs", + "lens.slo_summary", + "lens.audit_report", + "chaos.run_experiment", + } <= names + + +def test_tool_registry_entries_carry_metadata() -> None: + for tool in list_tools(): + assert tool["category"] in {"core", "lens", "chaos", "spec"} + assert isinstance(tool["prerequisites"], list) + assert tool["expected_duration"] in {"instant", "fast", "slow"} + assert isinstance(tool["mutates_session"], bool) + + +def test_run_experiment_is_flagged_slow_and_mutating() -> None: + entries = {tool["name"]: tool for tool in list_tools()} + run_experiment = entries["chaos.run_experiment"] + assert run_experiment["expected_duration"] == "slow" + assert run_experiment["mutates_session"] is True + assert "agentic_chaos" in run_experiment["prerequisites"] + + def test_resource_registry_not_empty() -> None: assert list_resources() uris = {resource["uri"] for resource in list_resources()} @@ -26,6 +55,15 @@ def test_prompt_registry_not_empty() -> None: assert list_prompts() +def test_prompt_registry_entries_have_arguments() -> None: + prompts = {prompt["name"]: prompt for prompt in list_prompts()} + assert "chaos.experiment_brief" in prompts + faults_argument = next( + arg for arg in prompts["chaos.experiment_brief"]["arguments"] if arg["name"] == "faults" + ) + assert faults_argument["required"] is True + + def test_typed_descriptors_build() -> None: assert tool_descriptors() assert resource_descriptors() diff --git a/tests/test_server.py b/tests/test_server.py index a75f4fa..20266df 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -1,20 +1,55 @@ """Smoke tests for MCP server boot and tool discovery.""" import json +import time from pathlib import Path import pytest from deep_agentic_core_mcp.server import _TOOL_DISPATCH, TOOLS, server +from deep_agentic_core_mcp.services import session as session_service ROOT = Path(__file__).resolve().parents[2] SPEC_V04 = ROOT / "ai-operations-spec" / "specification" / "v0.4" / "examples" +AGENTICLENS_ARTIFACTS = ROOT / "agenticlens" / "examples" / "pitch_demo" / "artifacts" +CHAOS_TARGET_SCRIPT = "mcp-server/examples/chaos_target.py" + +WORKFLOW_ARTIFACT = { + "name": "Support workflow", + "start_time": "2026-08-07T12:00:00Z", + "end_time": "2026-08-07T12:00:03Z", + "steps": [ + { + "name": "Retriever", + "type": "retriever", + "metrics": { + "prompt_tokens": 100, + "completion_tokens": 20, + "total_tokens": 120, + "latency": 1.0, + "cost": 0.01, + }, + "metadata": { + "chunk_count": 6, + "retrieved_chunks": ["a", "b", "c", "d", "e", "f"], + }, + } + ], +} def _load_json(path: Path) -> dict: return json.loads(path.read_text(encoding="utf-8")) +@pytest.fixture(autouse=True) +def _isolated_default_session(): + """Keep the shared in-memory session from leaking state between tests.""" + session_service.reset_session("default") + yield + session_service.reset_session("default") + + def test_server_has_name() -> None: assert server.name == "io.github.DeepAgentLabs/deep-agentic-core-mcp" @@ -76,35 +111,24 @@ async def test_handle_call_tool_list_faults() -> None: async def test_handle_call_tool_analyze_workflow() -> None: from deep_agentic_core_mcp.server import handle_call_tool - artifact = { - "name": "Support workflow", - "start_time": "2026-08-07T12:00:00Z", - "end_time": "2026-08-07T12:00:03Z", - "steps": [ - { - "name": "Retriever", - "type": "retriever", - "metrics": { - "prompt_tokens": 100, - "completion_tokens": 20, - "total_tokens": 120, - "latency": 1.0, - "cost": 0.01, - }, - "metadata": { - "chunk_count": 6, - "retrieved_chunks": ["a", "b", "c", "d", "e", "f"], - }, - } - ], - } - result = await handle_call_tool("lens.analyze_workflow", {"artifact": artifact}) + result = await handle_call_tool("lens.analyze_workflow", {"artifact": WORKFLOW_ARTIFACT}) payload = json.loads(result[0].text) assert payload["ok"] is True assert payload["workflow"]["name"] == "Support workflow" assert "recommendations" in payload +@pytest.mark.asyncio +async def test_handle_call_tool_analyze_workflow_malformed_artifact() -> None: + """A validation failure deep in an adapter must not raise past handle_call_tool.""" + from deep_agentic_core_mcp.server import handle_call_tool + + result = await handle_call_tool("lens.analyze_workflow", {"artifact": {"not": "a workflow"}}) + payload = json.loads(result[0].text) + assert payload["ok"] is False + assert "error" in payload + + @pytest.mark.asyncio async def test_handle_call_tool_validate_valid_run() -> None: from deep_agentic_core_mcp.server import handle_call_tool @@ -134,3 +158,201 @@ async def test_handle_call_tool_unknown() -> None: result = await handle_call_tool("nonexistent.tool", None) payload = json.loads(result[0].text) assert "error" in payload + + +# --------------------------------------------------------------------------- +# Phase 2: session state, verify +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_handle_call_tool_verify() -> None: + from deep_agentic_core_mcp.server import handle_call_tool + + result = await handle_call_tool("core.verify", {}) + payload = json.loads(result[0].text) + assert payload["ok"] is True + assert set(payload["adapters"]) == {"agenticlens", "agentic_chaos", "ai_operations_spec"} + + +@pytest.mark.asyncio +async def test_handle_call_tool_session_state_tracks_calls() -> None: + from deep_agentic_core_mcp.server import handle_call_tool + + empty = json.loads((await handle_call_tool("core.session_state", {}))[0].text) + assert empty["has_workflow"] is False + assert empty["history"] == [] + + await handle_call_tool("lens.analyze_workflow", {"artifact": WORKFLOW_ARTIFACT}) + after = json.loads((await handle_call_tool("core.session_state", {}))[0].text) + assert after["has_workflow"] is True + assert after["has_analysis"] is True + assert after["history"][-1]["tool"] == "lens.analyze_workflow" + + +# --------------------------------------------------------------------------- +# Phase 3a: AgenticLens additions +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_handle_call_tool_report_summary_reuses_session_workflow() -> None: + from deep_agentic_core_mcp.server import handle_call_tool + + await handle_call_tool("lens.analyze_workflow", {"artifact": WORKFLOW_ARTIFACT}) + result = await handle_call_tool("lens.report_summary", {}) + payload = json.loads(result[0].text) + assert payload["ok"] is True + assert "Support workflow" in payload["markdown_report"] + + +@pytest.mark.asyncio +async def test_handle_call_tool_report_summary_without_artifact_or_session_fails() -> None: + from deep_agentic_core_mcp.server import handle_call_tool + + result = await handle_call_tool("lens.report_summary", {}) + payload = json.loads(result[0].text) + assert payload["ok"] is False + assert "error" in payload + + +@pytest.mark.asyncio +async def test_handle_call_tool_compare_runs() -> None: + from deep_agentic_core_mcp.server import handle_call_tool + + trace = _load_json(AGENTICLENS_ARTIFACTS / "trace.json") + result = await handle_call_tool( + "lens.compare_runs", {"baseline": [trace], "candidate": [trace]} + ) + payload = json.loads(result[0].text) + assert payload["ok"] is True + assert payload["regressions"] == [] + + +@pytest.mark.asyncio +async def test_handle_call_tool_slo_summary() -> None: + from deep_agentic_core_mcp.server import handle_call_tool + + report = _load_json(AGENTICLENS_ARTIFACTS / "evaluation.json") + result = await handle_call_tool( + "lens.slo_summary", {"report": report, "thresholds": {"min_pass_rate": 0.0}} + ) + payload = json.loads(result[0].text) + assert payload["ok"] is True + assert payload["decision"]["passed"] is True + + +@pytest.mark.asyncio +async def test_handle_call_tool_audit_report() -> None: + from deep_agentic_core_mcp.server import handle_call_tool + + report = _load_json(AGENTICLENS_ARTIFACTS / "evaluation.json") + result = await handle_call_tool("lens.audit_report", {"report": report, "include_html": True}) + payload = json.loads(result[0].text) + assert payload["ok"] is True + assert payload["cases"] + assert " None: + from deep_agentic_core_mcp.server import handle_call_tool + + result = await handle_call_tool( + "chaos.run_experiment", + {"script": CHAOS_TARGET_SCRIPT, "faults": ["silent_degradation"]}, + ) + payload = json.loads(result[0].text) + assert payload["ok"] is True + assert payload["timed_out"] is False + fault_types = {event["fault_type"] for event in payload["chaos_events"]} + assert fault_types == {"silent_degradation"} + + +@pytest.mark.asyncio +async def test_handle_call_tool_run_experiment_honors_timeout() -> None: + """timeout_seconds must bound wall-clock time, not just the reported result. + + `token_timeout`'s default hang is 2s; a 0.3s timeout must make this call + return promptly rather than block for the full 2s regardless. + """ + from deep_agentic_core_mcp.server import handle_call_tool + + started = time.monotonic() + result = await handle_call_tool( + "chaos.run_experiment", + {"script": CHAOS_TARGET_SCRIPT, "faults": ["token_timeout"], "timeout_seconds": 0.3}, + ) + elapsed = time.monotonic() - started + payload = json.loads(result[0].text) + assert elapsed < 1.5, f"call blocked for {elapsed:.2f}s despite a 0.3s timeout" + assert payload["ok"] is False + assert payload["timed_out"] is True + + +@pytest.mark.asyncio +async def test_handle_call_tool_run_experiment_rejects_path_outside_workspace() -> None: + from deep_agentic_core_mcp.server import handle_call_tool + + result = await handle_call_tool( + "chaos.run_experiment", + {"script": "/etc/passwd", "faults": ["silent_degradation"]}, + ) + payload = json.loads(result[0].text) + assert payload["ok"] is False + assert "workspace" in payload["error"] + + +@pytest.mark.asyncio +async def test_handle_call_tool_run_experiment_rejects_unknown_fault() -> None: + from deep_agentic_core_mcp.server import handle_call_tool + + result = await handle_call_tool( + "chaos.run_experiment", + {"script": CHAOS_TARGET_SCRIPT, "faults": ["not_a_real_fault"]}, + ) + payload = json.loads(result[0].text) + assert payload["ok"] is False + assert "error" in payload + + +# --------------------------------------------------------------------------- +# Prompts +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_handle_list_prompts() -> None: + from deep_agentic_core_mcp.server import handle_list_prompts + + prompts = await handle_list_prompts() + names = {p.name for p in prompts} + assert "chaos.experiment_brief" in names + assert "lens.workflow_summary" in names + + +@pytest.mark.asyncio +async def test_handle_get_prompt_renders_arguments() -> None: + from deep_agentic_core_mcp.server import handle_get_prompt + + result = await handle_get_prompt( + "chaos.experiment_brief", {"faults": "silent_degradation", "script": "chaos_target.py"} + ) + assert len(result.messages) == 1 + text = result.messages[0].content.text + assert "silent_degradation" in text + assert "chaos_target.py" in text + + +@pytest.mark.asyncio +async def test_handle_get_prompt_unknown_name() -> None: + from deep_agentic_core_mcp.server import handle_get_prompt + + result = await handle_get_prompt("nonexistent.prompt", None) + assert result.messages == [] + assert result.description is not None and "Unknown" in result.description diff --git a/tests/test_session.py b/tests/test_session.py new file mode 100644 index 0000000..71cc2b2 --- /dev/null +++ b/tests/test_session.py @@ -0,0 +1,69 @@ +from deep_agentic_core_mcp.services import session + + +def setup_function() -> None: + # Each test gets a clean slate for the sessions it touches. + session.reset_session("default") + session.reset_session("custom") + + +def test_get_session_creates_and_reuses_default() -> None: + first = session.get_session() + first.workflow = {"name": "example"} + assert session.get_session() is first + assert session.get_session("default") is first + + +def test_get_session_is_isolated_per_id() -> None: + default_state = session.get_session("default") + custom_state = session.get_session("custom") + assert default_state is not custom_state + + +def test_record_call_appends_history_entry() -> None: + session.record_call("default", "core.health", ok=True) + history = session.get_session("default").history + assert len(history) == 1 + assert history[0]["tool"] == "core.health" + assert history[0]["ok"] is True + assert "at" in history[0] + + +def test_record_call_with_note_on_failure() -> None: + session.record_call("default", "lens.analyze_workflow", ok=False, note="boom") + entry = session.get_session("default").history[-1] + assert entry["ok"] is False + assert entry["note"] == "boom" + + +def test_history_is_capped() -> None: + limit = session._HISTORY_LIMIT + for i in range(limit + 10): + session.record_call("default", f"tool.{i}", ok=True) + history = session.get_session("default").history + assert len(history) == limit + assert history[-1]["tool"] == f"tool.{limit + 9}" + + +def test_last_successful_calls_ignores_failures() -> None: + session.record_call("default", "chaos.run_experiment", ok=False, note="timed out") + session.record_call("default", "chaos.run_experiment", ok=True) + latest = session.last_successful_calls("default") + assert "chaos.run_experiment" in latest + + +def test_reset_session_drops_state() -> None: + session.get_session("default").workflow = {"name": "example"} + session.reset_session("default") + assert session.get_session("default").workflow is None + + +def test_summary_reports_stored_artifacts() -> None: + state = session.get_session("default") + state.workflow = {"name": "example"} + state.baseline_runs = [{}] + state.candidate_runs = [{}, {}] + summary = state.summary() + assert summary["has_workflow"] is True + assert summary["baseline_run_count"] == 1 + assert summary["candidate_run_count"] == 2 diff --git a/uv.lock b/uv.lock index 288d2bc..8a03799 100644 --- a/uv.lock +++ b/uv.lock @@ -557,7 +557,7 @@ wheels = [ [[package]] name = "deep-agentic-core-mcp" -version = "0.1.3" +version = "0.2.0" source = { editable = "." } dependencies = [ { name = "jsonschema" },