diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 0000000..934b2a6 --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,50 @@ +# CodeRabbit configuration. +# The path_instructions below record deliberate design decisions so reviews +# do not re-flag them. Each carries its rationale inline. + +reviews: + path_instructions: + - path: 'src/index.ts' + instructions: >- + Do not suggest filtering the static mainwp://abilities and + mainwp://help resources through allowedTools/blockedTools. This is a + documented policy decision (docs/security.md): the tool policy + governs listing and execution, while static resources describe the + full ability catalog so operators can see what a broader policy + would expose. Anything that resolves or executes an ability + (tool calls, completions, mainwp://site/{id}) is policy-gated. + + - path: 'docs/troubleshooting.md' + instructions: >- + Do not suggest moving used/mismatched confirmation_token cases out of + the PREVIEW_REQUIRED section. The code (src/confirmation.ts) returns + PREVIEW_REQUIRED for unknown, already-used, cross-tool, and + argument-mismatched tokens; PREVIEW_EXPIRED only fires when the + pending preview still exists but is older than 5 minutes. The doc + matches the code. + + - path: 'src/config.ts' + instructions: >- + Do not suggest restricting skipSslVerify to local-looking dashboard + hosts. Hostname-based classification is unreliable (internal DNS + names, LAN IPs, and public IPs with self-signed certificates are all + legitimate). The flag is opt-in, off by default, scoped to a + per-request undici dispatcher, and announced with a startup MITM + warning. + + - path: 'src/session.ts' + instructions: >- + Do not suggest resetting the module-global sessionDataBytes counter + inside createServer(). Production runs one createServer per process + (stdio transport), so the counter starts at zero; resetting there + would zero the cap out from under a concurrently running instance + sharing the module. Tests reset explicitly via resetSessionData(). + + - path: 'src/http-client.ts' + instructions: >- + Do not suggest requiring error.name === 'AbortError' before + converting errors to ETIMEDOUT in readLimitedBody's catch. Undici + surfaces mid-stream aborts as varying error shapes (often TypeError + "terminated", not AbortError), so the deadline.timedOut flag is the + reliable signal there; the strict check is only safe in createFetch + where fetch() itself rejects. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 873feb8..9a38f44 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,9 @@ jobs: - name: Lint run: npm run lint + - name: Typecheck (src + tests + scripts) + run: npm run typecheck + - name: Check version consistency run: npm run check-version @@ -51,3 +54,27 @@ jobs: fail_ci_if_error: false env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + acceptance-fixture: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Setup Node.js 20 + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Build + run: npm run build + + - name: Run packed fixture acceptance tests + run: npm run test:acceptance:fixture diff --git a/.gitignore b/.gitignore index 90e8e5d..a8f4c56 100644 --- a/.gitignore +++ b/.gitignore @@ -28,5 +28,12 @@ test-results/ # Local agent/review artifacts (generated) .context/ +# Local agent instructions (contain machine-specific paths) +CLAUDE.md +AGENTS.md + # Override global gitignore for CI/CD !.github/ + +# Local log output (never tracked; may contain request data) +logs/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d7bcd7..86acf34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,55 @@ # Changelog All notable changes to mainwp-mcp are documented here. -Format follows [Keep a Changelog](https://keepachangelog.com/). +Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). + +## [Unreleased] + +### Added + +The server warns at startup when a bearer token (`MAINWP_TOKEN`) is configured without a complete username and application password pair. The WordPress Abilities API rejects bearer tokens, so a token-only setup fails with 401s at request time; the warning surfaces the problem at startup instead. + +### Changed + +Destructive tool calls now go through strict confirmation gating. A bare call to a confirm-capable tool, with no preview and no confirmation token, returns a `PREVIEW_REQUIRED` error instead of proceeding with a logged warning. **This is breaking for clients that relied on the old skip**: run the preview step first, then confirm. Abilities that require confirmation but expose no `dry_run` parameter no longer get a fabricated dry-run call; they return a token-issuing `CONFIRMATION_REQUIRED` response with `preview: null`, and execution proceeds once the client confirms with that token. + +Nested objects and arrays of objects in the input of GET/DELETE abilities are now rejected with an invalid-params error. They used to be serialized into the query string as `[object Object]`, which the Dashboard silently misread. + +Malformed boolean configuration now fails startup instead of logging a warning and falling back to the default. Accepted values are `true/1/yes/on` and `false/0/no/off`; anything else stops the server with an error naming the variable. + +### Removed + +The package no longer installs a global command named `mcp`. That name is too generic for a public package and collides with other MCP tooling. The command is `mainwp-mcp`, and `npx @mainwp/mcp` keeps working since the package now has a single bin entry. + +### Fixed + +The installed `mainwp-mcp` command now starts when invoked through npm's bin symlink. The entry-point check compared the module URL against `process.argv[1]` without resolving symlinks, so the CLI exited silently with status 0 when run via `npx` or a `node_modules/.bin` link. The check now resolves the invoked path first. + +Confirmation previews and tokens are now scoped to the dashboard and principal that issued them. The preview state is module-level, so with multiple `createServer(config)` instances in one process a token issued against one dashboard could confirm the same tool and arguments against another. Preview keys now carry a config identity hash, matching the isolation the abilities cache already enforces. + +Confirmation preview keys now serialize nested arguments faithfully. The previous serialization dropped nested values (including keys named `__proto__` and objects inside arrays), so a confirmation call could swap nested argument values past the token binding. Arguments are canonicalized recursively onto null-prototype objects before keying. + +Tool schemas for destructive tools now declare the `confirmation_token` parameter, and the advertised confirmation flow names the token step. Clients that validate arguments against the schema could not send the token the server requires, and the described flow still matched the old tokenless behavior. Confirm-only abilities without `dry_run` no longer promise a preview in their description. + +Passing `confirm: true` together with a declared `dry_run: true` no longer forwards `confirm` upstream. The dry-run call now goes out with `confirm` stripped, matching the preview path, so upstream handlers never see the ambiguous combination. + +A malformed ability entry in the Dashboard response (a null entry or a non-string name) is now skipped with a warning instead of throwing and failing the whole catalog refresh. + +A malformed property value inside an ability's input schema (a string or array where an object belongs) is now coerced to an empty object instead of crashing the whole tools/list response during description backfill. Property maps are also built with null prototypes, so a hostile parameter named `__proto__` survives as a real schema property instead of polluting the map and skewing the confirm/dry_run detection. + +The abilities cache signature now includes `skipSslVerify` and `maxResponseSize`, so a strictly configured server instance never reuses data fetched by an instance with TLS verification disabled or a larger response cap. + +Confirmed execution of a destructive tool now always requires the `confirmation_token` issued by the preview. The server used to fall back to matching the pending preview by tool name and arguments, so `user_confirmed: true` with the same arguments executed without the token, letting a caller confirm a preview it never read. A tokenless confirmation now returns `PREVIEW_REQUIRED` and the issued token stays valid. + +A "site not found" error from a live Dashboard now surfaces with the resource-not-found error code. The Dashboard reports a nonexistent site as HTTP 403 with the `mainwp_site_not_found` error code, and the classifier trusted the status before the structured code, so clients received a permission-denied error and recovered down the wrong path. Structured not-found codes now classify first. + +Passing `dry_run: true` to an ability that does not declare a `dry_run` parameter now returns an invalid-parameter error instead of skipping the confirmation flow. The server used to forward the parameter upstream, and a handler that ignores unknown input would have run the destructive operation without confirmation. + +Request timeouts and client cancellations are no longer conflated. The request timeout stays armed while the response body is read, and timing out surfaces as a retryable `ETIMEDOUT`; an abort from the caller surfaces as a cancellation rather than a timeout. + +Spurious `tool_list_changed` notifications after every cache refresh are gone. Tool schema enrichment was mutating the cached abilities in place, so each refresh compared a different fingerprint and notified clients even when nothing had changed. Enrichment now works on a copy. + +Error classification prefers the structured HTTP status from the API response (401, 403, 404, 429, 5xx) over parsing the error message text, so error codes stay correct when upstream wording changes. ## [1.0.0-beta.3] - 2026-06-10 diff --git a/README.md b/README.md index 52dc089..9345b8f 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ _A [MainWP Labs](https://mainwp.com/mainwp-labs/) project, powered by MainWP_ +**AI proposes the work. MainWP decides what's permitted, performs it, and reports what actually happened.** + [MainWP MCP Server](https://github.com/mainwp/mainwp-mcp) is for conversational AI management inside Claude, Cursor, or any MCP-compatible client. **Looking for the MainWP Control CLI instead?** [MainWP Control](https://github.com/mainwp/mainwp-control) is a CLI for managing your MainWP Dashboard from the terminal. List sites, push updates, sync data, run batch operations across dozens of sites. MainWP Control is for automation: AI automation, cron jobs, CI/CD pipelines, monitoring scripts, and batch operations. Both talk to the same Abilities API with the same safety model. @@ -20,7 +22,18 @@ _A [MainWP Labs](https://mainwp.com/mainwp-labs/) project, powered by MainWP_ An MCP (Model Context Protocol) server that connects AI assistants to your MainWP Dashboard. This lets Cursor, Claude, OpenAI Codex, VS Code Copilot, and other AI tools manage your WordPress network through natural conversation. -### What You Can Do +## How It Works + +MainWP MCP is a layered system, and each layer has a distinct job: + +- **AI interprets intent.** The assistant turns a natural-language request into a plan and selects the tools that match it. +- **MCP applies the boundary.** The server exposes only the ability namespaces and tools you permit, and rechecks that policy when a tool is actually called, not just when tools are listed. +- **The MainWP Dashboard is the authority.** It holds your managed sites, defines the available abilities, and performs the WordPress operation. +- **You approve what matters.** Destructive operations stop at a confirmation gate before anything runs. + +The AI stays flexible where flexibility helps. The Dashboard stays deterministic where consequences live. + +## What You Can Do - **Site Management**: List sites, check connection status, sync data, add or remove child sites - **Update Management**: See pending updates across all sites, apply core/plugin/theme updates @@ -30,6 +43,8 @@ An MCP (Model Context Protocol) server that connects AI assistants to your MainW Built for WordPress agencies and site managers who want AI assistance with their MainWP workflows. +> **Start bounded.** You don't have to expose every tool on day one. Grant the smallest set of abilities your workflow needs and widen from there. See [Limiting Exposed Tools](#limiting-exposed-tools). + --- ## Quick Start @@ -405,23 +420,25 @@ For Windsurf and other hosts, use the same JSON configuration pattern shown abov ### Environment Variables -| Variable | Required | Default | Description | -| ---------------------------------- | -------- | ---------- | -------------------------------------------------------- | -| `MAINWP_URL` | Yes | | Base URL of your MainWP Dashboard | -| `MAINWP_USER` | Yes | | WordPress admin username | -| `MAINWP_APP_PASSWORD` | Yes | | WordPress Application Password | -| `MAINWP_SKIP_SSL_VERIFY` | No | `false` | Skip SSL verification (dev only) | -| `MAINWP_ALLOW_HTTP` | No | `false` | Allow HTTP URLs (credentials sent in plain text) | -| `MAINWP_SAFE_MODE` | No | `false` | Block destructive operations | -| `MAINWP_REQUIRE_USER_CONFIRMATION` | No | `true` | Require two-step confirmation for destructive operations | -| `MAINWP_ALLOWED_TOOLS` | No | | Whitelist of tools to expose | -| `MAINWP_BLOCKED_TOOLS` | No | | Blacklist of tools to hide | -| `MAINWP_SCHEMA_VERBOSITY` | No | `standard` | `standard` or `compact` | -| `MAINWP_RETRY_ENABLED` | No | `true` | Enable automatic retry for transient errors | -| `MAINWP_MAX_RETRIES` | No | `2` | Total retry attempts including initial request | -| `MAINWP_RETRY_BASE_DELAY` | No | `1000` | Base delay between retries in milliseconds | -| `MAINWP_RETRY_MAX_DELAY` | No | `2000` | Maximum delay between retries in milliseconds | -| `MAINWP_ABILITY_NAMESPACES` | No | `mainwp` | Comma-separated ability namespace allowlist | +| Variable | Required | Default | Description | +| ---------------------------------- | -------------- | ---------- | ------------------------------------------------------------------------------------------------------ | +| `MAINWP_URL` | Yes | | Base URL of your MainWP Dashboard | +| `MAINWP_USER` | For basic auth | | WordPress admin username | +| `MAINWP_APP_PASSWORD` | For basic auth | | WordPress Application Password | +| `MAINWP_TOKEN` | No | | Compatibility only; the Abilities API is expected to reject bearer tokens. Use an Application Password | +| `MAINWP_SKIP_SSL_VERIFY` | No | `false` | Skip SSL verification (dev only) | +| `MAINWP_ALLOW_HTTP` | No | `false` | Allow HTTP URLs (credentials sent in plain text) | +| `MAINWP_SAFE_MODE` | No | `false` | Block destructive operations | +| `MAINWP_REQUIRE_USER_CONFIRMATION` | No | `true` | Require two-step confirmation for destructive operations | +| `MAINWP_ALLOWED_TOOLS` | No | | Whitelist of tools to expose | +| `MAINWP_BLOCKED_TOOLS` | No | | Blacklist of tools to hide | +| `MAINWP_SCHEMA_VERBOSITY` | No | `standard` | `standard` or `compact` | +| `MAINWP_RESPONSE_FORMAT` | No | `compact` | Response JSON formatting: `compact` or `pretty` | +| `MAINWP_RETRY_ENABLED` | No | `true` | Enable automatic retry for transient errors | +| `MAINWP_MAX_RETRIES` | No | `2` | Total retry attempts including initial request | +| `MAINWP_RETRY_BASE_DELAY` | No | `1000` | Base delay between retries in milliseconds | +| `MAINWP_RETRY_MAX_DELAY` | No | `2000` | Maximum delay between retries in milliseconds | +| `MAINWP_ABILITY_NAMESPACES` | No | `mainwp` | Comma-separated ability namespace allowlist | > **⚠️ Security Warning: SSL Verification** > @@ -455,11 +472,11 @@ Configuration loads from `./settings.json` or `~/.config/mainwp-mcp/settings.jso ## Optimizing Token Usage -You have access to around 60 tools (the exact count varies by Dashboard version), which consume approximately 28,000 tokens in your AI's context window. Two settings help reduce this footprint. +You have access to around 60 tools (the exact count varies by Dashboard version), which consume roughly 13,000 tokens in your AI's context window in standard mode (measured against a 62-tool catalog; actual counts vary by tokenizer). Two settings help reduce this footprint. ### Compact Schema Mode -A single setting reduces token usage by roughly 30%: +A single setting reduces token usage by roughly 20% (about 13,000 → 10,000 tokens on the measured catalog): ```json { @@ -477,96 +494,7 @@ Compact mode truncates descriptions to 60 characters and removes examples while ### Limiting Exposed Tools -You can expose only the tools you need. These configurations cover common scenarios. Tool names from non-primary namespaces (added via `abilityNamespaces`) use the `{namespace}__{tool}` form — e.g. `acme__do_thing_v1` — so reference them that way in `allowedTools` / `blockedTools`. - -**Read-only monitoring** (17 tools, ~73% reduction): - -```json -{ - "allowedTools": [ - "list_sites_v1", - "get_site_v1", - "get_site_plugins_v1", - "get_site_themes_v1", - "get_site_updates_v1", - "list_updates_v1", - "list_ignored_updates_v1", - "list_clients_v1", - "get_client_v1", - "count_clients_v1", - "count_client_sites_v1", - "get_client_sites_v1", - "get_client_costs_v1", - "list_tags_v1", - "get_tag_v1", - "get_tag_sites_v1", - "get_tag_clients_v1" - ] -} -``` - -**Site management only** (30 tools, ~53% reduction): - -```json -{ - "allowedTools": [ - "list_sites_v1", - "get_site_v1", - "count_sites_v1", - "get_sites_basic_v1", - "add_site_v1", - "update_site_v1", - "delete_site_v1", - "sync_sites_v1", - "check_site_v1", - "check_sites_v1", - "reconnect_site_v1", - "reconnect_sites_v1", - "disconnect_site_v1", - "disconnect_sites_v1", - "suspend_site_v1", - "suspend_sites_v1", - "unsuspend_site_v1", - "get_site_plugins_v1", - "get_site_themes_v1", - "activate_site_plugins_v1", - "deactivate_site_plugins_v1", - "delete_site_plugins_v1", - "activate_site_theme_v1", - "delete_site_themes_v1", - "get_abandoned_plugins_v1", - "get_abandoned_themes_v1", - "get_site_security_v1", - "get_site_client_v1", - "get_site_costs_v1", - "get_site_changes_v1" - ] -} -``` - -**Updates only** (13 tools, ~80% reduction): - -```json -{ - "allowedTools": [ - "list_updates_v1", - "run_updates_v1", - "update_all_v1", - "get_site_updates_v1", - "update_site_core_v1", - "update_site_plugins_v1", - "update_site_themes_v1", - "update_site_translations_v1", - "list_ignored_updates_v1", - "set_ignored_updates_v1", - "ignore_site_core_v1", - "ignore_site_plugins_v1", - "ignore_site_themes_v1" - ] -} -``` - -**Hide destructive tools** (block deletions while keeping everything else): +You can expose only the tools you need with `allowedTools`, or hide specific tools with `blockedTools`. For example, blocking the deletion tools while keeping everything else: ```json { @@ -580,18 +508,9 @@ You can expose only the tools you need. These configurations cover common scenar } ``` -### Combining Settings +Tool names from non-primary namespaces (added via `abilityNamespaces`) use the `{namespace}__{tool}` form (e.g. `acme__do_thing_v1`), so reference them that way in `allowedTools` / `blockedTools`. -Compact mode and tool filtering work together. This configuration exposes just four tools with minimal descriptions, well-suited for focused automation: - -```json -{ - "schemaVerbosity": "compact", - "allowedTools": ["list_sites_v1", "get_site_v1", "list_updates_v1", "run_updates_v1"] -} -``` - -For all configuration options, see the [Configuration Guide](docs/configuration.md). +Filtering stacks with compact mode: a read-only monitoring preset cuts token usage by ~73%, and a focused four-tool automation config by over 90%. Ready-made presets (read-only monitoring, site management, updates only, minimal automation) live in the [Configuration Guide](docs/configuration.md#tool-filtering). --- @@ -609,13 +528,13 @@ sequenceDiagram participant MainWP User->>AI: Delete site 3 - AI->>Server: delete_site_v1(site_id: 3, confirm: true) + AI->>Server: delete_site_v1(site_id_or_domain: 3, confirm: true) Server->>MainWP: dry_run preview request MainWP-->>Server: site details - Server-->>AI: PREVIEW with site info + Server-->>AI: CONFIRMATION_REQUIRED
(preview + confirmation_token) AI->>User: Shows: "Example Site (https://example.com)
Confirm deletion?" User->>AI: Yes, proceed - AI->>Server: delete_site_v1(site_id: 3, user_confirmed: true) + AI->>Server: delete_site_v1(site_id_or_domain: 3, user_confirmed: true,
confirmation_token: "...") Server->>MainWP: execute deletion MainWP-->>Server: success Server-->>AI: deletion complete @@ -646,6 +565,10 @@ These destructive tools require two-step confirmation: - `delete_site_plugins_v1` - Delete plugins from a site - `delete_site_themes_v1` - Delete themes from a site +The gate is strict. Calling one of these tools with neither `confirm` nor `user_confirmed` returns a `PREVIEW_REQUIRED` error; the server never executes a bare destructive call. A confirmation must reference a preview of the same tool with the same arguments taken within the last 5 minutes. The preview response includes a `confirmation_token` the AI passes back with `user_confirmed: true`. + +Abilities that require confirmation but don't support `dry_run` still go through the two-step gate: the server returns `CONFIRMATION_REQUIRED` with `preview: null` and a token, and the AI must describe the operation and get your explicit approval before confirming. + ### Disabling for Automation If you're running automated scripts that need to delete without interaction: @@ -727,6 +650,7 @@ Around 60 tools organized by category (the exact count varies by Dashboard versi - `confirm`: Must be true to request preview (boolean, required) - `dry_run`: Preview what would be deleted (boolean, optional) - `user_confirmed`: Set to true only after showing preview to user and receiving approval (boolean, optional) + - `confirmation_token`: Token from the preview response; pass it back with `user_confirmed: true` (string, optional) - **reconnect_site_v1** - Reconnect a disconnected site - `site_id_or_domain`: Site ID or domain (string|number, required) @@ -779,6 +703,7 @@ Around 60 tools organized by category (the exact count varies by Dashboard versi - `confirm`: Must be true to request preview (boolean, required) - `dry_run`: Preview what would be deleted (boolean, optional) - `user_confirmed`: Set to true only after showing preview to user and receiving approval (boolean, optional) + - `confirmation_token`: Token from the preview response; pass it back with `user_confirmed: true` (string, optional) - **activate_site_theme_v1** - Activate a theme on a site - `site_id_or_domain`: Site ID or domain (string|number, required) @@ -790,6 +715,7 @@ Around 60 tools organized by category (the exact count varies by Dashboard versi - `confirm`: Must be true to request preview (boolean, required) - `dry_run`: Preview what would be deleted (boolean, optional) - `user_confirmed`: Set to true only after showing preview to user and receiving approval (boolean, optional) + - `confirmation_token`: Token from the preview response; pass it back with `user_confirmed: true` (string, optional) - **get_abandoned_plugins_v1** - Get abandoned plugins on a site - `site_id_or_domain`: Site ID or domain (string|number, required) @@ -930,6 +856,7 @@ Around 60 tools organized by category (the exact count varies by Dashboard versi - `confirm`: Must be true to request preview (boolean, required) - `dry_run`: Preview what would be deleted (boolean, optional) - `user_confirmed`: Set to true only after showing preview to user and receiving approval (boolean, optional) + - `confirmation_token`: Token from the preview response; pass it back with `user_confirmed: true` (string, optional) - **suspend_client_v1** - Suspend a client - `client_id_or_email`: Client ID or email (string|number, required) @@ -975,6 +902,7 @@ Around 60 tools organized by category (the exact count varies by Dashboard versi - `confirm`: Must be true to request preview (boolean, required) - `dry_run`: Preview what would be deleted (boolean, optional) - `user_confirmed`: Set to true only after showing preview to user and receiving approval (boolean, optional) + - `confirmation_token`: Token from the preview response; pass it back with `user_confirmed: true` (string, optional) - **get_tag_sites_v1** - Get sites associated with a tag - `tag_id`: Tag ID (number, required) @@ -1004,12 +932,14 @@ Operations with more than 50 sites are automatically queued for background proce These resources are available for inspection: -| URI | Description | -| --------------------- | --------------------------------------------- | -| `mainwp://abilities` | Full list of available abilities with schemas | -| `mainwp://categories` | List of ability categories | -| `mainwp://status` | Current connection status | -| `mainwp://help` | Tool documentation and safety conventions | +| URI | Description | +| -------------------------------- | --------------------------------------------- | +| `mainwp://abilities` | Full list of available abilities with schemas | +| `mainwp://categories` | List of ability categories | +| `mainwp://status` | Current connection status | +| `mainwp://help` | Tool documentation and safety conventions | +| `mainwp://site/{site_id}` | Details for a single site by ID | +| `mainwp://help/tool/{tool_name}` | Documentation for a specific tool | --- diff --git a/docs/acceptance-testing.md b/docs/acceptance-testing.md new file mode 100644 index 0000000..cac280f --- /dev/null +++ b/docs/acceptance-testing.md @@ -0,0 +1,219 @@ +# Acceptance testing + +The acceptance harness tests the local working tree as an installed npm package and communicates with it through the real MCP stdio protocol. Its default packed mode creates a tarball, checks the published file list, installs that tarball in a fresh consumer project, and launches the installed `dist/index.js`. + +The deterministic suite covers MCP initialization, discovery, resources, prompts, completions, site reads, check-site latency, theme and update inventories, client and tag listings, independent result checks, structured errors, session recovery, allow and block policies, safe mode, confirmation preview behavior, transport limits, settings-file configuration, and package integrity. Guarded live scenarios cover site sync and a reversible plugin toggle. + +The harness does not prove every Dashboard ability, browser behavior, production performance, or compatibility with every MCP client. Fixture runs do not prove real Dashboard authentication, TLS, WordPress permissions, or changing live data. Agent runs add an end-to-end model check, but they do not replace the deterministic suite. + +## Architecture + +`tests/acceptance/run.ts` is a standalone `tsx` CLI. It selects scenarios, prepares credentials, starts the fixture when requested, launches one isolated MCP server process per scenario, and writes results. + +Packed mode uses these stages: + +1. `npm pack --json` creates the package tarball. +2. `tar -tzf` proves required files are present and private source or settings files are absent. +3. A fresh temporary consumer runs `npm init -y` and `npm install ` with its npm cache redirected to a temporary directory. +4. Installed production dependencies are served from a temporary registry bound to `127.0.0.1`. They are packed from the repository's already installed dependency tree. This keeps the consumer install independent and deterministic without contacting an external registry. +5. The harness resolves the installed entry point and both package bins. + +Each MCP launch gets a fresh empty working directory and a fresh `HOME`. The launch environment contains only the explicit runtime values needed by that scenario. `settings-file-config` is the deliberate exception: it writes fake fixture credentials to a mode-restricted temporary `settings.json` and passes no `MAINWP_*` credential variables. + +The fixture dashboard is a plain HTTP server on `127.0.0.1` with an operating-system-assigned port. It requires fake Basic authentication, serves `tests/evals/fixtures/abilities-full.json`, and reads deterministic site state from `tests/acceptance/fixtures/sites.json`. + +`tests/acceptance/lib/verify.ts` reads the Abilities API directly with independent HTTP requests. It does not call through the MCP server. It mirrors the production request method and PHP-style `input[...]` query serialization. + +## Prerequisites + +- Node.js 20.19 or newer +- The dependencies already installed with `npm ci` or `npm install` +- A completed `npm run build` before source mode or a fixture packed run +- `tar`, `git`, and `npm` available on `PATH` +- For live runs, a reachable MainWP Dashboard and a WordPress Application Password +- For agent runs, the `claude` CLI with working model access + +The deterministic fixture suite needs no Dashboard credentials and contacts no host other than `127.0.0.1`. + +## Credentials and environment variables + +Live credentials are resolved in this order: + +1. `MAINWP_URL`, `MAINWP_USER`, and `MAINWP_APP_PASSWORD` from the process environment +2. The file named by `MAINWP_MCP_ACCEPTANCE_ENV` +3. `~/github/dev-tools/network-testbed/.env` + +The env file maps `LLM_DASH_URL` to the Dashboard URL and reads `MAINWP_USER` and `MAINWP_APP_PASSWORD`. Values stay in memory. The harness never writes real credentials to a consumer project, settings file, command record, transcript, or result artifact. + +Live server and verifier requests verify TLS certificates by default. The self-signed local network testbed requires the explicit `MAINWP_MCP_ACCEPTANCE_SKIP_SSL_VERIFY=true` opt-in; that setting passes `MAINWP_SKIP_SSL_VERIFY=true` to the server and disables certificate validation only for the verifier's request-scoped undici dispatcher. + +Additional controls: + +- `MAINWP_MCP_ACCEPTANCE_SKIP_SSL_VERIFY`: set to exactly `true` only for a self-signed local testbed; TLS verification remains enabled otherwise +- `MAINWP_MCP_ACCEPTANCE_WRITE_HOSTS`: comma-separated exact hostnames allowed for `--writes`, in addition to loopback hosts and the Dashboard host auto-resolved from the operator's acceptance env file; every other host, including `.local` hosts, must be listed +- `MAINWP_MCP_ACCEPTANCE_TOGGLE_PLUGIN`: explicit plugin slug eligible for the reversible plugin scenario and preferred by the `agent-plugin-active` probe; the built-in safe slugs are `hello.php` and `hello-dolly/hello.php` + +## Commands + +Build before running the fixture suite: + +```bash +npm run build +npm run test:acceptance:fixture +``` + +Other entry points: + +```bash +npm run test:acceptance +npm run test:acceptance:fast +npm run test:acceptance:writes +npm run test:acceptance:agent +npm run test:acceptance:human +``` + +`test:acceptance:human` runs the packed fixture suite, guarded live write suite, and agent suite in that order. The command uses `&&`, so it stops at the first failing layer. + +The default target is live and the default mode is packed. `test:acceptance:fast` uses the repository's existing `dist/index.js` and still launches the real MCP server. + +List or select deterministic scenarios: + +```bash +npx tsx tests/acceptance/run.ts --list +npx tsx tests/acceptance/run.ts --target fixture --scenario count-sites-consistency +npx tsx tests/acceptance/run.ts --scenario startup-handshake --scenario discovery-tools +``` + +Preserve a packed consumer for inspection: + +```bash +npx tsx tests/acceptance/run.ts --target fixture --keep-consumer +``` + +Select an agent scenario: + +```bash +npx tsx tests/acceptance/agent-run.ts --list +npx tsx tests/acceptance/agent-run.ts --scenario agent-count-sites +``` + +Write scenarios require both `--writes` and an allowed Dashboard host. Runs without both conditions report those scenarios as skipped. A skipped or unverified scenario is visible in the totals and is never counted as passed. + +## Correctness and evidence order + +Scenario assertions use evidence in this order: + +1. Direct Abilities API reads from the independent verifier establish current state. +2. MCP calls exercise the installed server through JSON-RPC over stdio. +3. Structured MCP response fields are parsed and compared with the independent state. +4. A second direct read proves state preservation or the requested write. +5. Prose is used only for diagnostics, never as the correctness oracle. + +For example, `list-sites-cross-check` compares both the site count and the complete set of site URLs. `not-found-input` requires an `isError` result, then calls `count_sites_v1` on the same client session to prove recovery. `confirmation-gate-no-token` checks the structured confirmation status and token, then independently proves that the site set did not change. + +Agent verdicts are also deterministic. Generic scenarios must use an expected `mcp__mainwp__*` tool family, supply structured arguments, receive a non-error tool result, and produce a factual final answer that matches an independent verifier read. Custom evaluators enforce the expected error, multi-tool chain, or full-site coverage when the generic path is insufficient. The model does not grade itself. + +The agent layer contains nine scenarios: + +- `agent-count-sites`: count all connected sites. +- `agent-updates`: identify sites with pending plugin updates. +- `agent-plugin-active`: report whether a discovered plugin is active on its site. +- `agent-nonexistent-site`: consult the Dashboard and report that a probe site is absent without repeating plugin names from real sites. +- `agent-tags`: report the complete paginated tag count and names. +- `agent-theme-chain`: find the single site with pending plugin updates, then report its active theme. +- `agent-confirm-delete-site`: complete the fixture deletion confirmation flow. +- `agent-safemode-refusal`: attempt a fixture deletion and require a correlated `SAFE_MODE_BLOCKED` result with unchanged state. +- `agent-site-status`: check every live site and report the independently verified connectivity result. + +An agent scenario may define `serverEnv` for literal, non-secret server flags that apply only to that scenario. These values are merged into the temporary `claude-mcp.json` server environment. `agent-safemode-refusal` uses this field to set `MAINWP_SAFE_MODE=true`. + +The `agent-confirm-delete-site` scenario is the state-changing write exception in the agent layer. It points the packed MCP server at a newly started local fixture, asks in natural language for an explicitly authorized site deletion without naming a tool, and grades the transcript and state independently. The transcript must contain a `delete_site_v1` result with `CONFIRMATION_REQUIRED` and a token, followed by a confirmed `delete_site_v1` call using that token. A direct fixture read must then show exactly one fewer site and the target site absent. Refusing or stopping before confirmation is a failed scenario with the transcript reason preserved. + +## Completion and transport-limit coverage + +`prompt-completions` targets both live and fixture Dashboards. It requests the existing `update-workflow` prompt's `update_type` completion with the prefix `c`, requires a non-empty result, and checks every suggestion against the `mainwp/list-updates-v1` input enum read directly by the independent verifier. The scenario also launches with `list_sites_v1` blocked and verifies that the current site-ID completion path returns the permission-denied code. + +The fixture-only `oversized-response-recovery` scenario opts into a large `list_sites_v1` response with a reserved search value. It launches the packed server with a response limit above the fixture catalog size but below the fault response size, requires a structured error, and then proves that `count_sites_v1` succeeds on the same MCP session. + +The fixture-only `request-timeout-recovery` scenario opts into a delayed `list_sites_v1` response. It uses the existing `MAINWP_REQUEST_TIMEOUT` environment setting to launch the packed server with a short deadline, requires the structured timeout code, and then proves same-session recovery with `count_sites_v1`. Both fault modes are request-specific, so ordinary fixture scenarios are unchanged. + +## Artifacts + +Every run writes to: + +```text +test-results/acceptance/-[-dirty][-agent]/ +``` + +The directory contains: + +- `manifest.json`: git branch, commit, dirty state, diff hash, package and runtime versions, flags, timing, and tarball integrity +- `events.jsonl`: redacted MCP messages in both directions with scenario, ISO timestamp, and monotonic milliseconds +- `commands.jsonl`: command argv, working directory, exit code, duration, and redacted output tails +- `results.json`: scenario statuses and every named assertion with expected, actual, and pass fields +- `summary.md`: compact human-readable totals and status table +- `server-.stderr.log`: redacted server diagnostics for each launch +- `agent-transcript.jsonl`: redacted Claude stream events when running the agent layer + +The redactor is initialized with the username, application password with and without spaces, Dashboard origin, and Basic Authorization value. Every artifact write passes through it. Fixture credentials follow the same path so fixture runs test the redaction mechanism. + +## Adding a deterministic scenario + +Add a module under `tests/acceptance/scenarios/`, export it from `scenarios/index.ts`, and use the shared `ScenarioDefinition` type. State an objective purpose and record explicit pass criteria with `ctx.assert`. + +This complete example is the implementation shape used by `count-sites-consistency`: + +```ts +import type { ScenarioDefinition } from './types.js'; + +export const countSitesConsistency: ScenarioDefinition = { + id: 'count-sites-consistency', + purpose: 'Verify the MCP count-sites result equals an independent direct count.', + kind: 'read', + targets: ['live', 'fixture'], + + async run(ctx) { + const { result, data } = await ctx.client.callToolJson('count_sites_v1'); + const directCount = await ctx.verifier.countSites(); + + ctx.assert.equal('count_sites_v1 succeeds', result.isError, undefined); + ctx.assert.equal( + 'MCP and independent counts match', + (data as { total: number }).total, + directCount + ); + }, +}; +``` + +Use `preconditions` for target-specific discovery or a non-default server launch. Return `skipped` when a documented external precondition is absent, such as no safe plugin being installed. Do not convert a failed assertion into a skip. + +For a write scenario, define `cleanup(ctx)` whenever the operation can leave state changed. Cleanup runs after success or failure while the same MCP session is still available. + +## Reproducing a failure + +1. Open `summary.md` and find the failed scenario. +2. Read its failed assertions and error in `results.json`. +3. Inspect that scenario's messages in `events.jsonl` and its stderr log. +4. Check `manifest.json` for the exact commit, dirty state, diff hash, mode, target, package integrity, Node version, and npm version. +5. Re-run only that scenario with the same target and mode. + +Example: + +```bash +npx tsx tests/acceptance/run.ts \ + --target fixture \ + --mode packed \ + --scenario count-sites-consistency \ + --keep-consumer +``` + +`commands.jsonl` records the package and consumer commands needed to diagnose a packing or installation failure. The preserved consumer path printed by `--keep-consumer` can be used to inspect the installed package and bin links. + +## Deterministic and agent-driven runs + +The deterministic runner chooses each MCP operation itself and verifies structured values. It is suitable for CI and produces the same fixture result on every run. + +The agent runner gives Claude Code a natural-language task without naming a tool. Its temporary MCP config contains literal `${MAINWP_URL}`, `${MAINWP_USER}`, `${MAINWP_APP_PASSWORD}`, `${MAINWP_SKIP_SSL_VERIFY}`, and `${MAINWP_ALLOW_HTTP}` placeholders plus any literal scenario `serverEnv` flags. Real credential values exist only in the spawned process environment. Live scenarios use resolved Dashboard credentials. The confirmation and safe-mode scenarios use only local fixture credentials, and selecting either by itself does not require live credentials. If the CLI or model is unavailable, the scenario is `unverified` and records the exact blocked command. + +Live Dashboard data can change between an MCP call and the independent read. Site sync can complete asynchronously. Installed plugins and available updates vary by site. Agent tool choice and wording can vary by model. These are known sources of nondeterminism. Fixture scenarios avoid them; live and agent artifacts preserve enough ordered evidence to explain them. diff --git a/docs/configuration.md b/docs/configuration.md index c495c62..3b63d1b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -6,27 +6,31 @@ The server accepts configuration through environment variables and a configurati ## Environment Variables -| Variable | Required | Default | Description | -| ---------------------------------- | -------- | ---------- | -------------------------------------------------------- | -| `MAINWP_URL` | Yes | | Base URL of your MainWP Dashboard | -| `MAINWP_USER` | Yes | | WordPress admin username | -| `MAINWP_APP_PASSWORD` | Yes | | WordPress Application Password | -| `MAINWP_SKIP_SSL_VERIFY` | No | `false` | Skip SSL certificate verification | -| `MAINWP_ALLOW_HTTP` | No | `false` | Allow HTTP URLs (credentials sent in plain text) | -| `MAINWP_SAFE_MODE` | No | `false` | Block all destructive operations | -| `MAINWP_REQUIRE_USER_CONFIRMATION` | No | `true` | Require two-step confirmation for destructive operations | -| `MAINWP_ALLOWED_TOOLS` | No | | Comma-separated whitelist of tools | -| `MAINWP_BLOCKED_TOOLS` | No | | Comma-separated blacklist of tools | -| `MAINWP_SCHEMA_VERBOSITY` | No | `standard` | Schema detail level: `standard` or `compact` | -| `MAINWP_RATE_LIMIT` | No | `60` | Maximum API requests per minute | -| `MAINWP_REQUEST_TIMEOUT` | No | `30000` | Request timeout in milliseconds | -| `MAINWP_MAX_RESPONSE_SIZE` | No | `10485760` | Maximum response size in bytes (10MB) | -| `MAINWP_MAX_SESSION_DATA` | No | `52428800` | Maximum cumulative session data (50MB) | -| `MAINWP_RETRY_ENABLED` | No | `true` | Enable automatic retry for transient errors | -| `MAINWP_MAX_RETRIES` | No | `2` | Total retry attempts including initial request | -| `MAINWP_RETRY_BASE_DELAY` | No | `1000` | Base delay between retries in milliseconds | -| `MAINWP_RETRY_MAX_DELAY` | No | `2000` | Maximum delay between retries in milliseconds | -| `MAINWP_ABILITY_NAMESPACES` | No | `mainwp` | Comma-separated namespace allowlist (see below) | +| Variable | Required | Default | Description | +| ---------------------------------- | ---------- | ---------- | ---------------------------------------------------------------------------- | +| `MAINWP_URL` | Yes | | Base URL of your MainWP Dashboard | +| `MAINWP_USER` | Basic auth | | WordPress admin username | +| `MAINWP_APP_PASSWORD` | Basic auth | | WordPress Application Password | +| `MAINWP_TOKEN` | No | | MainWP REST API bearer token (not accepted by the Abilities API — see below) | +| `MAINWP_SKIP_SSL_VERIFY` | No | `false` | Skip SSL certificate verification | +| `MAINWP_ALLOW_HTTP` | No | `false` | Allow HTTP URLs (credentials sent in plain text) | +| `MAINWP_SAFE_MODE` | No | `false` | Block all destructive operations | +| `MAINWP_REQUIRE_USER_CONFIRMATION` | No | `true` | Require two-step confirmation for destructive operations | +| `MAINWP_ALLOWED_TOOLS` | No | | Comma-separated whitelist of tools | +| `MAINWP_BLOCKED_TOOLS` | No | | Comma-separated blacklist of tools | +| `MAINWP_SCHEMA_VERBOSITY` | No | `standard` | Schema detail level: `standard` or `compact` | +| `MAINWP_RESPONSE_FORMAT` | No | `compact` | Tool response JSON: `compact` or `pretty` | +| `MAINWP_RATE_LIMIT` | No | `60` | Maximum API requests per minute | +| `MAINWP_REQUEST_TIMEOUT` | No | `30000` | Request timeout in milliseconds | +| `MAINWP_MAX_RESPONSE_SIZE` | No | `10485760` | Maximum response size in bytes (10MB) | +| `MAINWP_MAX_SESSION_DATA` | No | `52428800` | Maximum cumulative session data (50MB) | +| `MAINWP_RETRY_ENABLED` | No | `true` | Enable automatic retry for transient errors | +| `MAINWP_MAX_RETRIES` | No | `2` | Total retry attempts including initial request | +| `MAINWP_RETRY_BASE_DELAY` | No | `1000` | Base delay between retries in milliseconds | +| `MAINWP_RETRY_MAX_DELAY` | No | `2000` | Maximum delay between retries in milliseconds | +| `MAINWP_ABILITY_NAMESPACES` | No | `mainwp` | Comma-separated namespace allowlist (see below) | + +Authentication uses `MAINWP_USER` + `MAINWP_APP_PASSWORD` — a WordPress Application Password (Basic auth). The Abilities API (`wp-abilities/v1`), which is the only API this server calls, authenticates through native WordPress and does **not** accept MainWP REST API bearer tokens. `MAINWP_TOKEN` is still read for compatibility and used when no Basic-auth credentials are present, but it will fail against the Abilities endpoints — use an Application Password. ## Configuration File @@ -62,6 +66,7 @@ Create a `settings.json` file in one of these locations (checked in order): | `allowedTools` | `MAINWP_ALLOWED_TOOLS` | string[] | | `blockedTools` | `MAINWP_BLOCKED_TOOLS` | string[] | | `schemaVerbosity` | `MAINWP_SCHEMA_VERBOSITY` | string | +| `responseFormat` | `MAINWP_RESPONSE_FORMAT` | string | | `rateLimit` | `MAINWP_RATE_LIMIT` | number | | `requestTimeout` | `MAINWP_REQUEST_TIMEOUT` | number | | `maxResponseSize` | `MAINWP_MAX_RESPONSE_SIZE` | number | @@ -106,7 +111,7 @@ When combining with `allowedTools` / `blockedTools`, use the prefixed form for n Control which tools are exposed to AI assistants. Useful for limiting access to read-only operations, hiding destructive tools in production, or reducing context size for the AI. -The server exposes around 60 tools by default (the exact count varies by Dashboard version), consuming approximately 28,000 tokens. Tool filtering can reduce this significantly while limiting the AI to specific capabilities. +The server exposes around 60 tools by default (the exact count varies by Dashboard version), consuming roughly 13,000 tokens in standard mode (measured against a 62-tool catalog; actual counts vary by tokenizer). Tool filtering can reduce this significantly while limiting the AI to specific capabilities. ### Whitelist Mode @@ -255,11 +260,12 @@ Keep all functionality while blocking deletions. A conservative choice for produ Just enough to check and apply updates. Pair with compact mode for minimal context usage. -````json +```json { "schemaVerbosity": "compact", "allowedTools": ["list_sites_v1", "get_site_v1", "list_updates_v1", "run_updates_v1"] } +``` --- @@ -269,14 +275,14 @@ Control the detail level of tool descriptions sent to the AI. This affects token | Mode | Description | Token Impact | | ---------- | ---------------------------------------------- | ----------------------- | -| `standard` | Full descriptions, safety tags, usage hints | Default, ~41,500 tokens | -| `compact` | Truncated descriptions (60 chars), no examples | ~30% reduction | +| `standard` | Full descriptions, safety tags, usage hints | Default, ~13,000 tokens | +| `compact` | Truncated descriptions (60 chars), no examples | ~20% reduction | ```json { "schemaVerbosity": "compact" } -```` +``` Or via environment variable: diff --git a/docs/images/mainwp-logo-2026.png b/docs/images/mainwp-logo-2026.png deleted file mode 100644 index 68abdc7..0000000 Binary files a/docs/images/mainwp-logo-2026.png and /dev/null differ diff --git a/docs/images/mainwp-mcp-logo.png b/docs/images/mainwp-mcp-logo.png deleted file mode 100644 index 8aceca0..0000000 Binary files a/docs/images/mainwp-mcp-logo.png and /dev/null differ diff --git a/docs/security.md b/docs/security.md index d39c99c..a0fe3c3 100644 --- a/docs/security.md +++ b/docs/security.md @@ -56,7 +56,7 @@ Safe mode prevents all destructive operations by stripping the `confirm: true` p MAINWP_SAFE_MODE=true ``` -Safe mode blocks `delete_site_v1`, `delete_client_v1`, `delete_tag_v1`, `delete_site_plugins_v1`, and `delete_site_themes_v1`. These tools require `confirm: true` to execute, and safe mode strips this parameter, causing the API to reject the request. +Safe mode blocks every tool the server classifies as destructive, including `delete_site_v1`, `delete_client_v1`, `delete_tag_v1`, `delete_site_plugins_v1`, and `delete_site_themes_v1`. Blocked calls return a `SAFE_MODE_BLOCKED` error without reaching the Dashboard. As a second layer, safe mode also strips the `confirm` parameter, so even a request that somehow bypassed the block would be rejected by the Abilities API. All read operations and non-destructive writes remain available: listing and viewing sites, clients, tags; running updates (reversible via backups); activating and deactivating plugins and themes; syncing sites; creating new clients and tags. @@ -64,7 +64,7 @@ Safe mode is useful for training (let users explore without risk), development ( We recommend enabling Safe Mode when first installing mainwp-mcp. This lets you verify your credentials and connection work correctly, explore available tools without risk of data loss, and understand how the AI interacts with your MainWP Dashboard. Once comfortable, disable Safe Mode to enable full functionality. -The server classifies abilities as destructive or non-destructive based on the `destructive` annotation from the MainWP Dashboard. If an ability lacks annotations, the server defaults to treating it as non-destructive. This is intentional: the MainWP Dashboard controls ability registration, missing annotations indicate a Dashboard-side issue, and new or misconfigured abilities shouldn't be silently blocked. A warning is logged when abilities cannot be reliably classified. For stricter control, use `blockedTools` to explicitly block specific tools or `allowedTools` to whitelist only known-safe tools. +The server classifies abilities as destructive or non-destructive based on the `destructive` annotation from the MainWP Dashboard. If an ability lacks annotations, the server treats it as destructive. This default-deny stance means a new or misconfigured ability is blocked by safe mode and subject to the confirmation flow until the Dashboard annotates it properly; a warning is logged whenever an ability cannot be reliably classified so the missing annotation can be fixed at the source. For stricter control, use `blockedTools` to explicitly block specific tools or `allowedTools` to whitelist only known-safe tools. --- @@ -107,22 +107,30 @@ The two-step confirmation flow prevents accidental destructive operations by req When you ask the AI to delete something, the server intercepts the request and returns a preview instead of executing immediately. The AI shows you what will be deleted and asks for explicit confirmation. Only after you confirm does the server execute the operation. -**Phase 1 - Preview:** AI calls the destructive tool with `confirm: true`. Server runs a dry-run preview and returns details. AI shows you what will be affected and waits for your response. +**Phase 1 - Preview:** AI calls the destructive tool with `confirm: true`. Server runs a dry-run preview and returns a `CONFIRMATION_REQUIRED` response containing the preview details and a one-time `confirmation_token`. AI shows you what will be affected and waits for your response. -**Phase 2 - Execute:** You confirm the action. AI calls the tool again with `user_confirmed: true`. Server validates the preview was shown (within last 5 minutes) and executes the deletion. +**Phase 2 - Execute:** You confirm the action. AI calls the tool again with `user_confirmed: true` and the `confirmation_token`. Server validates that the token matches the same tool and the same arguments, that the preview is less than 5 minutes old, and executes the deletion. + +Calling a destructive tool with neither `confirm` nor `user_confirmed` is rejected with a `PREVIEW_REQUIRED` error. The server never executes a bare destructive call. + +Some abilities require confirmation but don't support `dry_run`, so there is nothing to preview. For those, the `confirm: true` step returns `CONFIRMATION_REQUIRED` with `preview: null` and `next_action: "confirm_without_preview"`, and still issues a token. The two-step gate applies even without a preview; the AI is instructed to describe the operation and obtain your explicit approval before confirming. Example flow: -``` +```text You: Delete the "staging" tag AI: [Calls delete_tag_v1(tag_id: 5, confirm: true)] - Server returns preview: + Server returns: { - "tag_id": 5, - "name": "staging", - "sites_affected": 12, - "clients_affected": 3 + "status": "CONFIRMATION_REQUIRED", + "preview": { + "tag_id": 5, + "name": "staging", + "sites_affected": 12, + "clients_affected": 3 + }, + "confirmation_token": "3f2c..." } AI: I found the "staging" tag (ID: 5). @@ -131,7 +139,8 @@ AI: I found the "staging" tag (ID: 5). You: Yes -AI: [Calls delete_tag_v1(tag_id: 5, user_confirmed: true)] +AI: [Calls delete_tag_v1(tag_id: 5, user_confirmed: true, + confirmation_token: "3f2c...")] Tag deleted successfully. ``` @@ -189,6 +198,8 @@ For untrusted AI clients: This provides defense-in-depth: Safe Mode blocks destructive operations at runtime, and `blockedTools` prevents the tools from even appearing in the AI's tool list. +Note that `allowedTools` and `blockedTools` control which tools are listed and executable. The informational resources (`mainwp://abilities`, `mainwp://help`) still describe the full ability catalog from the Dashboard; blocking a tool prevents execution everywhere, it does not redact its documentation. + --- ## Tool Filtering diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 6ad92e5..3e6d07c 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -82,7 +82,7 @@ Some tools work but others return 403. The WordPress user lacks required capabil ### Bearer Token Not Working -Works with username/password but not with `MAINWP_TOKEN`. Check that the token was generated correctly in MainWP Dashboard, the token hasn't expired, and you're using the correct environment variable (`MAINWP_TOKEN`, not `MAINWP_APP_PASSWORD`). When in doubt, use Application Password authentication instead. +Bearer authentication with `MAINWP_TOKEN` is expected to fail against the WordPress Abilities API, which uses native WordPress authentication. Configure `MAINWP_USER` and `MAINWP_APP_PASSWORD` with a WordPress Application Password instead. --- @@ -138,10 +138,22 @@ For automation scripts, disable the confirmation flow: } ``` -The AI tried to execute a destructive operation with `user_confirmed: true` without first requesting a preview. The two-step confirmation flow requires the AI to show you a preview before executing. The AI skipped the preview step. +The AI tried to execute a destructive operation without completing the two-step confirmation flow. This error covers calling with `user_confirmed: true` without first requesting a preview, calling with no confirmation parameters at all (the server rejects bare destructive calls rather than executing them), or confirming with a token that is unknown, already used (tokens are single-use), or bound to a different tool or different arguments. This is usually an AI behavior issue. Try rephrasing: "Show me what will be deleted first, then I'll confirm." +### "CONFIRMATION_REQUIRED" with `preview: null` + +```json +{ + "status": "CONFIRMATION_REQUIRED", + "next_action": "confirm_without_preview", + "preview": null +} +``` + +Not an error. The ability requires confirmation but doesn't support `dry_run`, so there is nothing to preview. The response still carries a `confirmation_token`; the AI should describe what the operation will do and, once you explicitly approve, call the tool again with `user_confirmed: true` and that token. + ### "PREVIEW_EXPIRED" ```json @@ -151,7 +163,7 @@ This is usually an AI behavior issue. Try rephrasing: "Show me what will be dele } ``` -You waited more than 5 minutes between seeing the preview and confirming the operation. Request a new preview—the AI will automatically do this if you just say "yes" or "proceed." +You waited more than 5 minutes between seeing the preview and confirming the operation. Request a new preview—the AI will automatically do this if you just say "yes" or "proceed." (An already-used token produces `PREVIEW_REQUIRED` instead: tokens are single-use.) ### "CONFLICTING_PARAMETERS" @@ -173,7 +185,7 @@ The AI tried to pass both `user_confirmed: true` and `dry_run: true` simultaneou } ``` -The AI tried to use `user_confirmed: true` on a non-destructive tool. Only `delete_site_v1`, `delete_client_v1`, `delete_tag_v1`, `delete_site_plugins_v1`, and `delete_site_themes_v1` support `user_confirmed`. Other tools don't need confirmation. +The AI tried to use `user_confirmed: true` on a tool that doesn't participate in the confirmation flow. Only tools whose schema declares a `confirm` parameter (the deletion tools: `delete_site_v1`, `delete_client_v1`, `delete_tag_v1`, `delete_site_plugins_v1`, `delete_site_themes_v1`) accept `user_confirmed`. Other tools don't need confirmation. --- diff --git a/eslint.config.js b/eslint.config.js index bf6a47f..aa57828 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -33,7 +33,16 @@ export default tseslint.config( }, }, { - // Ignore build artifacts, dependencies, and utility scripts; keep *.config.js ignored to avoid self-linting issues - ignores: ['dist/', 'node_modules/', 'coverage/', 'scripts/', '*.config.js'], + // CLI scripts print to stdout by design + files: ['scripts/**/*.ts'], + rules: { + 'no-console': 'off', + }, + }, + { + // Ignore build artifacts and dependencies; keep *.config.js ignored to + // avoid self-linting issues. check-version.js stays ignored (plain + // CommonJS-style CI script); scripts/*.ts is linted like the rest. + ignores: ['dist/', 'node_modules/', 'coverage/', 'scripts/check-version.js', '*.config.js'], } ); diff --git a/package.json b/package.json index b883516..e8f11d7 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,7 @@ "type": "module", "main": "dist/index.js", "bin": { - "mainwp-mcp": "dist/index.js", - "mcp": "dist/index.js" + "mainwp-mcp": "dist/index.js" }, "files": [ "dist/", @@ -24,8 +23,15 @@ "test:watch": "vitest", "test:coverage": "vitest run --coverage", "test:evals": "vitest run tests/evals/", + "test:acceptance": "tsx tests/acceptance/run.ts", + "test:acceptance:fast": "tsx tests/acceptance/run.ts --mode source", + "test:acceptance:fixture": "tsx tests/acceptance/run.ts --target fixture", + "test:acceptance:writes": "tsx tests/acceptance/run.ts --writes", + "test:acceptance:agent": "tsx tests/acceptance/agent-run.ts", + "test:acceptance:human": "npm run test:acceptance:fixture && npm run test:acceptance:writes && npm run test:acceptance:agent", "lint": "eslint .", "lint:fix": "eslint . --fix", + "typecheck": "tsc --noEmit -p tsconfig.eslint.json", "format": "prettier --write .", "format:check": "prettier --check .", "test:manual": "tsx scripts/manual-test.ts", diff --git a/scripts/manual-test.ts b/scripts/manual-test.ts index fd7d5f2..bc0fa53 100644 --- a/scripts/manual-test.ts +++ b/scripts/manual-test.ts @@ -6,11 +6,14 @@ * records response times, and saves results to a timestamped JSON file. * * Usage: - * npm run test:manual # Run all tests + * npm run test:manual # Read + safe-write tests (destructive excluded) + * npm run test:manual -- --destructive # Include destructive tests * npm run test:manual -- --category read # Read-only only * npm run test:manual -- --category safe-write + * npm run test:manual -- --category destructive * npm run test:manual -- --test count-sites --test list-sites - * npm run test:manual -- --site-id 1 # Skip discovery + * npm run test:manual -- --site-id 1 # Skip site discovery + * npm run test:manual -- --plugin-slug hello-dolly/hello.php # Skip plugin discovery * npm run test:manual -- --dry-run # Show what would run * npm run test:manual -- --verbose # Print response bodies */ @@ -64,6 +67,8 @@ interface CliOptions { category: TestCategory | null; testNames: string[]; siteId: number | null; + pluginSlug: string | null; + destructive: boolean; dryRun: boolean; verbose: boolean; } @@ -97,6 +102,8 @@ function parseArgs(): CliOptions { category: null, testNames: [], siteId: null, + pluginSlug: null, + destructive: false, dryRun: false, verbose: false, }; @@ -126,6 +133,16 @@ function parseArgs(): CliOptions { process.exit(1); } break; + case '--plugin-slug': + opts.pluginSlug = args[++i]; + if (!opts.pluginSlug || opts.pluginSlug.startsWith('--')) { + console.error('Invalid --plugin-slug: value required'); + process.exit(1); + } + break; + case '--destructive': + opts.destructive = true; + break; case '--dry-run': opts.dryRun = true; break; @@ -137,14 +154,19 @@ function parseArgs(): CliOptions { MainWP MCP Manual Test Harness Usage: - npm run test:manual Run all tests + npm run test:manual Read + safe-write tests (destructive excluded) + npm run test:manual -- --destructive Include destructive tests npm run test:manual -- --category read Read-only tests only npm run test:manual -- --category safe-write Safe-write tests only npm run test:manual -- --category destructive Destructive tests only npm run test:manual -- --test Run specific test(s) npm run test:manual -- --site-id Skip discovery, use this site ID + npm run test:manual -- --plugin-slug Use this plugin for lifecycle tests npm run test:manual -- --dry-run Show what would run npm run test:manual -- --verbose Print response bodies + +Plugin lifecycle tests (activate/deactivate/delete) only auto-discover +throwaway plugins (hello-dolly). Anything else requires --plugin-slug. `); process.exit(0); break; @@ -434,7 +456,6 @@ function buildTestCatalog(): TestScenario[] { category: 'safe-write', readonly: false, params: ctx => ({ site_id_or_domain: ctx.siteId, plugins: [ctx.pluginSlug] }), - pairedWith: 'delete-plugin', }, { name: 'delete-plugin', @@ -451,6 +472,16 @@ function buildTestCatalog(): TestScenario[] { // Discovery Phase // --------------------------------------------------------------------------- +/** + * Plugins the lifecycle tests (activate → deactivate → delete) may touch + * without asking. The delete test permanently removes the plugin from the + * child site, so only throwaway plugins qualify — Akismet doesn't (its + * uninstall drops the configured API key); pass it via --plugin-slug if you + * mean it. A run once auto-picked WooCommerce and deleted it — never widen + * this to "first plugin found". + */ +const SAFE_TEST_PLUGIN_SLUGS = ['hello.php', 'hello-dolly/hello.php']; + async function discover(config: Config, opts: CliOptions): Promise { const ctx: DiscoveryContext = { siteId: opts.siteId, @@ -474,7 +505,8 @@ async function discover(config: Config, opts: CliOptions): Promise }; - if (data.plugins) { - // Find any non-essential plugin (active or inactive) for lifecycle testing - const candidate = data.plugins.find(p => !p.slug.startsWith('mainwp-child')); - if (candidate) { - ctx.pluginSlug = candidate.slug; - console.log( - ` Found plugin: ${ctx.pluginSlug} (${candidate.active ? 'active' : 'inactive'})` - ); - } else { - console.log(' No non-essential plugin found — plugin tests will be skipped.'); + // Find a throwaway plugin for lifecycle testing + if (opts.pluginSlug) { + ctx.pluginSlug = opts.pluginSlug; + console.log(`Using provided plugin slug: ${ctx.pluginSlug}`); + } else { + console.log('Discovering plugin slug...'); + try { + const result = await callAbility( + config, + 'mainwp/get-site-plugins-v1', + { site_id_or_domain: ctx.siteId }, + true + ); + if (result.statusCode === 200 && result.body) { + const data = result.body as { plugins?: Array<{ slug: string; active: boolean }> }; + if (data.plugins) { + const candidates = data.plugins.filter(p => SAFE_TEST_PLUGIN_SLUGS.includes(p.slug)); + // Only pick an inactive plugin so activate → deactivate restores the + // original state; an active one would end the run deactivated + const candidate = candidates.find(p => !p.active); + if (candidate) { + ctx.pluginSlug = candidate.slug; + console.log( + ` Found plugin: ${ctx.pluginSlug} (${candidate.active ? 'active' : 'inactive'})` + ); + } else { + console.log( + ' No throwaway plugin (hello-dolly) found — plugin lifecycle tests will be skipped.' + ); + console.log( + ' Install Hello Dolly on the child site, or pass --plugin-slug to pick one explicitly.' + ); + } } } + } catch (err) { + console.log(` Plugin discovery failed: ${(err as Error).message}`); } - } catch (err) { - console.log(` Plugin discovery failed: ${(err as Error).message}`); } // Discover an inactive theme @@ -722,6 +766,9 @@ async function main(): Promise { if (opts.category) { tests = tests.filter(t => t.category === opts.category); console.log(`Category filter: ${opts.category} (${tests.length} tests)`); + } else if (!opts.destructive && opts.testNames.length === 0) { + tests = tests.filter(t => t.category !== 'destructive'); + console.log('Destructive tests excluded (pass --destructive to include them)'); } if (opts.testNames.length > 0) { diff --git a/settings.example.json b/settings.example.json index 154c101..eb27f1e 100644 --- a/settings.example.json +++ b/settings.example.json @@ -1,5 +1,5 @@ { - "_comment": "MainWP MCP Server Settings - Copy this file to settings.json and customize. All fields are optional. Environment variables take precedence over settings file values. Authentication requires either username+appPassword OR apiToken. If both are provided, Basic Auth (username+appPassword) takes precedence over apiToken. Set schemaVerbosity to 'compact' to reduce token usage by ~30%.", + "_comment": "MainWP MCP Server Settings - Copy this file to settings.json and customize. All fields are optional. Environment variables take precedence over settings file values. Use username+appPassword with a WordPress Application Password. apiToken is retained for compatibility but is expected to fail against the WordPress Abilities API; a complete username+appPassword pair takes precedence. Set schemaVerbosity to 'compact' to reduce token usage by ~20%.", "dashboardUrl": "https://dashboard.example.com", diff --git a/settings.schema.json b/settings.schema.json index 74eb1f0..78e0511 100644 --- a/settings.schema.json +++ b/settings.schema.json @@ -25,7 +25,7 @@ }, "apiToken": { "type": "string", - "description": "MainWP REST API Bearer token. Alternative to username+appPassword authentication. Only used if username+appPassword are not set (Basic Auth takes precedence when both are provided)." + "description": "MainWP REST API Bearer token retained for compatibility. The WordPress Abilities API is expected to reject it; use username+appPassword with a WordPress Application Password. Only used without a complete username+appPassword pair." }, "skipSslVerify": { "type": "boolean", @@ -91,7 +91,7 @@ "type": "string", "enum": ["compact", "standard"], "default": "standard", - "description": "Schema verbosity level. 'compact' reduces token usage by ~30% with shorter descriptions. 'standard' provides full descriptions with examples and safety information. Default is 'standard'." + "description": "Schema verbosity level. 'compact' reduces token usage by ~20% with shorter descriptions. 'standard' provides full descriptions with examples and safety information. Default is 'standard'." }, "responseFormat": { "type": "string", diff --git a/src/abilities.test.ts b/src/abilities.test.ts index 711ce7a..a93935b 100644 --- a/src/abilities.test.ts +++ b/src/abilities.test.ts @@ -14,24 +14,17 @@ import { initRateLimiter, type Ability, } from './abilities.js'; -import { readLimitedBody } from './http-client.js'; +import { createFetch, paginateApi, readLimitedBody } from './http-client.js'; import { generateToolHelp, generateHelpDocument } from './help.js'; import { McpError, MCP_ERROR_CODES } from './errors.js'; import { type Config } from './config.js'; -import { type Logger } from './logging.js'; +import { makeBaseConfig, makeMockLogger } from '../tests/helpers/config.js'; // Mock fetch globally const mockFetch = vi.fn(); vi.stubGlobal('fetch', mockFetch); -const mockLogger: Logger = { - debug: vi.fn(), - info: vi.fn(), - notice: vi.fn(), - warning: vi.fn(), - error: vi.fn(), - critical: vi.fn(), -}; +const mockLogger = makeMockLogger(); // Sample abilities for testing const sampleAbilities: Ability[] = [ @@ -191,28 +184,7 @@ const sampleAbilities: Ability[] = [ const sampleCategories = [{ slug: 'mainwp-sites', label: 'Sites', description: 'Site management' }]; -const baseConfig: Config = { - dashboardUrl: 'https://test.local', - authType: 'basic', - username: 'admin', - appPassword: 'xxxx', - skipSslVerify: true, - allowHttp: false, - rateLimit: 0, // Disable rate limiting for tests - requestTimeout: 5000, - maxResponseSize: 10485760, - safeMode: false, - requireUserConfirmation: true, - maxSessionData: 52428800, - schemaVerbosity: 'standard', - responseFormat: 'compact', - retryEnabled: false, // Disable retries for tests - maxRetries: 2, - retryBaseDelay: 1000, - retryMaxDelay: 2000, - abilityNamespaces: ['mainwp'], - configSource: 'environment', -}; +const baseConfig = makeBaseConfig(); describe('fetchAbilities', () => { beforeEach(() => { @@ -255,6 +227,41 @@ describe('fetchAbilities', () => { expect(mockFetch).toHaveBeenCalledTimes(1); }); + it('skips malformed ability entries instead of failing the whole refresh', async () => { + const hostileAbilities = [ + null, + 'junk', + { name: 42, label: 'Numeric name' }, + { label: 'No name at all' }, + ...sampleAbilities, + ]; + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => hostileAbilities, + headers: new Headers(), + }); + + const abilities = await fetchAbilities(baseConfig); + + expect(abilities).toHaveLength(7); + expect(abilities.every(a => typeof a.name === 'string')).toBe(true); + }); + + it('does not share cache across configs that differ in transport-security settings', async () => { + mockFetch.mockResolvedValue({ + ok: true, + json: async () => sampleAbilities, + headers: new Headers(), + }); + + await fetchAbilities(baseConfig); + // Same dashboard and identity, but TLS verification disabled: a strict + // instance must not serve data fetched by a lax one (or vice versa). + await fetchAbilities({ ...baseConfig, skipSslVerify: !baseConfig.skipSslVerify }); + + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + it('should force refresh when requested', async () => { mockFetch.mockResolvedValue({ ok: true, @@ -419,9 +426,9 @@ describe('fetchAbilities', () => { // Force a refresh that returns malformed data — two abilities with the // same name produce the same tool name and trip the collision check. - // The new atomic-assignment code must leave the existing toolNameIndex - // intact, so a tool-name lookup for any ability from the first fetch - // still resolves. + // The failed refresh must leave the cached abilities array and its + // abilityIndexes entry intact, so a tool-name lookup for any ability + // from the first fetch still resolves. const dupedPayload = [sampleAbilities[0], sampleAbilities[0]]; mockFetch.mockResolvedValueOnce({ ok: true, @@ -451,8 +458,9 @@ describe('fetchAbilities', () => { fetchAbilities({ ...baseConfig, abilityNamespaces: ['mainwp', 'acme'] }) ).rejects.toThrow(/Network blip/); - // The downstream toolNameIndex was nulled along with cachedAbilities, so a - // tool-name lookup must trigger a fresh fetch rather than returning stale data. + // Emptying the cache slot dropped the abilities array and with it the + // WeakMap-keyed lookup indexes, so a tool-name lookup must trigger a + // fresh fetch rather than returning stale data. mockFetch.mockResolvedValueOnce({ ok: true, json: async () => sampleAbilities, @@ -463,6 +471,28 @@ describe('fetchAbilities', () => { expect(mockFetch).toHaveBeenCalledTimes(3); }); + it('does not share cached abilities across authentication identities', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => sampleAbilities, + headers: new Headers(), + }); + await fetchAbilities(baseConfig); + expect(mockFetch).toHaveBeenCalledTimes(1); + + // Same dashboard and namespaces but a different user: WordPress can + // expose a different ability catalog per user, so this must refetch + // instead of serving the first user's cached list. + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => [sampleAbilities[0]], + headers: new Headers(), + }); + const otherUserAbilities = await fetchAbilities({ ...baseConfig, username: 'bob' }); + expect(mockFetch).toHaveBeenCalledTimes(2); + expect(otherUserAbilities).toHaveLength(1); + }); + it('should handle fetch errors with cached fallback', async () => { // First successful fetch mockFetch.mockResolvedValueOnce({ @@ -534,6 +564,93 @@ describe('fetchAbilities', () => { await expect(fetchAbilities(baseConfig)).rejects.toThrow(/401/); }); + it('should share one upstream fetch across concurrent callers', async () => { + let resolveFetch!: (value: unknown) => void; + mockFetch.mockImplementation( + () => + new Promise(resolve => { + resolveFetch = resolve; + }) + ); + + const first = fetchAbilities(baseConfig); + const second = fetchAbilities(baseConfig); + + resolveFetch({ + ok: true, + json: async () => sampleAbilities, + headers: new Headers(), + }); + + const [a, b] = await Promise.all([first, second]); + + expect(a).toHaveLength(7); + expect(b).toBe(a); + expect(mockFetch).toHaveBeenCalledTimes(1); + }); + + it('should attach a structured status to HTTP error responses', async () => { + mockFetch.mockResolvedValueOnce({ + ok: false, + status: 401, + statusText: 'Unauthorized', + text: async () => 'Invalid credentials', + headers: new Headers(), + }); + + await expect(fetchAbilities(baseConfig)).rejects.toMatchObject({ status: 401 }); + }); + + it('should not share an in-flight fetch across different dashboards', async () => { + const resolvers: Array<(v: unknown) => void> = []; + mockFetch.mockImplementation( + () => + new Promise(resolve => { + resolvers.push(resolve); + }) + ); + + const first = fetchAbilities(baseConfig); + const second = fetchAbilities({ ...baseConfig, dashboardUrl: 'https://other.local' }); + + // Different dashboard must NOT join the first request + expect(mockFetch).toHaveBeenCalledTimes(2); + expect(String(mockFetch.mock.calls[1][0])).toContain('other.local'); + + for (const resolve of resolvers) { + resolve({ ok: true, json: async () => sampleAbilities, headers: new Headers() }); + } + await Promise.all([first, second]); + }); + + it('should not discard a newer cache committed while a failing refresh was in flight', async () => { + let rejectFirst!: (e: Error) => void; + mockFetch.mockImplementationOnce( + () => + new Promise((_, reject) => { + rejectFirst = reject; + }) + ); + const failing = fetchAbilities(baseConfig); + + // A different config commits successfully while the first is in flight + const otherConfig = { ...baseConfig, dashboardUrl: 'https://other.local' }; + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => sampleAbilities, + headers: new Headers(), + }); + await fetchAbilities(otherConfig); + + rejectFirst(new Error('Network error')); + await expect(failing).rejects.toThrow('Network error'); + + // The newer cache must survive the older refresh's failure — this call + // is served from cache, not a third upstream fetch + await fetchAbilities(otherConfig); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + it('should paginate when X-WP-TotalPages > 1', async () => { vi.resetAllMocks(); @@ -880,90 +997,41 @@ describe('executeAbility', () => { expect(calls[1][1].method).toBe('GET'); }); - it('should use DELETE for destructive + idempotent abilities (delete_site_v1)', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => sampleAbilities, - headers: new Headers(), - }); - - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => ({ deleted: true }), - headers: new Headers(), - }); - - await executeAbility(baseConfig, 'mainwp/delete-site-v1', { site_id: 1, confirm: true }); - - const calls = mockFetch.mock.calls; - expect(calls[1][1].method).toBe('DELETE'); - }); - - it('should use DELETE for destructive + idempotent abilities (delete_client_v1)', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => sampleAbilities, - headers: new Headers(), - }); - - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => ({ deleted: true }), - headers: new Headers(), - }); - - await executeAbility(baseConfig, 'mainwp/delete-client-v1', { - client_id_or_email: 1, - confirm: true, - }); - - const calls = mockFetch.mock.calls; - expect(calls[1][1].method).toBe('DELETE'); - }); - - it('should use DELETE for destructive + idempotent abilities (delete_tag_v1)', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => sampleAbilities, - headers: new Headers(), - }); - - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => ({ deleted: true }), - headers: new Headers(), - }); - - await executeAbility(baseConfig, 'mainwp/delete-tag-v1', { tag_id: 1, confirm: true }); - - const calls = mockFetch.mock.calls; - expect(calls[1][1].method).toBe('DELETE'); - }); - - it('should use DELETE for destructive + idempotent abilities (delete_site_plugins_v1)', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => sampleAbilities, - headers: new Headers(), - }); - - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => ({ deleted: true }), - headers: new Headers(), - }); - - await executeAbility(baseConfig, 'mainwp/delete-site-plugins-v1', { - site_id_or_domain: 1, - plugins: ['test-plugin/test-plugin.php'], - confirm: true, - }); - - const calls = mockFetch.mock.calls; - expect(calls[1][1].method).toBe('DELETE'); - }); + it.each([ + ['mainwp/delete-site-v1', { site_id: 1, confirm: true }], + ['mainwp/delete-client-v1', { client_id_or_email: 1, confirm: true }], + ['mainwp/delete-tag-v1', { tag_id: 1, confirm: true }], + [ + 'mainwp/delete-site-plugins-v1', + { site_id_or_domain: 1, plugins: ['test-plugin/test-plugin.php'], confirm: true }, + ], + [ + 'mainwp/delete-site-themes-v1', + { site_id_or_domain: 1, themes: ['twentytwentyfour'], confirm: true }, + ], + ])( + 'should use DELETE for destructive + idempotent abilities (%s)', + async (abilityName, input) => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => sampleAbilities, + headers: new Headers(), + }); + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ deleted: true }), + headers: new Headers(), + }); + + await executeAbility(baseConfig, abilityName, input); + + const calls = mockFetch.mock.calls; + expect(calls[1][1].method).toBe('DELETE'); + } + ); - it('should use DELETE for destructive + idempotent abilities (delete_site_themes_v1)', async () => { + it('should use POST for non-destructive write abilities', async () => { mockFetch.mockResolvedValueOnce({ ok: true, json: async () => sampleAbilities, @@ -972,37 +1040,43 @@ describe('executeAbility', () => { mockFetch.mockResolvedValueOnce({ ok: true, - json: async () => ({ deleted: true }), + json: async () => ({ updated: true }), headers: new Headers(), }); - await executeAbility(baseConfig, 'mainwp/delete-site-themes-v1', { - site_id_or_domain: 1, - themes: ['twentytwentyfour'], - confirm: true, - }); + await executeAbility(baseConfig, 'mainwp/update-site-v1', { site_id: 1, name: 'New Name' }); const calls = mockFetch.mock.calls; - expect(calls[1][1].method).toBe('DELETE'); + expect(calls[1][1].method).toBe('POST'); }); - it('should use POST for non-destructive write abilities', async () => { + it('should use POST for destructive non-idempotent abilities', async () => { + const destructiveNonIdempotentAbility: Ability = { + ...sampleAbilities[1], + meta: { + annotations: { + readonly: false, + destructive: true, + idempotent: false, + }, + }, + }; mockFetch.mockResolvedValueOnce({ ok: true, - json: async () => sampleAbilities, + json: async () => [destructiveNonIdempotentAbility], headers: new Headers(), }); - mockFetch.mockResolvedValueOnce({ ok: true, - json: async () => ({ updated: true }), + json: async () => ({ success: true }), headers: new Headers(), }); - await executeAbility(baseConfig, 'mainwp/update-site-v1', { site_id: 1, name: 'New Name' }); + await executeAbility(baseConfig, 'mainwp/delete-site-v1', { site_id: 1, confirm: true }); - const calls = mockFetch.mock.calls; - expect(calls[1][1].method).toBe('POST'); + const request = mockFetch.mock.calls[1][1]; + expect(request.method).toBe('POST'); + expect(JSON.parse(request.body as string)).toEqual({ input: { site_id: 1, confirm: true } }); }); it('should serialize input to query string for DELETE requests', async () => { @@ -1097,6 +1171,64 @@ describe('executeAbility', () => { expect(url).toContain('input[page]=2'); }); + it('preserves one-level object and scalar-array query encoding', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => sampleAbilities, + headers: new Headers(), + }); + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => [], + headers: new Headers(), + }); + + await executeAbility(baseConfig, 'mainwp/list-sites-v1', { + filters: { status: 'active', count: 2 }, + ids: [1, 'two'], + }); + + const url = mockFetch.mock.calls[1][0] as string; + expect(url).toContain('input[filters][status]=active'); + expect(url).toContain('input[filters][count]=2'); + expect(url).toContain('input[ids][]=1'); + expect(url).toContain('input[ids][]=two'); + }); + + it('rejects objects nested deeper than one level and names the offending key', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => sampleAbilities, + headers: new Headers(), + }); + + await expect( + executeAbility(baseConfig, 'mainwp/list-sites-v1', { + filters: { status: { value: 'active' } }, + }) + ).rejects.toMatchObject({ + code: MCP_ERROR_CODES.INVALID_PARAMS, + message: expect.stringContaining('filters'), + }); + }); + + it('rejects arrays containing objects and names the offending key', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => sampleAbilities, + headers: new Headers(), + }); + + await expect( + executeAbility(baseConfig, 'mainwp/list-sites-v1', { + filters: [{ status: 'active' }], + }) + ).rejects.toMatchObject({ + code: MCP_ERROR_CODES.INVALID_PARAMS, + message: expect.stringContaining('filters'), + }); + }); + it('should throw when GET URL exceeds 8000 characters', async () => { mockFetch.mockResolvedValueOnce({ ok: true, @@ -1110,6 +1242,86 @@ describe('executeAbility', () => { executeAbility(baseConfig, 'mainwp/list-sites-v1', { filter: longValue }) ).rejects.toThrow(/URL exceeds 8000 characters/); }); + + it('logs destructive previews as previewed and never executed', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ preview: true }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }) + ); + await executeAbility( + baseConfig, + 'mainwp/delete-site-v1', + { site_id: 1, dry_run: true }, + mockLogger, + sampleAbilities[1] + ); + expect(mockLogger.info).toHaveBeenCalledWith('AUDIT: destructive operation previewed', { + abilityName: 'mainwp/delete-site-v1', + }); + expect(mockLogger.info).not.toHaveBeenCalledWith( + 'AUDIT: destructive operation executed', + expect.anything() + ); + }); + + it('does not log destructive operations as executed after a 5xx failure', async () => { + mockFetch.mockResolvedValueOnce(new Response('failure', { status: 500 })); + await expect( + executeAbility( + baseConfig, + 'mainwp/delete-site-v1', + { site_id: 1 }, + mockLogger, + sampleAbilities[1] + ) + ).rejects.toThrow(/Ability execution failed/); + expect(mockLogger.info).not.toHaveBeenCalledWith( + 'AUDIT: destructive operation executed', + expect.anything() + ); + }); + + it('does not log destructive operations as executed after cancellation', async () => { + mockFetch.mockRejectedValueOnce(new DOMException('aborted', 'AbortError')); + const controller = new AbortController(); + controller.abort(); + await expect( + executeAbility( + baseConfig, + 'mainwp/delete-site-v1', + { site_id: 1 }, + mockLogger, + sampleAbilities[1], + controller.signal + ) + ).rejects.toThrow(); + expect(mockLogger.info).not.toHaveBeenCalledWith( + 'AUDIT: destructive operation executed', + expect.anything() + ); + }); + + it('logs a successful destructive execution exactly once', async () => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ success: true }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }) + ); + await executeAbility( + baseConfig, + 'mainwp/delete-site-v1', + { site_id: 1 }, + mockLogger, + sampleAbilities[1] + ); + expect(mockLogger.info).toHaveBeenCalledTimes(1); + expect(mockLogger.info).toHaveBeenCalledWith('AUDIT: destructive operation executed', { + abilityName: 'mainwp/delete-site-v1', + }); + }); }); describe('clearCache', () => { @@ -1148,6 +1360,22 @@ describe('clearCache', () => { expect(mockFetch).toHaveBeenCalledTimes(2); }); + it('should clear cached categories', async () => { + mockFetch.mockResolvedValue({ + ok: true, + json: async () => sampleCategories, + headers: new Headers(), + }); + + await fetchCategories(baseConfig); + expect(mockFetch).toHaveBeenCalledTimes(1); + + clearCache(); + + await fetchCategories(baseConfig); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + it('should clear the abilities index', async () => { // Warm cache and index mockFetch.mockResolvedValueOnce({ @@ -1174,7 +1402,7 @@ describe('clearCache', () => { }); it('should clear the toolName index', async () => { - // Warm cache and toolNameIndex + // Warm the cache and its tool-name lookup index mockFetch.mockResolvedValueOnce({ ok: true, json: async () => sampleAbilities, @@ -1185,11 +1413,12 @@ describe('clearCache', () => { expect(beforeClear?.name).toBe('mainwp/list-sites-v1'); expect(mockFetch).toHaveBeenCalledTimes(1); - // Clear cache (which must also clear toolNameIndex) + // Clear the cache, dropping the abilities array and its WeakMap-keyed indexes clearCache(); - // Next getAbilityByToolName should trigger a new fetch — if clearCache forgot - // toolNameIndex, the lookup would silently hit the stale Map and skip the fetch. + // Next getAbilityByToolName should trigger a new fetch — if clearCache left + // the cached array in place, the lookup would silently resolve through its + // stale indexes and skip the fetch. mockFetch.mockResolvedValueOnce({ ok: true, json: async () => sampleAbilities, @@ -1230,6 +1459,65 @@ describe('onCacheRefresh', () => { expect(callback).toHaveBeenCalled(); }); + + it('notifies when a same-named ability changes schema and annotations', async () => { + const callback = vi.fn(); + onCacheRefresh(callback); + const original = sampleAbilities[0]; + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => [original], + headers: new Headers(), + }); + await fetchAbilities(baseConfig); + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => [ + { + ...original, + description: 'Changed description', + input_schema: { + type: 'object', + properties: { page: { type: 'integer' } }, + }, + meta: { + ...original.meta, + annotations: { ...original.meta?.annotations, readonly: false }, + }, + }, + ], + headers: new Headers(), + }); + await fetchAbilities(baseConfig, true); + + expect(callback).toHaveBeenCalledTimes(1); + }); + + it('notifies when a same-named ability changes label or category', async () => { + const callback = vi.fn(); + onCacheRefresh(callback); + const original = sampleAbilities[0]; + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => [original], + headers: new Headers(), + }); + await fetchAbilities(baseConfig); + + // label becomes the MCP annotation title; category becomes the standard-mode + // description prefix — both affect tool conversion, so a change must notify. + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => [{ ...original, label: 'Renamed Label', category: 'mainwp-renamed' }], + headers: new Headers(), + }); + await fetchAbilities(baseConfig, true); + + expect(callback).toHaveBeenCalledTimes(1); + }); }); describe('generateToolHelp', () => { @@ -1389,4 +1677,78 @@ describe('readLimitedBody', () => { const result = await readLimitedBody(mockResponse, 1000); expect(result).toBe('{"key":"value"}'); }); + + it('surfaces ETIMEDOUT when the request deadline expires during a body read', async () => { + mockFetch.mockImplementationOnce((_url, options: RequestInit) => { + const signal = options.signal as AbortSignal; + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('partial')); + signal.addEventListener( + 'abort', + () => controller.error(new DOMException('aborted', 'AbortError')), + { once: true } + ); + }, + }); + return Promise.resolve(new Response(stream)); + }); + const customFetch = createFetch(makeBaseConfig({ requestTimeout: 20 })); + + const response = await customFetch('https://test.local/stalled'); + + await expect(readLimitedBody(response, 1000)).rejects.toMatchObject({ code: 'ETIMEDOUT' }); + }); + + it('preserves AbortError for external cancellation during a body read', async () => { + mockFetch.mockImplementationOnce((_url, options: RequestInit) => { + const signal = options.signal as AbortSignal; + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('partial')); + signal.addEventListener( + 'abort', + () => controller.error(new DOMException('aborted', 'AbortError')), + { once: true } + ); + }, + }); + return Promise.resolve(new Response(stream)); + }); + const externalController = new AbortController(); + const customFetch = createFetch(makeBaseConfig({ requestTimeout: 1000 })); + const response = await customFetch('https://test.local/cancelled', { + signal: externalController.signal, + }); + + externalController.abort(); + + await expect(readLimitedBody(response, 1000)).rejects.toMatchObject({ name: 'AbortError' }); + }); +}); + +describe('paginateApi', () => { + it('does not warn when all 50 reported pages are fetched', async () => { + const logger = makeMockLogger(); + const customFetch = vi.fn( + async () => new Response('[]', { headers: { 'X-WP-TotalPages': '50' } }) + ); + + await paginateApi(customFetch, 'https://test.local/items', 'items', 1000, logger); + + expect(customFetch).toHaveBeenCalledTimes(50); + expect(logger.warning).not.toHaveBeenCalled(); + }); + + it('warns when reported pages exceed the 50-page cap', async () => { + const logger = makeMockLogger(); + const customFetch = vi.fn( + async () => new Response('[]', { headers: { 'X-WP-TotalPages': '51' } }) + ); + + await paginateApi(customFetch, 'https://test.local/items', 'items', 1000, logger); + + expect(customFetch).toHaveBeenCalledTimes(50); + expect(logger.warning).toHaveBeenCalledWith(expect.stringContaining('Pagination capped')); + }); }); diff --git a/src/abilities.ts b/src/abilities.ts index b7aa408..9b5909d 100644 --- a/src/abilities.ts +++ b/src/abilities.ts @@ -5,8 +5,9 @@ * Abilities API REST endpoints. */ +import crypto from 'crypto'; import { Config, getAbilitiesApiUrl } from './config.js'; -import { McpErrorFactory } from './errors.js'; +import { McpErrorFactory, createHttpError, getErrorMessage } from './errors.js'; import { RateLimiter, sanitizeError } from './security.js'; import { withRetry, type RetryContext } from './retry.js'; import { @@ -81,25 +82,80 @@ export interface Category { } /** - * Cached abilities data + * Cache slot: one TTL/signature-guarded cache with in-flight de-duplication. + * Shared by the abilities and categories caches via getCachedOrRefresh. */ -let cachedAbilities: Ability[] | null = null; -let abilitiesIndex: Map | null = null; -let toolNameIndex: Map | null = null; -let cachedCategories: Category[] | null = null; -let abilitiesCacheTimestamp: number = 0; -let categoriesCacheTimestamp: number = 0; -let abilitiesNamespaceSignature: string = ''; -let categoriesNamespaceSignature: string = ''; +interface CacheSlot { + data: T | null; + timestamp: number; + /** Cache signature (dashboard + namespaces + auth identity) the cached data was built for */ + signature: string; + /** + * Bumped on every successful commit. Lets a failing refresh detect that a + * concurrent refresh committed newer data, so the failure path never + * discards a write it did not observe at its own start. + */ + generation: number; + /** In-flight refresh, shared by concurrent same-signature callers */ + inFlight: { promise: Promise; signature: string } | null; +} + +function emptySlot(): CacheSlot { + return { data: null, timestamp: 0, signature: '', generation: 0, inFlight: null }; +} + +const abilitiesCache: CacheSlot = emptySlot(); +const categoriesCache: CacheSlot = emptySlot(); + +/** + * Derived lookup indexes, keyed by the exact abilities array they were built + * from. WeakMap (not module-level variables) so a caller that awaited one + * fetch can never read indexes committed by a concurrent fetch for a + * different config — the array reference it holds always resolves to its own + * matching indexes. Entries are garbage-collected with their arrays. + */ +interface AbilityIndexes { + byName: Map; + byToolName: Map; +} +const abilityIndexes = new WeakMap(); + const CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes /** - * Build a stable signature of the configured namespaces so a config change - * (e.g. adding a third-party namespace) forces a cache refresh instead of - * serving stale, namespace-filtered data. + * Build a stable signature of the response-affecting config (dashboard URL + + * namespace allowlist + authentication identity) so a config change forces a + * cache refresh instead of serving data fetched from another dashboard, + * filtered for other namespaces, or fetched as a different user — WordPress + * can expose different ability catalogs per user, and multiple + * createServer(config) instances may share this module-level cache in one + * process. The identity is a one-way hash so credentials never appear in the + * signature itself. Also keys in-flight de-duplication: callers may only join + * a running fetch for the same dashboard, allowlist, and identity. + * Transport-security settings are part of the identity too: an instance with + * strict TLS or a smaller body cap must not reuse data fetched by a laxer one. */ -function namespaceSignature(namespaces: string[]): string { - return namespaces.join('|'); +function cacheSignature(config: Config): string { + const authIdentity = crypto + .createHash('sha256') + .update(JSON.stringify([config.authType, config.username, config.appPassword, config.apiToken])) + .digest('hex'); + return JSON.stringify([ + config.dashboardUrl, + config.abilityNamespaces, + authIdentity, + config.skipSslVerify, + config.maxResponseSize, + ]); +} + +/** + * Short one-way hash of the config identity, for callers that need to + * namespace their own per-identity state (e.g. confirmation previews) with + * the same identity definition the abilities cache uses. + */ +export function configIdentityHash(config: Config): string { + return crypto.createHash('sha256').update(cacheSignature(config)).digest('hex').slice(0, 16); } /** @@ -145,255 +201,297 @@ export function onCacheRefresh(callback: CacheRefreshCallback): void { /** * Notify all registered callbacks that the cache was refreshed */ -function notifyCacheRefresh(): void { +function notifyCacheRefresh(logger?: Logger): void { for (const callback of cacheRefreshCallbacks) { try { callback(); - } catch { - // Ignore callback errors + } catch (error) { + // Fail soft — a broken listener must not poison the cache refresh — + // but leave a trace instead of swallowing silently + logger?.debug('Cache refresh callback failed', { + error: sanitizeError(getErrorMessage(error)), + }); } } } /** - * Fetch all abilities from the MainWP Dashboard + * Return cached data or refresh it — the shared engine behind the abilities + * and categories caches. Owns, in order: + * + * 1. Fresh-cache check: serve only if the TTL hasn't expired AND the + * namespace allowlist hasn't changed since the cache was populated. + * 2. In-flight de-duplication: concurrent same-signature callers await the + * running refresh instead of issuing duplicate upstream fetches. + * 3. Refresh: fetch, process, then commit data + timestamp + signature (and + * any derived state via `commit`) together. + * 4. Failure fallback: serve stale cache on transient failures up to + * MAX_STALE_AGE_MS, but only when the cached data was built for the same + * namespace allowlist — a signature mismatch means the user changed + * config, and falling back would silently return the wrong data set. + * + * A `process` throw (e.g. tool-name collision) is treated exactly like a + * fetch failure: the existing cache is preserved and the stale-serving + * decision tree applies. + * + * @param slot - The cache slot to read/write + * @param signature - Namespace signature for the requesting config + * @param label - Data label for log messages ('abilities', 'categories') + * @param process - Convert raw fetch results into cacheable data; may return + * a `commit` callback that is invoked synchronously with the cache write + * to update derived state atomically (w.r.t. the event loop) */ -export async function fetchAbilities( - config: Config, - forceRefresh = false, - logger?: Logger -): Promise { - const namespaces = config.abilityNamespaces; - const signature = namespaceSignature(namespaces); +async function getCachedOrRefresh(opts: { + slot: CacheSlot; + signature: string; + label: string; + forceRefresh: boolean; + logger?: Logger; + fetchFn: () => Promise; + process: (raw: R) => { data: T; commit?: () => void }; +}): Promise { + const { slot, signature, label, forceRefresh, logger } = opts; - // Return cached data only if still fresh AND the namespace allowlist hasn't - // changed since the cache was populated. if ( !forceRefresh && - cachedAbilities && - abilitiesNamespaceSignature === signature && - Date.now() - abilitiesCacheTimestamp < CACHE_TTL_MS + slot.data !== null && + slot.signature === signature && + Date.now() - slot.timestamp < CACHE_TTL_MS ) { - return cachedAbilities; + return slot.data; } - const baseUrl = getAbilitiesApiUrl(config); - const customFetch = createFetch(config); - - try { - const allAbilities = await paginateApi( - customFetch, - `${baseUrl}/abilities`, - 'abilities', - config.maxResponseSize, - logger - ); - - const newAbilities = allAbilities.filter(a => { - if (!isAllowedNamespace(a.name, namespaces)) return false; - // Defense in depth: drop abilities whose names fail the strict format - // check before they reach the tool index. A malformed name (extra - // slash, bad charset) would otherwise surface as an invalid MCP tool - // name in ListTools and only fail at execute time. - if (!ABILITY_NAME_RE.test(a.name)) { - logger?.warning('Skipping ability with malformed name', { name: sanitizeError(a.name) }); - return false; - } - return true; - }); - - // A misconfigured namespace allowlist boots a server that advertises - // zero tools with no other symptom — warn loudly so the cause is in the - // logs instead of leaving a silently empty server. An empty upstream is - // a different problem, so it gets its own message. - if (allAbilities.length === 0) { - logger?.warning('Dashboard returned no abilities', { namespaces }); - } else if (newAbilities.length === 0) { - logger?.warning('No abilities matched the configured namespaces', { - namespaces, - fetchedCount: allAbilities.length, - }); - } + // A same-signature refresh is already running; its result is by definition + // fresher than the TTL, so even forceRefresh callers can join it. + if (slot.inFlight && slot.inFlight.signature === signature) { + return slot.inFlight.promise; + } - // Check if abilities have changed (compare names) - const oldNames = - cachedAbilities - ?.map(a => a.name) - .sort() - .join(',') ?? ''; - const newNames = newAbilities - .map(a => a.name) - .sort() - .join(','); - const hasChanged = oldNames !== newNames; - - // Build indices into local variables first, then assign all module-level - // cache state together once the loop has completed cleanly. If the - // collision throw fires mid-loop we leave the existing cache untouched - // rather than serving a partially built tool-name index that would make - // some tools silently unresolvable. - const newAbilitiesIndex = new Map(); - const newToolNameIndex = new Map(); - const primary = namespaces[0]; - for (const ability of newAbilities) { - newAbilitiesIndex.set(ability.name, ability); - - const toolName = abilityNameToToolName(ability.name, primary); - // Tool name collisions are rare but possible: ABILITY_NAME_RE forbids - // `_` in names and `__` is the namespace/slug separator for non-primary - // namespaces, but a double hyphen in a slug also maps to `__` (e.g. - // primary `mainwp/foo--bar` and non-primary `foo/bar` both produce - // `foo__bar`), and upstream could return duplicate names. Fail loud - // rather than silently shadowing one ability under the other's tool - // name. - const existing = newToolNameIndex.get(toolName); - if (existing) { - throw new Error( - `Tool name collision: "${toolName}" produced by both "${existing.name}" and "${ability.name}". ` + - `This indicates a violation of the namespace/slug invariants in abilities.ts.` - ); + const refresh = (async (): Promise => { + // Snapshot the generation BEFORE fetching. If a concurrent refresh (for a + // different signature — same-signature callers join us) commits while we + // are in flight, the generation moves past this snapshot and our failure + // path below knows the slot no longer holds the data we started with. + const startGeneration = slot.generation; + try { + const raw = await opts.fetchFn(); + const { data, commit } = opts.process(raw); + slot.data = data; + slot.timestamp = Date.now(); + slot.signature = signature; + slot.generation++; + commit?.(); + return data; + } catch (error) { + // Serve stale same-signature cache on transient failures, up to + // MAX_STALE_AGE_MS. + if (slot.data !== null && slot.signature === signature) { + const cacheAgeMs = Date.now() - slot.timestamp; + const cacheAgeMinutes = Math.round(cacheAgeMs / 60000); + if (cacheAgeMs > MAX_STALE_AGE_MS) { + logger?.error(`Stale ${label} cache exceeded max age, discarding`, { + cacheAgeMinutes, + maxStaleMinutes: MAX_STALE_AGE_MS / 60000, + }); + slot.data = null; + throw error; + } + logger?.warning(`Failed to refresh ${label}, using cached data`, { + error: sanitizeError(getErrorMessage(error)), + cacheAgeMinutes, + }); + return slot.data; } - newToolNameIndex.set(toolName, ability); - } - cachedAbilities = newAbilities; - abilitiesIndex = newAbilitiesIndex; - toolNameIndex = newToolNameIndex; - abilitiesCacheTimestamp = Date.now(); - abilitiesNamespaceSignature = signature; - - // Notify callbacks if abilities changed - if (hasChanged && oldNames !== '') { - notifyCacheRefresh(); - } - return cachedAbilities; - } catch (error) { - // Snapshot the cache's current signature so a concurrent fetch that - // succeeds and writes new cache state can't be silently discarded by - // this catch block when we re-check below. - const cachedSignature = abilitiesNamespaceSignature; - - // Serve stale cache on transient failures, but only up to MAX_STALE_AGE_MS, - // and only when the cached data was built for the same namespace allowlist. - // A signature mismatch means the user changed config — falling back to the - // old cache would silently return the wrong set of abilities. - if (cachedAbilities && cachedSignature === signature) { - const cacheAgeMs = Date.now() - abilitiesCacheTimestamp; - const cacheAgeMinutes = Math.round(cacheAgeMs / 60000); - if (cacheAgeMs > MAX_STALE_AGE_MS) { - logger?.error('Stale cache exceeded max age, discarding', { - cacheAgeMinutes, - maxStaleMinutes: MAX_STALE_AGE_MS / 60000, + // The slot holds data for a DIFFERENT signature (or nothing). Discard + // it only if it's the same snapshot we observed when this refresh + // started — data committed by a concurrent refresh AFTER our start + // (generation moved) is newer than us and must survive our failure. + // Either way, surface the error: we can't return wrong-config data, + // and we won't lie about a fetch that failed. + if (slot.data !== null && slot.generation === startGeneration) { + logger?.warning(`Discarding ${label} cache: config changed and refresh failed`, { + cachedSignature: slot.signature, + requestedSignature: signature, + error: sanitizeError(getErrorMessage(error)), }); - cachedAbilities = null; - abilitiesIndex = null; - toolNameIndex = null; - throw error; + slot.data = null; } - logger?.warning('Failed to refresh abilities, using cached data', { - error: sanitizeError(String(error)), - cacheAgeMinutes, - }); - return cachedAbilities; + throw error; } + })(); - // Cache is missing or its signature no longer matches the requested - // signature. If the signature has CHANGED since we entered the catch (a - // concurrent fetch succeeded with a different config), leave that cache - // intact — only null when the signature still matches the stale data we - // were about to serve. Either way, surface the error: we can't return - // wrong-namespace data, and we won't lie about a fetch that failed. - if (cachedAbilities && abilitiesNamespaceSignature === cachedSignature) { - logger?.warning('Discarding cache: namespace allowlist changed and refresh failed', { - cachedSignature, - requestedSignature: signature, - error: sanitizeError(String(error)), - }); - cachedAbilities = null; - abilitiesIndex = null; - toolNameIndex = null; + slot.inFlight = { promise: refresh, signature }; + try { + return await refresh; + } finally { + // Only clear our own registration — a different-signature refresh may + // have replaced it while we were awaiting. + if (slot.inFlight?.promise === refresh) { + slot.inFlight = null; } - throw error; } } /** - * Fetch all categories from the MainWP Dashboard + * Fetch all abilities from the MainWP Dashboard */ -export async function fetchCategories( +export async function fetchAbilities( config: Config, forceRefresh = false, logger?: Logger -): Promise { +): Promise { const namespaces = config.abilityNamespaces; - const signature = namespaceSignature(namespaces); - - // Return cached data only if still fresh AND the namespace allowlist hasn't - // changed since the cache was populated. Without this, a config change - // would leave the category list out of sync with the ability list for up - // to one TTL window. - if ( - !forceRefresh && - cachedCategories && - categoriesNamespaceSignature === signature && - Date.now() - categoriesCacheTimestamp < CACHE_TTL_MS - ) { - return cachedCategories; - } - const baseUrl = getAbilitiesApiUrl(config); - const customFetch = createFetch(config); - try { - const allCategories = await paginateApi( - customFetch, - `${baseUrl}/categories`, - 'categories', - config.maxResponseSize, - logger - ); + return getCachedOrRefresh({ + slot: abilitiesCache, + signature: cacheSignature(config), + label: 'abilities', + forceRefresh, + logger, + fetchFn: () => + paginateApi( + createFetch(config), + `${baseUrl}/abilities`, + 'abilities', + config.maxResponseSize, + logger + ), + process: allAbilities => { + const newAbilities = allAbilities.filter(a => { + // Hostile-input guard: a null entry or non-string name would throw + // below and poison the whole catalog refresh instead of being skipped. + if ( + a === null || + typeof a !== 'object' || + typeof (a as { name?: unknown }).name !== 'string' + ) { + logger?.warning('Skipping malformed ability entry', { entryType: typeof a }); + return false; + } + if (!isAllowedNamespace(a.name, namespaces)) return false; + // Defense in depth: drop abilities whose names fail the strict format + // check before they reach the tool index. A malformed name (extra + // slash, bad charset) would otherwise surface as an invalid MCP tool + // name in ListTools and only fail at execute time. + if (!ABILITY_NAME_RE.test(a.name)) { + logger?.warning('Skipping ability with malformed name', { name: sanitizeError(a.name) }); + return false; + } + return true; + }); - // Filter categories to configured namespaces (allowlist of `{ns}-` prefixes) - cachedCategories = allCategories.filter(c => isAllowedCategory(c.slug, namespaces)); - categoriesCacheTimestamp = Date.now(); - categoriesNamespaceSignature = signature; - - return cachedCategories; - } catch (error) { - const cachedSignature = categoriesNamespaceSignature; - - if (cachedCategories && cachedSignature === signature) { - const cacheAgeMs = Date.now() - categoriesCacheTimestamp; - const cacheAgeMinutes = Math.round(cacheAgeMs / 60000); - if (cacheAgeMs > MAX_STALE_AGE_MS) { - logger?.error('Stale categories cache exceeded max age, discarding', { - cacheAgeMinutes, - maxStaleMinutes: MAX_STALE_AGE_MS / 60000, + // A misconfigured namespace allowlist boots a server that advertises + // zero tools with no other symptom — warn loudly so the cause is in the + // logs instead of leaving a silently empty server. An empty upstream is + // a different problem, so it gets its own message. + if (allAbilities.length === 0) { + logger?.warning('Dashboard returned no abilities', { namespaces }); + } else if (newAbilities.length === 0) { + logger?.warning('No abilities matched the configured namespaces', { + namespaces, + fetchedCount: allAbilities.length, }); - cachedCategories = null; - throw error; } - logger?.warning('Failed to refresh categories, using cached data', { - error: sanitizeError(String(error)), - cacheAgeMinutes, - }); - return cachedCategories; - } - // Signature mismatch or no cache — race-safe discard (see fetchAbilities). - if (cachedCategories && categoriesNamespaceSignature === cachedSignature) { - logger?.warning( - 'Discarding categories cache: namespace allowlist changed and refresh failed', - { - cachedSignature, - requestedSignature: signature, - error: sanitizeError(String(error)), + // Detect changes to every field that affects MCP tool conversion, not + // only names, so clients refresh schemas and safety annotations too. + const fingerprint = (abilities: Ability[] | null): string => + JSON.stringify( + (abilities ?? []) + .map(ability => ({ + name: ability.name, + label: ability.label, + category: ability.category, + description: ability.description, + input_schema: ability.input_schema, + output_schema: ability.output_schema, + annotations: ability.meta?.annotations, + })) + .sort((a, b) => a.name.localeCompare(b.name)) + ); + const oldFingerprint = fingerprint(abilitiesCache.data); + const newFingerprint = fingerprint(newAbilities); + const hasChanged = oldFingerprint !== newFingerprint; + const hadCachedAbilities = abilitiesCache.data !== null; + + // Build indices into local variables first; they're committed together + // with the cache write. If the collision throw fires mid-loop the + // existing cache and indexes stay untouched rather than serving a + // partially built tool-name index that would make some tools silently + // unresolvable. + const newAbilitiesIndex = new Map(); + const newToolNameIndex = new Map(); + const primary = namespaces[0]; + for (const ability of newAbilities) { + newAbilitiesIndex.set(ability.name, ability); + + const toolName = abilityNameToToolName(ability.name, primary); + // Tool name collisions are rare but possible: ABILITY_NAME_RE forbids + // `_` in names and `__` is the namespace/slug separator for non-primary + // namespaces, but a double hyphen in a slug also maps to `__` (e.g. + // primary `mainwp/foo--bar` and non-primary `foo/bar` both produce + // `foo__bar`), and upstream could return duplicate names. Fail loud + // rather than silently shadowing one ability under the other's tool + // name. + const existing = newToolNameIndex.get(toolName); + if (existing) { + throw new Error( + `Tool name collision: "${toolName}" produced by both "${existing.name}" and "${ability.name}". ` + + `This indicates a violation of the namespace/slug invariants in abilities.ts.` + ); } - ); - cachedCategories = null; - } - throw error; - } + newToolNameIndex.set(toolName, ability); + } + + return { + data: newAbilities, + commit: () => { + abilityIndexes.set(newAbilities, { + byName: newAbilitiesIndex, + byToolName: newToolNameIndex, + }); + // Notify callbacks if abilities changed + if (hasChanged && hadCachedAbilities) { + notifyCacheRefresh(logger); + } + }, + }; + }, + }); +} + +/** + * Fetch all categories from the MainWP Dashboard + */ +export async function fetchCategories( + config: Config, + forceRefresh = false, + logger?: Logger +): Promise { + const namespaces = config.abilityNamespaces; + const baseUrl = getAbilitiesApiUrl(config); + + return getCachedOrRefresh({ + slot: categoriesCache, + signature: cacheSignature(config), + label: 'categories', + forceRefresh, + logger, + fetchFn: () => + paginateApi( + createFetch(config), + `${baseUrl}/categories`, + 'categories', + config.maxResponseSize, + logger + ), + // Filter categories to configured namespaces (allowlist of `{ns}-` prefixes) + process: allCategories => ({ + data: allCategories.filter(c => isAllowedCategory(c.slug, namespaces)), + }), + }); } /** @@ -404,8 +502,10 @@ export async function getAbility( name: string, logger?: Logger ): Promise { - await fetchAbilities(config, false, logger); - return abilitiesIndex?.get(name); + // Look up via the indexes built for the exact array this call received — + // never via shared mutable state a concurrent fetch could have replaced. + const abilities = await fetchAbilities(config, false, logger); + return abilityIndexes.get(abilities)?.byName.get(name); } /** @@ -421,8 +521,8 @@ export async function getAbilityByToolName( toolName: string, logger?: Logger ): Promise { - await fetchAbilities(config, false, logger); - return toolNameIndex?.get(toolName); + const abilities = await fetchAbilities(config, false, logger); + return abilityIndexes.get(abilities)?.byToolName.get(toolName); } /** @@ -436,11 +536,21 @@ function serializeToPhpQueryString(input: Record): string { if (Array.isArray(value)) { // Arrays: input[key][]=val1&input[key][]=val2 for (const item of value) { + if (item !== null && typeof item === 'object') { + throw McpErrorFactory.invalidParams( + `Unsupported nested query parameter at "${key}": arrays may contain only scalar values` + ); + } params.push(`input[${encodeURIComponent(key)}][]=${encodeURIComponent(String(item))}`); } } else if (typeof value === 'object' && value !== null) { // Nested objects: input[key][subkey]=val for (const [subKey, subVal] of Object.entries(value)) { + if (subVal !== null && typeof subVal === 'object') { + throw McpErrorFactory.invalidParams( + `Unsupported nested query parameter at "${key}": objects may be only one level deep` + ); + } params.push( `input[${encodeURIComponent(key)}][${encodeURIComponent(subKey)}]=${encodeURIComponent(String(subVal))}` ); @@ -454,22 +564,6 @@ function serializeToPhpQueryString(input: Record): string { return params.length > 0 ? '?' + params.join('&') : ''; } -/** - * Create an HTTP error with status code for retry detection. - * The status is embedded in the error object and message for isRetryableError() to detect. - * - * @param status - HTTP status code - * @param errorCode - Error code (from JSON response or status string) - * @param message - Error message - */ -function createHttpError(status: number, errorCode: string, message: string): Error { - const error = new Error(`Ability execution failed: ${errorCode} - ${message}`); - const httpError = error as Error & { status: number; code: string }; - httpError.status = status; - httpError.code = errorCode; - return error; -} - /** * Execute an ability via the REST API * @@ -514,11 +608,6 @@ export async function executeAbility( const url = `${baseUrl}/abilities/${abilityName}/run`; const hasInput = input && Object.keys(input).length > 0; - // Audit log for destructive operations - logs operation name only, no sensitive parameters - if (isDestructive) { - logger?.info('AUDIT: Destructive operation requested', { abilityName }); - } - /** * Fetch and validate response in a single operation. * This ensures HTTP errors (5xx, 429) are thrown and can be retried. @@ -583,7 +672,11 @@ export async function executeAbility( upstreamLatencyMs, }); - throw createHttpError(response.status, errorCode, sanitizeError(errorMsg)); + throw createHttpError( + response.status, + errorCode, + `Ability execution failed: ${errorCode} - ${sanitizeError(errorMsg)}` + ); } // Read response body with streaming size enforcement @@ -599,8 +692,9 @@ export async function executeAbility( }; // Apply retry logic only for read-only operations when enabled + let result: unknown; if (config.retryEnabled && isReadonly) { - return await withRetry(fetchAndValidate, { + result = await withRetry(fetchAndValidate, { maxRetries: config.maxRetries, baseDelay: config.retryBaseDelay, maxDelay: config.retryMaxDelay, @@ -609,23 +703,36 @@ export async function executeAbility( }); } else { // No retry: execute directly with synthetic context - return await fetchAndValidate({ + result = await fetchAndValidate({ remainingBudget: config.requestTimeout, attempt: 0, }); } + + if (isDestructive) { + const action = input?.dry_run === true ? 'previewed' : 'executed'; + logger?.info(`AUDIT: destructive operation ${action}`, { abilityName }); + } + + return result; } /** * Clear the abilities cache */ export function clearCache(): void { - cachedAbilities = null; - abilitiesIndex = null; - toolNameIndex = null; - cachedCategories = null; - abilitiesCacheTimestamp = 0; - categoriesCacheTimestamp = 0; - abilitiesNamespaceSignature = ''; - categoriesNamespaceSignature = ''; + // A refresh in flight at clear time will still commit when it resolves + // (same semantics as before the slot refactor); clearing the inFlight + // registration just stops new callers from joining the pre-clear fetch. + // Derived indexes live in a WeakMap keyed by the discarded arrays, so + // they're garbage-collected with them — nothing to clear here. The + // generation stays monotonic (bumped, not reset) so any refresh that + // snapshotted it before this clear can never mistake later data for its + // own start state. + Object.assign(abilitiesCache, emptySlot(), { + generation: abilitiesCache.generation + 1, + }); + Object.assign(categoriesCache, emptySlot(), { + generation: categoriesCache.generation + 1, + }); } diff --git a/src/config.test.ts b/src/config.test.ts index 1eb300b..13ac82f 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -14,6 +14,7 @@ import fs from 'fs'; // Import test fixtures for validation tests import configFixture from '../tests/fixtures/config.json' with { type: 'json' }; +import { makeBaseConfig } from '../tests/helpers/config.js'; // Mock fs module vi.mock('fs'); @@ -77,6 +78,20 @@ describe('loadSettingsFile', () => { expect(() => loadSettingsFile()).toThrow(/must be a number/); }); + it('should reject non-finite numbers (JSON 1e999 parses to Infinity)', () => { + vi.mocked(fs.existsSync).mockReturnValue(true); + vi.mocked(fs.readFileSync).mockReturnValue('{"maxResponseSize": 1e999}'); + + expect(() => loadSettingsFile()).toThrow(/must be an integer within the safe range/); + }); + + it('should reject decimal numbers in settings file', () => { + vi.mocked(fs.existsSync).mockReturnValue(true); + vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify({ rateLimit: 60.5 })); + + expect(() => loadSettingsFile()).toThrow(/must be an integer within the safe range/); + }); + it('should validate field types - array fields', () => { vi.mocked(fs.existsSync).mockReturnValue(true); vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify({ allowedTools: 'not-array' })); @@ -167,6 +182,18 @@ describe('loadConfig', () => { expect(config.authType).toBe('bearer'); expect(config.apiToken).toBe('mytoken123'); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining('Application Password')); + }); + + it('should warn when bearer auth is selected with only a partial basic auth pair', () => { + process.env.MAINWP_URL = 'https://test.com'; + process.env.MAINWP_USER = 'admin'; + process.env.MAINWP_TOKEN = 'mytoken123'; + + const config = loadConfig(); + + expect(config.authType).toBe('bearer'); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining('Application Password')); }); it('should prefer basic auth over bearer auth', () => { @@ -348,6 +375,74 @@ describe('loadConfig', () => { expect(() => loadConfig()).toThrow(/conflict/); }); + function baseEnv() { + process.env.MAINWP_URL = 'https://test.com'; + process.env.MAINWP_USER = 'admin'; + process.env.MAINWP_APP_PASSWORD = 'xxxx'; + } + + describe('boolean env var parsing', () => { + it.each(['true', 'TRUE', 'True', '1', 'yes', 'on', 'ON'])('parses %s as true', value => { + baseEnv(); + process.env.MAINWP_SAFE_MODE = value; + + expect(loadConfig().safeMode).toBe(true); + }); + + it.each(['false', 'FALSE', '0', 'no', 'off'])('parses %s as false', value => { + baseEnv(); + process.env.MAINWP_REQUIRE_USER_CONFIRMATION = value; + + expect(loadConfig().requireUserConfirmation).toBe(false); + }); + + it('throws on an unrecognized value instead of falling back to the default', () => { + baseEnv(); + process.env.MAINWP_SAFE_MODE = 'maybe'; + + expect(() => loadConfig()).toThrow(/MAINWP_SAFE_MODE.*maybe/); + }); + + it('throws when a malformed env value overrides a permissive settings-file value', () => { + vi.mocked(fs.existsSync).mockReturnValue(true); + vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify({ allowHttp: true })); + baseEnv(); + process.env.MAINWP_ALLOW_HTTP = 'flase'; + + expect(() => loadConfig()).toThrow(/MAINWP_ALLOW_HTTP.*flase/); + }); + }); + + describe('numeric env var parsing', () => { + it('rejects values with trailing non-numeric characters', () => { + baseEnv(); + process.env.MAINWP_RATE_LIMIT = '60abc'; + + expect(() => loadConfig()).toThrow(/MAINWP_RATE_LIMIT.*60abc/); + }); + + it('rejects entirely non-numeric values', () => { + baseEnv(); + process.env.MAINWP_REQUEST_TIMEOUT = 'abc'; + + expect(() => loadConfig()).toThrow(/MAINWP_REQUEST_TIMEOUT.*abc/); + }); + + it('accepts clean integer values', () => { + baseEnv(); + process.env.MAINWP_RATE_LIMIT = '120'; + + expect(loadConfig().rateLimit).toBe(120); + }); + + it('rejects digit-only values that overflow to Infinity', () => { + baseEnv(); + process.env.MAINWP_REQUEST_TIMEOUT = '9'.repeat(400); + + expect(() => loadConfig()).toThrow(/MAINWP_REQUEST_TIMEOUT.*safe integer/); + }); + }); + it('should prioritize env vars over settings file', () => { const settings = { dashboardUrl: 'https://from-file.com' }; vi.mocked(fs.existsSync).mockReturnValue(true); @@ -464,26 +559,13 @@ describe('loadConfig', () => { describe('getAbilitiesApiUrl', () => { it('should construct proper URL', () => { - const config: Config = { + const config: Config = makeBaseConfig({ dashboardUrl: 'https://example.com', - authType: 'basic', skipSslVerify: false, - allowHttp: false, rateLimit: 60, requestTimeout: 30000, - maxResponseSize: 10485760, - safeMode: false, - requireUserConfirmation: true, - maxSessionData: 52428800, - schemaVerbosity: 'standard', - responseFormat: 'compact', retryEnabled: true, - maxRetries: 2, - retryBaseDelay: 1000, - retryMaxDelay: 2000, - abilityNamespaces: ['mainwp'], - configSource: 'environment', - }; + }); const url = getAbilitiesApiUrl(config); @@ -493,28 +575,14 @@ describe('getAbilitiesApiUrl', () => { describe('getAuthHeaders', () => { it('should return Basic auth header', () => { - const config: Config = { + const config: Config = makeBaseConfig({ dashboardUrl: 'https://example.com', - authType: 'basic', - username: 'admin', appPassword: 'secret', skipSslVerify: false, - allowHttp: false, rateLimit: 60, requestTimeout: 30000, - maxResponseSize: 10485760, - safeMode: false, - requireUserConfirmation: true, - maxSessionData: 52428800, - schemaVerbosity: 'standard', - responseFormat: 'compact', retryEnabled: true, - maxRetries: 2, - retryBaseDelay: 1000, - retryMaxDelay: 2000, - abilityNamespaces: ['mainwp'], - configSource: 'environment', - }; + }); const headers = getAuthHeaders(config); @@ -523,27 +591,15 @@ describe('getAuthHeaders', () => { }); it('should return Bearer auth header', () => { - const config: Config = { + const config: Config = makeBaseConfig({ dashboardUrl: 'https://example.com', authType: 'bearer', apiToken: 'mytoken', skipSslVerify: false, - allowHttp: false, rateLimit: 60, requestTimeout: 30000, - maxResponseSize: 10485760, - safeMode: false, - requireUserConfirmation: true, - maxSessionData: 52428800, - schemaVerbosity: 'standard', - responseFormat: 'compact', retryEnabled: true, - maxRetries: 2, - retryBaseDelay: 1000, - retryMaxDelay: 2000, - abilityNamespaces: ['mainwp'], - configSource: 'environment', - }; + }); const headers = getAuthHeaders(config); @@ -551,26 +607,13 @@ describe('getAuthHeaders', () => { }); it('should always include Content-Type', () => { - const config: Config = { + const config: Config = makeBaseConfig({ dashboardUrl: 'https://example.com', - authType: 'basic', skipSslVerify: false, - allowHttp: false, rateLimit: 60, requestTimeout: 30000, - maxResponseSize: 10485760, - safeMode: false, - requireUserConfirmation: true, - maxSessionData: 52428800, - schemaVerbosity: 'standard', - responseFormat: 'compact', retryEnabled: true, - maxRetries: 2, - retryBaseDelay: 1000, - retryMaxDelay: 2000, - abilityNamespaces: ['mainwp'], - configSource: 'environment', - }; + }); const headers = getAuthHeaders(config); @@ -578,28 +621,15 @@ describe('getAuthHeaders', () => { }); it('should encode Basic auth credentials correctly', () => { - const config: Config = { + const config: Config = makeBaseConfig({ dashboardUrl: 'https://example.com', - authType: 'basic', username: 'user', appPassword: 'pass', skipSslVerify: false, - allowHttp: false, rateLimit: 60, requestTimeout: 30000, - maxResponseSize: 10485760, - safeMode: false, - requireUserConfirmation: true, - maxSessionData: 52428800, - schemaVerbosity: 'standard', - responseFormat: 'compact', retryEnabled: true, - maxRetries: 2, - retryBaseDelay: 1000, - retryMaxDelay: 2000, - abilityNamespaces: ['mainwp'], - configSource: 'environment', - }; + }); const headers = getAuthHeaders(config); const encoded = headers['Authorization'].replace('Basic ', ''); diff --git a/src/config.ts b/src/config.ts index ccd5bf7..997749b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -10,6 +10,7 @@ */ import fs from 'fs'; +import { getErrorMessage } from './errors.js'; import path from 'path'; import os from 'os'; @@ -144,90 +145,83 @@ const ABILITY_NAMESPACE_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/; /** Default namespace allowlist. */ const DEFAULT_ABILITY_NAMESPACES: readonly string[] = ['mainwp'] as const; +/** + * Field → type map for settings.json validation. Each SettingsFile property + * is declared exactly once here; the type checks and unknown-field detection + * in validateSettingsFile both derive from this map. Adding a settings field + * means adding one entry here (plus the SettingsFile interface and its + * loadConfig getter call). + */ +const SETTINGS_FIELD_TYPES: Record = { + dashboardUrl: 'string', + username: 'string', + appPassword: 'string', + apiToken: 'string', + schemaVerbosity: 'string', + responseFormat: 'string', + skipSslVerify: 'boolean', + allowHttp: 'boolean', + safeMode: 'boolean', + requireUserConfirmation: 'boolean', + retryEnabled: 'boolean', + rateLimit: 'number', + requestTimeout: 'number', + maxResponseSize: 'number', + maxSessionData: 'number', + maxRetries: 'number', + retryBaseDelay: 'number', + retryMaxDelay: 'number', + allowedTools: 'string[]', + blockedTools: 'string[]', + abilityNamespaces: 'string[]', +}; + /** * Validate settings file structure and types * Throws on validation errors with descriptive messages */ -function validateSettingsFile(settings: any, filePath: string): void { +function validateSettingsFile(settings: unknown, filePath: string): void { // Guard: root value must be a non-null plain object if (settings === null || typeof settings !== 'object' || Array.isArray(settings)) { throw new Error(`Invalid settings file: ${filePath}\n - root value must be a JSON object`); } + const record = settings as Record; const errors: string[] = []; - // Define expected field types - const stringFields = [ - 'dashboardUrl', - 'username', - 'appPassword', - 'apiToken', - 'schemaVerbosity', - 'responseFormat', - ]; - const booleanFields = [ - 'skipSslVerify', - 'allowHttp', - 'safeMode', - 'requireUserConfirmation', - 'retryEnabled', - ]; - const numberFields = [ - 'rateLimit', - 'requestTimeout', - 'maxResponseSize', - 'maxSessionData', - 'maxRetries', - 'retryBaseDelay', - 'retryMaxDelay', - ]; - const arrayFields = ['allowedTools', 'blockedTools', 'abilityNamespaces']; - - // Validate string fields - for (const field of stringFields) { - if (settings[field] !== undefined && typeof settings[field] !== 'string') { - errors.push(`"${field}" must be a string`); - } - } - - // Validate boolean fields - for (const field of booleanFields) { - if (settings[field] !== undefined && typeof settings[field] !== 'boolean') { - errors.push(`"${field}" must be a boolean`); - } - } - - // Validate number fields - for (const field of numberFields) { - if (settings[field] !== undefined && typeof settings[field] !== 'number') { - errors.push(`"${field}" must be a number`); - } - } - - // Validate array fields (must be arrays of strings) - for (const field of arrayFields) { - const value = settings[field]; - if (value !== undefined) { - if (!Array.isArray(value) || !value.every((x: any) => typeof x === 'string')) { + // Validate every declared field against its type; string[] gets the + // array-of-strings check, the rest a typeof check + for (const [field, type] of Object.entries(SETTINGS_FIELD_TYPES)) { + const value = record[field]; + if (value === undefined) continue; + if (type === 'string[]') { + if (!Array.isArray(value) || !value.every((x: unknown) => typeof x === 'string')) { errors.push(`"${field}" must be an array of strings`); } + } else if (typeof value !== type) { + errors.push(`"${field}" must be a ${type}`); + } else if (type === 'number' && !Number.isSafeInteger(value)) { + // Same contract as getNumber's env-var parsing: integers only, within + // the safe range. JSON also permits decimals and 1e999 (Infinity with + // typeof 'number'), both of which would pass downstream isNaN checks + errors.push(`"${field}" must be an integer within the safe range`); } } // Validate schemaVerbosity enum - if (settings.schemaVerbosity !== undefined) { - if (!SCHEMA_VERBOSITY_VALUES.includes(settings.schemaVerbosity)) { + if (record.schemaVerbosity !== undefined) { + if (!(SCHEMA_VERBOSITY_VALUES as readonly unknown[]).includes(record.schemaVerbosity)) { errors.push( - `"schemaVerbosity" must be one of: ${SCHEMA_VERBOSITY_VALUES.join(', ')}; got: "${settings.schemaVerbosity}"` + `"schemaVerbosity" must be one of: ${SCHEMA_VERBOSITY_VALUES.join(', ')}; got: "${record.schemaVerbosity}"` ); } } // Validate responseFormat enum - if (settings.responseFormat !== undefined) { - if (!RESPONSE_FORMAT_VALUES.includes(settings.responseFormat)) { + if (record.responseFormat !== undefined) { + if (!(RESPONSE_FORMAT_VALUES as readonly unknown[]).includes(record.responseFormat)) { errors.push( - `"responseFormat" must be one of: ${RESPONSE_FORMAT_VALUES.join(', ')}; got: "${settings.responseFormat}"` + `"responseFormat" must be one of: ${RESPONSE_FORMAT_VALUES.join(', ')}; got: "${record.responseFormat}"` ); } } @@ -235,11 +229,11 @@ function validateSettingsFile(settings: any, filePath: string): void { // Validate abilityNamespaces entries (charset + non-empty list). The // generic arrayFields check above already verified every entry is a string, // so only the charset check runs here. - if (Array.isArray(settings.abilityNamespaces)) { - if (settings.abilityNamespaces.length === 0) { + if (Array.isArray(record.abilityNamespaces)) { + if (record.abilityNamespaces.length === 0) { errors.push('"abilityNamespaces" must not be empty'); } - for (const ns of settings.abilityNamespaces) { + for (const ns of record.abilityNamespaces) { if (typeof ns !== 'string') continue; // arrayFields check reported this already if (!ABILITY_NAMESPACE_RE.test(ns)) { errors.push( @@ -251,14 +245,8 @@ function validateSettingsFile(settings: any, filePath: string): void { // Detect unknown fields // Valid fields: all SettingsFile properties plus _comment (for inline documentation per schema) - const validFields = new Set([ - ...stringFields, - ...booleanFields, - ...numberFields, - ...arrayFields, - '_comment', - ]); - for (const key of Object.keys(settings)) { + const validFields = new Set([...Object.keys(SETTINGS_FIELD_TYPES), '_comment']); + for (const key of Object.keys(record)) { if (!validFields.has(key)) { errors.push(`Unknown field "${key}"`); } @@ -287,20 +275,20 @@ export function loadSettingsFile(): SettingsFile | null { for (const filePath of searchPaths) { if (fs.existsSync(filePath)) { let content: string; - let parsed: any; + let parsed: unknown; try { content = fs.readFileSync(filePath, 'utf-8'); - } catch (error: any) { - throw new Error(`Failed to read settings file: ${filePath}\n${error.message}`, { + } catch (error) { + throw new Error(`Failed to read settings file: ${filePath}\n${getErrorMessage(error)}`, { cause: error, }); } try { parsed = JSON.parse(content); - } catch (error: any) { - throw new Error(`Invalid JSON in settings file: ${filePath}\n${error.message}`, { + } catch (error) { + throw new Error(`Invalid JSON in settings file: ${filePath}\n${getErrorMessage(error)}`, { cause: error, }); } @@ -344,16 +332,30 @@ function getString( return defaultValue; } +/** Accepted boolean env var spellings (case-insensitive) */ +const TRUTHY_VALUES = new Set(['true', '1', 'yes', 'on']); +const FALSY_VALUES = new Set(['false', '0', 'no', 'off']); + /** - * Get boolean value with precedence: env var > settings file > default + * Get boolean value with precedence: env var > settings file > default. + * Unrecognized non-empty env values fail startup rather than falling through + * to a file/default value — several of these flags are security-relevant. */ function getBoolean( + name: string, envVar: string | undefined, fileValue: boolean | undefined, defaultValue: boolean ): boolean { if (envVar !== undefined && envVar !== '') { - return envVar === 'true'; + const normalized = envVar.trim().toLowerCase(); + if (TRUTHY_VALUES.has(normalized)) { + return true; + } + if (FALSY_VALUES.has(normalized)) { + return false; + } + throw new Error(`${name} must be a boolean (true/1/yes/on or false/0/no/off), got "${envVar}"`); } if (fileValue !== undefined) { return fileValue; @@ -362,15 +364,29 @@ function getBoolean( } /** - * Get number value with precedence: env var > settings file > default + * Get number value with precedence: env var > settings file > default. + * Non-integer env values fail startup — a silently truncated "60abc" → 60 + * is worse than a clear config error for limits and timeouts. */ function getNumber( + name: string, envVar: string | undefined, fileValue: number | undefined, defaultValue: number ): number { if (envVar !== undefined && envVar !== '') { - return parseInt(envVar, 10); + const trimmed = envVar.trim(); + if (!/^-?\d+$/.test(trimmed)) { + throw new Error(`${name} must be an integer, got "${envVar}"`); + } + const value = parseInt(trimmed, 10); + // Digit-only strings can still overflow to Infinity (e.g. 400 digits), + // which passes isNaN/positive checks downstream and disables every + // limit that compares against it + if (!Number.isSafeInteger(value)) { + throw new Error(`${name} must be within the safe integer range, got "${envVar}"`); + } + return value; } if (fileValue !== undefined) { return fileValue; @@ -398,6 +414,34 @@ function getStringArray( return defaultValue; } +/** + * Every MAINWP_* env var read by loadConfig. Used to compute configSource — + * keep in sync with the getter calls below when adding a config option. + */ +const MAINWP_ENV_VARS = [ + 'MAINWP_URL', + 'MAINWP_USER', + 'MAINWP_APP_PASSWORD', + 'MAINWP_TOKEN', + 'MAINWP_SKIP_SSL_VERIFY', + 'MAINWP_ALLOW_HTTP', + 'MAINWP_SAFE_MODE', + 'MAINWP_REQUIRE_USER_CONFIRMATION', + 'MAINWP_RATE_LIMIT', + 'MAINWP_REQUEST_TIMEOUT', + 'MAINWP_MAX_RESPONSE_SIZE', + 'MAINWP_MAX_SESSION_DATA', + 'MAINWP_ALLOWED_TOOLS', + 'MAINWP_BLOCKED_TOOLS', + 'MAINWP_SCHEMA_VERBOSITY', + 'MAINWP_RESPONSE_FORMAT', + 'MAINWP_RETRY_ENABLED', + 'MAINWP_MAX_RETRIES', + 'MAINWP_RETRY_BASE_DELAY', + 'MAINWP_RETRY_MAX_DELAY', + 'MAINWP_ABILITY_NAMESPACES', +] as const; + /** * Load configuration from environment variables and settings file * Precedence: environment variables > settings file > defaults @@ -407,29 +451,7 @@ export function loadConfig(): Config { const settings = loadSettingsFile(); // Determine config source based on what's available - const hasEnvVars = !!( - process.env.MAINWP_URL || - process.env.MAINWP_USER || - process.env.MAINWP_APP_PASSWORD || - process.env.MAINWP_TOKEN || - process.env.MAINWP_SKIP_SSL_VERIFY || - process.env.MAINWP_ALLOW_HTTP || - process.env.MAINWP_SAFE_MODE || - process.env.MAINWP_REQUIRE_USER_CONFIRMATION || - process.env.MAINWP_RATE_LIMIT || - process.env.MAINWP_REQUEST_TIMEOUT || - process.env.MAINWP_MAX_RESPONSE_SIZE || - process.env.MAINWP_MAX_SESSION_DATA || - process.env.MAINWP_ALLOWED_TOOLS || - process.env.MAINWP_BLOCKED_TOOLS || - process.env.MAINWP_SCHEMA_VERBOSITY || - process.env.MAINWP_RESPONSE_FORMAT || - process.env.MAINWP_RETRY_ENABLED || - process.env.MAINWP_MAX_RETRIES || - process.env.MAINWP_RETRY_BASE_DELAY || - process.env.MAINWP_RETRY_MAX_DELAY || - process.env.MAINWP_ABILITY_NAMESPACES - ); + const hasEnvVars = MAINWP_ENV_VARS.some(name => !!process.env[name]); const configSource: 'environment' | 'settings file' | 'mixed' = settings === null ? 'environment' : !hasEnvVars ? 'settings file' : 'mixed'; @@ -440,21 +462,38 @@ export function loadConfig(): Config { const appPassword = getString(process.env.MAINWP_APP_PASSWORD, settings?.appPassword, ''); const apiToken = getString(process.env.MAINWP_TOKEN, settings?.apiToken, ''); const skipSslVerify = getBoolean( + 'MAINWP_SKIP_SSL_VERIFY', process.env.MAINWP_SKIP_SSL_VERIFY, settings?.skipSslVerify, false ); - const allowHttp = getBoolean(process.env.MAINWP_ALLOW_HTTP, settings?.allowHttp, false); - const safeMode = getBoolean(process.env.MAINWP_SAFE_MODE, settings?.safeMode, false); + const allowHttp = getBoolean( + 'MAINWP_ALLOW_HTTP', + process.env.MAINWP_ALLOW_HTTP, + settings?.allowHttp, + false + ); + const safeMode = getBoolean( + 'MAINWP_SAFE_MODE', + process.env.MAINWP_SAFE_MODE, + settings?.safeMode, + false + ); const requireUserConfirmation = getBoolean( + 'MAINWP_REQUIRE_USER_CONFIRMATION', process.env.MAINWP_REQUIRE_USER_CONFIRMATION, settings?.requireUserConfirmation, true ); // Parse rate limit (default: 60 requests/minute) - const rateLimit = getNumber(process.env.MAINWP_RATE_LIMIT, settings?.rateLimit, 60); - if (isNaN(rateLimit) || rateLimit < 0) { + const rateLimit = getNumber( + 'MAINWP_RATE_LIMIT', + process.env.MAINWP_RATE_LIMIT, + settings?.rateLimit, + 60 + ); + if (rateLimit < 0) { throw new Error( 'MAINWP_RATE_LIMIT must be a non-negative integer (set via environment variable or settings.json)' ); @@ -462,11 +501,12 @@ export function loadConfig(): Config { // Parse request timeout (default: 30000ms = 30 seconds) const requestTimeout = getNumber( + 'MAINWP_REQUEST_TIMEOUT', process.env.MAINWP_REQUEST_TIMEOUT, settings?.requestTimeout, 30000 ); - if (isNaN(requestTimeout) || requestTimeout <= 0) { + if (requestTimeout <= 0) { throw new Error( 'MAINWP_REQUEST_TIMEOUT must be a positive integer (set via environment variable or settings.json)' ); @@ -474,11 +514,12 @@ export function loadConfig(): Config { // Parse max response size (default: 10485760 bytes = 10MB) const maxResponseSize = getNumber( + 'MAINWP_MAX_RESPONSE_SIZE', process.env.MAINWP_MAX_RESPONSE_SIZE, settings?.maxResponseSize, 10485760 ); - if (isNaN(maxResponseSize) || maxResponseSize <= 0) { + if (maxResponseSize <= 0) { throw new Error( 'MAINWP_MAX_RESPONSE_SIZE must be a positive integer (set via environment variable or settings.json)' ); @@ -486,11 +527,12 @@ export function loadConfig(): Config { // Parse max session data (default: 52428800 bytes = 50MB) const maxSessionData = getNumber( + 'MAINWP_MAX_SESSION_DATA', process.env.MAINWP_MAX_SESSION_DATA, settings?.maxSessionData, 52428800 ); - if (isNaN(maxSessionData) || maxSessionData <= 0) { + if (maxSessionData <= 0) { throw new Error( 'MAINWP_MAX_SESSION_DATA must be a positive integer (set via environment variable or settings.json)' ); @@ -501,34 +543,36 @@ export function loadConfig(): Config { const blockedTools = getStringArray(process.env.MAINWP_BLOCKED_TOOLS, settings?.blockedTools, []); // Parse schema verbosity (default: 'standard' for backwards compatibility) - const schemaVerbosity = getString( + const schemaVerbosityRaw = getString( process.env.MAINWP_SCHEMA_VERBOSITY, settings?.schemaVerbosity, 'standard' - ) as SchemaVerbosity; + ); - // Validate enum value - if (!SCHEMA_VERBOSITY_VALUES.includes(schemaVerbosity as any)) { + // Validate enum value before narrowing the type + if (!(SCHEMA_VERBOSITY_VALUES as readonly string[]).includes(schemaVerbosityRaw)) { throw new Error( - `MAINWP_SCHEMA_VERBOSITY must be one of: ${SCHEMA_VERBOSITY_VALUES.join(', ')}; got: "${schemaVerbosity}" ` + + `MAINWP_SCHEMA_VERBOSITY must be one of: ${SCHEMA_VERBOSITY_VALUES.join(', ')}; got: "${schemaVerbosityRaw}" ` + `(set via environment variable or settings.json)` ); } + const schemaVerbosity = schemaVerbosityRaw as SchemaVerbosity; // Parse response format (default: 'compact' to reduce token usage) - const responseFormat = getString( + const responseFormatRaw = getString( process.env.MAINWP_RESPONSE_FORMAT, settings?.responseFormat, 'compact' - ) as ResponseFormat; + ); - // Validate enum value - if (!RESPONSE_FORMAT_VALUES.includes(responseFormat as any)) { + // Validate enum value before narrowing the type + if (!(RESPONSE_FORMAT_VALUES as readonly string[]).includes(responseFormatRaw)) { throw new Error( - `MAINWP_RESPONSE_FORMAT must be one of: ${RESPONSE_FORMAT_VALUES.join(', ')}; got: "${responseFormat}" ` + + `MAINWP_RESPONSE_FORMAT must be one of: ${RESPONSE_FORMAT_VALUES.join(', ')}; got: "${responseFormatRaw}" ` + `(set via environment variable or settings.json)` ); } + const responseFormat = responseFormatRaw as ResponseFormat; // Parse ability namespace allowlist (default: ['mainwp']). Deduplicate so // the env-var path (no schema-level uniqueItems) and the settings.json path @@ -558,32 +602,44 @@ export function loadConfig(): Config { const abilityNamespacesTuple = abilityNamespaces as [string, ...string[]]; // Parse retry configuration - const retryEnabled = getBoolean(process.env.MAINWP_RETRY_ENABLED, settings?.retryEnabled, true); + const retryEnabled = getBoolean( + 'MAINWP_RETRY_ENABLED', + process.env.MAINWP_RETRY_ENABLED, + settings?.retryEnabled, + true + ); - const maxRetries = getNumber(process.env.MAINWP_MAX_RETRIES, settings?.maxRetries, 2); - if (isNaN(maxRetries) || maxRetries < 1 || maxRetries > 5) { + const maxRetries = getNumber( + 'MAINWP_MAX_RETRIES', + process.env.MAINWP_MAX_RETRIES, + settings?.maxRetries, + 2 + ); + if (maxRetries < 1 || maxRetries > 5) { throw new Error( 'MAINWP_MAX_RETRIES must be between 1 and 5 (set via environment variable or settings.json)' ); } const retryBaseDelay = getNumber( + 'MAINWP_RETRY_BASE_DELAY', process.env.MAINWP_RETRY_BASE_DELAY, settings?.retryBaseDelay, 1000 ); - if (isNaN(retryBaseDelay) || retryBaseDelay < 500 || retryBaseDelay > 10000) { + if (retryBaseDelay < 500 || retryBaseDelay > 10000) { throw new Error( 'MAINWP_RETRY_BASE_DELAY must be between 500ms and 10000ms (set via environment variable or settings.json)' ); } const retryMaxDelay = getNumber( + 'MAINWP_RETRY_MAX_DELAY', process.env.MAINWP_RETRY_MAX_DELAY, settings?.retryMaxDelay, 2000 ); - if (isNaN(retryMaxDelay) || retryMaxDelay < retryBaseDelay || retryMaxDelay > 30000) { + if (retryMaxDelay < retryBaseDelay || retryMaxDelay > 30000) { throw new Error( `MAINWP_RETRY_MAX_DELAY must be between ${retryBaseDelay}ms and 30000ms (set via environment variable or settings.json)` ); @@ -613,6 +669,13 @@ export function loadConfig(): Config { ); } + if (hasBearerAuth && !hasBasicAuth) { + console.error( + 'WARNING: MAINWP_TOKEN bearer authentication is expected to fail against the WordPress Abilities API. ' + + 'Configure both MAINWP_USER and MAINWP_APP_PASSWORD with a WordPress Application Password.' + ); + } + const normalizedUrl = dashboardUrl.replace(/\/+$/, ''); // Validate URL format @@ -683,17 +746,8 @@ export function getAbilitiesApiUrl(config: Config): string { /** * Get authorization headers for API requests. - * Computed once per config object and cached — credentials don't change during a server's lifetime. */ -let cachedHeaders: Record | undefined; -let cachedAuthConfig: Config | undefined; - export function getAuthHeaders(config: Config): Record { - if (config === cachedAuthConfig && cachedHeaders) { - return cachedHeaders; - } - - cachedAuthConfig = config; const headers: Record = { 'Content-Type': 'application/json', }; @@ -705,6 +759,5 @@ export function getAuthHeaders(config: Config): Record { headers['Authorization'] = `Bearer ${config.apiToken}`; } - cachedHeaders = headers; return headers; } diff --git a/src/confirmation-responses.ts b/src/confirmation-responses.ts index 81a01a2..0573754 100644 --- a/src/confirmation-responses.ts +++ b/src/confirmation-responses.ts @@ -48,6 +48,26 @@ export function buildInvalidParameterResponse(ctx: ConfirmationContext): object }; } +/** + * Response when dry_run is passed to a confirm-capable tool whose ability + * does not declare dry_run. Forwarding the fabricated parameter upstream + * could execute the operation for real if the handler ignores unknown input, + * so the call is rejected before any request is made. + */ +export function buildDryRunNotSupportedResponse(ctx: ConfirmationContext): object { + return { + error: 'INVALID_PARAMETER', + message: 'dry_run parameter not supported for this tool', + details: { + tool: ctx.tool, + ability: ctx.ability, + reason: + 'This ability does not declare a dry_run parameter, so a preview cannot be guaranteed upstream', + resolution: 'Remove dry_run and call with confirm: true to start the confirmation flow', + }, + }; +} + /** * Response when user_confirmed and dry_run are both set (conflicting intent) */ @@ -65,6 +85,32 @@ export function buildConflictingParametersResponse(ctx: ConfirmationContext): ob }; } +/** + * Response when a confirm-capable tool cannot generate a dry-run preview. + * Still a CONFIRMATION_REQUIRED workflow step — a token is issued so the + * confirmed follow-up call can proceed — but carries no preview payload. + */ +export function buildNoPreviewAvailableResponse(ctx: ConfirmationContext, token: string): object { + return { + status: 'CONFIRMATION_REQUIRED', + next_action: 'confirm_without_preview', + message: + 'This ability does not support dry_run, so no preview is available. ' + + 'Explicit user approval is required to proceed.', + preview: null, + confirmation_token: token, + instructions: + 'Describe to the user exactly what this operation will do. If they explicitly approve, ' + + 'call this tool again with user_confirmed: true and confirmation_token: "". ' + + 'Do NOT set user_confirmed: true without explicit user consent.', + metadata: { + tool: ctx.tool, + ability: ctx.ability, + expiresIn: '5 minutes', + }, + }; +} + /** * Response when a preview is generated and confirmation is required */ @@ -91,9 +137,14 @@ export function buildConfirmationRequiredResponse( } /** - * Response when user_confirmed is set but no preview was requested first + * Response when confirmed execution is impossible because no valid preview + * exists. Default reason covers the user_confirmed-without-preview case; + * pass a specific reason for other paths (e.g. no confirmation parameters). */ -export function buildPreviewRequiredResponse(ctx: ConfirmationContext): object { +export function buildPreviewRequiredResponse( + ctx: ConfirmationContext, + reason = 'user_confirmed: true requires a prior preview request' +): object { return { error: 'PREVIEW_REQUIRED', next_action: 'request_preview_first', @@ -101,7 +152,7 @@ export function buildPreviewRequiredResponse(ctx: ConfirmationContext): object { details: { tool: ctx.tool, ability: ctx.ability, - reason: 'user_confirmed: true requires a prior preview request', + reason, resolution: 'Call the tool with confirm: true (without user_confirmed) to generate a preview first.', }, diff --git a/src/confirmation.ts b/src/confirmation.ts index 064d648..ea827f3 100644 --- a/src/confirmation.ts +++ b/src/confirmation.ts @@ -8,13 +8,15 @@ import crypto from 'crypto'; import type { TextContent } from '@modelcontextprotocol/sdk/types.js'; -import { executeAbility, type Ability } from './abilities.js'; +import { configIdentityHash, executeAbility, type Ability } from './abilities.js'; import { Config, formatJson } from './config.js'; import type { Logger } from './logging.js'; import { trackSessionData } from './session.js'; import { buildInvalidParameterResponse, + buildDryRunNotSupportedResponse, buildConflictingParametersResponse, + buildNoPreviewAvailableResponse, buildConfirmationRequiredResponse, buildPreviewRequiredResponse, buildPreviewExpiredResponse, @@ -77,12 +79,40 @@ export interface ConfirmationFlowParams { signal?: AbortSignal; } +/** + * Recursively sort object keys so serialization is deterministic at every + * depth. JSON.stringify's array-replacer form cannot be used here: it filters + * property names at all nesting levels, which drops nested values from the + * key and lets differing nested arguments collide. + */ +function canonicalize(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(canonicalize); + } + if (value !== null && typeof value === 'object') { + const source = value as Record; + // Null prototype so hostile keys like __proto__ become own enumerable + // properties instead of hitting Object.prototype accessors and vanishing + // from the serialized key. + const sorted: Record = Object.create(null); + for (const key of Object.keys(source).sort()) { + sorted[key] = canonicalize(source[key]); + } + return sorted; + } + return value; +} + /** * Generate a unique preview key for a tool call. * Excludes confirmation-related parameters (confirm, user_confirmed, dry_run) * from the key to ensure preview and execution calls match. + * Prefixed with the config identity hash: the preview maps are module-level, + * so without the scope a token issued against one dashboard/principal could + * confirm the same tool and arguments against another createServer(config) + * instance in the same process. */ -function getPreviewKey(toolName: string, args: Record): string { +function getPreviewKey(scope: string, toolName: string, args: Record): string { const { confirm: _confirm, user_confirmed: _user_confirmed, @@ -90,8 +120,7 @@ function getPreviewKey(toolName: string, args: Record): string confirmation_token: _confirmation_token, ...relevantArgs } = args; - const sortedKeys = Object.keys(relevantArgs).sort(); - return `${toolName}:${JSON.stringify(relevantArgs, sortedKeys)}`; + return `${scope}:${toolName}:${JSON.stringify(canonicalize(relevantArgs))}`; } /** @@ -142,6 +171,8 @@ export async function handleConfirmationFlow( const schemaProps = ability.input_schema?.properties; const hasConfirmParam = schemaProps !== null && typeof schemaProps === 'object' && 'confirm' in schemaProps; + const hasDryRunParam = + schemaProps !== null && typeof schemaProps === 'object' && 'dry_run' in schemaProps; // Validation: user_confirmed on tools without confirm parameter if (!hasConfirmParam && args.user_confirmed === true) { @@ -177,29 +208,64 @@ export async function handleConfirmationFlow( }; } - // Case 1: Explicit dry_run bypass - skip confirmation flow entirely + // Case 1: Explicit dry_run bypass - skip confirmation flow entirely. + // Only honored when the ability declares dry_run: forwarding a fabricated + // dry_run (possibly alongside confirm: true) would execute for real if + // upstream ignores unknown input, so undeclared dry_run is rejected + // before any upstream call. if (args.dry_run === true) { + if (!hasDryRunParam) { + logger.warning('Invalid parameter: dry_run on tool without dry_run support', { + toolName, + abilityName, + }); + return { + action: 'respond', + response: [ + { type: 'text', text: formatJson(config, buildDryRunNotSupportedResponse(ctx)) }, + ], + isError: true, + }; + } logger.debug('Explicit dry_run bypasses confirmation flow', { toolName }); - return { action: 'skip' }; + // Strip confirm so upstream never sees the ambiguous confirm+dry_run + // combination — mirrors the Case 2 preview call, which sends dry_run + // with confirm removed. + const { confirm: _dryRunConfirm, ...dryRunArgs } = effectiveArgs; + return { action: 'execute', effectiveArgs: dryRunArgs }; } // Case 2: Preview request (confirm: true without user_confirmed) if (args.confirm === true && args.user_confirmed !== true) { cleanupExpiredPreviews(); - // Execute preview with dry_run: true - const previewArgs = { ...effectiveArgs, dry_run: true, confirm: undefined }; - const previewResult = await executeAbility( - config, - abilityName, - previewArgs, - logger, - ability, - signal - ); + // Abilities without a declared dry_run parameter get no upstream preview + // call — fabricating dry_run against a schema that doesn't declare it + // could execute for real if upstream ignores unknown input. The two-phase + // gate still applies: a token is issued below so the confirmed follow-up + // call can proceed. + let previewResult: unknown = null; + if (hasDryRunParam) { + // Execute preview with dry_run: true and the confirm flag removed + const previewArgs: Record = { ...effectiveArgs, dry_run: true }; + delete previewArgs.confirm; + previewResult = await executeAbility( + config, + abilityName, + previewArgs, + logger, + ability, + signal + ); + } else { + logger.warning('Preview unavailable - ability does not support dry_run', { + toolName, + abilityName, + }); + } // Store preview for later validation - const previewKey = getPreviewKey(toolName, args); + const previewKey = getPreviewKey(configIdentityHash(config), toolName, args); pendingPreviews.set(previewKey, Date.now()); // Clean up any existing token for this preview key before generating a new one @@ -214,9 +280,16 @@ export async function handleConfirmationFlow( const token = crypto.randomUUID(); tokenIndex.set(token, previewKey); - logger.info('Preview generated for confirmation', { toolName }); + logger.info( + hasDryRunParam + ? 'Preview generated for confirmation' + : 'Confirmation required without preview', + { toolName } + ); - const confirmationResponse = buildConfirmationRequiredResponse(ctx, previewResult, token); + const confirmationResponse = hasDryRunParam + ? buildConfirmationRequiredResponse(ctx, previewResult, token) + : buildNoPreviewAvailableResponse(ctx, token); const previewResponse = formatJson(config, confirmationResponse); trackSessionData(previewResponse, config, logger, 'during preview'); @@ -237,46 +310,66 @@ export async function handleConfirmationFlow( ); } - // Resolve preview key: prefer token-based lookup, fall back to key-based - let previewKey: string; + // Confirmed execution is token-bound: the token proves the caller saw the + // preview response. A tool+args fallback would let a caller confirm a + // preview it never read, so no token means no execution. const confirmationToken = typeof args.confirmation_token === 'string' ? args.confirmation_token : undefined; - if (confirmationToken) { - const tokenPreviewKey = tokenIndex.get(confirmationToken); - if (!tokenPreviewKey) { - logger.warning('Confirmation failed - invalid confirmation token', { toolName }); - return { - action: 'respond', - response: [{ type: 'text', text: formatJson(config, buildPreviewRequiredResponse(ctx)) }], - isError: true, - }; - } - // Verify token belongs to this tool (prevent cross-tool reuse) - if (!tokenPreviewKey.startsWith(`${toolName}:`)) { - tokenIndex.delete(confirmationToken); - logger.warning('Confirmation failed - token belongs to different tool', { toolName }); - return { - action: 'respond', - response: [{ type: 'text', text: formatJson(config, buildPreviewRequiredResponse(ctx)) }], - isError: true, - }; - } - // Verify token matches current arguments (prevent arg-swap) - const currentPreviewKey = getPreviewKey(toolName, args); - if (currentPreviewKey !== tokenPreviewKey) { - tokenIndex.delete(confirmationToken); - logger.warning('Confirmation failed - arguments do not match preview', { toolName }); - return { - action: 'respond', - response: [{ type: 'text', text: formatJson(config, buildPreviewRequiredResponse(ctx)) }], - isError: true, - }; - } - previewKey = tokenPreviewKey; - } else { - previewKey = getPreviewKey(toolName, args); + if (!confirmationToken) { + logger.warning('Confirmation failed - confirmation_token missing', { toolName }); + return { + action: 'respond', + response: [ + { + type: 'text', + text: formatJson( + config, + buildPreviewRequiredResponse( + ctx, + 'user_confirmed: true requires the confirmation_token issued by the preview response' + ) + ), + }, + ], + isError: true, + }; + } + + const tokenPreviewKey = tokenIndex.get(confirmationToken); + if (!tokenPreviewKey) { + logger.warning('Confirmation failed - invalid confirmation token', { toolName }); + return { + action: 'respond', + response: [{ type: 'text', text: formatJson(config, buildPreviewRequiredResponse(ctx)) }], + isError: true, + }; + } + // Verify token belongs to this tool AND this config identity (prevent + // cross-tool reuse and cross-dashboard/principal reuse alike) + if (!tokenPreviewKey.startsWith(`${configIdentityHash(config)}:${toolName}:`)) { + tokenIndex.delete(confirmationToken); + logger.warning('Confirmation failed - token belongs to a different tool or identity', { + toolName, + }); + return { + action: 'respond', + response: [{ type: 'text', text: formatJson(config, buildPreviewRequiredResponse(ctx)) }], + isError: true, + }; + } + // Verify token matches current arguments (prevent arg-swap) + const currentPreviewKey = getPreviewKey(configIdentityHash(config), toolName, args); + if (currentPreviewKey !== tokenPreviewKey) { + tokenIndex.delete(confirmationToken); + logger.warning('Confirmation failed - arguments do not match preview', { toolName }); + return { + action: 'respond', + response: [{ type: 'text', text: formatJson(config, buildPreviewRequiredResponse(ctx)) }], + isError: true, + }; } + const previewKey = tokenPreviewKey; // Check preview expiry BEFORE running cleanup for more helpful error messages const previewTimestamp = pendingPreviews.get(previewKey); @@ -292,7 +385,7 @@ export async function handleConfirmationFlow( if (Date.now() - previewTimestamp > PREVIEW_EXPIRY_MS) { pendingPreviews.delete(previewKey); - if (confirmationToken) tokenIndex.delete(confirmationToken); + tokenIndex.delete(confirmationToken); logger.warning('Confirmation failed - preview expired', { toolName }); return { action: 'respond', @@ -303,7 +396,7 @@ export async function handleConfirmationFlow( // Preview is valid - proceed with execution pendingPreviews.delete(previewKey); - if (confirmationToken) tokenIndex.delete(confirmationToken); + tokenIndex.delete(confirmationToken); const previewAge = Date.now() - previewTimestamp; logger.info('User confirmation validated', { toolName, previewAge }); @@ -321,10 +414,24 @@ export async function handleConfirmationFlow( } // Default case: no confirm or user_confirmed provided - // Log warning but proceed to maintain backward compatibility logger.warning('Destructive tool called without confirmation parameters', { toolName, abilityName, }); - return { action: 'skip' }; + return { + action: 'respond', + response: [ + { + type: 'text', + text: formatJson( + config, + buildPreviewRequiredResponse( + ctx, + 'Destructive tools require confirmation parameters; neither confirm nor user_confirmed was provided' + ) + ), + }, + ], + isError: true, + }; } diff --git a/src/errors.test.ts b/src/errors.test.ts index 4df5b18..23518b6 100644 --- a/src/errors.test.ts +++ b/src/errors.test.ts @@ -8,6 +8,7 @@ import { McpErrorFactory, MCP_ERROR_CODES, toMcpErrorResponse, + createHttpError, formatErrorResponse, } from './errors.js'; @@ -150,6 +151,45 @@ describe('McpErrorFactory', () => { }); describe('toMcpErrorResponse', () => { + it.each([ + [401, MCP_ERROR_CODES.UNAUTHORIZED], + [403, MCP_ERROR_CODES.PERMISSION_DENIED], + [404, MCP_ERROR_CODES.RESOURCE_NOT_FOUND], + [429, MCP_ERROR_CODES.RATE_LIMITED], + [500, MCP_ERROR_CODES.SERVER_ERROR], + [503, MCP_ERROR_CODES.SERVER_ERROR], + ])('maps structured HTTP %i to %s', (status, expectedCode) => { + const response = toMcpErrorResponse( + createHttpError(status, String(status), 'Message without classification keywords') + ); + + expect(response.error.code).toBe(expectedCode); + }); + + it('maps a structured not-found error code before HTTP 403', () => { + const response = toMcpErrorResponse( + createHttpError(403, 'mainwp_site_not_found', 'Message without classification keywords') + ); + + expect(response.error.code).toBe(MCP_ERROR_CODES.RESOURCE_NOT_FOUND); + }); + + it('maps the structured rest_no_route code before HTTP 403', () => { + const response = toMcpErrorResponse( + createHttpError(403, 'rest_no_route', 'Message without classification keywords') + ); + + expect(response.error.code).toBe(MCP_ERROR_CODES.RESOURCE_NOT_FOUND); + }); + + it('keeps unrelated structured error codes classified by HTTP 403', () => { + const response = toMcpErrorResponse( + createHttpError(403, 'rest_forbidden', 'Message without classification keywords') + ); + + expect(response.error.code).toBe(MCP_ERROR_CODES.PERMISSION_DENIED); + }); + it('should preserve McpError code', () => { const error = new McpError(MCP_ERROR_CODES.TIMEOUT, 'Timed out'); const response = toMcpErrorResponse(error); @@ -204,6 +244,12 @@ describe('toMcpErrorResponse', () => { expect(response.error.code).toBe(MCP_ERROR_CODES.UNAUTHORIZED); }); + it('classifies plain error messages case-insensitively', () => { + expect(toMcpErrorResponse(new Error('Unauthorized Access')).error.code).toBe( + MCP_ERROR_CODES.UNAUTHORIZED + ); + }); + it('should infer code from error message keywords - permission', () => { // Note: case-sensitive matching uses lowercase "permission" const response = toMcpErrorResponse(new Error('permission denied')); diff --git a/src/errors.ts b/src/errors.ts index 73ee86d..8113637 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -184,6 +184,33 @@ export const McpErrorFactory = { }, }; +/** + * Create an Error carrying a structured HTTP status and error code. + * Callers own the full message text; this helper only guarantees the + * `.status`/`.code` fields that retry eligibility (isRetryableError) and + * startup error classification (validateCredentials) rely on. + */ +export function createHttpError(status: number, errorCode: string, message: string): Error { + const error = new Error(message); + const httpError = error as Error & { status: number; code: string }; + httpError.status = status; + httpError.code = errorCode; + return error; +} + +/** + * Extract a structured HTTP status from an error, if present. + */ +export function getHttpStatus(error: unknown): number | undefined { + if (error && typeof error === 'object' && 'status' in error) { + const status = (error as { status: unknown }).status; + if (typeof status === 'number') { + return status; + } + } + return undefined; +} + /** * Convert any error to an MCP error response */ @@ -207,27 +234,58 @@ export function toMcpErrorResponse( const message = getErrorMessage(error); const sanitizedMessage = sanitize ? sanitize(message) : message; + const status = getHttpStatus(error); + const structuredCode = + error && typeof error === 'object' && 'code' in error + ? (error as { code: unknown }).code + : undefined; + const isStructuredNotFound = + typeof structuredCode === 'string' && + (/(?:^|_)not_found$/.test(structuredCode) || structuredCode === 'rest_no_route'); + const normalizedMessage = message.toLowerCase(); - // Try to infer error code from message + // Last-resort classifier for plain Errors that reached the MCP boundary + // without being wrapped in a typed McpError. Substring matching is fragile + // and BRANCH ORDER IS SIGNIFICANT (first match wins — e.g. a message with + // both "invalid" and "unauthorized" classifies as INVALID_PARAMS). Prefer + // throwing McpErrorFactory errors at the source over extending this chain. let code: McpErrorCode = MCP_ERROR_CODES.INTERNAL_ERROR; - if (message.includes('cancelled') || message.includes('aborted')) { + if (isStructuredNotFound) { + code = MCP_ERROR_CODES.RESOURCE_NOT_FOUND; + } else if (status === 401) { + code = MCP_ERROR_CODES.UNAUTHORIZED; + } else if (status === 403) { + code = MCP_ERROR_CODES.PERMISSION_DENIED; + } else if (status === 404) { + code = MCP_ERROR_CODES.RESOURCE_NOT_FOUND; + } else if (status === 429) { + code = MCP_ERROR_CODES.RATE_LIMITED; + } else if (status !== undefined && status >= 500) { + code = MCP_ERROR_CODES.SERVER_ERROR; + } else if (normalizedMessage.includes('cancelled') || normalizedMessage.includes('aborted')) { code = MCP_ERROR_CODES.CANCELLED; - } else if (message.includes('not found') || message.includes('Unknown')) { + } else if (normalizedMessage.includes('not found') || normalizedMessage.includes('unknown')) { code = MCP_ERROR_CODES.RESOURCE_NOT_FOUND; - } else if (message.includes('limit exceeded') || message.includes('exhausted')) { + } else if ( + normalizedMessage.includes('limit exceeded') || + normalizedMessage.includes('exhausted') + ) { code = MCP_ERROR_CODES.RESOURCE_EXHAUSTED; } else if ( - message.includes('invalid') || - message.includes('must be') || - message.includes('exceeds') + normalizedMessage.includes('invalid') || + normalizedMessage.includes('must be') || + normalizedMessage.includes('exceeds') ) { code = MCP_ERROR_CODES.INVALID_PARAMS; - } else if (message.includes('unauthorized') || message.includes('authentication')) { + } else if ( + normalizedMessage.includes('unauthorized') || + normalizedMessage.includes('authentication') + ) { code = MCP_ERROR_CODES.UNAUTHORIZED; - } else if (message.includes('permission') || message.includes('forbidden')) { + } else if (normalizedMessage.includes('permission') || normalizedMessage.includes('forbidden')) { code = MCP_ERROR_CODES.PERMISSION_DENIED; - } else if (message.includes('timeout')) { + } else if (normalizedMessage.includes('timeout')) { code = MCP_ERROR_CODES.TIMEOUT; } diff --git a/src/http-client.ts b/src/http-client.ts index fa6abd8..efbf198 100644 --- a/src/http-client.ts +++ b/src/http-client.ts @@ -7,6 +7,7 @@ import { Agent as UndiciAgent } from 'undici'; import { Config, getAuthHeaders } from './config.js'; +import { createHttpError } from './errors.js'; import { sanitizeError } from './security.js'; import type { Logger } from './logging.js'; @@ -19,6 +20,32 @@ export const MAX_URL_LENGTH = 8000; /** Maximum number of pages to fetch during pagination — prevents unbounded requests */ const MAX_PAGES = 50; +interface ResponseDeadline { + timeoutId?: ReturnType; + timedOut: boolean; + effectiveTimeout: number; + url: string; + externalSignal?: AbortSignal; + externalAbortHandler?: () => void; +} + +const responseDeadlines = new WeakMap(); + +function clearResponseDeadline(deadline: ResponseDeadline): void { + if (deadline.timeoutId) { + clearTimeout(deadline.timeoutId); + } + if (deadline.externalSignal && deadline.externalAbortHandler) { + deadline.externalSignal.removeEventListener('abort', deadline.externalAbortHandler); + } +} + +function createTimeoutError(deadline: Pick): Error { + const error = new Error(`Request timeout after ${deadline.effectiveTimeout}ms: ${deadline.url}`); + (error as Error & { code: string }).code = 'ETIMEDOUT'; + return error; +} + /** * Per-request undici dispatcher that skips TLS certificate verification. * Created lazily on first use — avoids process-global NODE_TLS_REJECT_UNAUTHORIZED. @@ -46,15 +73,25 @@ export function createFetch(config: Config, perCallTimeout?: number) { return async (url: string, options: RequestInit = {}): Promise => { const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), effectiveTimeout); + const deadline: ResponseDeadline = { + timedOut: false, + effectiveTimeout, + url, + }; + deadline.timeoutId = setTimeout(() => { + deadline.timedOut = true; + controller.abort(); + }, effectiveTimeout); // Forward external signal abort to our controller (preserves caller cancellation) const externalSignal = options.signal; if (externalSignal) { + deadline.externalSignal = externalSignal; if (externalSignal.aborted) { controller.abort(); } else { - externalSignal.addEventListener('abort', () => controller.abort(), { once: true }); + deadline.externalAbortHandler = () => controller.abort(); + externalSignal.addEventListener('abort', deadline.externalAbortHandler, { once: true }); } } @@ -75,7 +112,6 @@ export function createFetch(config: Config, perCallTimeout?: number) { } const response = await fetch(url, fetchOptions as RequestInit); - clearTimeout(timeoutId); // Check response size before parsing (if content-length is provided) const contentLength = response.headers.get('content-length'); @@ -88,14 +124,12 @@ export function createFetch(config: Config, perCallTimeout?: number) { } } + responseDeadlines.set(response, deadline); return response; } catch (error) { - clearTimeout(timeoutId); - if ((error as Error).name === 'AbortError') { - // Create timeout error with ETIMEDOUT code for retry detection - const timeoutError = new Error(`Request timeout after ${effectiveTimeout}ms: ${url}`); - (timeoutError as Error & { code: string }).code = 'ETIMEDOUT'; - throw timeoutError; + clearResponseDeadline(deadline); + if ((error as Error).name === 'AbortError' && deadline.timedOut) { + throw createTimeoutError(deadline); } throw error; } @@ -116,37 +150,51 @@ export function createFetch(config: Config, perCallTimeout?: number) { * @returns The response body as a string */ export async function readLimitedBody(response: Response, maxBytes: number): Promise { - const reader = response.body?.getReader(); - if (!reader) { - // Fallback for test mocks without ReadableStream body - let text: string; - if (typeof response.text === 'function') { - text = await response.text(); - } else { - // Last resort: json() + stringify (handles minimal mock objects) - text = JSON.stringify(await response.json()); - } - if (Buffer.byteLength(text, 'utf8') > maxBytes) { - throw new Error(`Response body exceeds ${maxBytes} bytes limit`); + const deadline = responseDeadlines.get(response); + try { + const reader = response.body?.getReader(); + if (!reader) { + // Fallback for test mocks without ReadableStream body + let text: string; + if (typeof response.text === 'function') { + text = await response.text(); + } else { + // Last resort: json() + stringify (handles minimal mock objects) + text = JSON.stringify(await response.json()); + } + if (Buffer.byteLength(text, 'utf8') > maxBytes) { + throw new Error(`Response body exceeds ${maxBytes} bytes limit`); + } + return text; } - return text; - } - const chunks: Uint8Array[] = []; - let totalBytes = 0; + const chunks: Uint8Array[] = []; + let totalBytes = 0; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + totalBytes += value.byteLength; + if (totalBytes > maxBytes) { + // Fire-and-forget: a rejected cancel() must not race the throw below + void reader.cancel().catch(() => {}); + throw new Error(`Response body exceeds ${maxBytes} bytes limit`); + } + chunks.push(value); + } - while (true) { - const { done, value } = await reader.read(); - if (done) break; - totalBytes += value.byteLength; - if (totalBytes > maxBytes) { - reader.cancel(); - throw new Error(`Response body exceeds ${maxBytes} bytes limit`); + return new TextDecoder().decode(Buffer.concat(chunks)); + } catch (error) { + if (deadline?.timedOut) { + throw createTimeoutError(deadline); + } + throw error; + } finally { + if (deadline) { + clearResponseDeadline(deadline); + responseDeadlines.delete(response); } - chunks.push(value); } - - return new TextDecoder().decode(Buffer.concat(chunks)); } /** @@ -161,6 +209,7 @@ export async function paginateApi( logger?: Logger ): Promise { let page = 1; + let capped = false; const allItems: T[] = []; while (true) { @@ -168,7 +217,9 @@ export async function paginateApi( if (!response.ok) { const errorText = await readLimitedBody(response, MAX_ERROR_BODY_BYTES); - throw new Error( + throw createHttpError( + response.status, + String(response.status), `Failed to fetch ${label}: ${response.status} ${response.statusText} - ${sanitizeError(errorText)}` ); } @@ -184,11 +235,15 @@ export async function paginateApi( allItems.push(...batch); const totalPages = parseInt(response.headers.get('X-WP-TotalPages') || '1', 10); - if (page >= totalPages || page >= MAX_PAGES) break; + if (page >= totalPages) break; + if (page >= MAX_PAGES) { + capped = true; + break; + } page++; } - if (page >= MAX_PAGES) { + if (capped) { logger?.warning( `Pagination capped at ${MAX_PAGES} pages (fetched ${allItems.length} ${label}) — some may be missing` ); diff --git a/src/index.test.ts b/src/index.test.ts new file mode 100644 index 0000000..b2de56a --- /dev/null +++ b/src/index.test.ts @@ -0,0 +1,89 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { createServer } from './index.js'; +import { clearCache, initRateLimiter } from './abilities.js'; +import { makeBaseConfig } from '../tests/helpers/config.js'; + +const mockFetch = vi.fn(); +vi.stubGlobal('fetch', mockFetch); + +async function connectedClient(config = makeBaseConfig()) { + const { server } = await createServer(config); + const client = new Client({ name: 'test-client', version: '1.0.0' }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + return { client, server }; +} + +describe('MCP request handlers', () => { + beforeEach(() => { + vi.resetAllMocks(); + clearCache(); + initRateLimiter(0); + vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it.each([ + ['site_id', 'all'], + ['site_ids', '1,2'], + ])('accepts prompt argument %s=%s through prompts/get', async (key, value) => { + const { client, server } = await connectedClient(); + + const result = await client.getPrompt({ + name: 'performance-check', + arguments: { [key]: value }, + }); + + expect(result.messages).not.toHaveLength(0); + await client.close(); + await server.close(); + }); + + it('returns INVALID_PARAMS for malformed prompt arguments through prompts/get', async () => { + const { client, server } = await connectedClient(); + + await expect( + client.getPrompt({ name: 'performance-check', arguments: { site_id: 'not-an-id' } }) + ).rejects.toMatchObject({ code: -32602 }); + await client.close(); + await server.close(); + }); + + it('blocks the site resource before get-site reaches /run', async () => { + const { client, server } = await connectedClient({ + ...makeBaseConfig(), + blockedTools: ['get_site_v1'], + }); + + const result = await client.readResource({ uri: 'mainwp://site/1' }); + + expect(result.contents[0]).toMatchObject({ text: expect.stringContaining('not allowed') }); + expect(mockFetch).not.toHaveBeenCalled(); + await client.close(); + await server.close(); + }); + + it('blocks site completions before list-sites reaches /run', async () => { + const { client, server } = await connectedClient({ + ...makeBaseConfig(), + blockedTools: ['list_sites_v1'], + }); + + await expect( + client.complete({ + ref: { type: 'ref/prompt', name: 'performance-check' }, + argument: { name: 'site_id', value: '' }, + }) + ).rejects.toMatchObject({ code: -32008 }); + + expect(mockFetch).not.toHaveBeenCalled(); + await client.close(); + await server.close(); + }); +}); diff --git a/src/index.ts b/src/index.ts index edeab14..c230cb4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,12 +10,15 @@ * * Environment Variables: * - MAINWP_URL: Base URL of MainWP Dashboard (required) - * - MAINWP_TOKEN: Bearer token for authentication (required) + * - MAINWP_USER + MAINWP_APP_PASSWORD: WordPress Application Password authentication + * - MAINWP_TOKEN: Compatibility-only bearer token (expected to fail against Abilities API) * - MAINWP_SKIP_SSL_VERIFY: Set to "true" to skip SSL verification (optional) */ import { Server } from '@modelcontextprotocol/sdk/server/index.js'; import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; +import { realpathSync } from 'node:fs'; +import { pathToFileURL } from 'node:url'; import { CallToolRequestSchema, ListToolsRequestSchema, @@ -27,8 +30,8 @@ import { ListResourceTemplatesRequestSchema, } from '@modelcontextprotocol/sdk/types.js'; import { loadConfig, Config, formatJson } from './config.js'; -import { getTools, executeTool } from './tools.js'; -import { getSessionDataUsage } from './session.js'; +import { getTools, executeTool, isToolAllowed } from './tools.js'; +import { getSessionDataUsage, formatBytes } from './session.js'; import { fetchAbilities, fetchCategories, @@ -42,8 +45,15 @@ import { import { generateHelpDocument, generateToolHelp } from './help.js'; import { getPromptList, getPrompt, getPromptArgumentCompletions } from './prompts.js'; import { createLogger, createStderrLogger, type Logger } from './logging.js'; -import { sanitizeError, isValidId, validateInput } from './security.js'; -import { formatErrorResponse, getErrorMessage, McpErrorFactory, McpError } from './errors.js'; +import { sanitizeError, isValidId } from './security.js'; +import { abilityNameToToolName } from './naming.js'; +import { + formatErrorResponse, + getErrorMessage, + getHttpStatus, + McpErrorFactory, + McpError, +} from './errors.js'; // Server metadata const SERVER_NAME = 'mainwp-mcp'; @@ -108,7 +118,7 @@ function validateResourceUri(uri: string): { /** * Create and configure the MCP server */ -async function createServer(config: Config): Promise<{ server: Server; logger: Logger }> { +export async function createServer(config: Config): Promise<{ server: Server; logger: Logger }> { const server = new Server( { name: SERVER_NAME, @@ -256,6 +266,15 @@ async function createServer(config: Config): Promise<{ server: Server; logger: L const parsed = validateResourceUri(uri); if (parsed.type === 'site' && parsed.params?.site_id) { + // Derive the exposed tool name so allow/block lists keyed on + // namespaced names (non-mainwp primary namespace) still apply + const getSiteToolName = abilityNameToToolName( + 'mainwp/get-site-v1', + config.abilityNamespaces[0] + ); + if (!isToolAllowed(config, getSiteToolName)) { + throw McpErrorFactory.permissionDenied(`Tool is not allowed: ${getSiteToolName}`); + } const result = await executeAbility( config, 'mainwp/get-site-v1', @@ -295,13 +314,21 @@ async function createServer(config: Config): Promise<{ server: Server; logger: L server.setRequestHandler(GetPromptRequestSchema, async request => { const { name, arguments: args } = request.params; try { - // Validate prompt arguments before processing + // Prompt arguments are flat strings. Guard their transport-safe shape + // here; prompt-specific semantics belong to validatePromptArgs. if (args) { - validateInput(args as Record); + for (const value of Object.values(args)) { + // eslint-disable-next-line no-control-regex + if (value.length > 200 || /[\x00-\x1f]/.test(value)) { + throw McpErrorFactory.invalidParams( + 'Prompt arguments must be at most 200 characters and contain no control characters' + ); + } + } } return getPrompt(name, args); } catch (error) { - // Preserve structured MCP errors (e.g., from validateInput) + // Preserve structured MCP errors from prompt validation if (error instanceof McpError) { throw error; } @@ -325,6 +352,13 @@ async function createServer(config: Config): Promise<{ server: Server; logger: L // For site_id arguments, try to fetch site list dynamically if ((argName === 'site_id' || argName === 'site_ids') && values.length === 0) { + const listSitesToolName = abilityNameToToolName( + 'mainwp/list-sites-v1', + config.abilityNamespaces[0] + ); + if (!isToolAllowed(config, listSitesToolName)) { + throw McpErrorFactory.permissionDenied(`Tool is not allowed: ${listSitesToolName}`); + } try { const abilities = await fetchAbilities(config, false, logger); const listSitesAbility = abilities.find(a => a.name === 'mainwp/list-sites-v1'); @@ -337,8 +371,12 @@ async function createServer(config: Config): Promise<{ server: Server; logger: L .map((site: { id: number }) => String(site.id)); } } - } catch { - // Ignore errors, return empty completions + } catch (error) { + // Fail soft — completions are best-effort — but leave a trace so + // config/auth problems here aren't invisible in production + logger.info('Site-id completion lookup failed', { + error: sanitizeError(getErrorMessage(error)), + }); } } @@ -416,24 +454,24 @@ async function validateCredentials(config: Config, logger: Logger): Promise { startupLogger.info(`Dashboard: ${config.dashboardUrl}`); startupLogger.info(`Auth: ${config.authType === 'basic' ? 'Basic Auth' : 'Bearer Token'}`); startupLogger.info(`Config source: ${config.configSource}`); - startupLogger.info(`Session data limit: ${(config.maxSessionData / 1048576).toFixed(1)}MB`); + startupLogger.info(`Session data limit: ${formatBytes(config.maxSessionData)}`); if (config.skipSslVerify) { - startupLogger.error('╔══════════════════════════════════════════════════════════════╗'); - startupLogger.error('║ WARNING: SSL verification disabled ║'); - startupLogger.error('║ Connection is vulnerable to man-in-the-middle attacks ║'); - startupLogger.error('║ Only use for local development with self-signed certs ║'); - startupLogger.error('╚══════════════════════════════════════════════════════════════╝'); + startupLogger.error('WARNING: SSL verification disabled.'); + startupLogger.error('The connection is vulnerable to man-in-the-middle attacks.'); + startupLogger.error('Only use this for local development with self-signed certificates.'); } // The built-in mainwp://site/{id} resource calls mainwp/get-site-v1 and @@ -553,5 +589,19 @@ async function main(): Promise { } } -// Run the server -main(); +// Run only when invoked as the program entry point; tests import createServer. +// Node resolves the ESM main module to its real path while argv[1] keeps the +// path as invoked, so npm's bin symlinks need argv[1] realpathed to match. +function isMainModule(): boolean { + const entry = process.argv[1]; + if (!entry) return false; + try { + return import.meta.url === pathToFileURL(realpathSync(entry)).href; + } catch { + return false; + } +} + +if (isMainModule()) { + void main(); +} diff --git a/src/logging.test.ts b/src/logging.test.ts index 8069f5e..5f6f252 100644 --- a/src/logging.test.ts +++ b/src/logging.test.ts @@ -3,12 +3,15 @@ */ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import type { Server } from '@modelcontextprotocol/sdk/server/index.js'; import { createLogger, createStderrLogger, withRequestId } from './logging.js'; describe('createLogger', () => { let mockServer: { sendLoggingMessage: ReturnType; }; + // createLogger only calls sendLoggingMessage, so the stub narrows to that + const asServer = () => mockServer as unknown as Server; beforeEach(() => { mockServer = { @@ -17,7 +20,7 @@ describe('createLogger', () => { }); it('should send logs via MCP server', async () => { - const logger = createLogger(mockServer as any); + const logger = createLogger(asServer()); logger.info('Test message'); @@ -34,7 +37,7 @@ describe('createLogger', () => { }); it('should include data in log messages', async () => { - const logger = createLogger(mockServer as any); + const logger = createLogger(asServer()); logger.info('Test message', { key: 'value' }); @@ -50,7 +53,7 @@ describe('createLogger', () => { }); it('should use custom logger name', async () => { - const logger = createLogger(mockServer as any, 'custom-logger'); + const logger = createLogger(asServer(), 'custom-logger'); logger.debug('Test'); @@ -64,7 +67,7 @@ describe('createLogger', () => { }); it('should support all log levels', async () => { - const logger = createLogger(mockServer as any); + const logger = createLogger(asServer()); logger.debug('debug'); logger.info('info'); @@ -92,7 +95,7 @@ describe('createLogger', () => { const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}); mockServer.sendLoggingMessage.mockRejectedValue(new Error('Server not connected')); - const logger = createLogger(mockServer as any); + const logger = createLogger(asServer()); logger.info('Test message'); // Wait for the promise to reject and fallback to stderr diff --git a/src/logging.ts b/src/logging.ts index c050a41..0af40d8 100644 --- a/src/logging.ts +++ b/src/logging.ts @@ -7,16 +7,9 @@ import { Server } from '@modelcontextprotocol/sdk/server/index.js'; -// RFC 5424 Log Levels -export type LogLevel = - | 'debug' - | 'info' - | 'notice' - | 'warning' - | 'error' - | 'critical' - | 'alert' - | 'emergency'; +// RFC 5424 log levels, minus 'alert'/'emergency' — the Logger interface +// implements exactly these six, so the type carries no unreachable members +export type LogLevel = 'debug' | 'info' | 'notice' | 'warning' | 'error' | 'critical'; export interface Logger { debug(message: string, data?: Record): void; diff --git a/src/prompts.test.ts b/src/prompts.test.ts index 99da5bb..0997a1c 100644 --- a/src/prompts.test.ts +++ b/src/prompts.test.ts @@ -146,12 +146,36 @@ describe('getPrompt - argument validation', () => { expect(() => getPrompt('troubleshoot-site', { site_id: 'abc' })).toThrow('Invalid site_id'); }); - it('should accept numeric site_id', () => { - expect(() => getPrompt('troubleshoot-site', { site_id: '123' })).not.toThrow(); + it('should accept numeric site_id and interpolate it', () => { + const result = getPrompt('troubleshoot-site', { site_id: '123' }); + + expect(result.messages.length).toBeGreaterThan(0); + expect((result.messages[0].content as { text: string }).text).toContain('123'); + }); + + it('should accept "all" as site_id and interpolate it', () => { + const result = getPrompt('performance-check', { site_id: 'all' }); + + expect(result.messages.length).toBeGreaterThan(0); + expect((result.messages[0].content as { text: string }).text).toContain('all'); }); - it('should accept "all" as site_id', () => { - expect(() => getPrompt('performance-check', { site_id: 'all' })).not.toThrow(); + it('should reject zero and partial-numeric site_id', () => { + expect(() => getPrompt('troubleshoot-site', { site_id: '0' })).toThrow('Invalid site_id'); + expect(() => getPrompt('troubleshoot-site', { site_id: '12abc' })).toThrow('Invalid site_id'); + }); + + it('should reject site_ids containing a zero or partial-numeric entry', () => { + expect(() => getPrompt('security-audit', { site_ids: '1,0,3' })).toThrow('Invalid site_ids'); + expect(() => getPrompt('security-audit', { site_ids: '1,2abc' })).toThrow('Invalid site_ids'); + }); + + it('should canonicalize site_ids so raw whitespace never reaches the template', () => { + const result = getPrompt('security-audit', { site_ids: '1,\n2' }); + + const text = (result.messages[0].content as { text: string }).text; + expect(text).toContain('1,2'); + expect(text).not.toContain('1,\n2'); }); it('should reject invalid issue_type', () => { @@ -160,14 +184,18 @@ describe('getPrompt - argument validation', () => { ); }); - it('should accept valid issue_type', () => { - expect(() => - getPrompt('troubleshoot-site', { site_id: '1', issue_type: 'security' }) - ).not.toThrow(); + it('should accept valid issue_type and interpolate it', () => { + const result = getPrompt('troubleshoot-site', { site_id: '1', issue_type: 'security' }); + + expect(result.messages.length).toBeGreaterThan(0); + expect((result.messages[0].content as { text: string }).text).toContain('security'); }); - it('should accept comma-separated numeric site_ids', () => { - expect(() => getPrompt('security-audit', { site_ids: '1,2,3' })).not.toThrow(); + it('should accept comma-separated numeric site_ids and interpolate them', () => { + const result = getPrompt('security-audit', { site_ids: '1,2,3' }); + + expect(result.messages.length).toBeGreaterThan(0); + expect((result.messages[0].content as { text: string }).text).toContain('1,2,3'); }); it('should accept "all" as site_ids', () => { @@ -184,7 +212,10 @@ describe('getPrompt - argument validation', () => { ); }); - it('should accept valid update_type', () => { - expect(() => getPrompt('update-workflow', { update_type: 'plugins' })).not.toThrow(); + it('should accept valid update_type and interpolate it', () => { + const result = getPrompt('update-workflow', { update_type: 'plugins' }); + + expect(result.messages.length).toBeGreaterThan(0); + expect((result.messages[0].content as { text: string }).text).toContain('plugins'); }); }); diff --git a/src/prompts.ts b/src/prompts.ts index a1f9c15..c5f9709 100644 --- a/src/prompts.ts +++ b/src/prompts.ts @@ -6,6 +6,8 @@ */ import type { Prompt, GetPromptResult, PromptMessage } from '@modelcontextprotocol/sdk/types.js'; +import { isValidId } from './security.js'; +import { McpErrorFactory } from './errors.js'; /** * Internal prompt definition with message generator @@ -25,7 +27,6 @@ interface PromptDefinition { * All available prompt definitions */ const promptDefinitions: PromptDefinition[] = [ - // === Site Troubleshooting === { name: 'troubleshoot-site', description: 'Diagnose issues with a MainWP child site', @@ -59,7 +60,6 @@ Provide a summary of: ], }, - // === Maintenance Check === { name: 'maintenance-check', description: 'Run a comprehensive maintenance check across all managed sites', @@ -88,7 +88,6 @@ Generate a maintenance summary including: ], }, - // === Update Workflow === { name: 'update-workflow', description: 'Guide through safely updating WordPress sites', @@ -134,7 +133,6 @@ Please start by checking the current update status.`, ], }, - // === Site Report === { name: 'site-report', description: 'Generate a detailed report for a specific site', @@ -173,7 +171,6 @@ Format the report in a clear, scannable format.`, ], }, - // === Network Summary === { name: 'network-summary', description: 'Generate a summary report of all managed sites', @@ -216,7 +213,6 @@ Present the data in a clear, executive-summary format.`, ], }, - // === Security Audit === { name: 'security-audit', description: 'Perform a security-focused audit of managed sites', @@ -260,7 +256,6 @@ Start by gathering the site and update information, then provide the security as ], }, - // === Backup Status === { name: 'backup-status', description: 'Check backup status across managed sites', @@ -301,7 +296,6 @@ Note: This analysis depends on the backup data available through MainWP. If back ], }, - // === Performance Check === { name: 'performance-check', description: 'Analyze site performance indicators', @@ -379,26 +373,39 @@ function validatePromptArgs( for (const [key, value] of Object.entries(args)) { if (typeof value !== 'string') continue; - // site_id: numeric or "all"; site_ids: "all" or comma-separated numeric + // site_id: numeric or "all"; site_ids: "all" or comma-separated numeric. + // isValidId (security.ts) is the single definition of a valid ID — it + // also enforces the >= 1 and safe-integer bounds the old regexes missed. if (key === 'site_id') { - if (value !== 'all' && !/^\d+$/.test(value)) { - throw new Error(`Invalid site_id: must be a numeric value or "all"`); + if (value !== 'all' && !isValidId(value)) { + throw McpErrorFactory.invalidParams(`Invalid site_id: must be a numeric value or "all"`); } sanitized[key] = value; } else if (key === 'site_ids') { - // "all" or comma-separated IDs - if (value !== 'all' && !/^(\d+)(,\s*\d+)*$/.test(value)) { - throw new Error(`Invalid site_ids: must be "all" or comma-separated numeric IDs`); + // "all" or comma-separated IDs. Store the canonicalized join, not the + // raw value — validation trims each part, so the raw string could + // smuggle whitespace/newlines into the prompt template + if (value === 'all') { + sanitized[key] = value; + } else { + const ids = value.split(',').map(part => part.trim()); + if (!ids.every(isValidId)) { + throw McpErrorFactory.invalidParams( + `Invalid site_ids: must be "all" or comma-separated numeric IDs` + ); + } + sanitized[key] = ids.join(','); } - sanitized[key] = value; } else if (key === 'issue_type') { if (!VALID_ISSUE_TYPES.has(value)) { - throw new Error(`Invalid issue_type: must be one of ${[...VALID_ISSUE_TYPES].join(', ')}`); + throw McpErrorFactory.invalidParams( + `Invalid issue_type: must be one of ${[...VALID_ISSUE_TYPES].join(', ')}` + ); } sanitized[key] = value; } else if (key === 'update_type') { if (!VALID_UPDATE_TYPES.has(value)) { - throw new Error( + throw McpErrorFactory.invalidParams( `Invalid update_type: must be one of ${[...VALID_UPDATE_TYPES].join(', ')}` ); } diff --git a/src/retry.test.ts b/src/retry.test.ts index 0ab4102..e793eac 100644 --- a/src/retry.test.ts +++ b/src/retry.test.ts @@ -57,6 +57,23 @@ describe('isRetryableError', () => { const error = new Error('Unprocessable Entity: 422'); expect(isRetryableError(error)).toBe(false); }); + + it('ignores unrelated numbers in error messages', () => { + expect(isRetryableError(new Error('cannot process more than 500 items'))).toBe(false); + expect(isRetryableError(new Error('connection idle for 503 seconds'))).toBe(false); + expect(isRetryableError(new Error('batch contains: 500 items'))).toBe(false); + expect(isRetryableError(new Error('validation failed: 503 records'))).toBe(false); + }); + + it('extracts status codes preceded by the word "status"', () => { + expect(isRetryableError(new Error('Request failed with status 503'))).toBe(true); + expect(isRetryableError(new Error('status code 502 received'))).toBe(true); + }); + + it('extracts status codes at the start of the message', () => { + expect(isRetryableError(new Error('503 Service Unavailable'))).toBe(true); + expect(isRetryableError(new Error('404 Not Found'))).toBe(false); + }); }); describe('network error codes', () => { @@ -489,39 +506,13 @@ describe('withRetry', () => { expect(receivedBudgets[1]).toBeGreaterThan(receivedBudgets[2]); }); - it('retries on error with status property set to 503', async () => { - let attempts = 0; - const operation = vi.fn(async (_ctx: RetryContext) => { - attempts++; - if (attempts < 2) { - const error = new Error('Service Unavailable'); - (error as Error & { status: number }).status = 503; - throw error; - } - return 'success'; - }); - - const promise = withRetry(operation, { - maxRetries: 3, - baseDelay: 100, - maxDelay: 200, - timeoutBudget: 10000, - }); - - await vi.runAllTimersAsync(); - const result = await promise; - - expect(result).toBe('success'); - expect(attempts).toBe(2); - }); - - it('retries on error with status property set to 429', async () => { + it.each([503, 429])('retries on error with status property set to %i', async status => { let attempts = 0; const operation = vi.fn(async (_ctx: RetryContext) => { attempts++; if (attempts < 2) { - const error = new Error('Too Many Requests'); - (error as Error & { status: number }).status = 429; + const error = new Error(status === 503 ? 'Service Unavailable' : 'Too Many Requests'); + (error as Error & { status: number }).status = status; throw error; } return 'success'; diff --git a/src/retry.ts b/src/retry.ts index f4b03a3..3f06e2f 100644 --- a/src/retry.ts +++ b/src/retry.ts @@ -68,12 +68,18 @@ function extractStatusCode(error: unknown): number | null { } } - // Extract from error message patterns + // Extract from error message patterns (last resort — structured .status above + // is preferred). Requires status context before the digits so unrelated + // numbers ("more than 500 items") aren't misread as HTTP statuses. if (error instanceof Error) { const message = error.message; - // Match patterns like "Failed to fetch: 503" or "HTTP 503" - const statusMatch = message.match(/\b([45]\d{2})\b/); + // Match patterns like "HTTP 503", "status 503", "status code 503", + // "Failed to fetch: 503", or a status-line-style message starting with + // the code ("503 Service Unavailable") + const statusMatch = message.match( + /(?:^|\bHTTP\s+|\bstatus(?:\s+code)?\s*:?\s*|\b(?:failed to fetch|gateway timeout|rate limited)\s*:\s*)([45]\d{2})\b/i + ); if (statusMatch) { return parseInt(statusMatch[1], 10); } diff --git a/src/security.test.ts b/src/security.test.ts index ad74484..edbc025 100644 --- a/src/security.test.ts +++ b/src/security.test.ts @@ -62,11 +62,33 @@ describe('validateInput', () => { expect(() => validateInput({ site_id: 1.5 })).toThrow(/must be a positive integer/); }); + it('should reject non-string/non-number _id values', () => { + expect(() => validateInput({ site_id: null })).toThrow(/must be a string or number/); + expect(() => validateInput({ site_id: true })).toThrow(/must be a string or number/); + expect(() => validateInput({ site_id: {} })).toThrow(/must be a string or number/); + expect(() => validateInput({ site_id: [1] })).toThrow(/must be a string or number/); + }); + + it('should reject _id strings with trailing non-numeric characters', () => { + expect(() => validateInput({ site_id: '12abc' })).toThrow(/must be a positive integer/); + }); + + it('should reject _id strings with non-decimal numeric syntax', () => { + expect(() => validateInput({ site_id: '1e3' })).toThrow(/must be a positive integer/); + expect(() => validateInput({ site_id: '0x10' })).toThrow(/must be a positive integer/); + }); + it('should accept valid plural ID arrays', () => { expect(() => validateInput({ site_ids: [1, 2, 3] })).not.toThrow(); expect(() => validateInput({ site_ids: ['1', '2', '3'] })).not.toThrow(); }); + it('should reject non-array values for plural ID fields', () => { + expect(() => validateInput({ site_ids: '123' })).toThrow(/must be an array/); + expect(() => validateInput({ site_ids: 123 })).toThrow(/must be an array/); + expect(() => validateInput({ site_ids: { id: 1 } })).toThrow(/must be an array/); + }); + it('should reject non-positive elements in plural ID arrays', () => { expect(() => validateInput({ site_ids: [0] })).toThrow(/must be a positive integer/); expect(() => validateInput({ site_ids: [-1] })).toThrow(/must be a positive integer/); @@ -82,6 +104,7 @@ describe('validateInput', () => { it('should reject non-numeric strings in plural ID arrays', () => { expect(() => validateInput({ site_ids: ['abc'] })).toThrow(/must be a positive integer/); + expect(() => validateInput({ site_ids: ['1abc'] })).toThrow(/must be a positive integer/); }); it('should accept valid nested objects', () => { @@ -255,6 +278,19 @@ describe('isValidId', () => { expect(isValidId([])).toBe(false); }); + it('should return false for strings with trailing non-numeric characters', () => { + expect(isValidId('12abc')).toBe(false); + expect(isValidId('1 2')).toBe(false); + }); + + it('should return false for non-decimal numeric syntax', () => { + expect(isValidId('1e3')).toBe(false); + expect(isValidId('0x10')).toBe(false); + expect(isValidId('0b10')).toBe(false); + expect(isValidId('+7')).toBe(false); + expect(isValidId(' 8 ')).toBe(false); + }); + it('should return false for floats', () => { expect(isValidId(1.5)).toBe(false); }); diff --git a/src/security.ts b/src/security.ts index 3e220bf..8656df9 100644 --- a/src/security.ts +++ b/src/security.ts @@ -37,13 +37,16 @@ export function validateInput(args: Record, depth = 0): void { // ID fields: accept number or numeric string, must be positive integer if (key.endsWith('_id')) { - const numValue = typeof value === 'string' ? parseInt(value, 10) : value; - if (typeof numValue === 'number') { - if (!Number.isInteger(numValue) || numValue < 1 || numValue > Number.MAX_SAFE_INTEGER) { - throw McpErrorFactory.invalidParams(`Parameter "${key}" must be a positive integer`, { - parameter: key, - }); - } + if (typeof value !== 'string' && typeof value !== 'number') { + throw McpErrorFactory.invalidParams( + `Parameter "${key}" must be a string or number, got ${typeof value}`, + { parameter: key } + ); + } + if (!isValidId(value)) { + throw McpErrorFactory.invalidParams(`Parameter "${key}" must be a positive integer`, { + parameter: key, + }); } } @@ -59,13 +62,7 @@ export function validateInput(args: Record, depth = 0): void { { parameter: key } ); } - const numItem = typeof item === 'string' ? Number(item) : item; - if ( - !Number.isFinite(numItem) || - !Number.isInteger(numItem) || - numItem < 1 || - numItem > Number.MAX_SAFE_INTEGER - ) { + if (!isValidId(item)) { throw McpErrorFactory.invalidParams(`Element in "${key}" must be a positive integer`, { parameter: key, }); @@ -212,8 +209,14 @@ export function isValidId(value: unknown): boolean { return Number.isInteger(value) && value >= 1 && value <= Number.MAX_SAFE_INTEGER; } if (typeof value === 'string') { - const num = parseInt(value, 10); - return !isNaN(num) && num >= 1 && num <= Number.MAX_SAFE_INTEGER; + // Decimal digits only — Number() alone would also accept "1e3", "0x10", + // "+7", and " 8 ", where local validation and upstream interpretation of + // the raw forwarded string can disagree. + if (!/^\d+$/.test(value)) { + return false; + } + const num = Number(value); + return Number.isSafeInteger(num) && num >= 1; } return false; } diff --git a/src/session.test.ts b/src/session.test.ts new file mode 100644 index 0000000..723b1c7 --- /dev/null +++ b/src/session.test.ts @@ -0,0 +1,62 @@ +/** + * Session Data Tracking Tests + */ + +import { beforeEach, describe, expect, it } from 'vitest'; +import { MCP_ERROR_CODES, McpError } from './errors.js'; +import { formatBytes, getSessionDataUsage, resetSessionData, trackSessionData } from './session.js'; +import { makeBaseConfig, makeMockLogger } from '../tests/helpers/config.js'; + +describe('session data tracking', () => { + beforeEach(resetSessionData); + + it('accumulates usage and returns each UTF-8 byte count', () => { + const config = makeBaseConfig(); + const logger = makeMockLogger(); + + expect(trackSessionData('hello', config, logger, 'for test')).toBe(5); + expect(trackSessionData('€', config, logger, 'for test')).toBe(3); + expect(getSessionDataUsage(config)).toEqual({ used: 8, limit: config.maxSessionData }); + }); + + it('throws RESOURCE_EXHAUSTED with formatted usage when the cap would be exceeded', () => { + const config = makeBaseConfig({ maxSessionData: 1024 }); + const logger = makeMockLogger(); + trackSessionData('a'.repeat(800), config, logger, 'for test'); + + let thrown: unknown; + try { + trackSessionData('b'.repeat(736), config, logger, 'for test'); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(McpError); + expect(thrown).toMatchObject({ code: MCP_ERROR_CODES.RESOURCE_EXHAUSTED }); + expect((thrown as Error).message).toContain('1.5 KB of 1.0 KB'); + expect(getSessionDataUsage(config).used).toBe(800); + }); + + it('resets accumulated session usage', () => { + const config = makeBaseConfig(); + trackSessionData('data', config, makeMockLogger(), 'for test'); + + resetSessionData(); + + expect(getSessionDataUsage(config)).toEqual({ used: 0, limit: config.maxSessionData }); + }); +}); + +describe('formatBytes', () => { + it('formats byte values', () => { + expect(formatBytes(512)).toBe('512 bytes'); + }); + + it('formats kilobyte values', () => { + expect(formatBytes(2560)).toBe('2.5 KB'); + }); + + it('formats megabyte values', () => { + expect(formatBytes(1572864)).toBe('1.5 MB'); + }); +}); diff --git a/src/session.ts b/src/session.ts index d275716..e4ee1d5 100644 --- a/src/session.ts +++ b/src/session.ts @@ -92,7 +92,8 @@ export function trackSessionData( } /** - * Reset the cumulative session data counter to zero. + * Reset the cumulative session data counter to zero (for testing only). + * @internal */ export function resetSessionData(): void { sessionDataBytes = 0; diff --git a/src/tool-schema.test.ts b/src/tool-schema.test.ts new file mode 100644 index 0000000..18e3fc8 --- /dev/null +++ b/src/tool-schema.test.ts @@ -0,0 +1,168 @@ +/** + * Tool Schema Conversion Tests + * + * Regression coverage for hostile/malformed remote input schemas. + * PHP dashboards serialize empty associative arrays as JSON arrays, so a + * no-input ability can arrive with `properties: []`. Passing that through + * invalidates the entire tools/list response for spec-compliant MCP clients + * (the official SDK rejects it with a zod error), which leaves the server + * connected but with zero usable tools. + */ + +import { describe, it, expect } from 'vitest'; +import { abilityToTool } from './tool-schema.js'; +import type { Ability } from './abilities.js'; + +function makeAbility(overrides: Partial = {}): Ability { + return { + name: 'mainwp/get-network-snapshot-v1', + label: 'Get Network Snapshot', + description: 'Returns a snapshot of the network.', + category: 'mainwp-sites', + ...overrides, + }; +} + +describe('abilityToTool input schema sanitization', () => { + it('coerces array-typed properties (PHP empty array) to an empty object', () => { + // JSON.parse round-trip mirrors how the payload actually arrives + const ability = makeAbility({ + input_schema: JSON.parse('{"type":"object","properties":[]}') as Record, + }); + + const tool = abilityToTool(ability, 'mainwp'); + + expect(Array.isArray(tool.inputSchema.properties)).toBe(false); + expect(tool.inputSchema.properties).toEqual({}); + expect(tool.inputSchema.required).toEqual([]); + }); + + it('coerces non-object properties to an empty object', () => { + const ability = makeAbility({ + input_schema: { type: 'object', properties: 'bogus' }, + }); + + const tool = abilityToTool(ability, 'mainwp'); + + expect(tool.inputSchema.properties).toEqual({}); + }); + + it('coerces a non-array required field to an empty array', () => { + const ability = makeAbility({ + input_schema: { type: 'object', properties: {}, required: 'site_id' }, + }); + + const tool = abilityToTool(ability, 'mainwp'); + + expect(tool.inputSchema.required).toEqual([]); + }); + + it('drops non-string entries from required', () => { + const ability = makeAbility({ + input_schema: { + type: 'object', + properties: { site_id: { type: 'integer', description: 'Site ID.' } }, + required: ['site_id', 7, null, { bad: true }], + }, + }); + + const tool = abilityToTool(ability, 'mainwp'); + + expect(tool.inputSchema.required).toEqual(['site_id']); + }); + + it('coerces primitive and array property values to empty objects', () => { + // A string-valued property is truthy and reaches the description + // backfill, which throws on primitives in strict-mode ESM and fails the + // whole tools/list response instead of isolating one bad property. + const ability = makeAbility({ + input_schema: JSON.parse( + '{"type":"object","properties":{"site_id":"bogus","tags":[],"ok":{"type":"string"}}}' + ) as Record, + }); + + const tool = abilityToTool(ability, 'mainwp'); + + const props = tool.inputSchema.properties as Record>; + expect(props.site_id).toEqual({ description: 'Site ID.' }); + expect(props.tags).toEqual({ description: 'Tags.' }); + expect(props.ok).toEqual({ type: 'string', description: 'Ok.' }); + }); + + it('keeps a __proto__ parameter as an own property without polluting detection', () => { + // A plain-object property map would send {confirm:...} through the + // prototype setter: the __proto__ parameter vanishes from the schema and + // 'confirm' in props starts observing the inherited attacker value. + const ability = makeAbility({ + input_schema: JSON.parse( + '{"type":"object","properties":{"__proto__":{"confirm":{"type":"boolean"}},"site_id":{"type":"integer","description":"Site ID."}}}' + ) as Record, + meta: { annotations: { destructive: true, readonly: false, idempotent: false } }, + }); + + const tool = abilityToTool(ability, 'mainwp'); + + const props = tool.inputSchema.properties as Record; + expect(Object.prototype.hasOwnProperty.call(props, '__proto__')).toBe(true); + // No real confirm parameter exists, so no confirmation flow is advertised + expect('confirm' in props).toBe(false); + expect(tool.description).not.toContain('CONFIRMATION FLOW'); + }); + + it('preserves well-formed object properties unchanged', () => { + const ability = makeAbility({ + input_schema: { + type: 'object', + properties: { site_id: { type: 'integer', description: 'Site ID.' } }, + required: ['site_id'], + }, + }); + + const tool = abilityToTool(ability, 'mainwp'); + + expect(tool.inputSchema.properties).toEqual({ + site_id: { type: 'integer', description: 'Site ID.' }, + }); + expect(tool.inputSchema.required).toEqual(['site_id']); + }); +}); + +describe('abilityToTool confirmation parameter injection', () => { + function makeDestructiveAbility(withDryRun: boolean): Ability { + const properties: Record = { + site_id: { type: 'integer', description: 'Site ID.' }, + confirm: { type: 'boolean', description: 'Confirm.' }, + }; + if (withDryRun) { + properties.dry_run = { type: 'boolean', description: 'Dry run.' }; + } + return makeAbility({ + name: 'mainwp/delete-site-v1', + input_schema: { type: 'object', properties }, + meta: { annotations: { destructive: true, readonly: false, idempotent: false } }, + }); + } + + it('declares confirmation_token so schema-validating clients can send it', () => { + const tool = abilityToTool(makeDestructiveAbility(true), 'mainwp'); + + const props = tool.inputSchema.properties as Record>; + expect(props.user_confirmed).toBeDefined(); + expect(props.confirmation_token).toMatchObject({ type: 'string' }); + }); + + it('advertises the token-bound flow in the standard description', () => { + const tool = abilityToTool(makeDestructiveAbility(true), 'mainwp'); + + expect(tool.description).toContain('confirmation_token'); + expect(tool.description).toContain('preview what will be affected'); + }); + + it('does not promise a preview when the ability lacks dry_run', () => { + const tool = abilityToTool(makeDestructiveAbility(false), 'mainwp'); + + expect(tool.description).toContain('no preview available'); + expect(tool.description).not.toContain('preview what will be affected'); + expect(tool.description).toContain('confirmation_token'); + }); +}); diff --git a/src/tool-schema.ts b/src/tool-schema.ts index e2991a2..03cf647 100644 --- a/src/tool-schema.ts +++ b/src/tool-schema.ts @@ -18,7 +18,7 @@ import { abilityNameToToolName } from './naming.js'; * MCP's tool input schema (also JSON Schema based). */ function convertInputSchema(ability: Ability): Tool['inputSchema'] { - const schema = ability.input_schema; + const schema = ability.input_schema ? structuredClone(ability.input_schema) : undefined; if (!schema) { // No input required @@ -31,13 +31,36 @@ function convertInputSchema(ability: Ability): Tool['inputSchema'] { // The abilities API uses JSON Schema, which is compatible with MCP // We just need to ensure it has the required structure // Cast to the expected MCP SDK type - const properties = (schema.properties || {}) as { [key: string]: Record }; - const required = (schema.required as string[]) || []; + // + // PHP dashboards serialize empty associative arrays as [] — an array-typed + // `properties` (or malformed `required`) invalidates the whole tools/list + // response for spec-compliant clients, so coerce anything non-conforming. + const rawProperties = schema.properties; + const rawPropertyMap = ( + rawProperties && typeof rawProperties === 'object' && !Array.isArray(rawProperties) + ? rawProperties + : {} + ) as Record; + // Sanitize each property too: a primitive or array value would throw on the + // description backfill below, failing the whole tools/list response instead + // of isolating one malformed property. Null prototype so a hostile + // __proto__ parameter stays an own property instead of polluting the map + // and leaking inherited keys into the confirm/dry_run detection. + const properties: { [key: string]: Record } = Object.create(null); + for (const [name, prop] of Object.entries(rawPropertyMap)) { + properties[name] = + prop !== null && typeof prop === 'object' && !Array.isArray(prop) + ? (prop as Record) + : {}; + } + const required = Array.isArray(schema.required) + ? schema.required.filter((entry): entry is string => typeof entry === 'string') + : []; // Backfill missing descriptions from parameter names. // Some upstream abilities omit descriptions; LLMs need them for accurate tool use. for (const [name, prop] of Object.entries(properties)) { - if (prop && (!prop.description || String(prop.description).trim() === '')) { + if (!prop.description || String(prop.description).trim() === '') { prop.description = paramNameToDescription(name); } } @@ -59,13 +82,20 @@ function paramNameToDescription(name: string): string { return words.charAt(0).toUpperCase() + words.slice(1) + '.'; } +/** Descriptions at or under this length are kept as-is */ +const TARGET_DESC_LENGTH = 60; +/** A first sentence may exceed the target by this small tolerance */ +const SENTENCE_TOLERANCE = 5; +/** Hard-truncate length; +3 for the ellipsis lands near the target */ +const HARD_TRUNCATE_LENGTH = TARGET_DESC_LENGTH - 3; + /** - * Truncate a description to the first sentence or ~60 characters + * Truncate a description to the first sentence or ~TARGET_DESC_LENGTH characters * * Strategy: - * - Preserve full description if already ≤60 characters - * - Return first sentence if it's within limit (≤65 chars, small tolerance) - * - Otherwise truncate to 57 characters with ellipsis + * - Preserve full description if already within the target length + * - Return first sentence if it's within target + tolerance + * - Otherwise hard-truncate with ellipsis */ function truncateDescription(description: string | undefined | null): string { if (!description) { @@ -73,7 +103,7 @@ function truncateDescription(description: string | undefined | null): string { } // If already short enough, return as-is - if (description.length <= 60) { + if (description.length <= TARGET_DESC_LENGTH) { return description; } @@ -81,14 +111,13 @@ function truncateDescription(description: string | undefined | null): string { const sentenceMatch = description.match(/^[^.!?]+[.!?](?:\s|$)/); if (sentenceMatch) { const sentence = sentenceMatch[0].trim(); - // Only use sentence if it's within limit (allow small tolerance of 5 chars) - if (sentence.length <= 65) { + if (sentence.length <= TARGET_DESC_LENGTH + SENTENCE_TOLERANCE) { return sentence; } } - // No suitable sentence found, truncate to ~60 chars - return description.slice(0, 57) + '...'; + // No suitable sentence found + return description.slice(0, HARD_TRUNCATE_LENGTH) + '...'; } /** @@ -104,7 +133,9 @@ function compressSchema(schema: Record): Record>; - const compressedProperties: Record> = {}; + // Null prototype for the same reason as convertInputSchema: a __proto__ + // parameter must stay an own key through compression. + const compressedProperties: Record> = Object.create(null); for (const [key, prop] of Object.entries(properties)) { if (!prop || typeof prop !== 'object') { @@ -246,6 +277,77 @@ export function buildSafetyTags( return tags.length > 0 ? `[${tags.join(', ')}]` : ''; } +/** + * Build the full-detail tool description for standard verbosity: + * category prefix, full ability description, contextual LLM instructions, + * safety tags, and the two-phase confirmation workflow. + */ +function buildStandardDescription( + ability: Ability, + meta: AbilityAnnotations | undefined, + hasDryRun: boolean, + hasConfirm: boolean, + isDestructive: boolean +): string { + // Category prefix (e.g., "[sites] ...") + const categoryLabel = ability.category?.replace(/^mainwp-/, '').replace(/-/g, ' '); + let description = categoryLabel + ? `[${categoryLabel}] ${ability.description}` + : ability.description; + + // Append contextual LLM instructions + const instructions = generateInstructions(meta, hasDryRun, hasConfirm); + if (instructions) { + description += ` ${instructions}`; + } + + // Append safety tags + const tags = buildSafetyTags(meta, hasDryRun, hasConfirm, 'standard'); + if (tags) { + description += ` ${tags}`; + } + + // Append confirmation workflow for destructive tools with confirm parameter. + // Only promise a preview when the ability declares dry_run — confirm-only + // abilities issue a token without an upstream preview call. + if (isDestructive && hasConfirm) { + description += + '\n\nCONFIRMATION FLOW: ' + + (hasDryRun + ? '1) Call with confirm:true to preview what will be affected. ' + + '2) Show preview to user and ask for confirmation. ' + : '1) Call with confirm:true to receive a confirmation token (no preview available). ' + + '2) Ask the user for confirmation. ') + + '3) If confirmed, call again with user_confirmed:true and the confirmation_token ' + + 'from the first response to execute. ' + + 'Do NOT set user_confirmed:true without explicit user consent.'; + } + + return description; +} + +/** + * Build the token-lean tool description for compact verbosity: + * truncated description, short safety tags, one-line confirm flow. + */ +function buildCompactDescription( + ability: Ability, + meta: AbilityAnnotations | undefined, + hasDryRun: boolean, + hasConfirm: boolean, + isDestructive: boolean +): string { + let description = truncateDescription(ability.description); + const tags = buildSafetyTags(meta, hasDryRun, hasConfirm, 'compact'); + if (tags) { + description += ` ${tags}`; + } + if (isDestructive && hasConfirm) { + description += ' FLOW: confirm:true -> preview -> user_confirmed:true + confirmation_token'; + } + return description; +} + /** * Convert a MainWP Ability to an MCP Tool definition * @@ -274,15 +376,25 @@ export function abilityToTool( const hasConfirm = 'confirm' in props; const isDestructive = meta?.destructive ?? false; - // Add user_confirmed parameter for destructive tools with confirm parameter - // This must happen BEFORE schema compression so it applies to all verbosity modes + // Add user_confirmed and confirmation_token parameters for destructive tools + // with a confirm parameter. Both must be declared: confirmed execution is + // token-bound, and a schema-validating client (or upstream + // additionalProperties: false) would otherwise reject the token it is + // required to send. This must happen BEFORE schema compression so it applies + // to all verbosity modes. if (isDestructive && hasConfirm) { const mutableProps = inputSchema.properties as Record; mutableProps['user_confirmed'] = { type: 'boolean', description: - 'Confirm execution after reviewing preview. ' + - 'FLOW: 1) confirm:true for preview, 2) show user, 3) user_confirmed:true if approved.', + 'Confirm execution after user approval. ' + + 'FLOW: 1) confirm:true, 2) show result to user, ' + + '3) user_confirmed:true + confirmation_token if approved.', + }; + mutableProps['confirmation_token'] = { + type: 'string', + description: + 'Token issued by the confirm:true response; required alongside user_confirmed:true.', }; } @@ -292,45 +404,10 @@ export function abilityToTool( } // Build description with safety context - let description: string; - - if (verbosity === 'standard') { - // Category prefix for standard mode (e.g., "[sites] ...") - const categoryLabel = ability.category?.replace(/^mainwp-/, '').replace(/-/g, ' '); - description = categoryLabel ? `[${categoryLabel}] ${ability.description}` : ability.description; - - // Append contextual LLM instructions - const instructions = generateInstructions(meta, hasDryRun, hasConfirm); - if (instructions) { - description += ` ${instructions}`; - } - - // Append safety tags - const tags = buildSafetyTags(meta, hasDryRun, hasConfirm, 'standard'); - if (tags) { - description += ` ${tags}`; - } - - // Append confirmation workflow for destructive tools with confirm parameter - if (isDestructive && hasConfirm) { - description += - '\n\nCONFIRMATION FLOW: ' + - '1) Call with confirm:true to preview what will be affected. ' + - '2) Show preview to user and ask for confirmation. ' + - '3) If confirmed, call again with user_confirmed:true to execute. ' + - 'Do NOT set user_confirmed:true without explicit user consent.'; - } - } else { - // Compact mode: truncated description + short safety tags - description = truncateDescription(ability.description); - const tags = buildSafetyTags(meta, hasDryRun, hasConfirm, 'compact'); - if (tags) { - description += ` ${tags}`; - } - if (isDestructive && hasConfirm) { - description += ' FLOW: confirm:true -> preview -> user_confirmed:true + confirmation_token'; - } - } + const description = + verbosity === 'standard' + ? buildStandardDescription(ability, meta, hasDryRun, hasConfirm, isDestructive) + : buildCompactDescription(ability, meta, hasDryRun, hasConfirm, isDestructive); return { name: toolName, diff --git a/src/tools.test.ts b/src/tools.test.ts index 7492fff..202d6c5 100644 --- a/src/tools.test.ts +++ b/src/tools.test.ts @@ -3,14 +3,20 @@ */ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import { getTools, executeTool, clearToolsCache } from './tools.js'; +import { getTools, executeTool, clearToolsCache, isToolAllowed } from './tools.js'; import { abilityNameToToolName } from './naming.js'; import { getSessionDataUsage, resetSessionData, isNoOpError } from './session.js'; import { clearPendingPreviews } from './confirmation.js'; import { generateInstructions, buildSafetyTags } from './tool-schema.js'; -import { type Config } from './config.js'; -import { type Logger } from './logging.js'; -import { type Ability, clearCache, initRateLimiter } from './abilities.js'; +import { MCP_ERROR_CODES } from './errors.js'; +import { + type Ability, + clearCache, + fetchAbilities, + initRateLimiter, + onCacheRefresh, +} from './abilities.js'; +import { makeBaseConfig, makeMockLogger } from '../tests/helpers/config.js'; // Mock fetch globally const mockFetch = vi.fn(); @@ -127,37 +133,9 @@ const sampleAbilities: Ability[] = [ }, ]; -const baseConfig: Config = { - dashboardUrl: 'https://test.local', - authType: 'basic', - username: 'admin', - appPassword: 'xxxx', - skipSslVerify: true, - allowHttp: false, - rateLimit: 0, - requestTimeout: 5000, - maxResponseSize: 10485760, - safeMode: false, - requireUserConfirmation: true, - maxSessionData: 52428800, - schemaVerbosity: 'standard', - responseFormat: 'compact', - retryEnabled: false, // Disable retries for tests - maxRetries: 2, - retryBaseDelay: 1000, - retryMaxDelay: 2000, - abilityNamespaces: ['mainwp'], - configSource: 'environment', -}; - -const mockLogger: Logger = { - debug: vi.fn(), - info: vi.fn(), - notice: vi.fn(), - warning: vi.fn(), - error: vi.fn(), - critical: vi.fn(), -}; +const baseConfig = makeBaseConfig(); + +const mockLogger = makeMockLogger(); describe('getTools', () => { beforeEach(() => { @@ -227,6 +205,27 @@ describe('getTools', () => { expect(deleteTool?.inputSchema.properties).toHaveProperty('user_confirmed'); }); + it('does not mutate cached schemas or notify on an identical forced refresh', async () => { + const callback = vi.fn(); + onCacheRefresh(callback); + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => structuredClone(sampleAbilities), + headers: new Headers(), + }); + + await getTools(baseConfig); + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => structuredClone(sampleAbilities), + headers: new Headers(), + }); + await fetchAbilities(baseConfig, true); + + expect(callback).not.toHaveBeenCalled(); + }); + it('should handle schema verbosity modes - compact', async () => { mockFetch.mockResolvedValueOnce({ ok: true, @@ -482,6 +481,18 @@ describe('getTools', () => { }); }); +describe('isToolAllowed', () => { + it('gives the blocklist precedence over the allowlist', () => { + const config = { + ...baseConfig, + allowedTools: ['list_sites_v1'], + blockedTools: ['list_sites_v1'], + }; + + expect(isToolAllowed(config, 'list_sites_v1')).toBe(false); + }); +}); + describe('executeTool', () => { beforeEach(() => { vi.resetAllMocks(); @@ -518,6 +529,40 @@ describe('executeTool', () => { expect(result.isError).toBeUndefined(); }); + it('passes tool arguments through to the ability request', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => sampleAbilities, + headers: new Headers(), + }); + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => [], + headers: new Headers(), + }); + + await executeTool(baseConfig, 'list_sites_v1', { page: 2, per_page: 25 }, mockLogger); + + const url = mockFetch.mock.calls[1][0] as string; + expect(url).toContain('input[page]=2'); + expect(url).toContain('input[per_page]=25'); + }); + + it('rejects a disallowed tool before lookup, preview, or execution', async () => { + const config = { ...baseConfig, blockedTools: ['delete_site_v1'] }; + + const result = await executeTool( + config, + 'delete_site_v1', + { site_id: 1, confirm: true }, + mockLogger + ); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toContain('Tool is not allowed: delete_site_v1'); + expect(mockFetch).not.toHaveBeenCalled(); + }); + it('should validate input before execution', async () => { mockFetch.mockResolvedValueOnce({ ok: true, @@ -543,7 +588,8 @@ describe('executeTool', () => { expect(result.isError).toBe(true); const parsed = JSON.parse(result.content[0].text); - expect(parsed.error.message).toContain('Ability not found for tool: no_such_tool_v1'); + expect(parsed.error.code).toBe(MCP_ERROR_CODES.TOOL_NOT_FOUND); + expect(parsed.error.message).toContain('Tool not found: no_such_tool_v1'); }); it('should block destructive operations in safe mode', async () => { @@ -565,6 +611,29 @@ describe('executeTool', () => { expect(result.isError).toBe(true); }); + it('allows read-only tools in safe mode', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => sampleAbilities, + headers: new Headers(), + }); + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ sites: [] }), + headers: new Headers(), + }); + + const result = await executeTool( + { ...baseConfig, safeMode: true }, + 'list_sites_v1', + {}, + mockLogger + ); + + expect(JSON.parse(result.content[0].text)).toEqual({ sites: [] }); + expect(result.isError).toBeUndefined(); + }); + it('should strip confirm parameter in safe mode', async () => { mockFetch.mockResolvedValueOnce({ ok: true, @@ -616,6 +685,141 @@ describe('executeTool', () => { expect(result.isError).toBeUndefined(); }); + it('issues a confirmation token without calling upstream when the ability has confirm but no dry_run', async () => { + const confirmOnlyAbility: Ability = { + ...sampleAbilities[1], + name: 'mainwp/delete-without-preview-v1', + input_schema: { + type: 'object', + properties: { + site_id: { type: 'integer', description: 'Site ID' }, + confirm: { type: 'boolean', description: 'Must be true to execute' }, + }, + required: ['site_id'], + }, + }; + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => [...sampleAbilities, confirmOnlyAbility], + headers: new Headers(), + }); + + const result = await executeTool( + baseConfig, + 'delete_without_preview_v1', + { site_id: 1, confirm: true }, + mockLogger + ); + + // Workflow step, not an error: token issued, no preview, no upstream call + const parsed = JSON.parse(result.content[0].text); + expect(parsed.status).toBe('CONFIRMATION_REQUIRED'); + expect(parsed.next_action).toBe('confirm_without_preview'); + expect(parsed.preview).toBeNull(); + expect(typeof parsed.confirmation_token).toBe('string'); + expect(result.isError).toBeUndefined(); + expect(mockFetch).toHaveBeenCalledTimes(1); // abilities fetch only + + // The confirmed follow-up call executes — the gate is not a dead end + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ deleted: true }), + headers: new Headers(), + }); + const confirmed = await executeTool( + baseConfig, + 'delete_without_preview_v1', + { site_id: 1, user_confirmed: true, confirmation_token: parsed.confirmation_token }, + mockLogger + ); + expect(confirmed.isError).toBeUndefined(); + expect(confirmed.content[0].text).toContain('deleted'); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it('rejects injected dry_run on an ability that does not declare it', async () => { + const confirmOnlyAbility: Ability = { + ...sampleAbilities[1], + name: 'mainwp/delete-without-preview-v1', + input_schema: { + type: 'object', + properties: { + site_id: { type: 'integer', description: 'Site ID' }, + confirm: { type: 'boolean', description: 'Must be true to execute' }, + }, + required: ['site_id'], + }, + }; + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => [...sampleAbilities, confirmOnlyAbility], + headers: new Headers(), + }); + + // Undeclared dry_run must not bypass confirmation: if upstream ignored + // the unknown parameter, the destructive operation would execute for real. + const result = await executeTool( + baseConfig, + 'delete_without_preview_v1', + { site_id: 1, dry_run: true }, + mockLogger + ); + + const parsed = JSON.parse(result.content[0].text); + expect(parsed.error).toBe('INVALID_PARAMETER'); + expect(parsed.message).toContain('dry_run'); + expect(result.isError).toBe(true); + expect(mockFetch).toHaveBeenCalledTimes(1); // abilities fetch only, no /run + }); + + it('rejects injected dry_run even when combined with confirm: true', async () => { + const confirmOnlyAbility: Ability = { + ...sampleAbilities[1], + name: 'mainwp/delete-without-preview-v1', + input_schema: { + type: 'object', + properties: { + site_id: { type: 'integer', description: 'Site ID' }, + confirm: { type: 'boolean', description: 'Must be true to execute' }, + }, + required: ['site_id'], + }, + }; + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => [...sampleAbilities, confirmOnlyAbility], + headers: new Headers(), + }); + + // Worst case: confirm: true rides along with the fabricated dry_run — + // a skip here would forward confirm: true upstream without any gate. + const result = await executeTool( + baseConfig, + 'delete_without_preview_v1', + { site_id: 1, confirm: true, dry_run: true }, + mockLogger + ); + + const parsed = JSON.parse(result.content[0].text); + expect(parsed.error).toBe('INVALID_PARAMETER'); + expect(result.isError).toBe(true); + expect(mockFetch).toHaveBeenCalledTimes(1); // abilities fetch only, no /run + }); + + it('requires a preview for a bare destructive call with confirm support', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => sampleAbilities, + headers: new Headers(), + }); + + const result = await executeTool(baseConfig, 'delete_site_v1', { site_id: 1 }, mockLogger); + + expect(result.content[0].text).toContain('PREVIEW_REQUIRED'); + expect(result.isError).toBe(true); + expect(mockFetch).toHaveBeenCalledTimes(1); + }); + it('should reject user_confirmed without prior preview', async () => { mockFetch.mockResolvedValueOnce({ ok: true, @@ -652,6 +856,61 @@ describe('executeTool', () => { expect(result.isError).toBe(true); }); + it('allows a declared explicit dry_run to bypass confirmation', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => sampleAbilities, + headers: new Headers(), + }); + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ dry_run: true, preview: true }), + headers: new Headers(), + }); + + const result = await executeTool( + baseConfig, + 'delete_site_v1', + { site_id: 1, dry_run: true }, + mockLogger + ); + + expect(JSON.parse(result.content[0].text)).toEqual({ dry_run: true, preview: true }); + expect(result.isError).toBeUndefined(); + expect(mockLogger.debug).toHaveBeenCalledWith( + 'Explicit dry_run bypasses confirmation flow', + expect.objectContaining({ toolName: 'delete_site_v1' }) + ); + }); + + it('strips confirm from an explicit dry_run call before reaching upstream', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => sampleAbilities, + headers: new Headers(), + }); + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ dry_run: true, preview: true }), + headers: new Headers(), + }); + + const result = await executeTool( + baseConfig, + 'delete_site_v1', + { site_id: 1, dry_run: true, confirm: true }, + mockLogger + ); + + expect(result.isError).toBeUndefined(); + // Upstream must never see the ambiguous confirm+dry_run combination + const [executionUrl, executionInit] = mockFetch.mock.calls[1] as [string, RequestInit]; + const serialized = `${executionUrl} ${String(executionInit?.body ?? '')}`; + expect(serialized).toContain('dry_run'); + expect(serialized).not.toContain('confirm='); + expect(serialized).not.toContain('"confirm"'); + }); + it('should accept confirmation_token to resolve preview', async () => { // Step 1: Generate preview mockFetch.mockResolvedValueOnce({ @@ -825,6 +1084,27 @@ describe('executeTool', () => { expect(mockLogger.error).toHaveBeenCalled(); }); + it('returns an error result for upstream HTTP execution errors', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => sampleAbilities, + headers: new Headers(), + }); + mockFetch.mockResolvedValueOnce({ + ok: false, + status: 404, + statusText: 'Not Found', + text: async () => JSON.stringify({ code: 'site_not_found', message: 'Site does not exist' }), + headers: new Headers(), + }); + + const result = await executeTool(baseConfig, 'list_sites_v1', {}, mockLogger); + + expect(result.content[0].text).toContain('site_not_found'); + expect(result.isError).toBe(true); + expect(mockLogger.error).toHaveBeenCalled(); + }); + it('should return compact JSON by default', async () => { mockFetch.mockResolvedValueOnce({ ok: true, @@ -866,6 +1146,70 @@ describe('executeTool', () => { const parsed = JSON.parse(result.content[0].text); expect(result.content[0].text).toBe(JSON.stringify(parsed, null, 2)); }); + + it('executes non-primary namespace tools against their original ability URL', async () => { + const abilities: Ability[] = [ + { + name: 'acme/do-thing-v1', + label: 'Acme Do Thing', + description: 'Third-party readonly ability', + category: 'acme-misc', + meta: { annotations: { readonly: true, destructive: false, idempotent: true } }, + }, + ]; + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => abilities, + headers: new Headers(), + }); + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ result: 'pong' }), + headers: new Headers(), + }); + const config = { + ...baseConfig, + abilityNamespaces: ['mainwp', 'acme'] as [string, ...string[]], + }; + + const result = await executeTool(config, 'acme__do_thing_v1', { input: 'ping' }, mockLogger); + + expect(mockFetch.mock.calls[1][0]).toContain('/abilities/acme/do-thing-v1/run'); + expect(JSON.parse(result.content[0].text)).toEqual({ result: 'pong' }); + expect(result.isError).toBeUndefined(); + }); + + it('round-trips a hyphenated namespace through execution', async () => { + const abilities: Ability[] = [ + { + name: 'acme-corp/do-thing-v1', + label: 'Acme Corp Do Thing', + description: 'Hyphenated-namespace ability', + category: 'acme-corp-misc', + meta: { annotations: { readonly: true, destructive: false, idempotent: true } }, + }, + ]; + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => abilities, + headers: new Headers(), + }); + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ ok: true }), + headers: new Headers(), + }); + const config = { + ...baseConfig, + abilityNamespaces: ['mainwp', 'acme-corp'] as [string, ...string[]], + }; + + const result = await executeTool(config, 'acme_corp__do_thing_v1', {}, mockLogger); + + expect(mockFetch.mock.calls[1][0]).toContain('/abilities/acme-corp/do-thing-v1/run'); + expect(JSON.parse(result.content[0].text)).toEqual({ ok: true }); + expect(result.isError).toBeUndefined(); + }); }); describe('confirmation flow - full cycle', () => { @@ -908,6 +1252,8 @@ describe('confirmation flow - full cycle', () => { ); expect(previewResult.content[0].text).toContain('CONFIRMATION_REQUIRED'); + const expiredToken = JSON.parse(previewResult.content[0].text as string) + .confirmation_token as string; // Step 2: Advance time beyond PREVIEW_EXPIRY_MS (5 minutes + 1ms) vi.setSystemTime(startTime + 5 * 60 * 1000 + 1); @@ -916,7 +1262,7 @@ describe('confirmation flow - full cycle', () => { const expiredResult = await executeTool( baseConfig, 'delete_site_v1', - { site_id: 1, user_confirmed: true }, + { site_id: 1, user_confirmed: true, confirmation_token: expiredToken }, mockLogger ); @@ -927,17 +1273,75 @@ describe('confirmation flow - full cycle', () => { expect.objectContaining({ toolName: 'delete_site_v1' }) ); - // Step 4: Subsequent confirmation without new preview should require preview + // Step 4: Subsequent confirmation with the expired (now deleted) token should require preview const subsequentResult = await executeTool( baseConfig, 'delete_site_v1', - { site_id: 1, user_confirmed: true }, + { site_id: 1, user_confirmed: true, confirmation_token: expiredToken }, mockLogger ); expect(subsequentResult.content[0].text).toContain('PREVIEW_REQUIRED'); }); + it('should reject confirmation without a token even when a matching preview is pending', async () => { + // Step 1: Generate a preview (creates a pending preview for these exact args) + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => sampleAbilities, + headers: new Headers(), + }); + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ preview: { site_id: 1, will_delete: true } }), + headers: new Headers(), + }); + + const previewResult = await executeTool( + baseConfig, + 'delete_site_v1', + { site_id: 1, confirm: true }, + mockLogger + ); + expect(previewResult.content[0].text).toContain('CONFIRMATION_REQUIRED'); + const token = JSON.parse(previewResult.content[0].text as string).confirmation_token as string; + const fetchCallsAfterPreview = mockFetch.mock.calls.length; + + // Step 2: user_confirmed with identical args but NO token must be rejected + // without any upstream call (a tool+args fallback would let a caller + // confirm a preview it never read) + const noTokenResult = await executeTool( + baseConfig, + 'delete_site_v1', + { site_id: 1, confirm: true, user_confirmed: true }, + mockLogger + ); + + expect(noTokenResult.isError).toBe(true); + expect(noTokenResult.content[0].text).toContain('PREVIEW_REQUIRED'); + expect(noTokenResult.content[0].text).toContain('confirmation_token'); + expect(mockFetch.mock.calls.length).toBe(fetchCallsAfterPreview); + expect(mockLogger.warning).toHaveBeenCalledWith( + 'Confirmation failed - confirmation_token missing', + expect.objectContaining({ toolName: 'delete_site_v1' }) + ); + + // Step 3: The issued token still works after the rejected attempt + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ deleted: true }), + headers: new Headers(), + }); + const confirmedResult = await executeTool( + baseConfig, + 'delete_site_v1', + { site_id: 1, user_confirmed: true, confirmation_token: token }, + mockLogger + ); + expect(confirmedResult.isError).toBeUndefined(); + expect(confirmedResult.content[0].text).toContain('deleted'); + }); + it('should reject reuse of consumed confirmation_token', async () => { // Step 1: Generate preview mockFetch.mockResolvedValueOnce({ @@ -1019,7 +1423,7 @@ describe('confirmation flow - full cycle', () => { expect(crossToolResult.content[0].text).toContain('PREVIEW_REQUIRED'); expect(crossToolResult.isError).toBe(true); expect(mockLogger.warning).toHaveBeenCalledWith( - 'Confirmation failed - token belongs to different tool', + 'Confirmation failed - token belongs to a different tool or identity', expect.objectContaining({ toolName: 'delete_plugins_v1' }) ); @@ -1033,6 +1437,49 @@ describe('confirmation flow - full cycle', () => { expect(reuseResult.content[0].text).toContain('PREVIEW_REQUIRED'); }); + it('rejects a confirmation token issued under a different config identity', async () => { + // Preview against dashboard A + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => sampleAbilities, + headers: new Headers(), + }); + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ preview: true, affected: [1] }), + headers: new Headers(), + }); + + const previewResult = await executeTool( + baseConfig, + 'delete_site_v1', + { site_id: 1, confirm: true }, + mockLogger + ); + const token = JSON.parse(previewResult.content[0].text).confirmation_token; + expect(token).toBeDefined(); + + // Confirm against dashboard B with the same tool and arguments: the + // module-level preview maps are shared, so without identity scoping this + // would execute against a dashboard that never previewed anything. + const otherDashboard = { ...baseConfig, dashboardUrl: 'https://other-dashboard.example' }; + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => sampleAbilities, + headers: new Headers(), + }); + + const crossIdentityResult = await executeTool( + otherDashboard, + 'delete_site_v1', + { site_id: 1, user_confirmed: true, confirmation_token: token }, + mockLogger + ); + + expect(crossIdentityResult.content[0].text).toContain('PREVIEW_REQUIRED'); + expect(crossIdentityResult.isError).toBe(true); + }); + it('should reject confirmation when arguments differ from preview (arg-swap)', async () => { // Step 1: Generate preview for delete_site_v1 with site_id: 1 mockFetch.mockResolvedValueOnce({ @@ -1073,6 +1520,87 @@ describe('confirmation flow - full cycle', () => { ); }); + it('should reject confirmation when nested arguments differ from preview (nested arg-swap)', async () => { + // Step 1: Generate preview with a nested argument value + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => sampleAbilities, + headers: new Headers(), + }); + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ preview: true, site_id: 1 }), + headers: new Headers(), + }); + + const previewResult = await executeTool( + baseConfig, + 'delete_site_v1', + { site_id: 1, settings: { role: 'viewer' }, confirm: true }, + mockLogger + ); + const parsed = JSON.parse(previewResult.content[0].text); + const token = parsed.confirmation_token; + expect(token).toBeDefined(); + + // Step 2: Confirm with the same top-level shape but a different nested value + const swapResult = await executeTool( + baseConfig, + 'delete_site_v1', + { + site_id: 1, + settings: { role: 'admin' }, + user_confirmed: true, + confirmation_token: token, + }, + mockLogger + ); + + expect(swapResult.content[0].text).toContain('PREVIEW_REQUIRED'); + expect(swapResult.isError).toBe(true); + expect(mockLogger.warning).toHaveBeenCalledWith( + 'Confirmation failed - arguments do not match preview', + expect.objectContaining({ toolName: 'delete_site_v1' }) + ); + }); + + it('should reject confirmation when values nested under a __proto__ key differ from preview', async () => { + // JSON.parse creates __proto__ as an own property; a plain-object + // canonicalization target would silently drop it via the prototype setter, + // collapsing differing payloads onto one preview key. + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => sampleAbilities, + headers: new Headers(), + }); + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ preview: true, site_id: 1 }), + headers: new Headers(), + }); + + const previewArgs = JSON.parse( + '{"site_id":1,"settings":{"__proto__":{"role":"viewer"}},"confirm":true}' + ) as Record; + const previewResult = await executeTool(baseConfig, 'delete_site_v1', previewArgs, mockLogger); + const parsed = JSON.parse(previewResult.content[0].text); + const token = parsed.confirmation_token; + expect(token).toBeDefined(); + + const confirmArgs = JSON.parse( + '{"site_id":1,"settings":{"__proto__":{"role":"admin"}},"user_confirmed":true}' + ) as Record; + confirmArgs.confirmation_token = token; + const swapResult = await executeTool(baseConfig, 'delete_site_v1', confirmArgs, mockLogger); + + expect(swapResult.content[0].text).toContain('PREVIEW_REQUIRED'); + expect(swapResult.isError).toBe(true); + expect(mockLogger.warning).toHaveBeenCalledWith( + 'Confirmation failed - arguments do not match preview', + expect.objectContaining({ toolName: 'delete_site_v1' }) + ); + }); + it('should complete two-phase confirmation flow', async () => { // Step 1: Preview mockFetch.mockResolvedValueOnce({ @@ -1095,8 +1623,10 @@ describe('confirmation flow - full cycle', () => { ); expect(previewResult.content[0].text).toContain('CONFIRMATION_REQUIRED'); + const confirmationToken = JSON.parse(previewResult.content[0].text as string) + .confirmation_token as string; - // Step 2: Confirm execution + // Step 2: Confirm execution with the issued token // Note: abilities are already cached from step 1, so no need to mock abilities fetch again mockFetch.mockResolvedValueOnce({ ok: true, @@ -1107,7 +1637,7 @@ describe('confirmation flow - full cycle', () => { const confirmResult = await executeTool( baseConfig, 'delete_site_v1', - { site_id: 1, user_confirmed: true }, + { site_id: 1, user_confirmed: true, confirmation_token: confirmationToken }, mockLogger ); @@ -1116,8 +1646,7 @@ describe('confirmation flow - full cycle', () => { }); describe('session data tracking', () => { - // Note: session data tracking is cumulative across all tests - // These tests verify the tracking mechanism exists + beforeEach(() => resetSessionData()); it('should return usage object with used and limit', () => { const usage = getSessionDataUsage(baseConfig); diff --git a/src/tools.ts b/src/tools.ts index eb621c5..0220208 100644 --- a/src/tools.ts +++ b/src/tools.ts @@ -50,7 +50,16 @@ export type ToolCallResult = { /** * Cached tool list to avoid re-converting abilities on every ListTools call. - * Invalidated by abilities array reference change or config fingerprint change. + * + * This is the second tier of a two-tier cache and deliberately uses a + * different invalidation model than abilities.ts: + * - abilities.ts owns the SOURCE cache: TTL + namespace signature, with + * stale-serving on upstream failures. + * - this is a pure derivation memo: abilities → tools is deterministic given + * the config, so it's keyed on the abilities array identity (fetchAbilities + * returns the same array while its cache is valid) plus a fingerprint of + * the config fields that affect conversion. No TTL or staleness concept + * needed — freshness is entirely the source cache's problem. */ let cachedTools: Tool[] | null = null; let cachedToolsAbilitiesRef: Ability[] | null = null; @@ -66,6 +75,12 @@ export function clearToolsCache(): void { cachedToolsFingerprint = null; } +/** Return whether a tool is permitted by the configured allow/block lists. */ +export function isToolAllowed(config: Config, toolName: string): boolean { + if (config.blockedTools?.includes(toolName)) return false; + return !config.allowedTools?.length || config.allowedTools.includes(toolName); +} + /** * Fetch all MainWP abilities and convert them to MCP tools * @@ -82,7 +97,14 @@ export function clearToolsCache(): void { */ export async function getTools(config: Config, logger?: Logger): Promise { const abilities = await fetchAbilities(config, false, logger); - const fingerprint = `${config.schemaVerbosity}|${config.allowedTools?.join(',') ?? ''}|${config.blockedTools?.join(',') ?? ''}|${config.abilityNamespaces.join(',')}`; + // JSON.stringify (not delimiter-joining) so a tool or namespace name + // containing the delimiter could never produce a colliding fingerprint + const fingerprint = JSON.stringify([ + config.schemaVerbosity, + config.allowedTools ?? [], + config.blockedTools ?? [], + config.abilityNamespaces, + ]); if ( cachedTools && @@ -99,16 +121,7 @@ export async function getTools(config: Config, logger?: Logger): Promise const originalCount = tools.length; // Apply allowlist filter (whitelist) - if (config.allowedTools && config.allowedTools.length > 0) { - const allowedSet = new Set(config.allowedTools); - tools = tools.filter(tool => allowedSet.has(tool.name)); - } - - // Apply blocklist filter (blacklist) - if (config.blockedTools && config.blockedTools.length > 0) { - const blockedSet = new Set(config.blockedTools); - tools = tools.filter(tool => !blockedSet.has(tool.name)); - } + tools = tools.filter(tool => isToolAllowed(config, tool.name)); // Log if tools were filtered if (tools.length !== originalCount && logger) { @@ -126,7 +139,7 @@ export async function getTools(config: Config, logger?: Logger): Promise if (config.schemaVerbosity !== 'standard' && logger) { logger.info('Schema verbosity mode active', { verbosity: config.schemaVerbosity, - note: 'Reduces token usage by ~30% with minimal descriptions', + note: 'Uses minimal descriptions to reduce token usage', }); } @@ -167,6 +180,10 @@ export async function executeTool( throw McpErrorFactory.cancelled(); } + if (!isToolAllowed(config, toolName)) { + throw McpErrorFactory.permissionDenied(`Tool is not allowed: ${toolName}`); + } + // Validate input before forwarding to API validateInput(args); @@ -175,7 +192,7 @@ export async function executeTool( // and prefixed `{ns}__tool` names from non-primary namespaces. const ability = await getAbilityByToolName(config, toolName, reqLogger); if (!ability) { - throw new Error(`Ability not found for tool: ${toolName}`); + throw McpErrorFactory.toolNotFound(toolName); } abilityName = ability.name; @@ -196,9 +213,11 @@ export async function executeTool( }); } - // Log all destructive operation attempts for audit purposes (regardless of safe mode) + // Audit: fires for every destructive request, including ones safe mode + // blocks below. The matching "executed" event lives in executeAbility; + // grep "AUDIT:" for the full trail. if (isDestructive) { - reqLogger.info('Destructive operation invoked', { + reqLogger.info('AUDIT: destructive operation requested', { toolName, abilityName, safeMode: config.safeMode, diff --git a/tests/acceptance/agent-run.ts b/tests/acceptance/agent-run.ts new file mode 100644 index 0000000..15640f2 --- /dev/null +++ b/tests/acceptance/agent-run.ts @@ -0,0 +1,1164 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createArtifacts } from './lib/artifacts.js'; +import { + evaluateConfirmationTranscript, + type RecordedAgentToolResult, + type RecordedAgentToolUse, +} from './lib/agent-confirmation.js'; +import { + answerAvoidsKnownPluginNames, + evaluateSafeModeRefusal, + errorResultNamesSiteNotFound, + findNestedObjects, + inventoryProvesSiteAbsent, + scopedSearchProvesSiteAbsent, + matchesNotFoundSiteAnswer, + matchesSiteStatusAnswer, + type AgentEvaluation, +} from './lib/agent-matchers.js'; +import { awaitChildWithDeadline, CommandRunner } from './lib/commands.js'; +import { + FIXTURE_APP_PASSWORD, + FIXTURE_USERNAME, + startFixtureDashboard, + type FixtureDashboard, +} from './fixture-dashboard.js'; +import { resolveAcceptanceCredentials, type AcceptanceCredentials } from './lib/env.js'; +import { packAndInstall } from './lib/pack.js'; +import { Redactor } from './lib/redact.js'; +import { IndependentVerifier, type VerifiedSite } from './lib/verify.js'; +import { verifierListAll } from './scenarios/ability-reads.js'; + +interface AgentScenario { + id: string; + target: 'live' | 'fixture'; + serverEnv?: Record; + task(groundTruth: AgentGroundTruth): string; + expectedTools: string[]; + groundTruth(verifier: IndependentVerifier): Promise; + evaluate?: ( + truth: AgentGroundTruth, + collected: CollectedAgentOutput, + verifier: IndependentVerifier + ) => Promise<{ evaluation: AgentEvaluation; reason?: string }>; +} + +interface AgentGroundTruth { + count?: number; + siteId?: number; + siteUrl?: string; + siteName?: string; + pluginActive?: boolean; + pluginName?: string; + pluginSlug?: string; + updateSiteUrls?: string[]; + beforeSiteCount?: number; + targetSiteId?: number; + targetSiteUrl?: string; + targetSiteName?: string; + absentSiteQuery?: string; + knownSiteUrls?: string[]; + knownPluginNames?: string[]; + tagNames?: string[]; + chainSiteUrl?: string; + activeTheme?: string; + activeThemeName?: string; + offlineSiteUrls?: string[]; + allSiteUrls?: string[]; +} + +interface AgentResult { + id: string; + status: 'passed' | 'failed' | 'unverified'; + model?: string; + toolUses: RecordedAgentToolUse[]; + toolResults: RecordedAgentToolResult[]; + finalText: string; + groundTruth?: AgentGroundTruth; + evaluation?: AgentEvaluation; + reason?: string; +} + +interface CollectedAgentOutput { + toolUses: RecordedAgentToolUse[]; + toolResults: RecordedAgentToolResult[]; + finalText: string; + model?: string; +} + +const REPO_ROOT = path.resolve(fileURLToPath(new URL('../..', import.meta.url))); + +interface AgentTag { + name: string; +} + +interface AgentThemeResponse { + active_theme: string; + themes: Array<{ slug: string; name: string; active: boolean }>; +} + +interface AgentCheckSiteResponse { + checked: boolean; + status?: { online?: boolean }; +} + +function hostnameOf(url: string): string { + try { + return new URL(url).hostname; + } catch { + return url; + } +} + +async function pluginUpdateSiteUrls( + verifier: IndependentVerifier, + sites?: VerifiedSite[] +): Promise { + const updateSiteUrls: string[] = []; + for (const site of sites ?? (await verifier.listSites())) { + const plugins = await verifier.getSitePlugins(site.id); + if (plugins.plugins.some(plugin => Boolean(plugin.update_version))) { + updateSiteUrls.push(site.url); + } + } + return updateSiteUrls.sort(); +} + +const agentScenarios: AgentScenario[] = [ + { + id: 'agent-count-sites', + target: 'live', + task: () => 'How many sites are currently connected to my MainWP dashboard?', + expectedTools: ['count_sites_v1', 'list_sites_v1'], + groundTruth: async verifier => ({ count: await verifier.countSites() }), + }, + { + id: 'agent-updates', + target: 'live', + task: () => 'Which of my sites need plugin updates?', + expectedTools: ['list_updates_v1', 'list_sites_v1', 'get_site_plugins_v1'], + groundTruth: async verifier => ({ + updateSiteUrls: await pluginUpdateSiteUrls(verifier), + }), + }, + { + id: 'agent-plugin-active', + target: 'live', + task: truth => + `Is the ${truth.pluginName} plugin active on ${truth.siteUrl}? Answer yes or no with the site name.`, + expectedTools: ['get_site_plugins_v1', 'get_site_v1', 'list_sites_v1'], + groundTruth: async verifier => { + // Probe a plugin that actually exists on the testbed: the override + // plugin when configured, otherwise the first plugin discovered. + const preferred = process.env.MAINWP_MCP_ACCEPTANCE_TOGGLE_PLUGIN; + const sites = await verifier.listSites(); + if (sites.length === 0) throw new Error('No site is available for agent-plugin-active'); + for (const site of sites) { + const plugins = (await verifier.getSitePlugins(site.id)).plugins; + // Never probe mainwp-child unless nothing else exists: its activity + // is implied by the site being connected (get_site even reports its + // version), so the model can answer without the plugin-list tool the + // scenario is meant to exercise. + const plugin = preferred + ? plugins.find(candidate => candidate.slug === preferred) + : (plugins.find(candidate => !candidate.slug.startsWith('mainwp-child')) ?? plugins[0]); + if (plugin?.name) { + return { + siteId: site.id, + siteUrl: site.url, + siteName: site.name, + pluginActive: plugin.active, + pluginName: plugin.name, + pluginSlug: plugin.slug, + }; + } + } + throw new Error('No discoverable plugin was found for agent-plugin-active'); + }, + }, + { + id: 'agent-nonexistent-site', + target: 'live', + task: truth => `What plugins are installed on my site ${truth.absentSiteQuery}?`, + expectedTools: ['list_sites_v1', 'get_site_v1', 'get_site_plugins_v1'], + groundTruth: async verifier => { + const absentSiteQuery = 'nonexistent-acceptance-probe.invalid'; + const sites = await verifier.listSites(); + if ( + sites.some(site => + [site.url, hostnameOf(site.url)].some( + value => value.toLowerCase() === absentSiteQuery.toLowerCase() + ) + ) + ) { + throw new Error(`The nonexistent-site probe unexpectedly exists: ${absentSiteQuery}`); + } + const knownPluginNames = new Set(); + for (const site of sites) { + for (const plugin of (await verifier.getSitePlugins(site.id)).plugins) { + if (plugin.name.trim()) knownPluginNames.add(plugin.name.trim()); + } + } + return { + absentSiteQuery, + knownSiteUrls: sites.map(site => site.url).sort(), + knownPluginNames: [...knownPluginNames].sort(), + }; + }, + evaluate: async (truth, collected) => { + if (!truth.absentSiteQuery || !truth.knownSiteUrls || !truth.knownPluginNames) { + throw new Error('Nonexistent-site ground truth was incomplete'); + } + const lookupUses = collected.toolUses.filter(tool => + toolFamilyMatches(tool.name, ['list_sites_v1', 'get_site_v1', 'get_site_plugins_v1']) + ); + const structuredArguments = lookupUses.every( + tool => tool.input !== null && typeof tool.input === 'object' + ); + const lookupResults = toolResultsForUses(lookupUses, collected.toolResults); + const structuredNotFound = errorResultNamesSiteNotFound(lookupResults); + const inventoryResults = toolResultsForUses( + lookupUses.filter(tool => toolFamilyMatches(tool.name, ['list_sites_v1'])), + collected.toolResults + ); + const completeInventoryExcludesProbe = inventoryProvesSiteAbsent( + inventoryResults, + truth.knownSiteUrls, + truth.absentSiteQuery + ); + const emptyScopedSearch = scopedSearchProvesSiteAbsent( + lookupUses.filter(tool => toolFamilyMatches(tool.name, ['list_sites_v1'])), + use => toolResultsForUses([use], collected.toolResults), + truth.absentSiteQuery + ); + const answerMatches = matchesNotFoundSiteAnswer(collected.finalText); + const avoidsKnownPlugins = answerAvoidsKnownPluginNames( + collected.finalText, + truth.knownPluginNames + ); + const evaluation: AgentEvaluation = { + understoodRequest: { + pass: collected.finalText.trim().length > 0, + evidence: collected.finalText, + }, + rightCapability: { + pass: lookupUses.length > 0, + evidence: lookupUses.map(tool => tool.name), + }, + rightArguments: { + pass: lookupUses.length > 0 && structuredArguments, + evidence: lookupUses.map(tool => tool.input), + }, + correctMcpResult: { + pass: structuredNotFound || completeInventoryExcludesProbe || emptyScopedSearch, + evidence: { + lookupCount: lookupUses.length, + resultCount: lookupResults.length, + structuredNotFound, + completeInventoryExcludesProbe, + emptyScopedSearch, + absentSiteQuery: truth.absentSiteQuery, + knownSiteUrls: truth.knownSiteUrls, + }, + }, + stateChange: { + pass: true, + evidence: 'Not applicable. The nonexistent-site scenario is read-only.', + }, + faithfulFinalAnswer: { + pass: answerMatches && avoidsKnownPlugins, + evidence: { + finalText: collected.finalText, + siteAbsenceMatched: answerMatches, + avoidedKnownPluginNames: avoidsKnownPlugins, + }, + }, + }; + return { evaluation }; + }, + }, + { + id: 'agent-tags', + target: 'live', + task: () => 'How many tags exist on my dashboard and what are their names?', + expectedTools: ['list_tags_v1'], + groundTruth: async verifier => { + const tags = await verifierListAll(verifier, 'mainwp/list-tags-v1'); + return { count: tags.length, tagNames: tags.map(tag => tag.name).sort() }; + }, + }, + { + id: 'agent-theme-chain', + target: 'live', + task: () => 'Which theme is active on the site that has plugin updates pending?', + expectedTools: ['list_updates_v1', 'get_site_plugins_v1', 'get_site_themes_v1'], + groundTruth: async verifier => { + const sites = await verifier.listSites(); + const updateSiteUrls = await pluginUpdateSiteUrls(verifier, sites); + if (updateSiteUrls.length !== 1) { + throw new Error( + `Expected exactly one site with pending plugin updates, found ${updateSiteUrls.length}` + ); + } + const site = sites.find(candidate => candidate.url === updateSiteUrls[0]); + if (!site) throw new Error('The update-pending site was absent from the site inventory'); + const themes = (await verifier.execute('mainwp/get-site-themes-v1', { + site_id_or_domain: site.id, + })) as AgentThemeResponse; + const activeTheme = themes.themes.find( + theme => theme.active || theme.slug === themes.active_theme + ); + return { + siteId: site.id, + siteUrl: site.url, + siteName: site.name, + chainSiteUrl: site.url, + activeTheme: themes.active_theme, + ...(activeTheme?.name ? { activeThemeName: activeTheme.name } : {}), + }; + }, + evaluate: async (truth, collected, verifier) => { + if ( + truth.siteId === undefined || + !truth.siteUrl || + !truth.siteName || + !truth.chainSiteUrl || + !truth.activeTheme + ) { + throw new Error('Theme-chain ground truth was incomplete'); + } + const updateUses = collected.toolUses.filter(tool => + toolFamilyMatches(tool.name, ['list_updates_v1', 'get_site_plugins_v1']) + ); + const themeUses = collected.toolUses.filter(tool => + toolFamilyMatches(tool.name, ['get_site_themes_v1']) + ); + const themeTargeted = themeUses.some(tool => + toolInputTargetsSite(tool.input, truth.siteId as number, truth.siteUrl as string) + ); + const updateResults = toolResultsForUses(updateUses, collected.toolResults); + const themeResults = toolResultsForUses(themeUses, collected.toolResults); + const relevantResults = [...updateResults, ...themeResults]; + const resultText = flattenStrings(relevantResults).join('\n').toLowerCase(); + const resultsMatch = + resultText.includes(hostnameOf(truth.chainSiteUrl).toLowerCase()) && + [truth.activeTheme, truth.activeThemeName] + .filter((value): value is string => Boolean(value)) + .some(value => resultText.includes(value.toLowerCase())); + const afterUpdateSiteUrls = await pluginUpdateSiteUrls(verifier); + const oracleStable = + afterUpdateSiteUrls.length === 1 && afterUpdateSiteUrls[0] === truth.chainSiteUrl; + const finalText = collected.finalText.toLowerCase(); + const finalNamesTheme = [truth.activeTheme, truth.activeThemeName] + .filter((value): value is string => Boolean(value)) + .some(value => finalText.includes(value.toLowerCase())); + const finalNamesSite = + finalText.includes(hostnameOf(truth.chainSiteUrl).toLowerCase()) || + finalText.includes(truth.siteName.toLowerCase()); + const evaluation: AgentEvaluation = { + understoodRequest: { + pass: collected.finalText.trim().length > 0, + evidence: collected.finalText, + }, + rightCapability: { + pass: updateUses.length > 0 && themeUses.length > 0, + evidence: collected.toolUses.map(tool => tool.name), + }, + rightArguments: { + pass: + updateUses.length > 0 && + themeUses.length > 0 && + [...updateUses, ...themeUses].every( + tool => tool.input !== null && typeof tool.input === 'object' + ) && + themeTargeted, + evidence: { + updateInputs: updateUses.map(tool => tool.input), + themeInputs: themeUses.map(tool => tool.input), + expectedSiteId: truth.siteId, + expectedSiteUrl: truth.siteUrl, + }, + }, + correctMcpResult: { + pass: + updateResults.length > 0 && + themeResults.length > 0 && + relevantResults.every(result => !result.isError) && + resultsMatch, + evidence: { + updateResultCount: updateResults.length, + themeResultCount: themeResults.length, + groundTruthMatched: resultsMatch, + }, + }, + stateChange: { + pass: oracleStable, + evidence: { + beforeUpdateSiteUrls: [truth.chainSiteUrl], + afterUpdateSiteUrls, + }, + }, + faithfulFinalAnswer: { + pass: finalNamesTheme && finalNamesSite, + evidence: { + finalText: collected.finalText, + activeTheme: truth.activeTheme, + activeThemeName: truth.activeThemeName, + siteUrl: truth.chainSiteUrl, + siteName: truth.siteName, + }, + }, + }; + return { + evaluation, + ...(!oracleStable + ? { reason: 'The pending plugin update inventory changed during the agent run.' } + : {}), + }; + }, + }, + { + id: 'agent-confirm-delete-site', + target: 'fixture', + task: truth => + `Delete the MainWP site named ${truth.targetSiteName} at ${truth.targetSiteUrl} (site ID ${truth.targetSiteId}). This deletion is explicitly authorized. Proceed through any confirmation step needed to complete it, then report the outcome.`, + expectedTools: ['delete_site_v1'], + groundTruth: async verifier => { + const sites = await verifier.listSites(); + const target = sites.find(site => site.id === 1) ?? sites[0]; + if (!target) throw new Error('No fixture site was available for agent confirmation'); + return { + beforeSiteCount: sites.length, + targetSiteId: target.id, + targetSiteUrl: target.url, + targetSiteName: target.name, + }; + }, + evaluate: async (truth, collected, verifier) => { + if ( + truth.beforeSiteCount === undefined || + truth.targetSiteId === undefined || + !truth.targetSiteName + ) { + throw new Error('Fixture confirmation ground truth was incomplete'); + } + const transcript = evaluateConfirmationTranscript( + collected.toolUses, + collected.toolResults, + truth.targetSiteId + ); + const after = await verifier.listSites(); + const deleteUses = collected.toolUses.filter(tool => + toolFamilyMatches(tool.name, ['delete_site_v1']) + ); + const finalText = collected.finalText.toLowerCase(); + const evaluation: AgentEvaluation = { + understoodRequest: { + pass: collected.finalText.trim().length > 0, + evidence: collected.finalText, + }, + rightCapability: { + pass: deleteUses.length >= 2, + evidence: deleteUses.map(tool => tool.name), + }, + rightArguments: { + pass: transcript.pass, + evidence: { + targetSiteId: truth.targetSiteId, + confirmationToken: transcript.confirmationToken, + previewCallId: transcript.previewCallId, + confirmedCallId: transcript.confirmedCallId, + }, + }, + correctMcpResult: { + pass: transcript.pass, + evidence: transcript, + }, + stateChange: { + pass: + after.length === truth.beforeSiteCount - 1 && + !after.some(site => site.id === truth.targetSiteId), + evidence: { + beforeCount: truth.beforeSiteCount, + afterCount: after.length, + targetStillPresent: after.some(site => site.id === truth.targetSiteId), + }, + }, + faithfulFinalAnswer: { + pass: + finalText.includes(truth.targetSiteName.toLowerCase()) && + /\b(deleted|removed)\b/.test(finalText), + evidence: collected.finalText, + }, + }; + return { evaluation, ...(transcript.reason ? { reason: transcript.reason } : {}) }; + }, + }, + { + id: 'agent-safemode-refusal', + target: 'fixture', + serverEnv: { MAINWP_SAFE_MODE: 'true' }, + task: truth => + `Attempt to delete the MainWP site named ${truth.targetSiteName} at ${truth.targetSiteUrl} (site ID ${truth.targetSiteId}). This deletion is explicitly authorized. Make the deletion request even if a safety policy may block it, then report the outcome.`, + expectedTools: ['delete_site_v1'], + groundTruth: async verifier => { + const sites = await verifier.listSites(); + const target = sites.find(site => site.id === 1) ?? sites[0]; + if (!target) throw new Error('No fixture site was available for the safe-mode scenario'); + return { + beforeSiteCount: sites.length, + targetSiteId: target.id, + targetSiteUrl: target.url, + targetSiteName: target.name, + }; + }, + evaluate: async (truth, collected, verifier) => { + if (truth.beforeSiteCount === undefined || truth.targetSiteId === undefined) { + throw new Error('Safe-mode ground truth was incomplete'); + } + const after = await verifier.listSites(); + return evaluateSafeModeRefusal({ + toolUses: collected.toolUses, + toolResults: collected.toolResults, + finalText: collected.finalText, + beforeSiteCount: truth.beforeSiteCount, + afterSiteIds: after.map(site => site.id), + targetSiteId: truth.targetSiteId, + }); + }, + }, + { + id: 'agent-site-status', + target: 'live', + task: () => 'Are any of my sites down right now?', + expectedTools: ['check_sites_v1', 'check_site_v1'], + groundTruth: async verifier => { + const sites = await verifier.listSites(); + const offlineSiteUrls: string[] = []; + for (const site of sites) { + const status = (await verifier.execute('mainwp/check-site-v1', { + site_id_or_domain: site.id, + })) as AgentCheckSiteResponse; + if (!status.checked) throw new Error(`The direct check did not complete for ${site.url}`); + if (status.status?.online !== true) offlineSiteUrls.push(site.url); + } + return { + offlineSiteUrls: offlineSiteUrls.sort(), + allSiteUrls: sites.map(site => site.url).sort(), + }; + }, + evaluate: async (truth, collected) => { + if (!truth.offlineSiteUrls || !truth.allSiteUrls) { + throw new Error('Site-status ground truth was incomplete'); + } + const bulkUses = collected.toolUses.filter(tool => + toolFamilyMatches(tool.name, ['check_sites_v1']) + ); + const singleUses = collected.toolUses.filter(tool => + toolFamilyMatches(tool.name, ['check_site_v1']) + ); + const bulkCoverage = bulkUses.some(tool => { + const bulkResultText = flattenStrings(toolResultsForUses([tool], collected.toolResults)) + .join('\n') + .toLowerCase(); + return ( + bulkCheckCoversAllSites(tool.input, truth.allSiteUrls as string[]) || + truth.allSiteUrls?.every(url => bulkResultText.includes(hostnameOf(url).toLowerCase())) + ); + }); + const singleCoverage = truth.allSiteUrls.every(siteUrl => + singleUses.some(tool => { + if (toolInputTargetsSite(tool.input, undefined, siteUrl)) return true; + const resultText = flattenStrings(toolResultsForUses([tool], collected.toolResults)) + .join('\n') + .toLowerCase(); + return resultText.includes(hostnameOf(siteUrl).toLowerCase()); + }) + ); + const relevantUses = [...bulkUses, ...singleUses]; + const relevantResults = toolResultsForUses(relevantUses, collected.toolResults); + const resultText = flattenStrings(relevantResults).join('\n').toLowerCase(); + const offlineResultsMatch = truth.offlineSiteUrls.every(url => + resultText.includes(hostnameOf(url).toLowerCase()) + ); + const fullCoverage = bulkCoverage || singleCoverage; + const evaluation: AgentEvaluation = { + understoodRequest: { + pass: collected.finalText.trim().length > 0, + evidence: collected.finalText, + }, + rightCapability: { + pass: relevantUses.length > 0, + evidence: relevantUses.map(tool => tool.name), + }, + rightArguments: { + pass: relevantUses.length > 0 && fullCoverage, + evidence: { + bulkInputs: bulkUses.map(tool => tool.input), + singleInputs: singleUses.map(tool => tool.input), + allSiteUrls: truth.allSiteUrls, + fullCoverage, + }, + }, + correctMcpResult: { + pass: + relevantResults.length > 0 && + relevantResults.every(result => !result.isError) && + offlineResultsMatch, + evidence: { + resultCount: relevantResults.length, + offlineSiteUrls: truth.offlineSiteUrls, + offlineResultsMatch, + }, + }, + stateChange: { + pass: true, + evidence: 'Not applicable. The site-status scenario is read-only.', + }, + faithfulFinalAnswer: { + pass: matchesSiteStatusAnswer(collected.finalText, truth.offlineSiteUrls), + evidence: { + finalText: collected.finalText, + offlineSiteUrls: truth.offlineSiteUrls, + }, + }, + }; + return { evaluation }; + }, + }, +]; + +function parseArgs(args: string[]): { + scenarioIds: string[]; + list: boolean; + keepConsumer: boolean; +} { + const options = { scenarioIds: [] as string[], list: false, keepConsumer: false }; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === '--scenario') { + const id = args[index + 1]; + if (!id || id.startsWith('--')) throw new Error('--scenario requires an ID'); + options.scenarioIds.push(id); + index += 1; + } else if (arg === '--list') { + options.list = true; + } else if (arg === '--keep-consumer') { + options.keepConsumer = true; + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + return options; +} + +function shellDisplay(argv: string[]): string { + return argv.map(value => (/[\s*]/.test(value) ? JSON.stringify(value) : value)).join(' '); +} + +function contentBlocks(event: unknown): unknown[] { + if (!event || typeof event !== 'object') return []; + const record = event as Record; + const message = record.message; + if (message && typeof message === 'object') { + const content = (message as Record).content; + if (Array.isArray(content)) return content; + } + return []; +} + +function collectEvent(event: unknown, accumulator: CollectedAgentOutput): void { + if (!event || typeof event !== 'object') return; + const record = event as Record; + if (typeof record.model === 'string') accumulator.model = record.model; + if (record.message && typeof record.message === 'object') { + const model = (record.message as Record).model; + if (typeof model === 'string') accumulator.model = model; + } + for (const block of contentBlocks(event)) { + if (!block || typeof block !== 'object') continue; + const content = block as Record; + if (content.type === 'tool_use' && typeof content.name === 'string') { + if (content.name.startsWith('mcp__mainwp__')) { + accumulator.toolUses.push({ + ...(typeof content.id === 'string' ? { id: content.id } : {}), + name: content.name, + input: content.input, + }); + } + } else if (content.type === 'tool_result') { + accumulator.toolResults.push({ + ...(typeof content.tool_use_id === 'string' ? { toolUseId: content.tool_use_id } : {}), + content: content.content, + ...((content.is_error === true || content.isError === true) && { isError: true }), + }); + } else if (content.type === 'text' && typeof content.text === 'string') { + accumulator.finalText = content.text; + } + } + if (record.type === 'result' && typeof record.result === 'string') { + accumulator.finalText = record.result; + } +} + +function toolFamilyMatches(toolName: string, expected: string[]): boolean { + return expected.some( + family => toolName === `mcp__mainwp__${family}` || toolName.endsWith(family) + ); +} + +function toolInputTargetsSite( + input: unknown, + siteId: number | undefined, + siteUrl: string +): boolean { + if (!input || typeof input !== 'object') return false; + const target = (input as Record).site_id_or_domain; + if (target === undefined || target === null) return false; + const normalizedTarget = String(target).toLowerCase(); + return ( + (siteId !== undefined && normalizedTarget === String(siteId)) || + normalizedTarget === siteUrl.toLowerCase() || + normalizedTarget === hostnameOf(siteUrl).toLowerCase() + ); +} + +function bulkCheckCoversAllSites(input: unknown, allSiteUrls: string[]): boolean { + if (input === null || input === undefined) return true; + if (typeof input !== 'object') return false; + const targets = (input as Record).site_ids_or_domains; + if (targets === undefined) return true; + if (!Array.isArray(targets)) return false; + if (targets.length === 0) return true; + const normalizedTargets = targets.map(target => String(target).toLowerCase()); + return allSiteUrls.every( + siteUrl => + normalizedTargets.includes(siteUrl.toLowerCase()) || + normalizedTargets.includes(hostnameOf(siteUrl).toLowerCase()) + ); +} + +function toolResultsForUses( + toolUses: RecordedAgentToolUse[], + toolResults: RecordedAgentToolResult[] +): RecordedAgentToolResult[] { + const callIds = new Set( + toolUses.map(toolUse => toolUse.id).filter((id): id is string => Boolean(id)) + ); + return toolResults.filter(result => Boolean(result.toolUseId && callIds.has(result.toolUseId))); +} + +function finalAnswerMatches(truth: AgentGroundTruth, text: string): boolean { + if (truth.count !== undefined || truth.tagNames) { + const countMatches = + truth.count === undefined || + [...text.matchAll(/\b\d+\b/g)].some(match => Number(match[0]) === truth.count); + const lower = text.toLowerCase(); + const tagsMatch = + !truth.tagNames || truth.tagNames.every(name => lower.includes(name.toLowerCase())); + return countMatches && tagsMatch; + } + if (truth.updateSiteUrls) { + if (truth.updateSiteUrls.length === 0) return /\b(no|none|zero|0)\b/i.test(text); + // Agents commonly name sites by hostname ("site-two.example") rather than + // full URL, so match on hostnames. + const lower = text.toLowerCase(); + return truth.updateSiteUrls.every(url => lower.includes(hostnameOf(url).toLowerCase())); + } + if (truth.pluginActive !== undefined) { + const answer = text.match(/\b(yes|no)\b/i)?.[1]?.toLowerCase(); + return ( + answer === (truth.pluginActive ? 'yes' : 'no') && + Boolean(truth.siteName && text.toLowerCase().includes(truth.siteName.toLowerCase())) + ); + } + return false; +} + +function flattenStrings(value: unknown): string[] { + if (typeof value === 'string') return [value]; + if (Array.isArray(value)) return value.flatMap(flattenStrings); + if (value && typeof value === 'object') { + return Object.values(value as Record).flatMap(flattenStrings); + } + return []; +} + +function mcpResultsMatchTruth(truth: AgentGroundTruth, toolResults: unknown[]): boolean { + const text = `${flattenStrings(toolResults).join('\n')}\n${JSON.stringify(toolResults)}`; + if (truth.count !== undefined || truth.tagNames) { + const countMatches = + truth.count === undefined || new RegExp(`"total"\\s*:\\s*${truth.count}(?:\\D|$)`).test(text); + const lower = text.toLowerCase(); + const tagsMatch = + !truth.tagNames || truth.tagNames.every(name => lower.includes(name.toLowerCase())); + return countMatches && tagsMatch; + } + if (truth.updateSiteUrls) { + return truth.updateSiteUrls.every(url => text.includes(url)); + } + if (truth.pluginActive !== undefined && truth.pluginSlug) { + return toolResults + .flatMap(findNestedObjects) + .some(record => record.slug === truth.pluginSlug && record.active === truth.pluginActive); + } + return false; +} + +function evaluate( + scenario: AgentScenario, + truth: AgentGroundTruth, + collected: Pick +): AgentEvaluation { + const appropriate = collected.toolUses.filter(tool => + toolFamilyMatches(tool.name, scenario.expectedTools) + ); + const rightArguments = appropriate.every( + tool => tool.input !== null && typeof tool.input === 'object' + ); + const hasTargetArgument = + truth.siteId === undefined || + appropriate.some(tool => { + const serialized = JSON.stringify(tool.input); + return ( + serialized.includes(String(truth.siteId)) || + Boolean(truth.siteUrl && serialized.includes(truth.siteUrl)) + ); + }); + const resultErrors = collected.toolResults.filter(result => + JSON.stringify(result).match(/"is_error"\s*:\s*true|"isError"\s*:\s*true/) + ); + const resultsMatch = mcpResultsMatchTruth(truth, collected.toolResults); + return { + understoodRequest: { + pass: collected.finalText.trim().length > 0, + evidence: collected.finalText, + }, + rightCapability: { + pass: appropriate.length > 0, + evidence: collected.toolUses.map(tool => tool.name), + }, + rightArguments: { + pass: appropriate.length > 0 && rightArguments && hasTargetArgument, + evidence: appropriate.map(tool => tool.input), + }, + correctMcpResult: { + pass: collected.toolResults.length > 0 && resultErrors.length === 0 && resultsMatch, + evidence: { + resultCount: collected.toolResults.length, + errorCount: resultErrors.length, + groundTruthMatched: resultsMatch, + }, + }, + stateChange: { + pass: true, + evidence: 'Not applicable. Agent scenarios are read-only.', + }, + faithfulFinalAnswer: { + pass: finalAnswerMatches(truth, collected.finalText), + evidence: { truth, finalText: collected.finalText }, + }, + }; +} + +async function runClaude( + argv: string[], + cwd: string, + env: NodeJS.ProcessEnv, + onLine: (line: string) => void +): Promise<{ + exitCode: number; + stdout: string; + stderr: string; + durationMs: number; + timedOut: boolean; +}> { + const started = performance.now(); + // detached: own process group, so a timeout kill takes the CLI's children + // (MCP servers) down with it instead of leaving them holding the pipes. + const child = spawn(argv[0], argv.slice(1), { + cwd, + env, + shell: false, + stdio: ['ignore', 'pipe', 'pipe'], + detached: true, + }); + const stdoutChunks: Buffer[] = []; + const stderrChunks: Buffer[] = []; + let pending = ''; + child.stdout.on('data', chunk => { + const buffer = Buffer.from(chunk); + stdoutChunks.push(buffer); + pending += buffer.toString('utf8'); + const lines = pending.split(/\r?\n/); + pending = lines.pop() ?? ''; + for (const line of lines) if (line.trim()) onLine(line); + }); + child.stderr.on('data', chunk => stderrChunks.push(Buffer.from(chunk))); + const { exitCode, timedOut, spawnError } = await awaitChildWithDeadline(child, 300_000); + if (spawnError) throw spawnError; + if (pending.trim()) onLine(pending); + return { + exitCode, + stdout: Buffer.concat(stdoutChunks).toString('utf8'), + stderr: Buffer.concat(stderrChunks).toString('utf8'), + durationMs: Math.round(performance.now() - started), + timedOut, + }; +} + +async function main(): Promise { + const options = parseArgs(process.argv.slice(2)); + if (options.list) { + for (const scenario of agentScenarios) process.stdout.write(`${scenario.id}\n`); + return; + } + const byId = new Map(agentScenarios.map(scenario => [scenario.id, scenario])); + const selected = + options.scenarioIds.length > 0 + ? options.scenarioIds.map(id => { + const scenario = byId.get(id); + if (!scenario) throw new Error(`Unknown agent scenario: ${id}`); + return scenario; + }) + : agentScenarios; + const needsLive = selected.some(scenario => scenario.target === 'live'); + const needsFixture = selected.some(scenario => scenario.target === 'fixture'); + const liveCredentials = needsLive ? resolveAcceptanceCredentials() : undefined; + let fixture: FixtureDashboard | undefined; + let fixtureCredentials: AcceptanceCredentials | undefined; + if (needsFixture) { + fixture = await startFixtureDashboard(); + fixtureCredentials = { + dashboardUrl: fixture.url, + username: FIXTURE_USERNAME, + appPassword: FIXTURE_APP_PASSWORD, + }; + } + const credentialsByTarget = new Map<'live' | 'fixture', AcceptanceCredentials>(); + if (liveCredentials) credentialsByTarget.set('live', liveCredentials); + if (fixtureCredentials) credentialsByTarget.set('fixture', fixtureCredentials); + const firstCredentials = liveCredentials ?? fixtureCredentials; + if (!firstCredentials) throw new Error('No agent acceptance target was selected'); + const authorization = `Basic ${Buffer.from( + `${firstCredentials.username}:${firstCredentials.appPassword}` + ).toString('base64')}`; + const redactor = new Redactor({ ...firstCredentials, authorization }); + if (fixtureCredentials && fixtureCredentials !== firstCredentials) { + redactor.add({ + ...fixtureCredentials, + authorization: `Basic ${Buffer.from( + `${fixtureCredentials.username}:${fixtureCredentials.appPassword}` + ).toString('base64')}`, + }); + } + const runner = new CommandRunner(); + const artifactTarget = needsLive && needsFixture ? 'mixed' : needsFixture ? 'fixture' : 'live'; + const artifacts = await createArtifacts( + REPO_ROOT, + redactor, + runner, + 'packed', + artifactTarget, + { agent: true, scenarios: options.scenarioIds, keepConsumer: options.keepConsumer }, + '-agent' + ); + const verifiers = new Map<'live' | 'fixture', IndependentVerifier>(); + if (liveCredentials) { + verifiers.set( + 'live', + new IndependentVerifier( + liveCredentials, + process.env.MAINWP_MCP_ACCEPTANCE_SKIP_SSL_VERIFY === 'true' + ) + ); + } + if (fixtureCredentials) { + verifiers.set('fixture', new IndependentVerifier(fixtureCredentials, false)); + } + const results: AgentResult[] = []; + let packed; + try { + packed = await packAndInstall(REPO_ROOT, runner, artifacts, options.keepConsumer); + const configPath = path.join(packed.tempRoot, 'claude-mcp.json'); + const which = await runner.run(['which', 'claude'], REPO_ROOT, { allowFailure: true }); + const claudeAvailable = which.exitCode === 0; + + for (const scenario of selected) { + const credentials = credentialsByTarget.get(scenario.target); + const verifier = verifiers.get(scenario.target); + if (!credentials || !verifier) { + throw new Error(`No ${scenario.target} credentials or verifier were prepared`); + } + fs.writeFileSync( + configPath, + `${JSON.stringify( + { + mcpServers: { + mainwp: { + command: 'node', + args: [packed.installedEntry], + env: { + MAINWP_URL: '${MAINWP_URL}', + MAINWP_USER: '${MAINWP_USER}', + MAINWP_APP_PASSWORD: '${MAINWP_APP_PASSWORD}', + MAINWP_SKIP_SSL_VERIFY: '${MAINWP_SKIP_SSL_VERIFY}', + MAINWP_ALLOW_HTTP: '${MAINWP_ALLOW_HTTP}', + MAINWP_RATE_LIMIT: '0', + ...scenario.serverEnv, + }, + }, + }, + }, + null, + 2 + )}\n`, + { mode: 0o600 } + ); + let truth: AgentGroundTruth; + try { + truth = await scenario.groundTruth(verifier); + } catch (error) { + results.push({ + id: scenario.id, + status: 'unverified', + toolUses: [], + toolResults: [], + finalText: '', + reason: `Independent verifier precondition failed: ${error instanceof Error ? error.message : String(error)}`, + }); + continue; + } + const task = scenario.task(truth); + const argv = [ + 'claude', + '-p', + task, + '--mcp-config', + configPath, + '--strict-mcp-config', + '--allowedTools', + 'mcp__mainwp__*', + '--output-format', + 'stream-json', + '--verbose', + '--max-turns', + '20', + ]; + if (!claudeAvailable) { + results.push({ + id: scenario.id, + status: 'unverified', + toolUses: [], + toolResults: [], + finalText: '', + groundTruth: truth, + reason: `Blocked command: ${shellDisplay(argv)}. The claude CLI was not found.`, + }); + continue; + } + const collected: CollectedAgentOutput = { + toolUses: [], + toolResults: [], + finalText: '', + }; + const command = await runClaude( + argv, + REPO_ROOT, + { + ...process.env, + MAINWP_URL: credentials.dashboardUrl, + MAINWP_USER: credentials.username, + MAINWP_APP_PASSWORD: credentials.appPassword, + MAINWP_SKIP_SSL_VERIFY: + scenario.target === 'live' && + process.env.MAINWP_MCP_ACCEPTANCE_SKIP_SSL_VERIFY === 'true' + ? 'true' + : 'false', + MAINWP_ALLOW_HTTP: scenario.target === 'fixture' ? 'true' : 'false', + }, + line => { + try { + const event = JSON.parse(line) as unknown; + artifacts.appendJsonLine('agent-transcript.jsonl', { scenario: scenario.id, event }); + collectEvent(event, collected); + } catch { + artifacts.appendJsonLine('agent-transcript.jsonl', { + scenario: scenario.id, + unparsed: line, + }); + } + } + ); + runner.record({ + argv, + cwd: REPO_ROOT, + exitCode: command.exitCode, + durationMs: command.durationMs, + stdoutTail: command.stdout.slice(-12_000), + stderrTail: command.stderr.slice(-12_000), + ...(command.timedOut ? { timedOut: true } : {}), + }); + if (command.exitCode !== 0) { + results.push({ + id: scenario.id, + status: 'unverified', + model: collected.model, + toolUses: collected.toolUses, + toolResults: collected.toolResults, + finalText: collected.finalText, + groundTruth: truth, + reason: `Blocked command: ${shellDisplay(argv)}. Exit ${command.exitCode}: ${command.stderr.slice(-2000)}`, + }); + continue; + } + const evaluated = scenario.evaluate + ? await scenario.evaluate(truth, collected, verifier) + : { evaluation: evaluate(scenario, truth, collected) }; + const evaluation = evaluated.evaluation; + const pass = Object.values(evaluation).every(field => field.pass); + results.push({ + id: scenario.id, + status: pass ? 'passed' : 'failed', + model: collected.model, + toolUses: collected.toolUses, + toolResults: collected.toolResults, + finalText: collected.finalText, + groundTruth: truth, + evaluation, + ...(!pass && evaluated.reason ? { reason: evaluated.reason } : {}), + }); + } + artifacts.writeJson('results.json', { scenarios: results }); + artifacts.write( + 'summary.md', + `# MainWP MCP agent acceptance results\n\n${results + .map( + result => + `- ${result.status.toUpperCase()} ${result.id}${result.reason ? `: ${result.reason}` : ''}` + ) + .join('\n')}\n` + ); + } finally { + artifacts.finish(); + await Promise.all([...verifiers.values()].map(verifier => verifier.close())); + await fixture?.close(); + packed?.cleanup(); + } + for (const result of results) + process.stdout.write(`${result.status.toUpperCase()} ${result.id}\n`); + process.stdout.write(`Artifacts: ${artifacts.runDir}\n`); + if (results.some(result => result.status === 'failed')) process.exitCode = 1; +} + +main().catch(error => { + process.stderr.write( + `${error instanceof Error ? (error.stack ?? error.message) : String(error)}\n` + ); + process.exitCode = 1; +}); diff --git a/tests/acceptance/fixture-dashboard.ts b/tests/acceptance/fixture-dashboard.ts new file mode 100644 index 0000000..6e8ca5f --- /dev/null +++ b/tests/acceptance/fixture-dashboard.ts @@ -0,0 +1,317 @@ +import fs from 'node:fs'; +import http, { type IncomingMessage, type ServerResponse } from 'node:http'; +import { fileURLToPath } from 'node:url'; + +export const FIXTURE_USERNAME = 'fixture-user'; +export const FIXTURE_APP_PASSWORD = 'fixture app password'; +export const FIXTURE_OVERSIZED_SEARCH = '__mainwp_acceptance_oversized_response__'; +export const FIXTURE_DELAY_SEARCH = '__mainwp_acceptance_delayed_response__'; + +const FIXTURE_OVERSIZED_BYTES = 256 * 1024; +const FIXTURE_DELAY_MS = 750; + +interface FixturePlugin { + slug: string; + name: string; + version: string; + active: boolean; + update_version: string | null; +} + +interface FixtureSite { + id: number; + url: string; + name: string; + status: string; + client_id: number | null; + wp_version: string; + php_version: string; + last_sync: string; + admin_username: string; + child_version: string; + notes: string; + plugins: FixturePlugin[]; +} + +export interface FixtureDashboard { + url: string; + close(): Promise; +} + +const ABILITIES_PATH = fileURLToPath( + new URL('../evals/fixtures/abilities-full.json', import.meta.url) +); +const SITES_PATH = fileURLToPath(new URL('./fixtures/sites.json', import.meta.url)); +const API_PREFIX = '/wp-json/wp-abilities/v1'; + +function json(response: ServerResponse, status: number, body: unknown): void { + const encoded = JSON.stringify(body); + response.writeHead(status, { + 'content-type': 'application/json; charset=utf-8', + 'content-length': Buffer.byteLength(encoded), + }); + response.end(encoded); +} + +function parseScalar(value: string): string | number | boolean { + if (/^-?\d+$/.test(value)) return Number(value); + if (value === 'true') return true; + if (value === 'false') return false; + return value; +} + +function parseQueryInput(url: URL): Record { + const input: Record = {}; + for (const [rawKey, rawValue] of url.searchParams) { + const arrayMatch = rawKey.match(/^input\[([^\]]+)\]\[\]$/); + if (arrayMatch) { + const current = input[arrayMatch[1]]; + const values = Array.isArray(current) ? current : []; + values.push(parseScalar(rawValue)); + input[arrayMatch[1]] = values; + continue; + } + const scalarMatch = rawKey.match(/^input\[([^\]]+)\]$/); + if (scalarMatch) input[scalarMatch[1]] = parseScalar(rawValue); + } + return input; +} + +async function parseInput(request: IncomingMessage, url: URL): Promise> { + if (request.method === 'GET' || request.method === 'DELETE') return parseQueryInput(url); + const chunks: Buffer[] = []; + for await (const chunk of request) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + if (chunks.reduce((size, current) => size + current.length, 0) > 1024 * 1024) { + throw new Error('Fixture request body too large'); + } + } + if (chunks.length === 0) return {}; + const parsed = JSON.parse(Buffer.concat(chunks).toString('utf8')) as { input?: unknown }; + return parsed.input && typeof parsed.input === 'object' + ? (parsed.input as Record) + : {}; +} + +function publicSite(site: FixtureSite): Omit { + const { plugins: _plugins, ...siteData } = site; + return siteData; +} + +function findSite(sites: FixtureSite[], identifier: unknown): FixtureSite | undefined { + const value = String(identifier ?? '') + .replace(/\/+$/, '') + .toLowerCase(); + return sites.find(site => { + if (String(site.id) === value) return true; + const url = site.url.replace(/\/+$/, '').toLowerCase(); + return url === value || new URL(url).hostname === value; + }); +} + +function notFound(response: ServerResponse, message: string): void { + json(response, 404, { + code: 'mainwp_site_not_found', + message, + data: { status: 404 }, + }); +} + +export function getFixtureFaultMode( + abilityName: string, + input: Record +): 'oversized' | 'delay' | null { + if (abilityName !== 'mainwp/list-sites-v1') return null; + if (input.search === FIXTURE_OVERSIZED_SEARCH) return 'oversized'; + if (input.search === FIXTURE_DELAY_SEARCH) return 'delay'; + return null; +} + +async function runAbility( + abilityName: string, + input: Record, + sites: FixtureSite[], + response: ServerResponse +): Promise { + const faultMode = getFixtureFaultMode(abilityName, input); + if (faultMode === 'oversized') { + json(response, 200, { payload: 'x'.repeat(FIXTURE_OVERSIZED_BYTES) }); + return; + } + if (faultMode === 'delay') { + await new Promise(resolve => setTimeout(resolve, FIXTURE_DELAY_MS)); + } + + if (abilityName === 'mainwp/list-sites-v1') { + const page = typeof input.page === 'number' ? input.page : 1; + const perPage = typeof input.per_page === 'number' ? input.per_page : 20; + const status = typeof input.status === 'string' ? input.status : 'any'; + const search = typeof input.search === 'string' ? input.search.toLowerCase() : ''; + const filtered = sites.filter(site => { + const statusMatches = status === 'any' || site.status === status; + const searchMatches = + !search || + site.name.toLowerCase().includes(search) || + site.url.toLowerCase().includes(search); + return statusMatches && searchMatches; + }); + const start = (page - 1) * perPage; + json(response, 200, { + items: filtered.slice(start, start + perPage).map(publicSite), + page, + per_page: perPage, + total: filtered.length, + }); + return; + } + + if (abilityName === 'mainwp/count-sites-v1') { + const status = typeof input.status === 'string' ? input.status : null; + json(response, 200, { total: sites.filter(site => !status || site.status === status).length }); + return; + } + + if (abilityName === 'mainwp/get-site-v1') { + const site = findSite(sites, input.site_id_or_domain); + if (!site) return notFound(response, 'The requested MainWP site was not found.'); + json(response, 200, publicSite(site)); + return; + } + + if (abilityName === 'mainwp/get-site-plugins-v1') { + const site = findSite(sites, input.site_id_or_domain); + if (!site) return notFound(response, 'The requested MainWP site was not found.'); + const status = typeof input.status === 'string' ? input.status : 'all'; + const hasUpdate = input.has_update === true; + const plugins = site.plugins.filter(plugin => { + const statusMatches = + status === 'all' || + (status === 'active' ? plugin.active : status === 'inactive' && !plugin.active); + return statusMatches && (!hasUpdate || plugin.update_version !== null); + }); + json(response, 200, { + site_id: site.id, + site_url: site.url, + plugins, + total: plugins.length, + }); + return; + } + + if (abilityName === 'mainwp/check-site-v1') { + const site = findSite(sites, input.site_id_or_domain); + if (!site) return notFound(response, 'The requested MainWP site was not found.'); + const online = site.status === 'connected'; + const responseTime = 0.01; + json(response, 200, { + site_id: site.id, + response_time: responseTime, + checked: true, + site: { id: site.id, url: site.url, name: site.name }, + status: { online, http_code: online ? 200 : 503, response_time: responseTime }, + }); + return; + } + + if (abilityName === 'mainwp/delete-site-v1') { + const site = findSite(sites, input.site_id_or_domain); + if (!site) return notFound(response, 'The requested MainWP site was not found.'); + if (input.dry_run === true) { + json(response, 200, { + dry_run: true, + would_affect: publicSite(site), + warnings: ['Fixture preview only. No site was changed.'], + deleted: false, + }); + return; + } + if (input.confirm === true) { + sites.splice(sites.indexOf(site), 1); + json(response, 200, { + dry_run: false, + deleted: true, + site: publicSite(site), + }); + return; + } + json(response, 403, { + code: 'fixture_write_disabled', + message: 'The fixture dashboard requires confirm: true for site deletion.', + data: { status: 403 }, + }); + return; + } + + json(response, 404, { + code: 'rest_no_route', + message: `No route was found for ability ${abilityName}.`, + data: { status: 404 }, + }); +} + +export async function startFixtureDashboard(): Promise { + const abilities = JSON.parse(fs.readFileSync(ABILITIES_PATH, 'utf8')) as unknown[]; + const sites = JSON.parse(fs.readFileSync(SITES_PATH, 'utf8')) as FixtureSite[]; + const expectedAuthorization = `Basic ${Buffer.from( + `${FIXTURE_USERNAME}:${FIXTURE_APP_PASSWORD}` + ).toString('base64')}`; + + const server = http.createServer(async (request, response) => { + try { + if (request.headers.authorization !== expectedAuthorization) { + response.setHeader('www-authenticate', 'Basic realm="MainWP fixture"'); + json(response, 401, { + code: 'rest_not_logged_in', + message: 'You are not currently logged in.', + data: { status: 401 }, + }); + return; + } + + const url = new URL(request.url ?? '/', 'http://127.0.0.1'); + if (request.method === 'GET' && url.pathname === `${API_PREFIX}/abilities`) { + json(response, 200, abilities); + return; + } + + const prefix = `${API_PREFIX}/abilities/`; + const suffix = '/run'; + if (url.pathname.startsWith(prefix) && url.pathname.endsWith(suffix)) { + const abilityName = decodeURIComponent(url.pathname.slice(prefix.length, -suffix.length)); + const input = await parseInput(request, url); + await runAbility(abilityName, input, sites, response); + return; + } + + json(response, 404, { + code: 'rest_no_route', + message: 'No route was found matching the URL and request method.', + data: { status: 404 }, + }); + } catch (error) { + json(response, 500, { + code: 'fixture_internal_error', + message: error instanceof Error ? error.message : 'Fixture error', + data: { status: 500 }, + }); + } + }); + + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', () => resolve()); + }); + const address = server.address(); + if (!address || typeof address === 'string') { + server.close(); + throw new Error('Fixture dashboard did not bind to an IP socket'); + } + + return { + url: `http://127.0.0.1:${address.port}`, + close: () => + new Promise((resolve, reject) => { + server.close(error => (error ? reject(error) : resolve())); + }), + }; +} diff --git a/tests/acceptance/fixtures/sites.json b/tests/acceptance/fixtures/sites.json new file mode 100644 index 0000000..b8139a6 --- /dev/null +++ b/tests/acceptance/fixtures/sites.json @@ -0,0 +1,82 @@ +[ + { + "id": 1, + "url": "https://alpine.example.test", + "name": "Alpine Bakery", + "status": "connected", + "client_id": 101, + "wp_version": "6.8.1", + "php_version": "8.3.8", + "last_sync": "2026-07-15T12:00:00Z", + "admin_username": "alpine-admin", + "child_version": "5.4.0", + "notes": "Fixture site with active Hello Dolly", + "plugins": [ + { + "slug": "hello.php", + "name": "Hello Dolly", + "version": "1.7.2", + "active": true, + "update_version": null + }, + { + "slug": "akismet/akismet.php", + "name": "Akismet Anti-spam", + "version": "5.3.6", + "active": true, + "update_version": "5.3.7" + } + ] + }, + { + "id": 2, + "url": "https://beacon.example.test", + "name": "Beacon Studio", + "status": "connected", + "client_id": null, + "wp_version": "6.8.1", + "php_version": "8.2.20", + "last_sync": "2026-07-15T12:05:00Z", + "admin_username": "beacon-admin", + "child_version": "5.4.0", + "notes": "Fixture site with inactive Hello Dolly", + "plugins": [ + { + "slug": "hello-dolly/hello.php", + "name": "Hello Dolly", + "version": "1.7.2", + "active": false, + "update_version": null + }, + { + "slug": "seo-pack/seo.php", + "name": "SEO Pack", + "version": "2.0.0", + "active": true, + "update_version": null + } + ] + }, + { + "id": 3, + "url": "https://cedar.example.test", + "name": "Cedar Nonprofit", + "status": "disconnected", + "client_id": 102, + "wp_version": "6.7.2", + "php_version": "8.1.28", + "last_sync": "2026-07-14T09:30:00Z", + "admin_username": "cedar-admin", + "child_version": "5.3.1", + "notes": "Disconnected fixture site", + "plugins": [ + { + "slug": "forms/forms.php", + "name": "Fixture Forms", + "version": "3.1.0", + "active": true, + "update_version": "3.2.0" + } + ] + } +] diff --git a/tests/acceptance/lib/agent-confirmation.ts b/tests/acceptance/lib/agent-confirmation.ts new file mode 100644 index 0000000..cde2d7b --- /dev/null +++ b/tests/acceptance/lib/agent-confirmation.ts @@ -0,0 +1,135 @@ +export interface RecordedAgentToolUse { + id?: string; + name: string; + input: unknown; +} + +export interface RecordedAgentToolResult { + toolUseId?: string; + content: unknown; + isError?: boolean; +} + +export interface ConfirmationTranscriptEvaluation { + pass: boolean; + reason?: string; + confirmationToken?: string; + previewCallId?: string; + confirmedCallId?: string; +} + +function isDeleteSiteTool(name: string): boolean { + return name === 'mcp__mainwp__delete_site_v1' || name.endsWith('delete_site_v1'); +} + +function inputTargetsSite(input: unknown, targetSiteId: number): boolean { + if (!input || typeof input !== 'object') return false; + const target = (input as Record).site_id_or_domain; + return String(target) === String(targetSiteId); +} + +function findObject( + value: unknown, + predicate: (record: Record) => boolean +): Record | undefined { + if (typeof value === 'string') { + try { + return findObject(JSON.parse(value) as unknown, predicate); + } catch { + return undefined; + } + } + if (Array.isArray(value)) { + for (const item of value) { + const found = findObject(item, predicate); + if (found) return found; + } + return undefined; + } + if (!value || typeof value !== 'object') return undefined; + const record = value as Record; + if (predicate(record)) return record; + for (const nested of Object.values(record)) { + const found = findObject(nested, predicate); + if (found) return found; + } + return undefined; +} + +export function evaluateConfirmationTranscript( + toolUses: RecordedAgentToolUse[], + toolResults: RecordedAgentToolResult[], + targetSiteId: number +): ConfirmationTranscriptEvaluation { + const resultByCallId = new Map( + toolResults + .filter(result => result.toolUseId) + .map(result => [result.toolUseId as string, result]) + ); + const relevantUses = toolUses + .map((toolUse, index) => ({ toolUse, index })) + .filter( + ({ toolUse }) => + isDeleteSiteTool(toolUse.name) && inputTargetsSite(toolUse.input, targetSiteId) + ); + + let preview: + | { toolUse: RecordedAgentToolUse; index: number; confirmationToken: string } + | undefined; + for (const candidate of relevantUses) { + const result = candidate.toolUse.id ? resultByCallId.get(candidate.toolUse.id) : undefined; + if (!result || result.isError) continue; + const payload = findObject( + result.content, + record => + record.status === 'CONFIRMATION_REQUIRED' && typeof record.confirmation_token === 'string' + ); + if (payload && typeof payload.confirmation_token === 'string') { + preview = { ...candidate, confirmationToken: payload.confirmation_token }; + break; + } + } + + if (!preview) { + return { + pass: false, + reason: + 'The transcript did not contain a delete_site_v1 result with CONFIRMATION_REQUIRED and a confirmation token for the target site.', + }; + } + + const confirmed = relevantUses.find(({ toolUse, index }) => { + if (index <= preview.index || !toolUse.input || typeof toolUse.input !== 'object') return false; + const input = toolUse.input as Record; + return input.user_confirmed === true && input.confirmation_token === preview.confirmationToken; + }); + if (!confirmed) { + return { + pass: false, + reason: + 'The agent received a confirmation token but did not make a confirmed delete_site_v1 call with that token.', + confirmationToken: preview.confirmationToken, + previewCallId: preview.toolUse.id, + }; + } + + const confirmedResult = confirmed.toolUse.id + ? resultByCallId.get(confirmed.toolUse.id) + : undefined; + if (!confirmedResult) { + return { + pass: false, + reason: 'The confirmed delete_site_v1 call did not have a correlated tool result.', + confirmationToken: preview.confirmationToken, + previewCallId: preview.toolUse.id, + confirmedCallId: confirmed.toolUse.id, + }; + } + + return { + pass: true, + confirmationToken: preview.confirmationToken, + previewCallId: preview.toolUse.id, + confirmedCallId: confirmed.toolUse.id, + }; +} diff --git a/tests/acceptance/lib/agent-matchers.ts b/tests/acceptance/lib/agent-matchers.ts new file mode 100644 index 0000000..3282252 --- /dev/null +++ b/tests/acceptance/lib/agent-matchers.ts @@ -0,0 +1,310 @@ +import type { RecordedAgentToolResult, RecordedAgentToolUse } from './agent-confirmation.js'; + +export interface AgentEvaluationField { + pass: boolean; + evidence: unknown; +} + +export interface AgentEvaluation { + understoodRequest: AgentEvaluationField; + rightCapability: AgentEvaluationField; + rightArguments: AgentEvaluationField; + correctMcpResult: AgentEvaluationField; + stateChange: AgentEvaluationField; + faithfulFinalAnswer: AgentEvaluationField; +} + +export interface SafeModeRefusalInput { + toolUses: RecordedAgentToolUse[]; + toolResults: RecordedAgentToolResult[]; + finalText: string; + beforeSiteCount: number; + afterSiteIds: number[]; + targetSiteId: number; +} + +function isDeleteSiteTool(name: string): boolean { + return name === 'mcp__mainwp__delete_site_v1' || name.endsWith('delete_site_v1'); +} + +function inputTargetsSite(input: unknown, targetSiteId: number): boolean { + if (!input || typeof input !== 'object') return false; + const target = (input as Record).site_id_or_domain; + return String(target) === String(targetSiteId); +} + +function containsSafeModeBlocked(value: unknown): boolean { + if (typeof value === 'string') return value.includes('SAFE_MODE_BLOCKED'); + if (Array.isArray(value)) return value.some(containsSafeModeBlocked); + if (value && typeof value === 'object') { + return Object.values(value as Record).some(containsSafeModeBlocked); + } + return false; +} + +function hostnameOf(url: string): string { + try { + return new URL(url).hostname; + } catch { + return url; + } +} + +export function findNestedObjects(value: unknown): Record[] { + if (typeof value === 'string') { + try { + return findNestedObjects(JSON.parse(value) as unknown); + } catch { + return []; + } + } + if (Array.isArray(value)) return value.flatMap(findNestedObjects); + if (!value || typeof value !== 'object') return []; + const record = value as Record; + return [record, ...Object.values(record).flatMap(findNestedObjects)]; +} + +function normalizedSiteIdentifiers(value: unknown): string[] { + if (!value || typeof value !== 'object') return []; + const site = value as Record; + return [site.url, site.site_url, site.domain] + .filter((identifier): identifier is string => typeof identifier === 'string') + .flatMap(identifier => [identifier, hostnameOf(identifier)]) + .map(identifier => identifier.replace(/\/+$/, '').toLowerCase()); +} + +/** + * True when an error result names the upstream mainwp_site_not_found code. + * The server surfaces the upstream code inside the sanitized error message + * (the structured `code` field carries the numeric JSON-RPC code, -32002), + * so match the string anywhere in an error result rather than requiring a + * structured field the wire shape does not have. + */ +export function errorResultNamesSiteNotFound(results: RecordedAgentToolResult[]): boolean { + return results.some( + result => + result.isError === true && JSON.stringify(result.content).includes('mainwp_site_not_found') + ); +} + +/** + * True when a dashboard-side scoped search for the probe returned zero + * matches. The search term must be a meaningful fragment of the probe (5+ + * characters) so an unrelated or empty search cannot count as proof, and the + * correlated result must report an empty page with total 0 — the server + * itself asserting no site matches. + */ +export function scopedSearchProvesSiteAbsent( + uses: RecordedAgentToolUse[], + resultsForUse: (use: RecordedAgentToolUse) => RecordedAgentToolResult[], + absentSiteQuery: string +): boolean { + // Normalize to the hostname so a URL-shaped probe can never correlate with + // a generic scheme fragment like "https". + const probe = hostnameOf(absentSiteQuery).toLowerCase(); + return uses.some(use => { + if (!use.input || typeof use.input !== 'object') return false; + const search = (use.input as Record).search; + if (typeof search !== 'string') return false; + const term = hostnameOf(search.trim().toLowerCase()); + if (term.length < 5 || !probe.includes(term)) return false; + return resultsForUse(use).some(result => + findNestedObjects(result.content).some( + record => + result.isError !== true && + Array.isArray(record.items) && + record.items.length === 0 && + record.total === 0 + ) + ); + }); +} + +export function inventoryProvesSiteAbsent( + results: RecordedAgentToolResult[], + knownSiteUrls: string[], + absentSiteQuery: string +): boolean { + if (results.some(result => result.isError === true)) return false; + const pages = results + .flatMap(result => findNestedObjects(result.content)) + .filter( + (record): record is Record & { items: unknown[]; total: number } => + Array.isArray(record.items) && + typeof record.total === 'number' && + Number.isInteger(record.total) && + record.total >= 0 + ); + if (pages.length === 0 || !pages.some(page => page.total === knownSiteUrls.length)) return false; + + const inventory = new Set(pages.flatMap(page => page.items.flatMap(normalizedSiteIdentifiers))); + const knownSitesCovered = knownSiteUrls.every(url => + [url, hostnameOf(url)].some(identifier => + inventory.has(identifier.replace(/\/+$/, '').toLowerCase()) + ) + ); + return knownSitesCovered && !inventory.has(absentSiteQuery.toLowerCase()); +} + +/** + * Matches an answer that says the requested site or domain is absent from the + * dashboard. This deliberately does not treat a missing plugin list as a + * missing site. + */ +export function matchesNotFoundSiteAnswer(text: string): boolean { + const answer = text.toLowerCase().replace(/\s+/g, ' '); + if ( + /\b(?:the |that )?(?:site|domain|website)\s+(?:exists|is registered|is connected|was found)\b/.test( + answer + ) + ) { + return false; + } + + // Relaying the structured error code is faithful absence reporting + // regardless of surrounding phrasing (the guard above already rejected + // answers that claim the site exists). + if (answer.includes('mainwp_site_not_found')) { + return true; + } + + // Gaps allow ~90 chars so a long quoted hostname (37+ chars plus backticks + // and words like "named") fits between the subject and the verdict. + return [ + /\b(?:site|domain|website)\b.{0,90}\b(?:is not|isn't|was not|wasn't|not)\s+(?:found|registered|connected|known|present|available)\b/, + /\b(?:site|domain|website)\b.{0,90}\b(?:does not|doesn't)\s+exist\b/, + /\b(?:no|zero)\s+(?:matching\s+)?(?:site|domain|website)s?\b.{0,90}\b(?:found|registered|connected|exists?)\b/, + /\b(?:could not|couldn't|cannot|can't|unable to)\s+(?:find|locate|identify)\b.{0,90}\b(?:site|domain|website)\b/, + /\b(?:site|domain|website)\b.{0,90}\b(?:is not|isn't|was not|wasn't|not)\s+(?:in|on|among|part of|listed (?:in|on))\b.{0,60}\b(?:dashboard|mainwp|network|managed sites|site list)\b/, + /\bno\s+(?:matching\s+)?(?:site|domain|website)s?\s+(?:named|called|matching|like|by that name)\b/, + ].some(pattern => pattern.test(answer)); +} + +/** + * Rejects plugin names harvested from real sites. A wholly invented plugin + * name cannot be detected by this check because it is absent from the oracle. + */ +export function answerAvoidsKnownPluginNames(text: string, knownPluginNames: string[]): boolean { + const answer = text.toLowerCase(); + return knownPluginNames.every( + name => !name.trim() || !answer.includes(name.trim().toLowerCase()) + ); +} + +export function matchesSafeModeRefusalAnswer(text: string): boolean { + const answer = text.toLowerCase().replace(/\s+/g, ' '); + // Reject only assertions that safe mode IS off ("safe mode is disabled"), + // not remedy suggestions ("restart the server with safe mode off") or + // conditional remedies, which contain a copula ("once safe mode is off, + // re-run"). + if ( + /(? pattern.test(answer)); +} + +export function matchesSiteStatusAnswer(text: string, offlineSiteUrls: string[]): boolean { + const answer = text.toLowerCase().replace(/\s+/g, ' '); + if (offlineSiteUrls.length > 0) { + return offlineSiteUrls.every(url => answer.includes(hostnameOf(url).toLowerCase())); + } + + if ( + /\bnot all\b.{0,40}\b(?:sites?|websites?)\b.{0,30}\b(?:up|online|connected|reachable)\b/.test( + answer + ) || + /\b(?:one|some|a|[1-9]\d*)\b.{0,30}\b(?:sites?|websites?)\b.{0,30}\b(?:down|offline|unreachable|disconnected)\b/.test( + answer + ) + ) { + return false; + } + + return [ + /\b(?:none|no|zero)\b.{0,40}\b(?:sites?|websites?)\b.{0,30}\b(?:down|offline|unreachable|disconnected)\b/, + /\b(?:all|every)\b.{0,30}\b(?:sites?|website)\b.{0,30}\b(?:up|online|connected|reachable)\b/, + /\bnot any\b.{0,30}\b(?:sites?|websites?)\b.{0,30}\b(?:down|offline|unreachable|disconnected)\b/, + ].some(pattern => pattern.test(answer)); +} + +export function evaluateSafeModeRefusal({ + toolUses, + toolResults, + finalText, + beforeSiteCount, + afterSiteIds, + targetSiteId, +}: SafeModeRefusalInput): { evaluation: AgentEvaluation; reason?: string } { + const deleteUses = toolUses.filter(toolUse => isDeleteSiteTool(toolUse.name)); + const targetDeleteUses = deleteUses.filter(toolUse => + inputTargetsSite(toolUse.input, targetSiteId) + ); + const resultByCallId = new Map( + toolResults + .filter(result => result.toolUseId) + .map(result => [result.toolUseId as string, result]) + ); + const blockedCall = targetDeleteUses.find(toolUse => { + const result = toolUse.id ? resultByCallId.get(toolUse.id) : undefined; + return result?.isError === true && containsSafeModeBlocked(result.content); + }); + const stateUnchanged = + afterSiteIds.length === beforeSiteCount && afterSiteIds.includes(targetSiteId); + const evaluation: AgentEvaluation = { + understoodRequest: { + pass: finalText.trim().length > 0, + evidence: finalText, + }, + rightCapability: { + pass: deleteUses.length > 0, + evidence: deleteUses.map(toolUse => toolUse.name), + }, + rightArguments: { + pass: targetDeleteUses.length > 0, + evidence: targetDeleteUses.map(toolUse => toolUse.input), + }, + correctMcpResult: { + pass: Boolean(blockedCall), + evidence: { + targetSiteId, + blockedCallId: blockedCall?.id, + }, + }, + stateChange: { + pass: stateUnchanged, + evidence: { + beforeSiteCount, + afterSiteCount: afterSiteIds.length, + targetStillPresent: afterSiteIds.includes(targetSiteId), + }, + }, + faithfulFinalAnswer: { + pass: matchesSafeModeRefusalAnswer(finalText), + evidence: finalText, + }, + }; + + return { + evaluation, + ...(!blockedCall + ? { + reason: + 'The target delete_site_v1 call did not have a correlated SAFE_MODE_BLOCKED result.', + } + : {}), + }; +} diff --git a/tests/acceptance/lib/artifacts.ts b/tests/acceptance/lib/artifacts.ts new file mode 100644 index 0000000..32f6e9a --- /dev/null +++ b/tests/acceptance/lib/artifacts.ts @@ -0,0 +1,173 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import type { JSONRPCMessage } from '@modelcontextprotocol/sdk/types.js'; +import type { CommandRecord, CommandRunner } from './commands.js'; +import type { Redactor } from './redact.js'; + +export const HARNESS_VERSION = '1.0.0'; + +export interface TarballManifest { + filename: string; + sha256: string; + integrity: string; +} + +export interface AcceptanceManifest { + git: { + branch: string; + commit: string; + dirty: boolean; + diffSha256: string; + }; + packageVersion: string; + tarball: TarballManifest | null; + nodeVersion: string; + npmVersion: string; + os: string; + arch: string; + harnessVersion: string; + mode: string; + target: string; + flags: Record; + startTime: string; + endTime: string | null; +} + +interface EventRecord { + scenario: string; + direction: 'client-to-server' | 'server-to-client'; + timestamp: string; + monotonicMs: number; + message: JSONRPCMessage; +} + +export class Artifacts { + readonly runDir: string; + readonly manifest: AcceptanceManifest; + private readonly monotonicStart = performance.now(); + private readonly serverStderrBuffers = new Map(); + + constructor( + readonly repoRoot: string, + readonly runId: string, + private readonly redactor: Redactor, + manifest: AcceptanceManifest + ) { + this.runDir = path.join(repoRoot, 'test-results', 'acceptance', runId); + this.manifest = manifest; + fs.mkdirSync(this.runDir, { recursive: true }); + this.writeJson('manifest.json', manifest); + fs.writeFileSync(path.join(this.runDir, 'events.jsonl'), ''); + fs.writeFileSync(path.join(this.runDir, 'commands.jsonl'), ''); + } + + writeJson(filename: string, value: unknown): void { + this.write(filename, `${JSON.stringify(value, null, 2)}\n`); + } + + write(filename: string, value: string): void { + fs.writeFileSync(path.join(this.runDir, filename), this.redactor.redact(value), 'utf8'); + } + + appendJsonLine(filename: string, value: unknown): void { + fs.appendFileSync( + path.join(this.runDir, filename), + `${this.redactor.stringify(value)}\n`, + 'utf8' + ); + } + + appendEvent( + scenario: string, + direction: EventRecord['direction'], + message: JSONRPCMessage + ): void { + this.appendJsonLine('events.jsonl', { + scenario, + direction, + timestamp: new Date().toISOString(), + monotonicMs: Math.round((performance.now() - this.monotonicStart) * 1000) / 1000, + message, + } satisfies EventRecord); + } + + appendServerStderr(scenario: string, value: string): void { + const filename = `server-${scenario.replace(/[^a-z0-9_-]/gi, '_')}.stderr.log`; + const buffered = `${this.serverStderrBuffers.get(filename) ?? ''}${value}`; + const { output, remainder } = this.redactor.redactStream(buffered); + this.serverStderrBuffers.set(filename, remainder); + if (output) fs.appendFileSync(path.join(this.runDir, filename), output, 'utf8'); + } + + flushServerStderr(scenario: string): void { + const filename = `server-${scenario.replace(/[^a-z0-9_-]/gi, '_')}.stderr.log`; + const remainder = this.serverStderrBuffers.get(filename); + if (remainder) { + fs.appendFileSync(path.join(this.runDir, filename), this.redactor.redact(remainder), 'utf8'); + } + this.serverStderrBuffers.delete(filename); + } + + recordCommand(record: CommandRecord): void { + this.appendJsonLine('commands.jsonl', record); + } + + setTarball(tarball: TarballManifest): void { + this.manifest.tarball = tarball; + this.writeJson('manifest.json', this.manifest); + } + + finish(): void { + this.manifest.endTime = new Date().toISOString(); + this.writeJson('manifest.json', this.manifest); + } +} + +export async function createArtifacts( + repoRoot: string, + redactor: Redactor, + runner: CommandRunner, + mode: string, + target: string, + flags: Record, + suffix = '' +): Promise { + const branch = (await runner.run(['git', 'branch', '--show-current'], repoRoot)).stdout.trim(); + const commit = (await runner.run(['git', 'rev-parse', 'HEAD'], repoRoot)).stdout.trim(); + const status = (await runner.run(['git', 'status', '--porcelain'], repoRoot)).stdout; + const diff = (await runner.run(['git', 'diff', 'HEAD'], repoRoot)).stdout; + const npmVersion = (await runner.run(['npm', '--version'], repoRoot)).stdout.trim(); + const packageJson = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8')) as { + version: string; + }; + const startTime = new Date().toISOString(); + const timestamp = startTime.replace(/[-:.]/g, '').replace('Z', 'Z'); + const dirty = status.length > 0; + const runId = `${timestamp}-${commit.slice(0, 8)}${dirty ? '-dirty' : ''}${suffix}`; + const manifest: AcceptanceManifest = { + git: { + branch, + commit, + dirty, + diffSha256: crypto.createHash('sha256').update(diff).digest('hex'), + }, + packageVersion: packageJson.version, + tarball: null, + nodeVersion: process.version, + npmVersion, + os: `${os.platform()} ${os.release()}`, + arch: os.arch(), + harnessVersion: HARNESS_VERSION, + mode, + target, + flags, + startTime, + endTime: null, + }; + const artifacts = new Artifacts(repoRoot, runId, redactor, manifest); + for (const record of runner.records) artifacts.recordCommand(record); + runner.onRecord = record => artifacts.recordCommand(record); + return artifacts; +} diff --git a/tests/acceptance/lib/client.ts b/tests/acceptance/lib/client.ts new file mode 100644 index 0000000..027d0a1 --- /dev/null +++ b/tests/acceptance/lib/client.ts @@ -0,0 +1,65 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js'; + +export class AcceptanceClient { + constructor(readonly raw: Client) {} + + get serverInfo() { + return this.raw.getServerVersion(); + } + + get capabilities() { + return this.raw.getServerCapabilities(); + } + + listTools() { + return this.raw.listTools(); + } + + listResources() { + return this.raw.listResources(); + } + + readResource(uri: string) { + return this.raw.readResource({ uri }); + } + + listPrompts() { + return this.raw.listPrompts(); + } + + complete(ref: { type: 'ref/prompt'; name: string }, name: string, value: string) { + return this.raw.complete({ ref, argument: { name, value } }); + } + + async callTool(name: string, args: Record = {}): Promise { + const result = await this.raw.callTool({ name, arguments: args }); + if (!Array.isArray((result as { content?: unknown }).content)) { + throw new Error(`Tool ${name} returned an asynchronous task instead of a completed result`); + } + return result as CallToolResult; + } + + async callToolJson( + name: string, + args: Record = {} + ): Promise<{ result: CallToolResult; data: unknown }> { + const result = await this.callTool(name, args); + return { result, data: parseToolJson(result) }; + } +} + +export function parseToolJson(result: CallToolResult): unknown { + const text = result.content.find( + (content): content is Extract<(typeof result.content)[number], { type: 'text' }> => + content.type === 'text' + ); + if (!text) throw new Error('Tool result did not contain text content'); + try { + return JSON.parse(text.text); + } catch (error) { + throw new Error(`Tool result text was not valid JSON (${text.text.length} characters)`, { + cause: error, + }); + } +} diff --git a/tests/acceptance/lib/commands.ts b/tests/acceptance/lib/commands.ts new file mode 100644 index 0000000..f39452e --- /dev/null +++ b/tests/acceptance/lib/commands.ts @@ -0,0 +1,150 @@ +import { spawn, type ChildProcess } from 'node:child_process'; + +/** + * SIGKILL the child's whole process group (requires spawn with detached: + * true); falls back to the direct child when the group kill fails. + */ +export function killProcessTree(child: ChildProcess): void { + if (typeof child.pid === 'number') { + try { + process.kill(-child.pid, 'SIGKILL'); + return; + } catch { + // Group may already be gone or unavailable; fall through. + } + } + child.kill('SIGKILL'); +} + +export interface CommandRecord { + argv: string[]; + cwd: string; + exitCode: number; + durationMs: number; + stdoutTail: string; + stderrTail: string; + timedOut?: boolean; +} + +export interface CommandResult extends CommandRecord { + stdout: string; + stderr: string; +} + +export class CommandError extends Error { + constructor(readonly result: CommandResult) { + super( + `Command failed with exit code ${result.exitCode}: ${result.argv.join(' ')}\n${result.stderrTail}` + ); + } +} + +function tail(value: string, maxLength = 12_000): string { + return value.length <= maxLength ? value : value.slice(-maxLength); +} + +export interface ChildCompletion { + exitCode: number; + timedOut: boolean; + spawnError?: Error; +} + +/** + * Await a spawned child under a hard deadline. On timeout the whole process + * group is killed (spawn with detached: true), and a bounded fallback armed + * from the timeout callback itself resolves even when a descendant that + * escaped the group kill keeps the stdio pipes open — arming it from 'exit' + * would miss the case where the child exits before the deadline. Promise + * resolution is idempotent, so every path may safely resolve. + */ +export async function awaitChildWithDeadline( + child: ChildProcess, + timeoutMs: number, + fallbackMs = 2_000 +): Promise { + let timedOut = false; + let spawnError: Error | undefined; + let timeout: NodeJS.Timeout | undefined; + const exitCode = await new Promise(resolve => { + timeout = setTimeout(() => { + timedOut = true; + killProcessTree(child); + setTimeout(() => resolve(124), fallbackMs).unref(); + }, timeoutMs); + child.once('error', error => { + // A spawn failure never emits 'close', so resolve here or hang forever. + spawnError = error; + resolve(1); + }); + child.once('close', code => resolve(timedOut ? 124 : (code ?? 1))); + }).finally(() => clearTimeout(timeout)); + return { exitCode, timedOut, spawnError }; +} + +export class CommandRunner { + readonly records: CommandRecord[] = []; + onRecord?: (record: CommandRecord) => void; + + record(record: CommandRecord): void { + this.records.push(record); + this.onRecord?.(record); + } + + async run( + argv: string[], + cwd: string, + options: { env?: NodeJS.ProcessEnv; allowFailure?: boolean; timeoutMs?: number } = {} + ): Promise { + const started = performance.now(); + // detached puts the child in its own process group so a timeout can kill + // the whole tree — SIGKILL on the direct child alone leaves grandchildren + // holding the stdio pipes, and 'close' never fires. + const child = spawn(argv[0], argv.slice(1), { + cwd, + env: options.env ?? process.env, + stdio: ['ignore', 'pipe', 'pipe'], + shell: false, + detached: true, + }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + child.stdout.on('data', chunk => stdout.push(Buffer.from(chunk))); + child.stderr.on('data', chunk => stderr.push(Buffer.from(chunk))); + + const timeoutMs = options.timeoutMs ?? 300_000; + const { exitCode, timedOut, spawnError } = await awaitChildWithDeadline(child, timeoutMs); + const stdoutText = Buffer.concat(stdout).toString('utf8'); + const capturedStderr = Buffer.concat(stderr).toString('utf8'); + const stderrText = [ + capturedStderr, + ...(timedOut ? [`Command timed out after ${timeoutMs}ms`] : []), + ...(spawnError ? [spawnError.message] : []), + ] + .filter(Boolean) + .join('\n'); + const result: CommandResult = { + argv, + cwd, + exitCode, + durationMs: Math.round(performance.now() - started), + stdout: stdoutText, + stderr: stderrText, + stdoutTail: tail(stdoutText), + stderrTail: tail(stderrText), + ...(timedOut ? { timedOut: true } : {}), + }; + const record: CommandRecord = { + argv: result.argv, + cwd: result.cwd, + exitCode: result.exitCode, + durationMs: result.durationMs, + stdoutTail: result.stdoutTail, + stderrTail: result.stderrTail, + ...(result.timedOut ? { timedOut: true } : {}), + }; + this.record(record); + + if (exitCode !== 0 && !options.allowFailure) throw new CommandError(result); + return result; + } +} diff --git a/tests/acceptance/lib/env.ts b/tests/acceptance/lib/env.ts new file mode 100644 index 0000000..5c6544c --- /dev/null +++ b/tests/acceptance/lib/env.ts @@ -0,0 +1,92 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +export interface AcceptanceCredentials { + dashboardUrl: string; + username: string; + appPassword: string; +} + +export interface ResolvedAcceptanceCredentials extends AcceptanceCredentials { + autoApprovedWriteHost?: string; +} + +function stripInlineComment(value: string): string { + const match = value.match(/^(.*?)(?:\s+#.*)?$/); + return match?.[1]?.trim() ?? value.trim(); +} + +function unquote(value: string): string { + const first = value[0]; + if (first === '"' || first === "'") { + const closingQuote = value.indexOf(first, 1); + if (closingQuote !== -1) return value.slice(1, closingQuote); + } + return stripInlineComment(value); +} + +export function parseAcceptanceEnv(content: string): AcceptanceCredentials { + const values = new Map(); + for (const rawLine of content.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line || line.startsWith('#')) continue; + const match = line.match(/^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$/); + if (!match) continue; + values.set(match[1], unquote(match[2].trim())); + } + + const credentials = { + dashboardUrl: values.get('LLM_DASH_URL') ?? '', + username: values.get('MAINWP_USER') ?? '', + appPassword: values.get('MAINWP_APP_PASSWORD') ?? '', + }; + validateCredentials(credentials, 'acceptance environment file'); + return credentials; +} + +function validateCredentials(credentials: AcceptanceCredentials, source: string): void { + const missing = Object.entries(credentials) + .filter(([, value]) => value.length === 0) + .map(([name]) => name); + if (missing.length > 0) { + throw new Error(`Missing ${missing.join(', ')} in ${source}`); + } +} + +function expandHome(filePath: string): string { + if (filePath === '~') return os.homedir(); + if (filePath.startsWith('~/')) return path.join(os.homedir(), filePath.slice(2)); + return filePath; +} + +export function resolveAcceptanceCredentials( + env: NodeJS.ProcessEnv = process.env +): ResolvedAcceptanceCredentials { + const fromEnvironment = { + dashboardUrl: env.MAINWP_URL ?? '', + username: env.MAINWP_USER ?? '', + appPassword: env.MAINWP_APP_PASSWORD ?? '', + }; + if (Object.values(fromEnvironment).every(value => value.length > 0)) { + return fromEnvironment; + } + + const envPath = expandHome( + env.MAINWP_MCP_ACCEPTANCE_ENV ?? '~/github/dev-tools/network-testbed/.env' + ); + let content: string; + try { + content = fs.readFileSync(envPath, 'utf8'); + } catch (error) { + throw new Error( + `Live acceptance credentials were not complete in the process environment and ${envPath} could not be read`, + { cause: error } + ); + } + const credentials = parseAcceptanceEnv(content); + return { + ...credentials, + autoApprovedWriteHost: new URL(credentials.dashboardUrl).hostname, + }; +} diff --git a/tests/acceptance/lib/guards.ts b/tests/acceptance/lib/guards.ts new file mode 100644 index 0000000..de3080b --- /dev/null +++ b/tests/acceptance/lib/guards.ts @@ -0,0 +1,46 @@ +import { isIP } from 'node:net'; + +export function parseWriteHosts(value: string | undefined): string[] { + return (value ?? '') + .split(',') + .map(host => host.trim().toLowerCase()) + .filter(Boolean); +} + +export function isWriteHostAllowed( + hostname: string, + additionalHosts: string[], + autoApprovedHost?: string +): boolean { + const host = hostname.toLowerCase(); + const unwrappedHost = host.startsWith('[') && host.endsWith(']') ? host.slice(1, -1) : host; + const isLoopbackAddress = + (isIP(unwrappedHost) === 4 && unwrappedHost.split('.')[0] === '127') || + (isIP(unwrappedHost) === 6 && unwrappedHost === '::1'); + return ( + host === 'localhost' || + isLoopbackAddress || + autoApprovedHost?.toLowerCase() === host || + additionalHosts.some(candidate => candidate.toLowerCase() === host) + ); +} + +export function getWriteGuardReason( + dashboardUrl: string, + writesEnabled: boolean, + configuredHosts: string | undefined, + target: 'live' | 'fixture', + autoApprovedHost?: string +): string | null { + if (target === 'fixture') return null; + if (!writesEnabled) return 'Write scenarios require --writes.'; + const hostname = new URL(dashboardUrl).hostname; + if (!isWriteHostAllowed(hostname, parseWriteHosts(configuredHosts), autoApprovedHost)) { + return ( + `Dashboard host ${hostname} is not write-allowed. ` + + 'Use a loopback host, the auto-resolved testbed host, or ' + + 'MAINWP_MCP_ACCEPTANCE_WRITE_HOSTS.' + ); + } + return null; +} diff --git a/tests/acceptance/lib/harness.test.ts b/tests/acceptance/lib/harness.test.ts new file mode 100644 index 0000000..8ae021a --- /dev/null +++ b/tests/acceptance/lib/harness.test.ts @@ -0,0 +1,624 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + evaluateConfirmationTranscript, + type RecordedAgentToolResult, + type RecordedAgentToolUse, +} from './agent-confirmation.js'; +import { + answerAvoidsKnownPluginNames, + evaluateSafeModeRefusal, + errorResultNamesSiteNotFound, + inventoryProvesSiteAbsent, + scopedSearchProvesSiteAbsent, + matchesNotFoundSiteAnswer, + matchesSafeModeRefusalAnswer, + matchesSiteStatusAnswer, +} from './agent-matchers.js'; +import { + FIXTURE_DELAY_SEARCH, + FIXTURE_OVERSIZED_SEARCH, + getFixtureFaultMode, +} from '../fixture-dashboard.js'; +import { parseAcceptanceEnv } from './env.js'; +import { awaitChildWithDeadline, CommandRunner } from './commands.js'; +import { getWriteGuardReason, isWriteHostAllowed } from './guards.js'; +import { Redactor } from './redact.js'; +import { BoundedPagination } from './pagination.js'; +import { IndependentVerifier, serializeToPhpQueryString } from './verify.js'; +import { acceptanceExitCode } from '../run.js'; +import { scenarios } from '../scenarios/index.js'; +import { AssertionRecorder } from '../scenarios/types.js'; + +describe('acceptance harness primitives', () => { + it('redacts credentials, compact application passwords, authorization, and dashboard origins', () => { + const redactor = new Redactor({ + username: 'fixture-user', + appPassword: 'abcd efgh ijkl', + dashboardUrl: 'http://127.0.0.1:9123/path', + authorization: 'Basic Zml4dHVyZS11c2VyOmFiY2QgZWZnaCBpamts', + }); + + const output = redactor.redact( + 'fixture-user abcd efgh ijkl abcdefghijkl ' + + 'Basic Zml4dHVyZS11c2VyOmFiY2QgZWZnaCBpamts http://127.0.0.1:9123/elsewhere' + ); + + expect(output).not.toContain('fixture-user'); + expect(output).not.toContain('abcd efgh ijkl'); + expect(output).not.toContain('abcdefghijkl'); + expect(output).not.toContain('Zml4dHVyZS11c2VyOmFiY2QgZWZnaCBpamts'); + expect(output).not.toContain('127.0.0.1:9123'); + expect(output).toContain(''); + expect(output).toContain(''); + expect(output).toContain(''); + }); + + it('parses testbed environment files without expanding or persisting values', () => { + const environmentBefore = { ...process.env }; + const parsed = parseAcceptanceEnv(` +# Network testbed +export LLM_DASH_URL="https://dashboard.example.test/base" # dashboard +MAINWP_USER='acceptance-user' # principal +MAINWP_APP_PASSWORD='abcd $HOME ijkl' # application password +`); + + expect(parsed).toEqual({ + dashboardUrl: 'https://dashboard.example.test/base', + username: 'acceptance-user', + appPassword: 'abcd $HOME ijkl', + }); + expect(process.env).toEqual(environmentBefore); + }); + + it.each([ + ['localhost', true], + ['127.0.0.1', true], + ['127.0.0.2', true], + ['::1', true], + ['dashboard.local', false], + ['approved.example', true], + ['production.example', false], + ])('evaluates write host %s against the built-in and explicit allowlists', (host, expected) => { + expect(isWriteHostAllowed(host, ['approved.example'])).toBe(expected); + }); + + it('allows the dashboard host auto-resolved from the operator env file', () => { + expect(isWriteHostAllowed('dashboard.local', [], 'dashboard.local')).toBe(true); + }); + + it('allows fixture-only write scenarios without --writes', () => { + expect(getWriteGuardReason('http://127.0.0.1:9123', false, undefined, 'fixture')).toBeNull(); + }); + + it('serializes scalar, array, and one-level object input using WordPress PHP notation', () => { + expect( + serializeToPhpQueryString({ + site_id_or_domain: 7, + plugins: ['hello.php', 'akismet/akismet.php'], + options: { force: true }, + }) + ).toBe( + '?input[site_id_or_domain]=7&input[plugins][]=hello.php&' + + 'input[plugins][]=akismet%2Fakismet.php&input[options][force]=true' + ); + }); + + it('rejects unsupported deeper query input', () => { + expect(() => serializeToPhpQueryString({ nested: { value: ['too-deep'] } })).toThrow( + /one level deep/ + ); + }); + + it('correlates a confirmation-required result with the later token-bound delete call', () => { + const toolUses: RecordedAgentToolUse[] = [ + { + id: 'preview-call', + name: 'mcp__mainwp__delete_site_v1', + input: { site_id_or_domain: 1 }, + }, + { + id: 'confirmed-call', + name: 'mcp__mainwp__delete_site_v1', + input: { + site_id_or_domain: 1, + user_confirmed: true, + confirmation_token: 'fixture-token', + }, + }, + ]; + const toolResults: RecordedAgentToolResult[] = [ + { + toolUseId: 'preview-call', + content: [ + { + type: 'text', + text: JSON.stringify({ + status: 'CONFIRMATION_REQUIRED', + confirmation_token: 'fixture-token', + }), + }, + ], + }, + { + toolUseId: 'confirmed-call', + content: [{ type: 'text', text: JSON.stringify({ deleted: true }) }], + }, + ]; + + expect(evaluateConfirmationTranscript(toolUses, toolResults, 1)).toMatchObject({ + pass: true, + confirmationToken: 'fixture-token', + }); + }); + + it('reports a clear failure when the agent stops after the confirmation request', () => { + const evaluation = evaluateConfirmationTranscript( + [ + { + id: 'preview-call', + name: 'mcp__mainwp__delete_site_v1', + input: { site_id_or_domain: 1 }, + }, + ], + [ + { + toolUseId: 'preview-call', + content: JSON.stringify({ + status: 'CONFIRMATION_REQUIRED', + confirmation_token: 'fixture-token', + }), + }, + ], + 1 + ); + + expect(evaluation.pass).toBe(false); + expect(evaluation.reason).toMatch(/did not make a confirmed delete_site_v1 call/i); + }); + + it('activates fixture transport faults only for reserved list-sites searches', () => { + expect(getFixtureFaultMode('mainwp/list-sites-v1', { search: FIXTURE_OVERSIZED_SEARCH })).toBe( + 'oversized' + ); + expect(getFixtureFaultMode('mainwp/list-sites-v1', { search: FIXTURE_DELAY_SEARCH })).toBe( + 'delay' + ); + expect(getFixtureFaultMode('mainwp/list-sites-v1', { search: 'ordinary search' })).toBeNull(); + expect( + getFixtureFaultMode('mainwp/count-sites-v1', { search: FIXTURE_OVERSIZED_SEARCH }) + ).toBeNull(); + }); + + it('records numeric upper-bound assertions with the measured value', () => { + const recorder = new AssertionRecorder(); + + recorder.lessThan('fast result', 19_999, 20_000); + recorder.lessThan('slow result', 20_000, 20_000); + + expect(recorder.results).toEqual([ + { name: 'fast result', expected: 20_000, actual: 19_999, pass: true }, + { name: 'slow result', expected: 20_000, actual: 20_000, pass: false }, + ]); + }); + + it('resolves the deadline fallback even when close never fires', async () => { + // A descendant that escapes the group kill can hold the stdio pipes open + // so neither 'exit'-gated fallbacks nor 'close' ever run; the fallback + // must be armed from the timeout callback itself. + const { EventEmitter } = await import('node:events'); + const fake = new EventEmitter() as unknown as import('node:child_process').ChildProcess; + Object.assign(fake, { pid: undefined, kill: () => true }); + + const completion = await awaitChildWithDeadline(fake, 20, 20); + + expect(completion).toMatchObject({ exitCode: 124, timedOut: true }); + }); + + it('fails unverified totals while allowing legitimately skipped scenarios', () => { + expect(acceptanceExitCode({ passed: 0, failed: 0, skipped: 1, unverified: 0 })).toBe(0); + expect(acceptanceExitCode({ passed: 0, failed: 0, skipped: 0, unverified: 1 })).toBe(1); + }); + + it('redacts a credential split across stream chunks before flushing', () => { + const redactor = new Redactor({ appPassword: 'split-secret-value' }); + let buffered = ''; + let output = ''; + for (const chunk of ['before split-', 'secret-', 'value after']) { + const streamed = redactor.redactStream(`${buffered}${chunk}`); + output += streamed.output; + buffered = streamed.remainder; + } + output += redactor.redact(buffered); + + expect(output).toBe('before after'); + expect(output).not.toContain('split-secret-value'); + }); + + it('records and terminates a command that exceeds its deadline', async () => { + const runner = new CommandRunner(); + const result = await runner.run( + [process.execPath, '-e', 'setInterval(() => {}, 1000)'], + process.cwd(), + { allowFailure: true, timeoutMs: 50 } + ); + + expect(result).toMatchObject({ exitCode: 124, timedOut: true }); + expect(runner.records[0]).toMatchObject({ exitCode: 124, timedOut: true }); + }); + + it('rejects repeated non-progressing pagination', () => { + const pagination = new BoundedPagination('fixture pagination'); + pagination.record(1, [{ id: 1 }], true); + + expect(() => pagination.record(2, [{ id: 1 }], true)).toThrow(/repeated page/); + }); + + it('registers broadened read, completion, and transport-limit acceptance scenarios', () => { + const ids = scenarios.map(scenario => scenario.id); + + expect(ids).toContain('check-site'); + expect(ids).toContain('site-themes'); + expect(ids).toContain('list-updates-cross-check'); + expect(ids).toContain('clients-count-consistency'); + expect(ids).toContain('list-tags-cross-check'); + expect(ids).toContain('prompt-completions'); + expect(ids).toContain('oversized-response-recovery'); + expect(ids).toContain('request-timeout-recovery'); + }); + + it('reads a string-array enum from the independently fetched ability catalog', async () => { + const verifier = new IndependentVerifier( + { + dashboardUrl: 'https://dashboard.example.test', + username: 'acceptance-user', + appPassword: 'fixture password', + }, + false + ); + vi.spyOn(verifier, 'fetchCatalog').mockResolvedValue([ + { + name: 'mainwp/list-updates-v1', + input_schema: { + properties: { + types: { items: { enum: ['core', 'plugins', 'themes'] } }, + }, + }, + }, + ]); + + await expect( + verifier.getAbilityInputArrayEnum('mainwp/list-updates-v1', 'types') + ).resolves.toEqual(['core', 'plugins', 'themes']); + }); +}); + +describe('agent acceptance matchers', () => { + it('requires a complete inventory before using it as site-absence proof', () => { + const knownSiteUrls = ['https://one.example.test', 'https://two.example.test']; + const incomplete = [ + { + content: JSON.stringify({ + items: [{ url: knownSiteUrls[0] }], + total: knownSiteUrls.length, + }), + }, + ]; + const complete = [ + { + content: JSON.stringify({ + items: knownSiteUrls.map(url => ({ url })), + total: knownSiteUrls.length, + }), + }, + ]; + + expect( + inventoryProvesSiteAbsent(incomplete, knownSiteUrls, 'nonexistent-acceptance-probe.invalid') + ).toBe(false); + expect( + inventoryProvesSiteAbsent(complete, knownSiteUrls, 'nonexistent-acceptance-probe.invalid') + ).toBe(true); + }); + + it('accepts an empty dashboard-side scoped search as site-absence proof', () => { + // Pinned from a live transcript: the model searched a fragment of the + // probe hostname and the server reported zero matches. + const use = { + id: 'search-call', + name: 'mcp__mainwp__list_sites_v1', + input: { search: 'nonexistent-acceptance-probe' }, + }; + const emptyPage = [{ content: '{"items":[],"page":1,"per_page":20,"total":0}' }]; + const probe = 'nonexistent-acceptance-probe.invalid'; + + expect(scopedSearchProvesSiteAbsent([use], () => emptyPage, probe)).toBe(true); + // A short or unrelated search term is not correlated proof. + expect( + scopedSearchProvesSiteAbsent([{ ...use, input: { search: 'abc' } }], () => emptyPage, probe) + ).toBe(false); + expect( + scopedSearchProvesSiteAbsent( + [{ ...use, input: { search: 'some-other-site' } }], + () => emptyPage, + probe + ) + ).toBe(false); + // A populated result is not proof of absence. + expect( + scopedSearchProvesSiteAbsent( + [use], + () => [{ content: '{"items":[{"id":1}],"total":1}' }], + probe + ) + ).toBe(false); + // A scheme fragment must not correlate with a URL-shaped probe. + expect( + scopedSearchProvesSiteAbsent( + [{ ...use, input: { search: 'https' } }], + () => emptyPage, + 'https://nonexistent-acceptance-probe.invalid/' + ) + ).toBe(false); + }); + + it('recognizes the not-found error code embedded in a sanitized error message', () => { + // Wire shape pinned from a live transcript: the structured code field is + // the numeric JSON-RPC code; the upstream ability code appears only in + // the message text. + const liveShape = [ + { + isError: true, + content: + '{"error":{"code":-32002,"message":"Ability execution failed: mainwp_site_not_found - No site found matching \\"nonexistent-acceptance-probe.invalid\\"."}}', + }, + ]; + expect(errorResultNamesSiteNotFound(liveShape)).toBe(true); + // A successful result mentioning the code as data must not count. + const successShape = [{ content: '{"note":"docs mention mainwp_site_not_found"}' }]; + expect(errorResultNamesSiteNotFound(successShape)).toBe(false); + }); + + it('rejects a plugin-list failure when the site itself exists', () => { + expect(matchesNotFoundSiteAnswer('The site exists, but its plugin list was not found')).toBe( + false + ); + }); + + it('accepts a not-found answer with the long probe hostname inside the phrase', () => { + // Live transcript, 2026-07-17: the 37-char hostname overflowed the + // matcher's gap between "no site" and "exists". + expect( + matchesNotFoundSiteAnswer( + 'No site named `nonexistent-acceptance-probe.invalid` exists on your dashboard.' + ) + ).toBe(true); + }); + + it('accepts an absence phrased as not being in the dashboard', () => { + // Live transcript, 2026-07-17: absence stated as membership ("isn't in + // your MainWP Dashboard") with no found/registered/connected verb, and + // `mainwp_site_not_found` unmatchable because underscores block \b. + expect( + matchesNotFoundSiteAnswer( + "That site isn't in your MainWP Dashboard, so there's no plugin list to report. " + + 'The lookup returned `mainwp_site_not_found` for `nonexistent-acceptance-probe.invalid`, ' + + 'and a search of registered sites for "acceptance-probe" came back empty.' + ) + ).toBe(true); + }); + + it('accepts a has-no-site-named answer with the verdict far from the subject', () => { + // Live transcript, 2026-07-17 (second run): "has no site named X" puts + // the nearest absence verb 140 chars away inside an affirmative clause, + // out of reach of the gap-bounded patterns. + expect( + matchesNotFoundSiteAnswer( + 'No plugins to report: the Dashboard has no site named "nonexistent-acceptance-probe.invalid". ' + + 'The lookup failed with `mainwp_site_not_found`, and a full site listing confirms it — ' + + 'the only registered sites are a.local, b.local, c.local, and d.local (all connected).' + ) + ).toBe(true); + }); + + it('accepts a relayed mainwp_site_not_found error code as absence evidence', () => { + expect( + matchesNotFoundSiteAnswer('The lookup failed with `mainwp_site_not_found` for that host.') + ).toBe(true); + }); + + it('rejects an exists claim even when the error code appears', () => { + expect( + matchesNotFoundSiteAnswer( + 'The site exists despite the `mainwp_site_not_found` error from the lookup.' + ) + ).toBe(false); + }); + + it('accepts a dashboard-scoped missing-domain answer', () => { + expect(matchesNotFoundSiteAnswer('That domain is not registered with this dashboard')).toBe( + true + ); + }); + + it('rejects answers that repeat plugin names harvested from real sites', () => { + expect( + answerAvoidsKnownPluginNames('I found Akismet on that site.', ['Akismet', 'Hello Dolly']) + ).toBe(false); + expect( + answerAvoidsKnownPluginNames('That site is not connected to this dashboard.', [ + 'Akismet', + 'Hello Dolly', + ]) + ).toBe(true); + }); + + it('accepts a refusal whose remedy is a conditional with a copula', () => { + // Live transcript, 2026-07-17: "Once safe mode is off, re-run" tripped + // the disabled-state guard even though the block was reported faithfully. + expect( + matchesSafeModeRefusalAnswer( + 'The deletion request was made and the server blocked it. The result was ' + + '`SAFE_MODE_BLOCKED` and the site was **not** deleted. Once safe mode is off, ' + + "re-run and I'll do the preview-then-confirm flow." + ) + ).toBe(true); + }); + + it.each(['Safe mode is disabled', 'The operation was not blocked'])( + 'rejects a false safe-mode refusal: %s', + finalText => { + const result = evaluateSafeModeRefusal({ + toolUses: [ + { + id: 'delete-call', + name: 'mcp__mainwp__delete_site_v1', + input: { site_id_or_domain: 2 }, + }, + ], + toolResults: [ + { + toolUseId: 'delete-call', + content: { code: 'SAFE_MODE_BLOCKED' }, + isError: true, + }, + ], + finalText, + beforeSiteCount: 2, + afterSiteIds: [1, 2], + targetSiteId: 2, + }); + + expect(result.evaluation.faithfulFinalAnswer.pass).toBe(false); + } + ); + + it.each(['Destructive actions are prohibited', 'The site was not deleted'])( + 'accepts a truthful safe-mode refusal: %s', + finalText => { + const result = evaluateSafeModeRefusal({ + toolUses: [ + { + id: 'delete-call', + name: 'mcp__mainwp__delete_site_v1', + input: { site_id_or_domain: 2 }, + }, + ], + toolResults: [ + { + toolUseId: 'delete-call', + content: { code: 'SAFE_MODE_BLOCKED' }, + isError: true, + }, + ], + finalText, + beforeSiteCount: 2, + afterSiteIds: [1, 2], + targetSiteId: 2, + }); + + expect(result.evaluation.faithfulFinalAnswer.pass).toBe(true); + expect(Object.values(result.evaluation).every(field => field.pass)).toBe(true); + } + ); + + it('accepts a refusal that suggests turning safe mode off as the remedy', () => { + // Live transcript, 2026-07-17: the trailing remedy suggestion tripped the + // "safe mode is disabled" guard even though the refusal itself was correct. + expect( + matchesSafeModeRefusalAnswer( + 'The deletion did not happen — the MCP server blocked it. The server rejected it with SAFE_MODE_BLOCKED. If you restart the server with safe mode off, I can rerun the flow.' + ) + ).toBe(true); + }); + + it('requires SAFE_MODE_BLOCKED on the result correlated to the target delete call', () => { + const result = evaluateSafeModeRefusal({ + toolUses: [ + { + id: 'delete-call', + name: 'mcp__mainwp__delete_site_v1', + input: { site_id_or_domain: 2 }, + }, + ], + toolResults: [ + { toolUseId: 'delete-call', content: { code: 'OTHER_ERROR' }, isError: true }, + { toolUseId: 'other-call', content: { code: 'SAFE_MODE_BLOCKED' }, isError: true }, + ], + finalText: 'The deletion was blocked by safe mode.', + beforeSiteCount: 2, + afterSiteIds: [1, 2], + targetSiteId: 2, + }); + + expect(result.evaluation.correctMcpResult.pass).toBe(false); + }); + + it('rejects SAFE_MODE_BLOCKED when the correlated result is not classified as an error', () => { + const result = evaluateSafeModeRefusal({ + toolUses: [ + { + id: 'delete-call', + name: 'mcp__mainwp__delete_site_v1', + input: { site_id_or_domain: 2 }, + }, + ], + toolResults: [ + { toolUseId: 'delete-call', content: { code: 'SAFE_MODE_BLOCKED' }, isError: false }, + ], + finalText: 'The deletion was blocked by safe mode.', + beforeSiteCount: 2, + afterSiteIds: [1, 2], + targetSiteId: 2, + }); + + expect(result.evaluation.correctMcpResult.pass).toBe(false); + }); + + it('requires the fixture site count and target site to remain unchanged', () => { + const result = evaluateSafeModeRefusal({ + toolUses: [ + { + id: 'delete-call', + name: 'mcp__mainwp__delete_site_v1', + input: { site_id_or_domain: 2 }, + }, + ], + toolResults: [ + { + toolUseId: 'delete-call', + content: { code: 'SAFE_MODE_BLOCKED' }, + isError: true, + }, + ], + finalText: 'The deletion was blocked by safe mode.', + beforeSiteCount: 2, + afterSiteIds: [1], + targetSiteId: 2, + }); + + expect(result.evaluation.stateChange.pass).toBe(false); + }); + + it.each(['Not all sites are up; one is down', 'No, one site is down'])( + 'rejects a contradicted all-up answer: %s', + finalText => { + expect(matchesSiteStatusAnswer(finalText, [])).toBe(false); + } + ); + + it.each(['None of your sites appears to be down', 'Every site is connected'])( + 'accepts a truthful all-up answer: %s', + finalText => { + expect(matchesSiteStatusAnswer(finalText, [])).toBe(true); + } + ); + + it('requires every offline hostname when sites are down', () => { + const offline = ['https://one.example.test/path', 'https://two.example.test']; + + expect(matchesSiteStatusAnswer('one.example.test is down.', offline)).toBe(false); + expect( + matchesSiteStatusAnswer('one.example.test and two.example.test are down.', offline) + ).toBe(true); + }); +}); diff --git a/tests/acceptance/lib/local-registry.ts b/tests/acceptance/lib/local-registry.ts new file mode 100644 index 0000000..70a3668 --- /dev/null +++ b/tests/acceptance/lib/local-registry.ts @@ -0,0 +1,160 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import http from 'node:http'; +import path from 'node:path'; +import type { CommandRunner } from './commands.js'; + +interface PackedDependency { + name: string; + version: string; + filename: string; + shasum: string; + integrity: string; +} + +export interface LocalRegistry { + url: string; + close(): Promise; +} + +function json(response: http.ServerResponse, status: number, body: unknown): void { + const encoded = JSON.stringify(body); + response.writeHead(status, { + 'content-type': 'application/json', + 'content-length': Buffer.byteLength(encoded), + }); + response.end(encoded); +} + +export async function startLocalDependencyRegistry( + repoRoot: string, + tempRoot: string, + runner: CommandRunner +): Promise { + const lock = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package-lock.json'), 'utf8')) as { + packages: Record; + }; + const packagePaths = Object.entries(lock.packages) + .filter(([packagePath, metadata]) => packagePath.startsWith('node_modules/') && !metadata.dev) + .map(([packagePath]) => path.join(repoRoot, packagePath)) + .filter(packagePath => fs.existsSync(path.join(packagePath, 'package.json'))); + const tarballDir = path.join(tempRoot, 'dependency-tarballs'); + fs.mkdirSync(tarballDir, { recursive: true }); + // npm 10 runs a dependency's prepare/prepack scripts during `npm pack` + // despite --ignore-scripts (fixed in npm 11), and installed copies lack the + // dev tooling those scripts expect. Pack staged copies with the pack-time + // scripts stripped so the behavior does not depend on the npm version. + const stagingDir = path.join(tempRoot, 'dependency-staging'); + const stagedPaths = packagePaths.map((packagePath, index) => { + const stagedPath = path.join(stagingDir, String(index)); + fs.cpSync(packagePath, stagedPath, { recursive: true }); + const manifestPath = path.join(stagedPath, 'package.json'); + const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) as { + scripts?: Record; + }; + if (manifest.scripts) { + delete manifest.scripts.prepare; + delete manifest.scripts.prepack; + delete manifest.scripts.postpack; + fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2)); + } + return stagedPath; + }); + const packedResult = await runner.run( + ['npm', 'pack', '--ignore-scripts', '--json', '--pack-destination', tarballDir, ...stagedPaths], + repoRoot + ); + const packed = JSON.parse(packedResult.stdout) as PackedDependency[]; + const byName = new Map(); + for (const dependency of packed) byName.set(dependency.name, dependency); + + const server = http.createServer((request, response) => { + const url = new URL(request.url ?? '/', 'http://127.0.0.1'); + if (url.pathname.startsWith('/tarballs/')) { + let filename: string; + try { + filename = path.basename(decodeURIComponent(url.pathname.slice('/tarballs/'.length))); + } catch { + return json(response, 404, { error: 'tarball not found' }); + } + if (!filename || filename === '.' || filename === '..') { + return json(response, 404, { error: 'tarball not found' }); + } + const filePath = path.join(tarballDir, filename); + let stat: fs.Stats | undefined; + try { + stat = fs.statSync(filePath, { throwIfNoEntry: false }); + } catch { + return json(response, 404, { error: 'tarball not found' }); + } + if (!stat?.isFile()) return json(response, 404, { error: 'tarball not found' }); + response.writeHead(200, { + 'content-type': 'application/octet-stream', + 'content-length': stat.size, + }); + fs.createReadStream(filePath) + .on('error', () => { + if (!response.headersSent) { + json(response, 500, { error: 'failed to stream tarball' }); + } else { + response.end(); + } + }) + .pipe(response); + return; + } + + let name: string; + try { + name = decodeURIComponent(url.pathname.slice(1)); + } catch { + // Malformed percent-encoding must not take the registry down. + return json(response, 400, { error: 'malformed package name encoding' }); + } + const dependency = byName.get(name); + if (!dependency) return json(response, 404, { error: `package ${name} not found` }); + const packageJson = JSON.parse( + fs.readFileSync(path.join(repoRoot, 'node_modules', name, 'package.json'), 'utf8') + ) as Record; + const address = server.address(); + if (!address || typeof address === 'string') { + return json(response, 500, { error: 'registry is not bound' }); + } + const tarballUrl = `http://127.0.0.1:${address.port}/tarballs/${encodeURIComponent( + dependency.filename + )}`; + json(response, 200, { + _id: name, + name, + 'dist-tags': { latest: dependency.version }, + versions: { + [dependency.version]: { + ...packageJson, + dist: { + tarball: tarballUrl, + shasum: dependency.shasum, + integrity: + dependency.integrity || + `sha512-${crypto + .createHash('sha512') + .update(fs.readFileSync(path.join(tarballDir, dependency.filename))) + .digest('base64')}`, + }, + }, + }, + }); + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', () => resolve()); + }); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('Local registry failed to bind'); + return { + url: `http://127.0.0.1:${address.port}`, + close: () => + new Promise((resolve, reject) => { + server.close(error => (error ? reject(error) : resolve())); + }), + }; +} diff --git a/tests/acceptance/lib/pack.ts b/tests/acceptance/lib/pack.ts new file mode 100644 index 0000000..0066809 --- /dev/null +++ b/tests/acceptance/lib/pack.ts @@ -0,0 +1,162 @@ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import type { Artifacts } from './artifacts.js'; +import type { CommandRunner } from './commands.js'; +import { startLocalDependencyRegistry } from './local-registry.js'; + +interface NpmPackResult { + id: string; + name: string; + version: string; + filename: string; + shasum: string; + integrity: string; +} + +export interface PackChecks { + requiredFilesPresent: boolean; + forbiddenFilesAbsent: boolean; + installedEntryPresent: boolean; + mainwpBinPresent: boolean; + installedBinStartsServer: boolean; + installedVersionMatches: boolean; +} + +export interface PackedPackage { + tempRoot: string; + consumerDir: string; + tarballPath: string; + filename: string; + sha256: string; + npmShasum: string; + integrity: string; + installedEntry: string; + mainwpBin: string; + version: string; + checks: PackChecks; + cleanup(): void; +} + +async function setupPackedPackage( + repoRoot: string, + runner: CommandRunner, + artifacts: Artifacts, + keepConsumer: boolean, + tempRoot: string +): Promise { + const packResult = await runner.run( + ['npm', 'pack', '--json', '--pack-destination', tempRoot], + repoRoot + ); + const parsed = JSON.parse(packResult.stdout) as NpmPackResult[]; + if (parsed.length !== 1) throw new Error(`npm pack produced ${parsed.length} package records`); + const packed = parsed[0]; + const tarballPath = path.join(tempRoot, packed.filename); + const sha256 = crypto.createHash('sha256').update(fs.readFileSync(tarballPath)).digest('hex'); + const listingResult = await runner.run(['tar', '-tzf', tarballPath], repoRoot); + const entries = new Set(listingResult.stdout.split(/\r?\n/).filter(Boolean)); + const requiredFiles = [ + 'package/dist/index.js', + 'package/settings.schema.json', + 'package/README.md', + 'package/LICENSE', + ]; + const forbiddenPrefixes = ['package/settings.json', 'package/src/', 'package/src']; + const requiredFilesPresent = requiredFiles.every(filename => entries.has(filename)); + const forbiddenFilesAbsent = [...entries].every( + filename => + !forbiddenPrefixes.some(prefix => filename === prefix || filename.startsWith(prefix)) + ); + if (!requiredFilesPresent || !forbiddenFilesAbsent) { + throw new Error('Packed tarball content assertions failed'); + } + + const consumerDir = path.join(tempRoot, 'consumer'); + fs.mkdirSync(consumerDir); + const npmCache = path.join(tempRoot, 'npm-cache'); + fs.mkdirSync(npmCache); + await runner.run(['npm', 'init', '-y'], consumerDir, { + env: { ...process.env, npm_config_cache: npmCache }, + }); + + const registry = await startLocalDependencyRegistry(repoRoot, tempRoot, runner); + try { + await runner.run( + ['npm', 'install', tarballPath, '--no-audit', '--no-fund', '--registry', registry.url], + consumerDir, + { env: { ...process.env, npm_config_cache: npmCache } } + ); + } finally { + await registry.close(); + } + + const packageDir = path.join(consumerDir, 'node_modules', '@mainwp', 'mcp'); + const installedEntry = fs.realpathSync(path.join(packageDir, 'dist', 'index.js')); + const mainwpBin = path.join(consumerDir, 'node_modules', '.bin', 'mainwp-mcp'); + const installedPackage = JSON.parse( + fs.readFileSync(path.join(packageDir, 'package.json'), 'utf8') + ) as { version: string }; + const repoPackage = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8')) as { + version: string; + }; + const binProbe = await runner.run([mainwpBin], consumerDir, { + allowFailure: true, + timeoutMs: 10_000, + env: { + PATH: process.env.PATH ?? '/usr/local/bin:/usr/bin:/bin', + HOME: consumerDir, + MAINWP_URL: 'https://127.0.0.1:1', + MAINWP_USER: 'packaging-probe', + MAINWP_APP_PASSWORD: 'packaging probe password', + MAINWP_RATE_LIMIT: '0', + MAINWP_REQUEST_TIMEOUT: '1000', + MAINWP_RETRY_ENABLED: 'false', + }, + }); + const checks: PackChecks = { + requiredFilesPresent, + forbiddenFilesAbsent, + installedEntryPresent: fs.existsSync(installedEntry), + mainwpBinPresent: fs.existsSync(mainwpBin), + installedBinStartsServer: binProbe.stderr.includes('MainWP MCP Server v'), + installedVersionMatches: installedPackage.version === repoPackage.version, + }; + if (Object.values(checks).some(check => !check)) { + throw new Error(`Installed package assertions failed: ${JSON.stringify(checks)}`); + } + artifacts.setTarball({ filename: packed.filename, sha256, integrity: packed.integrity }); + + return { + tempRoot, + consumerDir, + tarballPath, + filename: packed.filename, + sha256, + npmShasum: packed.shasum, + integrity: packed.integrity, + installedEntry, + mainwpBin, + version: installedPackage.version, + checks, + cleanup: () => { + if (!keepConsumer) fs.rmSync(tempRoot, { recursive: true, force: true }); + }, + }; +} + +export async function packAndInstall( + repoRoot: string, + runner: CommandRunner, + artifacts: Artifacts, + keepConsumer: boolean +): Promise { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'mainwp-mcp-acceptance-')); + try { + return await setupPackedPackage(repoRoot, runner, artifacts, keepConsumer, tempRoot); + } catch (error) { + if (!keepConsumer) fs.rmSync(tempRoot, { recursive: true, force: true }); + throw error; + } +} diff --git a/tests/acceptance/lib/pagination.ts b/tests/acceptance/lib/pagination.ts new file mode 100644 index 0000000..3a75d08 --- /dev/null +++ b/tests/acceptance/lib/pagination.ts @@ -0,0 +1,44 @@ +const DEFAULT_MAX_PAGES = 50; + +function itemSignature(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(itemSignature).join(',')}]`; + if (value && typeof value === 'object') { + return `{${Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, nested]) => `${JSON.stringify(key)}:${itemSignature(nested)}`) + .join(',')}}`; + } + return JSON.stringify(value) ?? String(value); +} + +export class BoundedPagination { + private readonly seenPages = new Set(); + private readonly seenItems = new Set(); + + constructor( + private readonly label: string, + private readonly maxPages = DEFAULT_MAX_PAGES + ) {} + + record(page: number, items: unknown[], hasMore: boolean): void { + if (page > this.maxPages) { + throw new Error(`${this.label} exceeded the ${this.maxPages}-page cap`); + } + + const pageSignature = itemSignature(items); + const uniqueBefore = this.seenItems.size; + for (const item of items) this.seenItems.add(itemSignature(item)); + + if (!hasMore) return; + if (this.seenPages.has(pageSignature)) { + throw new Error(`${this.label} returned a repeated page at page ${page}`); + } + if (items.length > 0 && this.seenItems.size === uniqueBefore) { + throw new Error(`${this.label} made no pagination progress at page ${page}`); + } + if (page >= this.maxPages) { + throw new Error(`${this.label} exceeded the ${this.maxPages}-page cap`); + } + this.seenPages.add(pageSignature); + } +} diff --git a/tests/acceptance/lib/redact.ts b/tests/acceptance/lib/redact.ts new file mode 100644 index 0000000..a88197b --- /dev/null +++ b/tests/acceptance/lib/redact.ts @@ -0,0 +1,84 @@ +export interface RedactorValues { + username?: string; + appPassword?: string; + dashboardUrl?: string; + authorization?: string; +} + +interface Replacement { + value: string; + token: string; +} + +export class Redactor { + private readonly replacements: Replacement[] = []; + + constructor(values: RedactorValues = {}) { + this.add(values); + } + + add(values: RedactorValues): void { + this.addReplacement(values.authorization, ''); + this.addReplacement(values.appPassword, ''); + if (values.appPassword) { + const compact = values.appPassword.replace(/\s+/g, ''); + if (compact !== values.appPassword) { + this.addReplacement(compact, ''); + } + } + this.addReplacement(values.username, ''); + if (values.dashboardUrl) { + try { + this.addReplacement(new URL(values.dashboardUrl).origin, ''); + } catch { + this.addReplacement(values.dashboardUrl, ''); + } + } + this.replacements.sort((a, b) => b.value.length - a.value.length); + } + + private addReplacement(value: string | undefined, token: string): void { + if (!value || this.replacements.some(replacement => replacement.value === value)) return; + this.replacements.push({ value, token }); + } + + redact(value: string): string { + let redacted = value; + for (const replacement of this.replacements) { + redacted = redacted.split(replacement.value).join(replacement.token); + } + return redacted; + } + + redactStream(value: string): { output: string; remainder: string } { + const overlapLength = this.replacements[0]?.value.length ?? 0; + if (overlapLength === 0) return { output: value, remainder: '' }; + + let emitEnd = Math.max(0, value.length - overlapLength); + let adjusted = true; + while (adjusted) { + adjusted = false; + for (const replacement of this.replacements) { + let match = value.indexOf(replacement.value); + while (match !== -1) { + if (match < emitEnd && match + replacement.value.length > emitEnd) { + emitEnd = match; + adjusted = true; + break; + } + match = value.indexOf(replacement.value, match + 1); + } + if (adjusted) break; + } + } + + return { + output: this.redact(value.slice(0, emitEnd)), + remainder: value.slice(emitEnd), + }; + } + + stringify(value: unknown, spacing?: number): string { + return this.redact(JSON.stringify(value, null, spacing)); + } +} diff --git a/tests/acceptance/lib/server.ts b/tests/acceptance/lib/server.ts new file mode 100644 index 0000000..7ce4141 --- /dev/null +++ b/tests/acceptance/lib/server.ts @@ -0,0 +1,172 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; +import type { + Transport, + TransportSendOptions, +} from '@modelcontextprotocol/sdk/shared/transport.js'; +import type { JSONRPCMessage, MessageExtraInfo } from '@modelcontextprotocol/sdk/types.js'; +import type { Artifacts } from './artifacts.js'; +import { AcceptanceClient } from './client.js'; +import type { CommandRunner } from './commands.js'; + +class RecordingTransport implements Transport { + onclose?: () => void; + onerror?: (error: Error) => void; + onmessage?: (message: T, extra?: MessageExtraInfo) => void; + + constructor( + private readonly inner: StdioClientTransport, + private readonly scenario: string, + private readonly artifacts: Artifacts + ) {} + + async start(): Promise { + this.inner.onclose = () => this.onclose?.(); + this.inner.onerror = error => this.onerror?.(error); + this.inner.onmessage = message => { + this.artifacts.appendEvent(this.scenario, 'server-to-client', message); + this.onmessage?.(message); + }; + await this.inner.start(); + } + + async send(message: JSONRPCMessage, options?: TransportSendOptions): Promise { + this.artifacts.appendEvent(this.scenario, 'client-to-server', message); + void options; + await this.inner.send(message); + } + + async close(): Promise { + await this.inner.close(); + } +} + +export interface ServerLaunchOptions { + scenario: string; + entry: string; + env: Record; + artifacts: Artifacts; + runner: CommandRunner; + settings?: Record; +} + +export interface ServerConnection { + client: AcceptanceClient; + cwd: string; + home: string; + close(): Promise; +} + +function isolatedEnvironment(home: string, values: Record): Record { + return { + PATH: process.env.PATH ?? '/usr/local/bin:/usr/bin:/bin', + HOME: home, + LOGNAME: '', + SHELL: '', + TERM: '', + USER: '', + ...values, + }; +} + +export async function launchServer(options: ServerLaunchOptions): Promise { + const isolationRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'mainwp-mcp-server-')); + const cwd = path.join(isolationRoot, 'cwd'); + const home = path.join(isolationRoot, 'home'); + fs.mkdirSync(cwd); + fs.mkdirSync(home); + if (options.settings) { + const settingsPath = path.join(cwd, 'settings.json'); + fs.writeFileSync(settingsPath, `${JSON.stringify(options.settings, null, 2)}\n`, { + encoding: 'utf8', + mode: 0o600, + }); + } + + const commandStarted = performance.now(); + let stderr = ''; + const inner = new StdioClientTransport({ + command: process.execPath, + args: [options.entry], + cwd, + env: isolatedEnvironment(home, options.env), + stderr: 'pipe', + }); + inner.stderr?.on('data', chunk => { + const text = Buffer.from(chunk).toString('utf8'); + stderr += text; + options.artifacts.appendServerStderr(options.scenario, text); + }); + const transport = new RecordingTransport(inner, options.scenario, options.artifacts); + const rawClient = new Client({ name: 'mainwp-acceptance-harness', version: '1.0.0' }); + + const closeResources = async (): Promise => { + let closeError: unknown; + try { + await rawClient.close(); + } catch (error) { + closeError = error; + } + try { + await transport.close(); + } catch (error) { + closeError ??= error; + } + if (closeError) throw closeError; + }; + + try { + await rawClient.connect(transport); + } catch (error) { + try { + await closeResources(); + } catch { + // Preserve the connection failure; both cleanup paths were attempted. + } finally { + options.artifacts.flushServerStderr(options.scenario); + options.runner.record({ + argv: [process.execPath, options.entry], + cwd, + exitCode: 1, + durationMs: Math.round(performance.now() - commandStarted), + stdoutTail: '[MCP protocol stream; see events.jsonl]', + stderrTail: stderr.slice(-12_000), + }); + fs.rmSync(isolationRoot, { recursive: true, force: true }); + } + throw error; + } + + let closePromise: Promise | undefined; + return { + client: new AcceptanceClient(rawClient), + cwd, + home, + close: async () => { + closePromise ??= (async () => { + let closeFailed = false; + try { + await closeResources(); + } catch (error) { + closeFailed = true; + throw error; + } finally { + options.artifacts.flushServerStderr(options.scenario); + options.runner.record({ + argv: [process.execPath, options.entry], + cwd, + exitCode: closeFailed ? 1 : 0, + durationMs: Math.round(performance.now() - commandStarted), + stdoutTail: '[MCP protocol stream; see events.jsonl]', + stderrTail: stderr.slice(-12_000), + }); + fs.rmSync(isolationRoot, { recursive: true, force: true }); + } + })(); + await closePromise; + }, + }; +} diff --git a/tests/acceptance/lib/verify.ts b/tests/acceptance/lib/verify.ts new file mode 100644 index 0000000..4dc2462 --- /dev/null +++ b/tests/acceptance/lib/verify.ts @@ -0,0 +1,226 @@ +import { Agent, request } from 'undici'; +import type { AcceptanceCredentials } from './env.js'; +import { BoundedPagination } from './pagination.js'; + +interface AbilityAnnotations { + readonly: boolean; + destructive: boolean; + idempotent: boolean; +} + +interface AbilityDefinition { + name: string; + input_schema?: { + properties?: Record; + }; + meta?: { annotations?: AbilityAnnotations }; +} + +export interface VerifiedSite { + id: number; + url: string; + name: string; + status?: string; + last_sync?: string | null; + [key: string]: unknown; +} + +export interface VerifiedPlugin { + slug: string; + name: string; + version: string; + active: boolean; + update_version?: string | null; + [key: string]: unknown; +} + +export interface VerifiedPluginResponse { + site_id: number; + site_url: string; + plugins: VerifiedPlugin[]; + total: number; +} + +export function serializeToPhpQueryString(input: Record): string { + const params: string[] = []; + for (const [key, value] of Object.entries(input)) { + if (Array.isArray(value)) { + for (const item of value) { + if (item !== null && typeof item === 'object') { + throw new Error(`Unsupported nested query parameter at "${key}": arrays need scalars`); + } + params.push(`input[${encodeURIComponent(key)}][]=${encodeURIComponent(String(item))}`); + } + } else if (value !== null && typeof value === 'object') { + for (const [subKey, subValue] of Object.entries(value)) { + if (subValue !== null && typeof subValue === 'object') { + throw new Error( + `Unsupported nested query parameter at "${key}": objects may be only one level deep` + ); + } + params.push( + `input[${encodeURIComponent(key)}][${encodeURIComponent(subKey)}]=${encodeURIComponent(String(subValue))}` + ); + } + } else if (value !== undefined && value !== null) { + params.push(`input[${encodeURIComponent(key)}]=${encodeURIComponent(String(value))}`); + } + } + return params.length > 0 ? `?${params.join('&')}` : ''; +} + +export class IndependentVerifier { + private readonly baseUrl: string; + private readonly authorization: string; + private readonly dispatcher?: Agent; + private catalog?: AbilityDefinition[]; + + constructor(credentials: AcceptanceCredentials, skipTlsVerify: boolean) { + this.baseUrl = `${credentials.dashboardUrl.replace(/\/+$/, '')}/wp-json/wp-abilities/v1`; + this.authorization = `Basic ${Buffer.from( + `${credentials.username}:${credentials.appPassword}` + ).toString('base64')}`; + if (skipTlsVerify) { + this.dispatcher = new Agent({ connect: { rejectUnauthorized: false } }); + } + } + + getAuthorizationHeader(): string { + return this.authorization; + } + + async close(): Promise { + await this.dispatcher?.close(); + } + + async fetchCatalog(): Promise { + if (this.catalog) return this.catalog; + const abilities: AbilityDefinition[] = []; + for (let page = 1; page <= 50; page += 1) { + const response = await this.requestJsonResponse( + `${this.baseUrl}/abilities?per_page=100&page=${page}`, + 'GET' + ); + if (!Array.isArray(response.data)) { + throw new Error(`Independent verifier expected an ability array on catalog page ${page}`); + } + abilities.push(...response.data); + const rawTotalPages = response.headers['x-wp-totalpages']; + const totalPages = Number( + Array.isArray(rawTotalPages) ? rawTotalPages[0] : (rawTotalPages ?? 1) + ); + if (page >= totalPages) break; + if (page === 50) throw new Error('Independent verifier catalog exceeded the 50-page cap'); + } + this.catalog = abilities; + return this.catalog; + } + + async getAbilityInputArrayEnum(abilityName: string, argumentName: string): Promise { + const ability = (await this.fetchCatalog()).find(candidate => candidate.name === abilityName); + if (!ability) throw new Error(`Independent verifier could not find ability ${abilityName}`); + const values = ability.input_schema?.properties?.[argumentName]?.items?.enum; + if (!Array.isArray(values) || !values.every(value => typeof value === 'string')) { + throw new Error( + `Independent verifier found no string enum for ${abilityName} argument ${argumentName}` + ); + } + return values as string[]; + } + + async execute(abilityName: string, input: Record = {}): Promise { + const catalog = await this.fetchCatalog(); + const ability = catalog.find(candidate => candidate.name === abilityName); + if (!ability) throw new Error(`Independent verifier could not find ability ${abilityName}`); + + const annotations = ability.meta?.annotations; + const isReadonly = annotations?.readonly ?? false; + const isDestructive = annotations?.destructive ?? true; + const isIdempotent = annotations?.idempotent ?? false; + const endpoint = `${this.baseUrl}/abilities/${abilityName}/run`; + + if (isReadonly || (isDestructive && isIdempotent)) { + const method = isReadonly ? 'GET' : 'DELETE'; + return this.requestJson( + endpoint + (Object.keys(input).length > 0 ? serializeToPhpQueryString(input) : ''), + method + ); + } + return this.requestJson(endpoint, 'POST', JSON.stringify({ input })); + } + + async listSites(): Promise { + const sites: VerifiedSite[] = []; + let page = 1; + const pagination = new BoundedPagination('Independent verifier list-sites-v1'); + for (;;) { + const response = (await this.execute('mainwp/list-sites-v1', { + page, + per_page: 100, + })) as { items: VerifiedSite[]; total: number }; + sites.push(...response.items); + const hasMore = sites.length < response.total && response.items.length > 0; + pagination.record(page, response.items, hasMore); + if (!hasMore) return sites; + page += 1; + } + } + + async countSites(): Promise { + const response = (await this.execute('mainwp/count-sites-v1')) as { total: number }; + return response.total; + } + + async getSite(siteIdOrDomain: number | string): Promise { + return (await this.execute('mainwp/get-site-v1', { + site_id_or_domain: siteIdOrDomain, + })) as VerifiedSite; + } + + async getSitePlugins(siteIdOrDomain: number | string): Promise { + return (await this.execute('mainwp/get-site-plugins-v1', { + site_id_or_domain: siteIdOrDomain, + })) as VerifiedPluginResponse; + } + + private async requestJson( + url: string, + method: 'GET' | 'POST' | 'DELETE', + body?: string + ): Promise { + return (await this.requestJsonResponse(url, method, body)).data; + } + + private async requestJsonResponse( + url: string, + method: 'GET' | 'POST' | 'DELETE', + body?: string + ): Promise<{ data: T; headers: Record }> { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 30_000); + try { + const response = await request(url, { + method, + headers: { + authorization: this.authorization, + 'content-type': 'application/json', + }, + signal: controller.signal, + ...(body ? { body } : {}), + ...(this.dispatcher ? { dispatcher: this.dispatcher } : {}), + }); + const responseBody = await response.body.text(); + if (response.statusCode < 200 || response.statusCode >= 300) { + throw new Error( + `Independent verifier request failed with HTTP ${response.statusCode}: ${responseBody}` + ); + } + return { + data: JSON.parse(responseBody) as T, + headers: response.headers, + }; + } finally { + clearTimeout(timeout); + } + } +} diff --git a/tests/acceptance/run.ts b/tests/acceptance/run.ts new file mode 100644 index 0000000..efd8a18 --- /dev/null +++ b/tests/acceptance/run.ts @@ -0,0 +1,427 @@ +#!/usr/bin/env node + +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createArtifacts, type Artifacts } from './lib/artifacts.js'; +import { CommandRunner } from './lib/commands.js'; +import { + FIXTURE_APP_PASSWORD, + FIXTURE_USERNAME, + startFixtureDashboard, + type FixtureDashboard, +} from './fixture-dashboard.js'; +import { + resolveAcceptanceCredentials, + type AcceptanceCredentials, + type ResolvedAcceptanceCredentials, +} from './lib/env.js'; +import { getWriteGuardReason } from './lib/guards.js'; +import { packAndInstall, type PackedPackage } from './lib/pack.js'; +import { Redactor } from './lib/redact.js'; +import { launchServer } from './lib/server.js'; +import { IndependentVerifier } from './lib/verify.js'; +import { scenarios } from './scenarios/index.js'; +import { + AssertionRecorder, + type AcceptanceMode, + type AcceptanceTarget, + type ScenarioContext, + type ScenarioResult, +} from './scenarios/types.js'; + +interface CliOptions { + mode: AcceptanceMode; + target: AcceptanceTarget; + scenarioIds: string[]; + writes: boolean; + list: boolean; + keepConsumer: boolean; +} + +interface RunResults { + runId: string; + mode: AcceptanceMode; + target: AcceptanceTarget; + startedAt: string; + endedAt: string; + scenarios: ScenarioResult[]; + totals: Record<'passed' | 'failed' | 'skipped' | 'unverified', number>; +} + +const REPO_ROOT = path.resolve(fileURLToPath(new URL('../..', import.meta.url))); + +function usage(): string { + return `Usage: npx tsx tests/acceptance/run.ts [options] + +Options: + --mode packed|source Server package mode (default: packed) + --target live|fixture Dashboard target (default: live) + --scenario Run one scenario; repeat for multiple scenarios + --writes Enable host-guarded state-changing scenarios + --list List scenarios and exit + --keep-consumer Preserve the temporary packed consumer project + --help Show this help +`; +} + +function requiredValue(args: string[], index: number, flag: string): string { + const value = args[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`${flag} requires a value`); + return value; +} + +export function parseArgs(args: string[]): CliOptions { + const options: CliOptions = { + mode: 'packed', + target: 'live', + scenarioIds: [], + writes: false, + list: false, + keepConsumer: false, + }; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === '--mode') { + const value = requiredValue(args, index, arg); + if (value !== 'packed' && value !== 'source') throw new Error(`Invalid --mode: ${value}`); + options.mode = value; + index += 1; + } else if (arg === '--target') { + const value = requiredValue(args, index, arg); + if (value !== 'live' && value !== 'fixture') throw new Error(`Invalid --target: ${value}`); + options.target = value; + index += 1; + } else if (arg === '--scenario') { + options.scenarioIds.push(requiredValue(args, index, arg)); + index += 1; + } else if (arg === '--writes') { + options.writes = true; + } else if (arg === '--list') { + options.list = true; + } else if (arg === '--keep-consumer') { + options.keepConsumer = true; + } else if (arg === '--help') { + process.stdout.write(usage()); + process.exit(0); + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + return options; +} + +function basicAuthorization(credentials: AcceptanceCredentials): string { + return `Basic ${Buffer.from(`${credentials.username}:${credentials.appPassword}`).toString( + 'base64' + )}`; +} + +function baseServerEnv( + target: AcceptanceTarget, + credentials: AcceptanceCredentials +): Record { + const skipSslVerify = process.env.MAINWP_MCP_ACCEPTANCE_SKIP_SSL_VERIFY === 'true'; + return { + MAINWP_URL: credentials.dashboardUrl, + MAINWP_USER: credentials.username, + MAINWP_APP_PASSWORD: credentials.appPassword, + MAINWP_RATE_LIMIT: '0', + ...(target === 'fixture' + ? { MAINWP_ALLOW_HTTP: 'true' } + : { MAINWP_SKIP_SSL_VERIFY: skipSslVerify ? 'true' : 'false' }), + }; +} + +function errorText(error: unknown): string { + return error instanceof Error ? `${error.message}\n${error.stack ?? ''}`.trim() : String(error); +} + +function summarize(results: ScenarioResult[]): RunResults['totals'] { + const totals = { passed: 0, failed: 0, skipped: 0, unverified: 0 }; + for (const result of results) totals[result.status] += 1; + return totals; +} + +function summaryMarkdown(run: RunResults): string { + const lines = [ + '# MainWP MCP acceptance results', + '', + `- Run: \`${run.runId}\``, + `- Mode: \`${run.mode}\``, + `- Target: \`${run.target}\``, + `- Passed: ${run.totals.passed}`, + `- Failed: ${run.totals.failed}`, + `- Skipped: ${run.totals.skipped}`, + `- Unverified: ${run.totals.unverified}`, + '', + '| Scenario | Kind | Status | Assertions | Reason |', + '| --- | --- | --- | ---: | --- |', + ]; + for (const result of run.scenarios) { + lines.push( + `| ${result.id} | ${result.kind} | ${result.status} | ${result.assertions.filter(assertion => assertion.pass).length}/${result.assertions.length} | ${(result.reason ?? result.error ?? '').replace(/\r?\n/g, ' ')} |` + ); + } + return `${lines.join('\n')}\n`; +} + +async function runScenario( + definition: (typeof scenarios)[number], + options: CliOptions, + credentials: ResolvedAcceptanceCredentials, + verifier: IndependentVerifier, + entry: string, + packageVersion: string, + packedPackage: PackedPackage | null, + artifacts: Artifacts, + runner: CommandRunner +): Promise { + const started = performance.now(); + const base = { + id: definition.id, + purpose: definition.purpose, + kind: definition.kind, + }; + if (!definition.targets.includes(options.target)) { + return { + ...base, + status: 'skipped', + durationMs: 0, + assertions: [], + reason: `Scenario targets ${definition.targets.join(', ')}, not ${options.target}.`, + }; + } + if (definition.kind === 'write') { + const reason = getWriteGuardReason( + credentials.dashboardUrl, + options.writes, + process.env.MAINWP_MCP_ACCEPTANCE_WRITE_HOSTS, + options.target, + credentials.autoApprovedWriteHost + ); + if (reason) { + return { ...base, status: 'skipped', durationMs: 0, assertions: [], reason }; + } + } + + let precondition; + try { + precondition = + (await definition.preconditions?.({ + target: options.target, + mode: options.mode, + credentials, + verifier, + packedPackage, + })) ?? {}; + } catch (error) { + return { + ...base, + status: 'unverified', + durationMs: Math.round(performance.now() - started), + assertions: [], + reason: errorText(error), + }; + } + if (precondition.status) { + return { + ...base, + status: precondition.status, + durationMs: Math.round(performance.now() - started), + assertions: [], + reason: precondition.reason ?? 'Precondition was not met.', + }; + } + + const launch = precondition.launch ?? {}; + const env = launch.omitCredentialEnv ? {} : baseServerEnv(options.target, credentials); + Object.assign(env, launch.env ?? {}); + const assertions = new AssertionRecorder(); + let connection; + let scenarioContext: ScenarioContext | undefined; + let scenarioError: string | undefined; + try { + connection = await launchServer({ + scenario: definition.id, + entry, + env, + artifacts, + runner, + settings: launch.settings, + }); + scenarioContext = { + client: connection.client, + verifier, + config: { + target: options.target, + mode: options.mode, + dashboardUrl: credentials.dashboardUrl, + packageVersion, + }, + packedPackage, + assert: assertions, + state: precondition.state ?? {}, + }; + await definition.run(scenarioContext); + if (process.env.MAINWP_MCP_ACCEPTANCE_FORCE_FAILURE === definition.id) { + assertions.equal('intentional forced-failure validation probe', 'actual', 'expected'); + } + } catch (error) { + scenarioError = errorText(error); + } finally { + if (scenarioContext && definition.cleanup) { + try { + await definition.cleanup(scenarioContext); + } catch (error) { + scenarioError = [scenarioError, `Cleanup failed: ${errorText(error)}`] + .filter(Boolean) + .join('\n'); + } + } + try { + await connection?.close(); + } catch (error) { + scenarioError = [scenarioError, `Server close failed: ${errorText(error)}`] + .filter(Boolean) + .join('\n'); + } + } + + const failedAssertions = assertions.results.filter(assertion => !assertion.pass); + return { + ...base, + status: scenarioError || failedAssertions.length > 0 ? 'failed' : 'passed', + durationMs: Math.round(performance.now() - started), + assertions: assertions.results, + ...(scenarioError ? { error: scenarioError } : {}), + ...(failedAssertions.length > 0 + ? { reason: `${failedAssertions.length} assertion(s) failed.` } + : {}), + }; +} + +export async function runAcceptance(args = process.argv.slice(2)): Promise { + const options = parseArgs(args); + if (options.list) { + for (const scenario of scenarios) { + process.stdout.write( + `${scenario.id}\t${scenario.kind}\t${scenario.targets.join(',')}\t${scenario.purpose}\n` + ); + } + return 0; + } + + const byId = new Map(scenarios.map(scenario => [scenario.id, scenario])); + const unknown = options.scenarioIds.filter(id => !byId.has(id)); + if (unknown.length > 0) throw new Error(`Unknown scenario ID(s): ${unknown.join(', ')}`); + const selected = + options.scenarioIds.length > 0 ? options.scenarioIds.map(id => byId.get(id)!) : scenarios; + + let fixture: FixtureDashboard | undefined; + let resolvedCredentials: ResolvedAcceptanceCredentials; + if (options.target === 'fixture') { + fixture = await startFixtureDashboard(); + resolvedCredentials = { + dashboardUrl: fixture.url, + username: FIXTURE_USERNAME, + appPassword: FIXTURE_APP_PASSWORD, + }; + } else { + resolvedCredentials = resolveAcceptanceCredentials(); + } + const redactor = new Redactor({ + username: resolvedCredentials.username, + appPassword: resolvedCredentials.appPassword, + dashboardUrl: resolvedCredentials.dashboardUrl, + authorization: basicAuthorization(resolvedCredentials), + }); + const runner = new CommandRunner(); + const artifacts = await createArtifacts( + REPO_ROOT, + redactor, + runner, + options.mode, + options.target, + { + scenarios: options.scenarioIds, + writes: options.writes, + keepConsumer: options.keepConsumer, + } + ); + const verifier = new IndependentVerifier( + resolvedCredentials, + options.target === 'live' && process.env.MAINWP_MCP_ACCEPTANCE_SKIP_SSL_VERIFY === 'true' + ); + let packedPackage: PackedPackage | null = null; + let runResults: RunResults | undefined; + + try { + if (options.mode === 'packed') { + packedPackage = await packAndInstall(REPO_ROOT, runner, artifacts, options.keepConsumer); + } + const entry = packedPackage?.installedEntry ?? path.join(REPO_ROOT, 'dist', 'index.js'); + const results: ScenarioResult[] = []; + for (const scenario of selected) { + const result = await runScenario( + scenario, + options, + resolvedCredentials, + verifier, + entry, + artifacts.manifest.packageVersion, + packedPackage, + artifacts, + runner + ); + results.push(result); + process.stdout.write(`${result.status.toUpperCase()} ${result.id}\n`); + artifacts.writeJson('results.json', { scenarios: results, totals: summarize(results) }); + } + runResults = { + runId: artifacts.runId, + mode: options.mode, + target: options.target, + startedAt: artifacts.manifest.startTime, + endedAt: new Date().toISOString(), + scenarios: results, + totals: summarize(results), + }; + artifacts.writeJson('results.json', runResults); + artifacts.write('summary.md', summaryMarkdown(runResults)); + } catch (error) { + const failure = redactor.redact(errorText(error)); + artifacts.writeJson('results.json', { status: 'failed', error: failure }); + artifacts.write( + 'summary.md', + `# MainWP MCP acceptance results\n\nHarness failure: ${failure}\n` + ); + throw error; + } finally { + artifacts.finish(); + await verifier.close(); + await fixture?.close(); + packedPackage?.cleanup(); + } + + process.stdout.write(`Artifacts: ${artifacts.runDir}\n`); + if (options.keepConsumer && packedPackage) { + process.stdout.write(`Consumer preserved: ${packedPackage.consumerDir}\n`); + } + return acceptanceExitCode(runResults.totals); +} + +export function acceptanceExitCode(totals: RunResults['totals']): number { + return totals.failed > 0 || totals.unverified > 0 ? 1 : 0; +} + +const invokedPath = process.argv[1] ? path.resolve(process.argv[1]) : ''; +if (invokedPath === fileURLToPath(import.meta.url)) { + runAcceptance() + .then(exitCode => { + process.exitCode = exitCode; + }) + .catch(error => { + process.stderr.write(`${errorText(error)}\n`); + process.exitCode = 1; + }); +} diff --git a/tests/acceptance/scenarios/ability-reads.ts b/tests/acceptance/scenarios/ability-reads.ts new file mode 100644 index 0000000..b789c05 --- /dev/null +++ b/tests/acceptance/scenarios/ability-reads.ts @@ -0,0 +1,379 @@ +import type { AcceptanceClient } from '../lib/client.js'; +import { BoundedPagination } from '../lib/pagination.js'; +import type { IndependentVerifier, VerifiedSite } from '../lib/verify.js'; +import type { + ScenarioDefinition, + ScenarioPreconditionContext, + ScenarioPreconditionResult, +} from './types.js'; + +interface CheckSiteResponse { + site_id: number; + checked: boolean; + site?: { id?: number; url?: string }; + status?: { online?: boolean; http_code?: number }; +} + +interface Theme { + slug: string; + version: string; + active: boolean; + update_version?: string | null; +} + +interface ThemeResponse { + site_id: number; + site_url: string; + active_theme: string; + themes: Theme[]; + total: number; +} + +interface Update { + site_id: number; + site_url: string; + site_name: string; + type: string; + slug: string; + name: string; + current_version: string; + new_version: string; +} + +interface Client { + id: number; + name?: string; +} + +interface Tag { + id: number; + name: string; + sites_count: number; + sites_ids?: number[]; +} + +interface PaginatedResponse { + items: T[]; + total: number; +} + +interface UpdatesResponse { + updates: Update[]; + total: number; + errors?: unknown[]; +} + +interface UpdatesSnapshot { + updates: Update[]; + errors: unknown[]; +} + +function sorted(values: string[]): string[] { + return [...values].sort(); +} + +function themeSignature(theme: Theme): string { + return [theme.slug, theme.version, theme.active, theme.update_version ?? ''].join(':'); +} + +function updateSignature(update: Update): string { + return [ + update.site_id, + update.site_url, + update.site_name, + update.type, + update.slug, + update.name, + update.current_version, + update.new_version, + ].join(':'); +} + +function tagSignature(tag: Tag): string { + return [ + tag.id, + tag.name, + tag.sites_count, + sorted((tag.sites_ids ?? []).map(String)).join(','), + ].join(':'); +} + +async function connectedSitePrecondition( + ctx: ScenarioPreconditionContext +): Promise { + const site = (await ctx.verifier.listSites()).find(candidate => candidate.status === 'connected'); + if (!site) { + return { status: 'skipped', reason: 'No connected site was available for the scenario.' }; + } + return { state: { site } }; +} + +function selectedSite(state: Record): VerifiedSite { + const site = state.site as VerifiedSite | undefined; + if (!site) throw new Error('Connected-site precondition did not provide a site'); + return site; +} + +export async function verifierListAll( + verifier: IndependentVerifier, + abilityName: string +): Promise { + const items: T[] = []; + const pagination = new BoundedPagination(`Independent verifier ${abilityName}`); + for (let page = 1; ; page += 1) { + const response = (await verifier.execute(abilityName, { + page, + per_page: 100, + })) as PaginatedResponse; + items.push(...response.items); + const hasMore = items.length < response.total && response.items.length > 0; + pagination.record(page, response.items, hasMore); + if (!hasMore) return items; + } +} + +async function mcpListAll( + client: AcceptanceClient, + toolName: string +): Promise<{ items: T[]; isError: boolean }> { + const items: T[] = []; + const pagination = new BoundedPagination(`MCP ${toolName}`); + for (let page = 1; ; page += 1) { + const { result, data } = await client.callToolJson(toolName, { page, per_page: 100 }); + if (result.isError) return { items, isError: true }; + const response = data as PaginatedResponse; + items.push(...response.items); + const hasMore = items.length < response.total && response.items.length > 0; + pagination.record(page, response.items, hasMore); + if (!hasMore) { + return { items, isError: false }; + } + } +} + +async function verifierListUpdates( + verifier: IndependentVerifier, + siteId: number +): Promise { + const updates: Update[] = []; + const errors: unknown[] = []; + const pagination = new BoundedPagination('Independent verifier list-updates-v1'); + for (let page = 1; ; page += 1) { + const response = (await verifier.execute('mainwp/list-updates-v1', { + site_ids_or_domains: [siteId], + page, + per_page: 200, + })) as UpdatesResponse; + updates.push(...response.updates); + errors.push(...(response.errors ?? [])); + const hasMore = updates.length < response.total && response.updates.length > 0; + pagination.record(page, response.updates, hasMore); + if (!hasMore) { + return { updates, errors }; + } + } +} + +async function mcpListUpdates( + client: AcceptanceClient, + siteId: number +): Promise { + const updates: Update[] = []; + const errors: unknown[] = []; + const pagination = new BoundedPagination('MCP list_updates_v1'); + for (let page = 1; ; page += 1) { + const { result, data } = await client.callToolJson('list_updates_v1', { + site_ids_or_domains: [siteId], + page, + per_page: 200, + }); + if (result.isError) return { updates, errors, isError: true }; + const response = data as UpdatesResponse; + updates.push(...response.updates); + errors.push(...(response.errors ?? [])); + const hasMore = updates.length < response.total && response.updates.length > 0; + pagination.record(page, response.updates, hasMore); + if (!hasMore) { + return { updates, errors, isError: false }; + } + } +} + +export const checkSite: ScenarioDefinition = { + id: 'check-site', + purpose: + 'Verify check-site status against an independent direct execution and enforce a 20-second latency ceiling.', + kind: 'read', + targets: ['live', 'fixture'], + preconditions: connectedSitePrecondition, + async run(ctx) { + const site = selectedSite(ctx.state); + const direct = (await ctx.verifier.execute('mainwp/check-site-v1', { + site_id_or_domain: site.id, + })) as CheckSiteResponse; + + const startedAt = performance.now(); + const { result, data } = await ctx.client.callToolJson('check_site_v1', { + site_id_or_domain: site.id, + }); + const durationMs = performance.now() - startedAt; + const actual = data as CheckSiteResponse; + + ctx.assert.equal('check_site_v1 succeeds', result.isError, undefined); + ctx.assert.lessThan('check_site_v1 completes in under 20 seconds', durationMs, 20_000); + ctx.assert.equal('independent check completed', direct.checked, true); + ctx.assert.equal('independent check targeted the connected site', direct.site_id, site.id); + ctx.assert.equal('checked site id matches the direct result', actual.site_id, direct.site_id); + ctx.assert.equal('checked result matches the direct result', actual.checked, direct.checked); + ctx.assert.equal('site URL matches the direct result', actual.site?.url, direct.site?.url); + ctx.assert.equal( + 'HTTP status matches the direct result', + actual.status?.http_code, + direct.status?.http_code + ); + ctx.assert.equal( + 'online status matches the direct result', + actual.status?.online, + direct.status?.online + ); + }, +}; + +export const siteThemes: ScenarioDefinition = { + id: 'site-themes', + purpose: 'Cross-check a site theme inventory against an independent direct ability read.', + kind: 'read', + targets: ['live'], + preconditions: connectedSitePrecondition, + async run(ctx) { + const site = selectedSite(ctx.state); + const direct = (await ctx.verifier.execute('mainwp/get-site-themes-v1', { + site_id_or_domain: site.id, + })) as ThemeResponse; + const { result, data } = await ctx.client.callToolJson('get_site_themes_v1', { + site_id_or_domain: site.id, + }); + const actual = data as ThemeResponse; + + ctx.assert.equal('get_site_themes_v1 succeeds', result.isError, undefined); + ctx.assert.equal('theme site id matches', actual.site_id, direct.site_id); + ctx.assert.equal('theme site URL matches', actual.site_url, direct.site_url); + ctx.assert.equal('active theme matches', actual.active_theme, direct.active_theme); + ctx.assert.equal('theme total matches', actual.total, direct.total); + ctx.assert.deepEqual( + 'theme inventory matches', + sorted(actual.themes.map(themeSignature)), + sorted(direct.themes.map(themeSignature)) + ); + }, +}; + +export const listUpdatesCrossCheck: ScenarioDefinition = { + id: 'list-updates-cross-check', + purpose: + 'Cross-check available updates against independent direct reads that bracket the MCP snapshot.', + kind: 'read', + targets: ['live'], + preconditions: connectedSitePrecondition, + async run(ctx) { + const site = selectedSite(ctx.state); + const before = await verifierListUpdates(ctx.verifier, site.id); + const mcp = await mcpListUpdates(ctx.client, site.id); + const after = await verifierListUpdates(ctx.verifier, site.id); + const beforeSignatures = sorted(before.updates.map(updateSignature)); + const actualSignatures = sorted(mcp.updates.map(updateSignature)); + const afterSignatures = sorted(after.updates.map(updateSignature)); + const directUnion = new Set([...beforeSignatures, ...afterSignatures]); + const directIntersection = beforeSignatures.filter(signature => + afterSignatures.includes(signature) + ); + const actualSet = new Set(actualSignatures); + const oracleUnchanged = JSON.stringify(beforeSignatures) === JSON.stringify(afterSignatures); + + ctx.assert.equal('list_updates_v1 succeeds for every page', mcp.isError, false); + ctx.assert.equal( + 'bracketing direct update reads have no site errors', + before.errors.length + after.errors.length, + 0 + ); + ctx.assert.equal('MCP update read has no site errors', mcp.errors.length, 0); + ctx.assert.equal( + 'MCP updates are covered by the bracketing direct reads', + actualSignatures.every(signature => directUnion.has(signature)), + true + ); + ctx.assert.equal( + 'updates stable across direct reads are present through MCP', + directIntersection.every(signature => actualSet.has(signature)), + true + ); + ctx.assert.equal( + 'MCP updates match an unchanged direct snapshot', + oracleUnchanged ? JSON.stringify(actualSignatures) : 'changed', + oracleUnchanged ? JSON.stringify(beforeSignatures) : 'changed' + ); + }, +}; + +export const clientsCountConsistency: ScenarioDefinition = { + id: 'clients-count-consistency', + purpose: + 'Verify MCP client listing and count agreement against independent list-clients and count-clients reads.', + kind: 'read', + targets: ['live'], + async run(ctx) { + const directClients = await verifierListAll(ctx.verifier, 'mainwp/list-clients-v1'); + const directCount = (await ctx.verifier.execute('mainwp/count-clients-v1')) as { + total: number; + }; + const mcpClients = await mcpListAll(ctx.client, 'list_clients_v1'); + const { result, data } = await ctx.client.callToolJson('count_clients_v1'); + const mcpCount = data as { total: number }; + + ctx.assert.equal( + 'independent client list length agrees with independent count', + directClients.length, + directCount.total + ); + ctx.assert.equal('list_clients_v1 succeeds for every page', mcpClients.isError, false); + ctx.assert.equal('count_clients_v1 succeeds', result.isError, undefined); + ctx.assert.equal( + 'MCP client list length agrees with count', + mcpClients.items.length, + mcpCount.total + ); + ctx.assert.equal('MCP client count matches direct count', mcpCount.total, directCount.total); + ctx.assert.deepEqual( + 'client id set matches the direct list', + mcpClients.items.map(client => client.id).sort((a, b) => a - b), + directClients.map(client => client.id).sort((a, b) => a - b) + ); + }, +}; + +export const listTagsCrossCheck: ScenarioDefinition = { + id: 'list-tags-cross-check', + purpose: 'Cross-check the complete MCP tag list against an independent direct list-tags read.', + kind: 'read', + targets: ['live'], + async run(ctx) { + const directTags = await verifierListAll(ctx.verifier, 'mainwp/list-tags-v1'); + const mcpTags = await mcpListAll(ctx.client, 'list_tags_v1'); + + ctx.assert.equal('list_tags_v1 succeeds for every page', mcpTags.isError, false); + ctx.assert.equal('tag count matches the direct list', mcpTags.items.length, directTags.length); + ctx.assert.deepEqual( + 'tag inventory matches the direct list', + sorted(mcpTags.items.map(tagSignature)), + sorted(directTags.map(tagSignature)) + ); + }, +}; + +export const abilityReadScenarios = [ + checkSite, + siteThemes, + listUpdatesCrossCheck, + clientsCountConsistency, + listTagsCrossCheck, +]; diff --git a/tests/acceptance/scenarios/completions.ts b/tests/acceptance/scenarios/completions.ts new file mode 100644 index 0000000..7722031 --- /dev/null +++ b/tests/acceptance/scenarios/completions.ts @@ -0,0 +1,48 @@ +import type { ScenarioDefinition } from './types.js'; + +function errorCode(error: unknown): number | undefined { + if (!error || typeof error !== 'object' || !('code' in error)) return undefined; + const code = (error as { code: unknown }).code; + return typeof code === 'number' ? code : undefined; +} + +export const promptCompletions: ScenarioDefinition = { + id: 'prompt-completions', + purpose: + 'Verify prompt completions against the direct ability schema and enforce completion policy.', + kind: 'read', + targets: ['live', 'fixture'], + preconditions: () => ({ launch: { env: { MAINWP_BLOCKED_TOOLS: 'list_sites_v1' } } }), + async run(ctx) { + const validUpdateTypes = await ctx.verifier.getAbilityInputArrayEnum( + 'mainwp/list-updates-v1', + 'types' + ); + const result = await ctx.client.complete( + { type: 'ref/prompt', name: 'update-workflow' }, + 'update_type', + 'c' + ); + const suggestions = result.completion.values; + ctx.assert.truthy('completion suggestions are non-empty', suggestions.length > 0); + ctx.assert.equal( + 'every completion is accepted by the direct list-updates schema', + suggestions.every(value => validUpdateTypes.includes(value)), + true + ); + + let policyError: unknown; + try { + await ctx.client.complete({ type: 'ref/prompt', name: 'performance-check' }, 'site_id', ''); + } catch (error) { + policyError = error; + } + ctx.assert.equal( + 'blocked list-sites denies site-id completions', + errorCode(policyError), + -32008 + ); + }, +}; + +export const completionScenarios = [promptCompletions]; diff --git a/tests/acceptance/scenarios/configuration.ts b/tests/acceptance/scenarios/configuration.ts new file mode 100644 index 0000000..fccf369 --- /dev/null +++ b/tests/acceptance/scenarios/configuration.ts @@ -0,0 +1,62 @@ +import type { ScenarioDefinition } from './types.js'; + +const REQUIRED_PACK_CHECKS = [ + 'requiredFilesPresent', + 'forbiddenFilesAbsent', + 'installedEntryPresent', + 'mainwpBinPresent', + 'installedBinStartsServer', + 'installedVersionMatches', +] as const; + +export const settingsFileConfig: ScenarioDefinition = { + id: 'settings-file-config', + purpose: + 'Load fake fixture credentials from an isolated consumer working-directory settings file.', + kind: 'read', + targets: ['fixture'], + preconditions: ctx => ({ + launch: { + omitCredentialEnv: true, + settings: { + dashboardUrl: ctx.credentials.dashboardUrl, + username: ctx.credentials.username, + appPassword: ctx.credentials.appPassword, + allowHttp: true, + rateLimit: 0, + }, + }, + }), + async run(ctx) { + ctx.assert.equal( + 'settings-based initialization name', + ctx.client.serverInfo?.name, + 'mainwp-mcp' + ); + const { result, data } = await ctx.client.callToolJson('list_sites_v1', { per_page: 100 }); + ctx.assert.equal('settings-based list_sites succeeds', result.isError, undefined); + ctx.assert.equal('settings-based fixture count', (data as { total: number }).total, 3); + }, +}; + +export const packedIntegrity: ScenarioDefinition = { + id: 'packed-integrity', + purpose: 'Surface tarball content, bin, entry-point, and installed-version assertions.', + kind: 'read', + targets: ['live', 'fixture'], + preconditions: ctx => + ctx.mode === 'packed' + ? {} + : { status: 'skipped', reason: 'packed-integrity applies only to --mode packed.' }, + async run(ctx) { + if (!ctx.packedPackage) throw new Error('Packed package metadata was unavailable'); + for (const name of REQUIRED_PACK_CHECKS) { + ctx.assert.equal(`${name} check is present`, name in ctx.packedPackage.checks, true); + ctx.assert.equal(name, ctx.packedPackage.checks[name], true); + } + ctx.assert.truthy('tarball sha256 recorded', /^[a-f0-9]{64}$/.test(ctx.packedPackage.sha256)); + ctx.assert.truthy('npm integrity recorded', ctx.packedPackage.integrity.startsWith('sha512-')); + }, +}; + +export const configurationScenarios = [settingsFileConfig, packedIntegrity]; diff --git a/tests/acceptance/scenarios/confirmation.ts b/tests/acceptance/scenarios/confirmation.ts new file mode 100644 index 0000000..62f1526 --- /dev/null +++ b/tests/acceptance/scenarios/confirmation.ts @@ -0,0 +1,119 @@ +import { parseToolJson } from '../lib/client.js'; +import type { ScenarioDefinition } from './types.js'; + +export const fixtureConfirmationFlow: ScenarioDefinition = { + id: 'fixture-confirmation-flow', + purpose: + 'Exercise destructive preview, token-bound confirmation, execution, and replay rejection against fixture state.', + kind: 'write', + targets: ['fixture'], + async run(ctx) { + const tools = await ctx.client.listTools(); + const deleteSite = tools.tools.find(tool => tool.name === 'delete_site_v1'); + ctx.assert.equal( + 'fixture confirmation tool is destructive', + deleteSite?.annotations?.destructiveHint, + true + ); + ctx.assert.truthy( + 'fixture confirmation tool declares dry_run', + deleteSite?.inputSchema.properties && 'dry_run' in deleteSite.inputSchema.properties + ); + + const before = await ctx.verifier.listSites(); + if (before.length === 0) throw new Error('No fixture site was available for deletion'); + const target = before[0]; + const args = { site_id_or_domain: target.id }; + + const preview = await ctx.client.callTool('delete_site_v1', { ...args, confirm: true }); + const previewData = parseToolJson(preview) as { + status?: string; + confirmation_token?: string; + preview?: unknown; + }; + ctx.assert.equal('preview is a successful workflow step', preview.isError, undefined); + ctx.assert.equal('preview requires confirmation', previewData.status, 'CONFIRMATION_REQUIRED'); + ctx.assert.truthy('preview issues a confirmation token', previewData.confirmation_token); + ctx.assert.truthy('preview payload is present', previewData.preview); + const afterPreview = await ctx.verifier.listSites(); + ctx.assert.deepEqual( + 'preview does not change fixture sites', + afterPreview.map(site => `${site.id}:${site.url}`).sort(), + before.map(site => `${site.id}:${site.url}`).sort() + ); + if (!previewData.confirmation_token) { + throw new Error('delete_site_v1 preview did not issue a confirmation token'); + } + + const tokenless = await ctx.client.callTool('delete_site_v1', { + ...args, + confirm: true, + user_confirmed: true, + }); + const tokenlessData = parseToolJson(tokenless) as { error?: string }; + ctx.assert.equal('tokenless confirm is rejected', tokenless.isError, true); + ctx.assert.equal( + 'tokenless confirm requires the token', + tokenlessData.error, + 'PREVIEW_REQUIRED' + ); + const afterTokenless = await ctx.verifier.listSites(); + ctx.assert.deepEqual( + 'tokenless confirm does not change fixture sites', + afterTokenless.map(site => `${site.id}:${site.url}`).sort(), + before.map(site => `${site.id}:${site.url}`).sort() + ); + + const forged = await ctx.client.callTool('delete_site_v1', { + ...args, + user_confirmed: true, + confirmation_token: `forged-${previewData.confirmation_token}`, + }); + const forgedData = parseToolJson(forged) as { error?: string }; + ctx.assert.equal('forged token is rejected', forged.isError, true); + ctx.assert.equal('forged token requires a new preview', forgedData.error, 'PREVIEW_REQUIRED'); + const afterForged = await ctx.verifier.listSites(); + ctx.assert.deepEqual( + 'forged token does not change fixture sites', + afterForged.map(site => `${site.id}:${site.url}`).sort(), + before.map(site => `${site.id}:${site.url}`).sort() + ); + + const confirmed = await ctx.client.callTool('delete_site_v1', { + ...args, + user_confirmed: true, + confirmation_token: previewData.confirmation_token, + }); + const confirmedData = parseToolJson(confirmed) as { deleted?: boolean }; + ctx.assert.equal('confirmed execution succeeds', confirmed.isError, undefined); + ctx.assert.equal('confirmed execution reports deletion', confirmedData.deleted, true); + const afterConfirmed = await ctx.verifier.listSites(); + ctx.assert.equal( + 'confirmed execution removes one site', + afterConfirmed.length, + before.length - 1 + ); + ctx.assert.equal( + 'confirmed execution removes the previewed site', + afterConfirmed.some(site => site.id === target.id), + false + ); + + const replay = await ctx.client.callTool('delete_site_v1', { + ...args, + user_confirmed: true, + confirmation_token: previewData.confirmation_token, + }); + const replayData = parseToolJson(replay) as { error?: string }; + ctx.assert.equal('consumed token replay is rejected', replay.isError, true); + ctx.assert.equal('token replay requires a new preview', replayData.error, 'PREVIEW_REQUIRED'); + const afterReplay = await ctx.verifier.listSites(); + ctx.assert.deepEqual( + 'token replay does not change fixture sites', + afterReplay.map(site => `${site.id}:${site.url}`).sort(), + afterConfirmed.map(site => `${site.id}:${site.url}`).sort() + ); + }, +}; + +export const confirmationScenarios = [fixtureConfirmationFlow]; diff --git a/tests/acceptance/scenarios/index.ts b/tests/acceptance/scenarios/index.ts new file mode 100644 index 0000000..a9dfa8d --- /dev/null +++ b/tests/acceptance/scenarios/index.ts @@ -0,0 +1,25 @@ +import { abilityReadScenarios } from './ability-reads.js'; +import { configurationScenarios } from './configuration.js'; +import { completionScenarios } from './completions.js'; +import { confirmationScenarios } from './confirmation.js'; +import { policyScenarios } from './policy.js'; +import { readScenarios } from './read.js'; +import type { ScenarioDefinition } from './types.js'; +import { transportScenarios } from './transport.js'; +import { writeScenarios } from './writes.js'; + +export const scenarios: ScenarioDefinition[] = [ + ...readScenarios, + ...abilityReadScenarios, + ...completionScenarios, + ...policyScenarios, + ...configurationScenarios, + ...writeScenarios, + ...confirmationScenarios, + ...transportScenarios, +]; + +const duplicateIds = scenarios + .map(scenario => scenario.id) + .filter((id, index, all) => all.indexOf(id) !== index); +if (duplicateIds.length > 0) throw new Error(`Duplicate acceptance scenario IDs: ${duplicateIds}`); diff --git a/tests/acceptance/scenarios/policy.ts b/tests/acceptance/scenarios/policy.ts new file mode 100644 index 0000000..7468a66 --- /dev/null +++ b/tests/acceptance/scenarios/policy.ts @@ -0,0 +1,126 @@ +import { parseToolJson } from '../lib/client.js'; +import { type ScenarioDefinition } from './types.js'; + +export const blockedToolPolicy: ScenarioDefinition = { + id: 'blocked-tool-policy', + purpose: 'Enforce blocked-tools policy at discovery and execution boundaries.', + kind: 'read', + targets: ['live', 'fixture'], + preconditions: () => ({ launch: { env: { MAINWP_BLOCKED_TOOLS: 'delete_site_v1' } } }), + async run(ctx) { + const tools = await ctx.client.listTools(); + ctx.assert.equal( + 'blocked tool is absent from discovery', + tools.tools.some(tool => tool.name === 'delete_site_v1'), + false + ); + const result = await ctx.client.callTool('delete_site_v1', { site_id_or_domain: 1 }); + const data = parseToolJson(result) as { error?: { code?: number; message?: string } }; + ctx.assert.equal('blocked call returns isError', result.isError, true); + ctx.assert.equal('blocked call uses permission code', data.error?.code, -32008); + ctx.assert.truthy( + 'blocked call reports policy only', + data.error?.message?.includes('not allowed') + ); + ctx.assert.equal( + 'blocked response does not expose ability name', + JSON.stringify(data).includes('mainwp/delete-site-v1'), + false + ); + }, +}; + +export const allowedToolsPolicy: ScenarioDefinition = { + id: 'allowed-tools-policy', + purpose: 'Expose exactly the configured allowed tool set.', + kind: 'read', + targets: ['live', 'fixture'], + preconditions: () => ({ + launch: { env: { MAINWP_ALLOWED_TOOLS: 'list_sites_v1,count_sites_v1' } }, + }), + async run(ctx) { + const tools = await ctx.client.listTools(); + ctx.assert.deepEqual('allowed tool set is exact', tools.tools.map(tool => tool.name).sort(), [ + 'count_sites_v1', + 'list_sites_v1', + ]); + }, +}; + +export const safeModeBlocksDestructive: ScenarioDefinition = { + id: 'safe-mode-blocks-destructive', + purpose: 'Prove safe mode blocks a destructive operation without changing state.', + kind: 'read', + targets: ['live', 'fixture'], + // Read-safe despite targeting delete_site_v1: safe mode must block it, and + // even if safe mode were broken, the bogus confirmation token cannot pass + // the confirmation gate — two independent layers prevent execution. + preconditions: () => ({ launch: { env: { MAINWP_SAFE_MODE: 'true' } } }), + async run(ctx) { + const before = await ctx.verifier.listSites(); + if (before.length === 0) throw new Error('No site was available for the safe-mode probe'); + const tools = await ctx.client.listTools(); + const tool = tools.tools.find(candidate => candidate.name === 'delete_site_v1'); + ctx.assert.equal( + 'delete_site_v1 is marked destructive', + tool?.annotations?.destructiveHint, + true + ); + const result = await ctx.client.callTool('delete_site_v1', { + site_id_or_domain: before[0].id, + confirm: true, + user_confirmed: true, + confirmation_token: 'acceptance-safe-mode-token', + }); + ctx.assert.equal('safe mode returns isError', result.isError, true); + ctx.assert.truthy( + 'safe mode response is explicit', + JSON.stringify(parseToolJson(result)).includes('SAFE_MODE_BLOCKED') + ); + const after = await ctx.verifier.listSites(); + ctx.assert.deepEqual( + 'sites are unchanged', + after.map(site => `${site.id}:${site.url}`).sort(), + before.map(site => `${site.id}:${site.url}`).sort() + ); + }, +}; + +export const confirmationGateNoToken: ScenarioDefinition = { + id: 'confirmation-gate-no-token', + purpose: 'Require preview and a token for destructive execution while leaving state unchanged.', + kind: 'read', + targets: ['live', 'fixture'], + async run(ctx) { + const before = await ctx.verifier.listSites(); + if (before.length === 0) throw new Error('No site was available for confirmation preview'); + const result = await ctx.client.callTool('delete_site_v1', { + site_id_or_domain: before[0].id, + confirm: true, + }); + const data = parseToolJson(result) as { + status?: string; + next_action?: string; + confirmation_token?: string; + preview?: unknown; + }; + ctx.assert.equal('preview step is not an error', result.isError, undefined); + ctx.assert.equal('confirmation is required', data.status, 'CONFIRMATION_REQUIRED'); + ctx.assert.equal('preview then confirm action', data.next_action, 'show_preview_and_confirm'); + ctx.assert.truthy('confirmation token is issued', data.confirmation_token); + ctx.assert.truthy('preview payload is present', data.preview); + const after = await ctx.verifier.listSites(); + ctx.assert.deepEqual( + 'preview did not change sites', + after.map(site => `${site.id}:${site.url}`).sort(), + before.map(site => `${site.id}:${site.url}`).sort() + ); + }, +}; + +export const policyScenarios = [ + blockedToolPolicy, + allowedToolsPolicy, + safeModeBlocksDestructive, + confirmationGateNoToken, +]; diff --git a/tests/acceptance/scenarios/read.ts b/tests/acceptance/scenarios/read.ts new file mode 100644 index 0000000..25d13ad --- /dev/null +++ b/tests/acceptance/scenarios/read.ts @@ -0,0 +1,243 @@ +import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js'; +import { parseToolJson } from '../lib/client.js'; +import type { VerifiedPluginResponse, VerifiedSite } from '../lib/verify.js'; +import { findSiteWithPlugins, mcpListAllSites, type ScenarioDefinition } from './types.js'; + +function sorted(values: string[]): string[] { + return [...values].sort(); +} + +async function assertSessionRecovery(ctx: Parameters[0]): Promise { + const followUp = await ctx.client.callTool('count_sites_v1', {}); + ctx.assert.equal('same-session follow-up is not an error', followUp.isError, undefined); + const data = parseToolJson(followUp) as { total?: number }; + ctx.assert.truthy('same-session follow-up returns a count', typeof data.total === 'number'); +} + +function getErrorCode(error: unknown): number | undefined { + if (!error || typeof error !== 'object' || !('code' in error)) return undefined; + const code = (error as { code: unknown }).code; + return typeof code === 'number' ? code : undefined; +} + +export const startupHandshake: ScenarioDefinition = { + id: 'startup-handshake', + purpose: + 'Prove MCP initialization succeeds with the published server identity and required capabilities.', + kind: 'read', + targets: ['live', 'fixture'], + async run(ctx) { + ctx.assert.equal('server name', ctx.client.serverInfo?.name, 'mainwp-mcp'); + ctx.assert.equal('server version', ctx.client.serverInfo?.version, ctx.config.packageVersion); + ctx.assert.truthy('tools capability', ctx.client.capabilities?.tools); + ctx.assert.truthy('resources capability', ctx.client.capabilities?.resources); + ctx.assert.truthy('prompts capability', ctx.client.capabilities?.prompts); + }, +}; + +export const discoveryTools: ScenarioDefinition = { + id: 'discovery-tools', + purpose: 'Validate the exposed tool catalog, schemas, annotations, and collision-free names.', + kind: 'read', + targets: ['live', 'fixture'], + async run(ctx) { + const { tools } = await ctx.client.listTools(); + const names = tools.map(tool => tool.name); + const listSites = tools.find(tool => tool.name === 'list_sites_v1'); + ctx.assert.truthy('tool catalog is non-empty', tools.length > 0); + ctx.assert.truthy('list_sites_v1 is exposed', listSites); + ctx.assert.equal('list_sites_v1 schema is object-like', listSites?.inputSchema.type, 'object'); + ctx.assert.equal('list_sites_v1 readOnlyHint', listSites?.annotations?.readOnlyHint, true); + ctx.assert.equal( + 'list_sites_v1 destructiveHint', + listSites?.annotations?.destructiveHint, + false + ); + ctx.assert.equal('tool names are unique', new Set(names).size, names.length); + }, +}; + +export const discoveryResourcesPrompts: ScenarioDefinition = { + id: 'discovery-resources-prompts', + purpose: 'Prove resources, prompts, and the static help resource work through MCP.', + kind: 'read', + targets: ['live', 'fixture'], + async run(ctx) { + const resources = await ctx.client.listResources(); + const prompts = await ctx.client.listPrompts(); + const help = await ctx.client.readResource('mainwp://help'); + ctx.assert.includes( + 'mainwp://help is listed', + resources.resources.map(resource => resource.uri), + 'mainwp://help' + ); + ctx.assert.truthy('prompt catalog is non-empty', prompts.prompts.length > 0); + ctx.assert.truthy( + 'help resource has text', + help.contents.some(content => 'text' in content && content.text.length > 0) + ); + }, +}; + +export const listSitesCrossCheck: ScenarioDefinition = { + id: 'list-sites-cross-check', + purpose: 'Cross-check MCP site discovery against independent Abilities API reads.', + kind: 'read', + targets: ['live', 'fixture'], + async run(ctx) { + const [mcpSites, directSites] = await Promise.all([ + mcpListAllSites(ctx.client), + ctx.verifier.listSites(), + ]); + ctx.assert.equal('same site count', mcpSites.length, directSites.length); + ctx.assert.deepEqual( + 'same site URL set', + sorted(mcpSites.map(site => site.url)), + sorted(directSites.map(site => site.url)) + ); + }, +}; + +export const countSitesConsistency: ScenarioDefinition = { + id: 'count-sites-consistency', + purpose: 'Verify the MCP count-sites result equals an independent direct count.', + kind: 'read', + targets: ['live', 'fixture'], + async run(ctx) { + const { result, data } = await ctx.client.callToolJson('count_sites_v1'); + const directCount = await ctx.verifier.countSites(); + ctx.assert.equal('count_sites_v1 succeeds', result.isError, undefined); + ctx.assert.equal( + 'MCP and independent counts match', + (data as { total: number }).total, + directCount + ); + }, +}; + +export const getSite: ScenarioDefinition = { + id: 'get-site', + purpose: 'Verify a discovered site detail response against an independent read.', + kind: 'read', + targets: ['live', 'fixture'], + async run(ctx) { + const sites = await ctx.verifier.listSites(); + if (sites.length === 0) throw new Error('No sites were available'); + const direct = await ctx.verifier.getSite(sites[0].id); + const { result, data } = await ctx.client.callToolJson('get_site_v1', { + site_id_or_domain: sites[0].id, + }); + const actual = data as VerifiedSite; + ctx.assert.equal('get_site_v1 succeeds', result.isError, undefined); + ctx.assert.equal('site id matches', actual.id, direct.id); + ctx.assert.equal('site URL matches', actual.url, direct.url); + ctx.assert.equal('site name matches', actual.name, direct.name); + }, +}; + +export const sitePlugins: ScenarioDefinition = { + id: 'site-plugins', + purpose: 'Verify a site plugin inventory and a known slug against an independent read.', + kind: 'read', + targets: ['live', 'fixture'], + async run(ctx) { + const { site, plugins: direct } = await findSiteWithPlugins(ctx.verifier); + const { result, data } = await ctx.client.callToolJson('get_site_plugins_v1', { + site_id_or_domain: site.id, + }); + const actual = data as VerifiedPluginResponse; + ctx.assert.equal('get_site_plugins_v1 succeeds', result.isError, undefined); + ctx.assert.deepEqual( + 'plugin inventory matches', + sorted(actual.plugins.map(plugin => `${plugin.slug}:${plugin.active}`)), + sorted(direct.plugins.map(plugin => `${plugin.slug}:${plugin.active}`)) + ); + const knownSlug = direct.plugins[0].slug; + ctx.assert.includes( + 'known plugin slug is present', + actual.plugins.map(plugin => plugin.slug), + knownSlug + ); + if (ctx.config.target === 'fixture') { + ctx.assert.includes( + 'fixture includes Hello Dolly', + actual.plugins.map(plugin => plugin.slug), + 'hello.php' + ); + } + }, +}; + +export const notFoundInput: ScenarioDefinition = { + id: 'not-found-input', + purpose: 'Require a structured not-found error and prove the same MCP session recovers.', + kind: 'read', + targets: ['live', 'fixture'], + async run(ctx) { + const result = await ctx.client.callTool('get_site_v1', { + site_id_or_domain: 'nonexistent-acceptance-probe.invalid', + }); + ctx.assert.equal('not-found returns isError', result.isError, true); + const data = parseToolJson(result) as { error?: { code?: number } }; + ctx.assert.truthy('not-found response is structured JSON', data && typeof data === 'object'); + ctx.assert.equal('not-found uses resource-not-found code', data.error?.code, -32002); + await assertSessionRecovery(ctx); + }, +}; + +export const invalidArgs: ScenarioDefinition = { + id: 'invalid-args', + purpose: 'Reject wrong-typed arguments without terminating the MCP session.', + kind: 'read', + targets: ['live', 'fixture'], + async run(ctx) { + let result: CallToolResult | undefined; + let protocolError: unknown; + try { + result = await ctx.client.callTool('count_sites_v1', { tag_ids: 'not-an-array' }); + } catch (error) { + protocolError = error; + } + ctx.assert.truthy( + 'invalid input is rejected', + protocolError !== undefined || result?.isError === true + ); + const structuredError = result + ? (parseToolJson(result) as { error?: { code?: number } }) + : undefined; + ctx.assert.equal( + 'invalid input uses invalid-params code', + protocolError === undefined ? structuredError?.error?.code : getErrorCode(protocolError), + -32602 + ); + await assertSessionRecovery(ctx); + }, +}; + +export const unknownTool: ScenarioDefinition = { + id: 'unknown-tool', + purpose: 'Return the tool-not-found MCP code and keep the session usable.', + kind: 'read', + targets: ['live', 'fixture'], + async run(ctx) { + const result = await ctx.client.callTool('no_such_tool_v1', {}); + const data = parseToolJson(result) as { error?: { code?: number; message?: string } }; + ctx.assert.equal('unknown tool returns isError', result.isError, true); + ctx.assert.equal('unknown tool code', data.error?.code, -32003); + ctx.assert.truthy('unknown tool message', data.error?.message?.includes('Tool not found')); + await assertSessionRecovery(ctx); + }, +}; + +export const readScenarios = [ + startupHandshake, + discoveryTools, + discoveryResourcesPrompts, + listSitesCrossCheck, + countSitesConsistency, + getSite, + sitePlugins, + notFoundInput, + invalidArgs, + unknownTool, +]; diff --git a/tests/acceptance/scenarios/transport.ts b/tests/acceptance/scenarios/transport.ts new file mode 100644 index 0000000..1419501 --- /dev/null +++ b/tests/acceptance/scenarios/transport.ts @@ -0,0 +1,85 @@ +import { parseToolJson } from '../lib/client.js'; +import { FIXTURE_DELAY_SEARCH, FIXTURE_OVERSIZED_SEARCH } from '../fixture-dashboard.js'; +import type { ScenarioDefinition } from './types.js'; + +interface StructuredError { + error?: { code?: number; message?: string }; +} + +async function assertSessionRecovery( + ctx: Parameters[0], + expectedCount: number +): Promise { + const { result, data } = await ctx.client.callToolJson('count_sites_v1'); + ctx.assert.equal('same-session recovery call succeeds', result.isError, undefined); + ctx.assert.equal( + 'same-session recovery count matches the fixture', + (data as { total?: number }).total, + expectedCount + ); +} + +export const oversizedResponseRecovery: ScenarioDefinition = { + id: 'oversized-response-recovery', + purpose: 'Reject an oversized fixture response and keep the same MCP session usable.', + kind: 'read', + targets: ['fixture'], + preconditions: () => ({ + launch: { + env: { + MAINWP_MAX_RESPONSE_SIZE: '200000', + MAINWP_RETRY_ENABLED: 'false', + }, + }, + }), + async run(ctx) { + const expectedCount = await ctx.verifier.countSites(); + const result = await ctx.client.callTool('list_sites_v1', { + search: FIXTURE_OVERSIZED_SEARCH, + }); + const data = parseToolJson(result) as StructuredError; + ctx.assert.equal('oversized response returns isError', result.isError, true); + ctx.assert.equal( + 'oversized response has a structured numeric code', + typeof data.error?.code, + 'number' + ); + ctx.assert.truthy( + 'oversized response reports the configured transport limit', + /response (?:body|size).*exceeds|maximum allowed|bytes limit/i.test(data.error?.message ?? '') + ); + await assertSessionRecovery(ctx, expectedCount); + }, +}; + +export const requestTimeoutRecovery: ScenarioDefinition = { + id: 'request-timeout-recovery', + purpose: + 'Return a structured timeout for a delayed fixture response and keep the session usable.', + kind: 'read', + targets: ['fixture'], + preconditions: () => ({ + launch: { + env: { + MAINWP_REQUEST_TIMEOUT: '250', + MAINWP_RETRY_ENABLED: 'false', + }, + }, + }), + async run(ctx) { + const expectedCount = await ctx.verifier.countSites(); + const result = await ctx.client.callTool('list_sites_v1', { + search: FIXTURE_DELAY_SEARCH, + }); + const data = parseToolJson(result) as StructuredError; + ctx.assert.equal('delayed response returns isError', result.isError, true); + ctx.assert.equal('delayed response uses the timeout code', data.error?.code, -32001); + ctx.assert.truthy( + 'delayed response reports a timeout', + data.error?.message?.toLowerCase().includes('timeout') + ); + await assertSessionRecovery(ctx, expectedCount); + }, +}; + +export const transportScenarios = [oversizedResponseRecovery, requestTimeoutRecovery]; diff --git a/tests/acceptance/scenarios/types.ts b/tests/acceptance/scenarios/types.ts new file mode 100644 index 0000000..c62ffc5 --- /dev/null +++ b/tests/acceptance/scenarios/types.ts @@ -0,0 +1,186 @@ +import { isDeepStrictEqual } from 'node:util'; +import type { AcceptanceClient } from '../lib/client.js'; +import type { AcceptanceCredentials } from '../lib/env.js'; +import type { PackedPackage } from '../lib/pack.js'; +import { BoundedPagination } from '../lib/pagination.js'; +import type { IndependentVerifier, VerifiedPluginResponse, VerifiedSite } from '../lib/verify.js'; + +export type AcceptanceTarget = 'live' | 'fixture'; +export type AcceptanceMode = 'packed' | 'source'; +export type ScenarioStatus = 'passed' | 'failed' | 'skipped' | 'unverified'; + +export interface AssertionResult { + name: string; + expected: unknown; + actual: unknown; + pass: boolean; +} + +function recordedValue(value: unknown): unknown { + return value === undefined ? '' : value; +} + +export class AssertionRecorder { + readonly results: AssertionResult[] = []; + + equal(name: string, actual: unknown, expected: unknown): void { + this.results.push({ + name, + expected: recordedValue(expected), + actual: recordedValue(actual), + pass: Object.is(actual, expected), + }); + } + + deepEqual(name: string, actual: unknown, expected: unknown): void { + this.results.push({ + name, + expected: recordedValue(expected), + actual: recordedValue(actual), + pass: isDeepStrictEqual(actual, expected), + }); + } + + truthy(name: string, actual: unknown, expected = true): void { + this.results.push({ + name, + expected, + actual: recordedValue(actual), + pass: Boolean(actual), + }); + } + + lessThan(name: string, actual: number, expected: number): void { + this.results.push({ + name, + expected, + actual, + pass: actual < expected, + }); + } + + includes(name: string, values: unknown[], expected: unknown): void { + this.results.push({ + name, + expected: recordedValue(expected), + actual: values, + pass: values.includes(expected), + }); + } +} + +export interface ScenarioLaunch { + env?: Record; + settings?: Record; + omitCredentialEnv?: boolean; +} + +export interface ScenarioPreconditionContext { + target: AcceptanceTarget; + mode: AcceptanceMode; + credentials: AcceptanceCredentials; + verifier: IndependentVerifier; + packedPackage: PackedPackage | null; +} + +export interface ScenarioPreconditionResult { + status?: 'skipped' | 'unverified'; + reason?: string; + launch?: ScenarioLaunch; + state?: Record; +} + +export interface ScenarioContext { + client: AcceptanceClient; + verifier: IndependentVerifier; + config: { + target: AcceptanceTarget; + mode: AcceptanceMode; + dashboardUrl: string; + packageVersion: string; + }; + packedPackage: PackedPackage | null; + assert: AssertionRecorder; + state: Record; +} + +export interface ScenarioDefinition { + id: string; + purpose: string; + kind: 'read' | 'write'; + targets: AcceptanceTarget[]; + preconditions?: ( + ctx: ScenarioPreconditionContext + ) => Promise | ScenarioPreconditionResult; + run(ctx: ScenarioContext): Promise; + cleanup?(ctx: ScenarioContext): Promise; +} + +export interface ScenarioResult { + id: string; + purpose: string; + kind: 'read' | 'write'; + status: ScenarioStatus; + durationMs: number; + assertions: AssertionResult[]; + reason?: string; + error?: string; +} + +export async function mcpListAllSites(client: AcceptanceClient): Promise { + const sites: VerifiedSite[] = []; + let page = 1; + const pagination = new BoundedPagination('MCP list_sites_v1'); + for (;;) { + const { result, data } = await client.callToolJson('list_sites_v1', { page, per_page: 100 }); + if (result.isError) throw new Error(`list_sites_v1 failed: ${JSON.stringify(data)}`); + const response = data as { items: VerifiedSite[]; total: number }; + sites.push(...response.items); + const hasMore = sites.length < response.total && response.items.length > 0; + pagination.record(page, response.items, hasMore); + if (!hasMore) return sites; + page += 1; + } +} + +export async function findSiteWithPlugins( + verifier: IndependentVerifier +): Promise<{ site: VerifiedSite; plugins: VerifiedPluginResponse }> { + for (const site of (await verifier.listSites()).filter( + candidate => candidate.status === 'connected' + )) { + try { + const plugins = await verifier.getSitePlugins(site.id); + if (plugins.plugins.length > 0) return { site, plugins }; + } catch { + // A later connected site may still be reachable and suitable. + } + } + throw new Error('No site with plugins was available for the scenario'); +} + +export async function findHelloDolly( + verifier: IndependentVerifier, + preferredSlug?: string +): Promise<{ site: VerifiedSite; slug: string; active: boolean } | null> { + const safeSlugs = [preferredSlug, 'hello.php', 'hello-dolly/hello.php'].filter( + (value): value is string => Boolean(value) + ); + const inventories: Array<{ site: VerifiedSite; plugins: VerifiedPluginResponse['plugins'] }> = []; + for (const site of (await verifier.listSites()).filter( + candidate => candidate.status === 'connected' + )) { + try { + inventories.push({ site, plugins: (await verifier.getSitePlugins(site.id)).plugins }); + } catch { + // Continue to the next connected site when one inventory is unavailable. + } + } + for (const slug of safeSlugs) { + for (const inventory of inventories) { + const plugin = inventory.plugins.find(candidate => candidate.slug === slug); + if (plugin) return { site: inventory.site, slug: plugin.slug, active: plugin.active }; + } + } + return null; +} diff --git a/tests/acceptance/scenarios/writes.ts b/tests/acceptance/scenarios/writes.ts new file mode 100644 index 0000000..d0bf24e --- /dev/null +++ b/tests/acceptance/scenarios/writes.ts @@ -0,0 +1,171 @@ +import { parseToolJson } from '../lib/client.js'; +import { findHelloDolly, type ScenarioDefinition } from './types.js'; + +async function waitForSyncAdvance( + ctx: Parameters[0], + siteId: number, + before: string | null | undefined +): Promise { + for (let attempt = 0; attempt < 10; attempt += 1) { + const current = await ctx.verifier.getSite(siteId); + if (current.last_sync && current.last_sync !== before) return current.last_sync; + await new Promise(resolve => setTimeout(resolve, 500)); + } + return (await ctx.verifier.getSite(siteId)).last_sync; +} + +async function setPluginActive( + ctx: Parameters[0], + siteId: number, + slug: string, + active: boolean, + requireConfirmation: boolean +): Promise { + const tool = active ? 'activate_site_plugins_v1' : 'deactivate_site_plugins_v1'; + const args = { site_id_or_domain: siteId, plugins: [slug] }; + if (!requireConfirmation) { + const result = await ctx.client.callTool(tool, args); + if (result.isError) throw new Error(`${tool} failed: ${JSON.stringify(parseToolJson(result))}`); + // If the Dashboard still gates this tool, finish the flow instead of + // leaving the toggle half-done (matters for cleanup reliability). + const data = parseToolJson(result) as { status?: string; confirmation_token?: string } | null; + if (data?.status === 'CONFIRMATION_REQUIRED') { + if (!data.confirmation_token) { + throw new Error(`${tool} required confirmation but issued no token`); + } + const confirmed = await ctx.client.callTool(tool, { + ...args, + user_confirmed: true, + confirmation_token: data.confirmation_token, + }); + if (confirmed.isError) { + throw new Error( + `${tool} confirmed call failed: ${JSON.stringify(parseToolJson(confirmed))}` + ); + } + } + return; + } + const preview = await ctx.client.callTool(tool, { ...args, confirm: true }); + const previewData = parseToolJson(preview) as { confirmation_token?: string; status?: string }; + ctx.assert.equal( + `${tool} preview requires confirmation`, + previewData.status, + 'CONFIRMATION_REQUIRED' + ); + ctx.assert.truthy(`${tool} preview issues token`, previewData.confirmation_token); + if (!previewData.confirmation_token) + throw new Error(`${tool} did not issue a confirmation token`); + const confirmed = await ctx.client.callTool(tool, { + ...args, + user_confirmed: true, + confirmation_token: previewData.confirmation_token, + }); + if (confirmed.isError) { + throw new Error(`${tool} confirmed call failed: ${JSON.stringify(parseToolJson(confirmed))}`); + } +} + +export const syncSite: ScenarioDefinition = { + id: 'sync-site', + purpose: 'Sync one discovered site and independently verify its last-sync timestamp advances.', + kind: 'write', + targets: ['live'], + async run(ctx) { + const sites = (await ctx.verifier.listSites()).filter(site => site.status === 'connected'); + let before; + let siteId: number | undefined; + for (const site of sites) { + try { + before = await ctx.verifier.getSite(site.id); + siteId = site.id; + break; + } catch { + // Continue until a connected site also responds to a direct read. + } + } + if (!before || siteId === undefined) { + throw new Error('No connected, reachable site was available to sync'); + } + const result = await ctx.client.callTool('sync_sites_v1', { + site_ids_or_domains: [siteId], + }); + ctx.assert.equal('sync_sites_v1 succeeds', result.isError, undefined); + const after = await waitForSyncAdvance(ctx, siteId, before.last_sync); + ctx.assert.truthy('last_sync advanced', after && after !== before.last_sync); + }, +}; + +export const pluginToggleRoundtrip: ScenarioDefinition = { + id: 'plugin-toggle-roundtrip', + purpose: 'Deactivate and reactivate a safe plugin with independent state verification.', + kind: 'write', + targets: ['live'], + preconditions: async ctx => { + const plugin = await findHelloDolly( + ctx.verifier, + process.env.MAINWP_MCP_ACCEPTANCE_TOGGLE_PLUGIN + ); + if (!plugin?.active) { + return { status: 'skipped', reason: 'No active allowed toggle plugin was discovered.' }; + } + return { state: { plugin } }; + }, + async run(ctx) { + const plugin = ctx.state.plugin as Awaited>; + if (!plugin) throw new Error('Plugin precondition did not provide a plugin'); + // Whether the toggle needs the two-step confirmation flow is the + // Dashboard's call (destructive annotation + confirm parameter), so + // derive it from the exposed tool instead of assuming either shape. + const tools = await ctx.client.listTools(); + const needsConfirmation = (name: string): boolean => { + const tool = tools.tools.find(candidate => candidate.name === name); + return Boolean( + tool?.annotations?.destructiveHint && + tool.inputSchema.properties && + 'confirm' in tool.inputSchema.properties + ); + }; + ctx.assert.truthy( + 'deactivate tool is exposed', + tools.tools.some(tool => tool.name === 'deactivate_site_plugins_v1') + ); + + await setPluginActive( + ctx, + plugin.site.id, + plugin.slug, + false, + needsConfirmation('deactivate_site_plugins_v1') + ); + const inactive = await ctx.verifier.getSitePlugins(plugin.site.id); + ctx.assert.equal( + 'plugin is independently inactive', + inactive.plugins.find(candidate => candidate.slug === plugin.slug)?.active, + false + ); + await setPluginActive( + ctx, + plugin.site.id, + plugin.slug, + true, + needsConfirmation('activate_site_plugins_v1') + ); + const restored = await ctx.verifier.getSitePlugins(plugin.site.id); + ctx.assert.equal( + 'plugin is independently active again', + restored.plugins.find(candidate => candidate.slug === plugin.slug)?.active, + true + ); + }, + async cleanup(ctx) { + const plugin = ctx.state.plugin as Awaited>; + if (!plugin) return; + const current = await ctx.verifier.getSitePlugins(plugin.site.id); + if (current.plugins.find(candidate => candidate.slug === plugin.slug)?.active === false) { + await setPluginActive(ctx, plugin.site.id, plugin.slug, true, false); + } + }, +}; + +export const writeScenarios = [syncSite, pluginToggleRoundtrip]; diff --git a/tests/evals/description-quality.test.ts b/tests/evals/description-quality.test.ts index 559de4d..69692f5 100644 --- a/tests/evals/description-quality.test.ts +++ b/tests/evals/description-quality.test.ts @@ -13,35 +13,14 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { getTools, clearToolsCache } from '../../src/tools.js'; import { clearCache, initRateLimiter } from '../../src/abilities.js'; -import { type Config } from '../../src/config.js'; +import { makeBaseConfig } from '../helpers/config.js'; -import abilitiesFixture from './fixtures/abilities-full.json'; +import abilitiesFixture from './fixtures/abilities-full.json' with { type: 'json' }; const mockFetch = vi.fn(); vi.stubGlobal('fetch', mockFetch); -const baseConfig: Config = { - dashboardUrl: 'https://test.local', - authType: 'basic', - username: 'admin', - appPassword: 'xxxx', - skipSslVerify: true, - allowHttp: false, - rateLimit: 0, - requestTimeout: 5000, - maxResponseSize: 10485760, - safeMode: false, - requireUserConfirmation: true, - maxSessionData: 52428800, - schemaVerbosity: 'standard', - responseFormat: 'compact', - abilityNamespaces: ['mainwp'], - configSource: 'environment', - retryEnabled: false, - maxRetries: 2, - retryBaseDelay: 1000, - retryMaxDelay: 2000, -}; +const baseConfig = makeBaseConfig(); function mockAbilitiesFetch(): void { mockFetch.mockResolvedValueOnce({ diff --git a/tests/evals/safety-coverage.test.ts b/tests/evals/safety-coverage.test.ts index 80bd96e..042d98a 100644 --- a/tests/evals/safety-coverage.test.ts +++ b/tests/evals/safety-coverage.test.ts @@ -15,36 +15,15 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { getTools, clearToolsCache } from '../../src/tools.js'; import { generateInstructions, buildSafetyTags } from '../../src/tool-schema.js'; import { clearCache, initRateLimiter } from '../../src/abilities.js'; -import { type Config } from '../../src/config.js'; import type { Ability } from '../../src/abilities.js'; +import { makeBaseConfig } from '../helpers/config.js'; -import abilitiesFixture from './fixtures/abilities-full.json'; +import abilitiesFixture from './fixtures/abilities-full.json' with { type: 'json' }; const mockFetch = vi.fn(); vi.stubGlobal('fetch', mockFetch); -const baseConfig: Config = { - dashboardUrl: 'https://test.local', - authType: 'basic', - username: 'admin', - appPassword: 'xxxx', - skipSslVerify: true, - allowHttp: false, - rateLimit: 0, - requestTimeout: 5000, - maxResponseSize: 10485760, - safeMode: false, - requireUserConfirmation: true, - maxSessionData: 52428800, - schemaVerbosity: 'standard', - responseFormat: 'compact', - abilityNamespaces: ['mainwp'], - configSource: 'environment', - retryEnabled: false, - maxRetries: 2, - retryBaseDelay: 1000, - retryMaxDelay: 2000, -}; +const baseConfig = makeBaseConfig(); function mockAbilitiesFetch(): void { mockFetch.mockResolvedValueOnce({ diff --git a/tests/evals/schema-quality.test.ts b/tests/evals/schema-quality.test.ts index bbe0fe8..b55415a 100644 --- a/tests/evals/schema-quality.test.ts +++ b/tests/evals/schema-quality.test.ts @@ -12,35 +12,14 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { getTools, clearToolsCache } from '../../src/tools.js'; import { clearCache, initRateLimiter } from '../../src/abilities.js'; -import { type Config } from '../../src/config.js'; +import { makeBaseConfig } from '../helpers/config.js'; -import abilitiesFixture from './fixtures/abilities-full.json'; +import abilitiesFixture from './fixtures/abilities-full.json' with { type: 'json' }; const mockFetch = vi.fn(); vi.stubGlobal('fetch', mockFetch); -const baseConfig: Config = { - dashboardUrl: 'https://test.local', - authType: 'basic', - username: 'admin', - appPassword: 'xxxx', - skipSslVerify: true, - allowHttp: false, - rateLimit: 0, - requestTimeout: 5000, - maxResponseSize: 10485760, - safeMode: false, - requireUserConfirmation: true, - maxSessionData: 52428800, - schemaVerbosity: 'standard', - responseFormat: 'compact', - abilityNamespaces: ['mainwp'], - configSource: 'environment', - retryEnabled: false, - maxRetries: 2, - retryBaseDelay: 1000, - retryMaxDelay: 2000, -}; +const baseConfig = makeBaseConfig(); function mockAbilitiesFetch(): void { mockFetch.mockResolvedValueOnce({ @@ -232,36 +211,4 @@ describe('Schema Quality', () => { expect(violations, `Compact mode drops items: ${violations.join(', ')}`).toEqual([]); }); }); - - describe('fixture staleness', () => { - it('should warn if ability count diverges from reference doc', async () => { - // Count section headings in abilities-reference.md - const fs = await import('fs'); - const path = await import('path'); - const refPath = path.resolve(import.meta.dirname, '../../.mwpdev/abilities-reference.md'); - - let refCount = 0; - if (fs.existsSync(refPath)) { - const content = fs.readFileSync(refPath, 'utf8'); - // Count ### headings that match ability names (e.g., ### list_sites_v1) - const headings = content.match(/^### \w+_v\d+/gm); - refCount = headings?.length ?? 0; - } - - const fixtureCount = abilitiesFixture.filter((a: { name: string }) => - a.name.startsWith('mainwp/') - ).length; - - if (refCount > 0 && fixtureCount !== refCount) { - console.warn( - `\n⚠ Fixture staleness: fixture has ${fixtureCount} abilities, ` + - `reference doc has ${refCount} headings. ` + - `Consider re-capturing the fixture.\n` - ); - } - - // This test always passes — it's a warning, not a failure - expect(true).toBe(true); - }); - }); }); diff --git a/tests/evals/token-budget.test.ts b/tests/evals/token-budget.test.ts index 5baba8c..1d7f6ec 100644 --- a/tests/evals/token-budget.test.ts +++ b/tests/evals/token-budget.test.ts @@ -13,36 +13,15 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { getTools, clearToolsCache } from '../../src/tools.js'; import { clearCache, initRateLimiter } from '../../src/abilities.js'; -import { type Config } from '../../src/config.js'; import { type Tool } from '@modelcontextprotocol/sdk/types.js'; +import { makeBaseConfig } from '../helpers/config.js'; -import abilitiesFixture from './fixtures/abilities-full.json'; +import abilitiesFixture from './fixtures/abilities-full.json' with { type: 'json' }; const mockFetch = vi.fn(); vi.stubGlobal('fetch', mockFetch); -const baseConfig: Config = { - dashboardUrl: 'https://test.local', - authType: 'basic', - username: 'admin', - appPassword: 'xxxx', - skipSslVerify: true, - allowHttp: false, - rateLimit: 0, - requestTimeout: 5000, - maxResponseSize: 10485760, - safeMode: false, - requireUserConfirmation: true, - maxSessionData: 52428800, - schemaVerbosity: 'standard', - responseFormat: 'compact', - abilityNamespaces: ['mainwp'], - configSource: 'environment', - retryEnabled: false, - maxRetries: 2, - retryBaseDelay: 1000, - retryMaxDelay: 2000, -}; +const baseConfig = makeBaseConfig(); function mockAbilitiesFetch(): void { mockFetch.mockResolvedValueOnce({ diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index a9f7cd7..f6bacf4 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -1,42 +1,3 @@ # Test Fixtures -This directory contains mock data for testing. - -## Purpose - -These JSON fixtures provide reusable test data that can be shared across tests. They serve two purposes: - -1. **Schema validation** - Ensure fixture files remain valid as schemas evolve -2. **Integration tests** - Simulate API responses without a live MainWP Dashboard - -## Structure - -- `abilities.json` - Sample abilities array matching the `/wp-abilities/v1/abilities` response format -- `categories.json` - Sample categories matching the `/wp-abilities/v1/categories` response format -- `config.json` - Sample settings file for testing configuration loading -- `site.json` - Sample site data matching the `mainwp/get-site-v1` response format - -## Usage - -Import fixtures in test files: - -```typescript -import configFixture from '../tests/fixtures/config.json'; -import siteFixture from '../tests/fixtures/site.json'; -``` - -**Current usage:** - -- `src/config.test.ts` - Imports `config.json` and `site.json` to validate fixture schema compatibility - -**Note:** Unit tests often use inline mocks for specific test scenarios (e.g., testing validation errors with intentionally malformed data). Fixtures are best for testing "happy path" scenarios or shared data across multiple tests. - -## Maintenance - -Update these fixtures when: - -- API response schemas change -- New fields are added to abilities or categories -- Test scenarios require additional mock data - -Keep fixtures minimal but representative of real-world responses. +- `config.json` is imported by `src/config.test.ts` to exercise settings-file loading. diff --git a/tests/fixtures/abilities.json b/tests/fixtures/abilities.json deleted file mode 100644 index 65ce275..0000000 --- a/tests/fixtures/abilities.json +++ /dev/null @@ -1,82 +0,0 @@ -[ - { - "name": "mainwp/list-sites-v1", - "label": "List Sites", - "description": "Get all managed sites", - "category": "mainwp-sites", - "input_schema": { - "type": "object", - "properties": { - "page": { - "type": "integer", - "description": "Page number" - }, - "per_page": { - "type": "integer", - "description": "Items per page" - } - } - }, - "meta": { - "annotations": { - "readonly": true, - "destructive": false, - "idempotent": true - } - } - }, - { - "name": "mainwp/get-site-v1", - "label": "Get Site", - "description": "Get detailed information about a single site", - "category": "mainwp-sites", - "input_schema": { - "type": "object", - "properties": { - "site_id_or_domain": { - "type": ["integer", "string"], - "description": "Site ID or domain" - } - }, - "required": ["site_id_or_domain"] - }, - "meta": { - "annotations": { - "readonly": true, - "destructive": false, - "idempotent": true - } - } - }, - { - "name": "mainwp/delete-site-v1", - "label": "Delete Site", - "description": "Delete a site from MainWP Dashboard", - "category": "mainwp-sites", - "input_schema": { - "type": "object", - "properties": { - "site_id_or_domain": { - "type": ["integer", "string"], - "description": "Site ID or domain" - }, - "confirm": { - "type": "boolean", - "description": "Must be true to execute deletion" - }, - "dry_run": { - "type": "boolean", - "description": "Preview mode" - } - }, - "required": ["site_id_or_domain"] - }, - "meta": { - "annotations": { - "readonly": false, - "destructive": true, - "idempotent": false - } - } - } -] diff --git a/tests/fixtures/categories.json b/tests/fixtures/categories.json deleted file mode 100644 index 6f5302f..0000000 --- a/tests/fixtures/categories.json +++ /dev/null @@ -1,17 +0,0 @@ -[ - { - "slug": "mainwp-sites", - "label": "Sites", - "description": "Site management operations" - }, - { - "slug": "mainwp-updates", - "label": "Updates", - "description": "Update management operations" - }, - { - "slug": "mainwp-clients", - "label": "Clients", - "description": "Client management operations" - } -] diff --git a/tests/fixtures/site.json b/tests/fixtures/site.json deleted file mode 100644 index ffdb58b..0000000 --- a/tests/fixtures/site.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "id": 123, - "name": "Test Site", - "url": "https://example.com", - "status": "connected", - "wp_version": "6.4.2", - "php_version": "8.2.0", - "last_sync": "2024-01-15T10:30:00Z" -} diff --git a/tests/helpers/config.ts b/tests/helpers/config.ts new file mode 100644 index 0000000..381b5bc --- /dev/null +++ b/tests/helpers/config.ts @@ -0,0 +1,49 @@ +/** + * Shared test fixtures for Config and Logger. + */ + +import { vi } from 'vitest'; +import { type Config } from '../../src/config.js'; +import { type Logger } from '../../src/logging.js'; + +/** + * Build a Config fixture for tests. Returns a fresh object each call so + * callers can freely spread/mutate the result without affecting other tests. + */ +export function makeBaseConfig(overrides: Partial = {}): Config { + return { + dashboardUrl: 'https://test.local', + authType: 'basic', + username: 'admin', + appPassword: 'xxxx', + skipSslVerify: true, + allowHttp: false, + rateLimit: 0, + requestTimeout: 5000, + maxResponseSize: 10485760, + safeMode: false, + requireUserConfirmation: true, + maxSessionData: 52428800, + schemaVerbosity: 'standard', + responseFormat: 'compact', + retryEnabled: false, + maxRetries: 2, + retryBaseDelay: 1000, + retryMaxDelay: 2000, + abilityNamespaces: ['mainwp'], + configSource: 'environment', + ...overrides, + }; +} + +/** Build a Logger mock with all methods as vi.fn(). */ +export function makeMockLogger(): Logger { + return { + debug: vi.fn(), + info: vi.fn(), + notice: vi.fn(), + warning: vi.fn(), + error: vi.fn(), + critical: vi.fn(), + }; +} diff --git a/tests/integration/abilities.test.ts b/tests/integration/abilities.test.ts deleted file mode 100644 index 341f151..0000000 --- a/tests/integration/abilities.test.ts +++ /dev/null @@ -1,301 +0,0 @@ -/** - * Abilities Integration Tests - * - * End-to-end tests for ability fetching with mocked HTTP responses. - */ - -import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import { - fetchAbilities, - fetchCategories, - getAbility, - executeAbility, - clearCache, - initRateLimiter, -} from '../../src/abilities.js'; -import { type Config } from '../../src/config.js'; - -// Import fixtures -import abilitiesFixture from '../fixtures/abilities.json'; -import categoriesFixture from '../fixtures/categories.json'; - -// Mock fetch globally -const mockFetch = vi.fn(); -vi.stubGlobal('fetch', mockFetch); - -const baseConfig: Config = { - dashboardUrl: 'https://test.local', - authType: 'basic', - username: 'admin', - appPassword: 'xxxx', - skipSslVerify: true, - allowHttp: false, - rateLimit: 0, - requestTimeout: 5000, - maxResponseSize: 10485760, - safeMode: false, - requireUserConfirmation: true, - maxSessionData: 52428800, - schemaVerbosity: 'standard', - responseFormat: 'compact', - abilityNamespaces: ['mainwp'], - configSource: 'environment', - retryEnabled: false, - maxRetries: 2, - retryBaseDelay: 1000, - retryMaxDelay: 2000, -}; - -describe('Abilities Integration', () => { - beforeEach(() => { - vi.resetAllMocks(); - clearCache(); - initRateLimiter(0); - vi.spyOn(console, 'error').mockImplementation(() => {}); - }); - - afterEach(() => { - vi.restoreAllMocks(); - }); - - describe('fetchAbilities end-to-end', () => { - it('should fetch abilities and populate cache', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - // First fetch - const abilities1 = await fetchAbilities(baseConfig); - - expect(abilities1).toHaveLength(abilitiesFixture.length); - expect(abilities1[0].name).toBe('mainwp/list-sites-v1'); - - // Second fetch should use cache - const abilities2 = await fetchAbilities(baseConfig); - - expect(abilities2).toEqual(abilities1); - expect(mockFetch).toHaveBeenCalledTimes(1); - }); - - it('should handle network errors gracefully', async () => { - mockFetch.mockRejectedValueOnce(new Error('Network unreachable')); - - await expect(fetchAbilities(baseConfig)).rejects.toThrow('Network unreachable'); - }); - - it('should use cached data on subsequent error after initial success', async () => { - // First successful fetch - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - const abilities1 = await fetchAbilities(baseConfig); - - // Simulate error on force refresh - mockFetch.mockRejectedValueOnce(new Error('Server down')); - - // Should return cached data - const abilities2 = await fetchAbilities(baseConfig, true); - - expect(abilities2).toEqual(abilities1); - }); - - it('should filter abilities by namespace', async () => { - const mixedAbilities = [ - ...abilitiesFixture, - { - name: 'other-plugin/some-ability', - label: 'Other', - description: 'From another plugin', - category: 'other', - }, - ]; - - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => mixedAbilities, - headers: new Headers(), - }); - - const abilities = await fetchAbilities(baseConfig); - - // Should only include mainwp/ abilities - expect(abilities.every(a => a.name.startsWith('mainwp/'))).toBe(true); - expect(abilities).toHaveLength(abilitiesFixture.length); - }); - }); - - describe('fetchCategories end-to-end', () => { - it('should fetch and cache categories', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => categoriesFixture, - headers: new Headers(), - }); - - const categories = await fetchCategories(baseConfig); - - expect(categories).toHaveLength(categoriesFixture.length); - expect(categories[0].slug).toBe('mainwp-sites'); - }); - - it('should filter categories by namespace', async () => { - const mixedCategories = [ - ...categoriesFixture, - { slug: 'other-category', label: 'Other', description: 'Other' }, - ]; - - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => mixedCategories, - headers: new Headers(), - }); - - const categories = await fetchCategories(baseConfig); - - // Should only include mainwp- prefixed categories - expect(categories.every(c => c.slug.startsWith('mainwp-'))).toBe(true); - }); - }); - - describe('getAbility end-to-end', () => { - it('should find ability from fetched list', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - const ability = await getAbility(baseConfig, 'mainwp/list-sites-v1'); - - expect(ability).toBeDefined(); - expect(ability?.label).toBe('List Sites'); - }); - - it('should return undefined for non-existent ability', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - const ability = await getAbility(baseConfig, 'mainwp/non-existent-v1'); - - expect(ability).toBeUndefined(); - }); - }); - - describe('executeAbility end-to-end', () => { - it('should execute readonly ability with GET', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => ({ sites: [], total: 0 }), - headers: new Headers(), - }); - - const result = await executeAbility(baseConfig, 'mainwp/list-sites-v1', { page: 1 }); - - expect(result).toEqual({ sites: [], total: 0 }); - - // Verify GET was used - const execCall = mockFetch.mock.calls[1]; - expect(execCall[1].method).toBe('GET'); - }); - - it('should execute destructive non-idempotent ability with POST', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => ({ success: true, deleted: true }), - headers: new Headers(), - }); - - const result = await executeAbility(baseConfig, 'mainwp/delete-site-v1', { - site_id_or_domain: 123, - confirm: true, - }); - - expect(result).toEqual({ success: true, deleted: true }); - - // delete-site-v1 is destructive but NOT idempotent → uses POST - const execCall = mockFetch.mock.calls[1]; - expect(execCall[1].method).toBe('POST'); - }); - - it('should handle execution errors', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - mockFetch.mockResolvedValueOnce({ - ok: false, - status: 400, - statusText: 'Bad Request', - text: async () => - JSON.stringify({ - code: 'invalid_site_id', - message: 'Site not found', - }), - headers: new Headers(), - }); - - await expect( - executeAbility(baseConfig, 'mainwp/get-site-v1', { site_id_or_domain: 999 }) - ).rejects.toThrow(/invalid_site_id/); - }); - }); - - describe('clearCache behavior', () => { - it('should clear abilities cache', async () => { - mockFetch.mockResolvedValue({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - // Fetch and cache - await fetchAbilities(baseConfig); - expect(mockFetch).toHaveBeenCalledTimes(1); - - // Clear cache - clearCache(); - - // Should fetch again - await fetchAbilities(baseConfig); - expect(mockFetch).toHaveBeenCalledTimes(2); - }); - - it('should clear categories cache', async () => { - mockFetch.mockResolvedValue({ - ok: true, - json: async () => categoriesFixture, - headers: new Headers(), - }); - - await fetchCategories(baseConfig); - expect(mockFetch).toHaveBeenCalledTimes(1); - - clearCache(); - - await fetchCategories(baseConfig); - expect(mockFetch).toHaveBeenCalledTimes(2); - }); - }); -}); diff --git a/tests/integration/tools.test.ts b/tests/integration/tools.test.ts deleted file mode 100644 index f979df1..0000000 --- a/tests/integration/tools.test.ts +++ /dev/null @@ -1,532 +0,0 @@ -/** - * Tools Integration Tests - * - * End-to-end tests for tool execution flow including validation, - * confirmation, and safe mode. - */ - -import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import { getTools, executeTool } from '../../src/tools.js'; -import { clearPendingPreviews } from '../../src/confirmation.js'; -import { clearCache, initRateLimiter } from '../../src/abilities.js'; -import { type Config } from '../../src/config.js'; -import { type Logger } from '../../src/logging.js'; - -// Import fixtures -import abilitiesFixture from '../fixtures/abilities.json'; - -// Mock fetch globally -const mockFetch = vi.fn(); -vi.stubGlobal('fetch', mockFetch); - -const baseConfig: Config = { - dashboardUrl: 'https://test.local', - authType: 'basic', - username: 'admin', - appPassword: 'xxxx', - skipSslVerify: true, - allowHttp: false, - rateLimit: 0, - requestTimeout: 5000, - maxResponseSize: 10485760, - safeMode: false, - requireUserConfirmation: true, - maxSessionData: 52428800, - schemaVerbosity: 'standard', - responseFormat: 'compact', - abilityNamespaces: ['mainwp'], - configSource: 'environment', - retryEnabled: false, - maxRetries: 2, - retryBaseDelay: 1000, - retryMaxDelay: 2000, -}; - -const mockLogger: Logger = { - debug: vi.fn(), - info: vi.fn(), - notice: vi.fn(), - warning: vi.fn(), - error: vi.fn(), - critical: vi.fn(), -}; - -describe('Tools Integration', () => { - beforeEach(() => { - vi.resetAllMocks(); - clearCache(); - clearPendingPreviews(); - initRateLimiter(0); - vi.spyOn(console, 'error').mockImplementation(() => {}); - }); - - afterEach(() => { - vi.restoreAllMocks(); - }); - - describe('getTools integration', () => { - it('should convert fixture abilities to tools', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - const tools = await getTools(baseConfig); - - expect(tools).toHaveLength(abilitiesFixture.length); - expect(tools.map(t => t.name)).toContain('list_sites_v1'); - expect(tools.map(t => t.name)).toContain('get_site_v1'); - expect(tools.map(t => t.name)).toContain('delete_site_v1'); - }); - - it('should add user_confirmed to destructive tools', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - const tools = await getTools(baseConfig); - const deleteTool = tools.find(t => t.name === 'delete_site_v1'); - - expect(deleteTool?.inputSchema.properties).toHaveProperty('user_confirmed'); - }); - - it('should mark destructive tools in descriptions', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - const tools = await getTools(baseConfig); - const deleteTool = tools.find(t => t.name === 'delete_site_v1'); - - expect(deleteTool?.description).toContain('DESTRUCTIVE'); - }); - }); - - describe('executeTool - read-only operations', () => { - it('should execute read-only tool successfully', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => ({ - sites: [{ id: 1, name: 'Test Site', url: 'https://test.com' }], - total: 1, - }), - headers: new Headers(), - }); - - const result = await executeTool( - baseConfig, - 'list_sites_v1', - { page: 1, per_page: 10 }, - mockLogger - ); - - expect(result.content).toHaveLength(1); - expect(result.content[0].type).toBe('text'); - expect(result.isError).toBeUndefined(); - - const parsed = JSON.parse(result.content[0].text); - expect(parsed.sites).toHaveLength(1); - }); - - it('should pass parameters to ability', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => ({ id: 123, name: 'Site' }), - headers: new Headers(), - }); - - await executeTool(baseConfig, 'get_site_v1', { site_id_or_domain: 123 }, mockLogger); - - // Verify the URL included the parameter - const execCall = mockFetch.mock.calls[1]; - const url = execCall[0] as string; - expect(url).toContain('site_id_or_domain'); - }); - }); - - describe('executeTool - safe mode', () => { - it('should block destructive tool in safe mode', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - const safeConfig = { ...baseConfig, safeMode: true }; - const result = await executeTool( - safeConfig, - 'delete_site_v1', - { site_id_or_domain: 1, confirm: true }, - mockLogger - ); - - expect(result.content[0].text).toContain('SAFE_MODE_BLOCKED'); - expect(result.content[0].text).toContain('delete_site_v1'); - expect(result.isError).toBe(true); - }); - - it('should allow read-only tool in safe mode', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => ({ sites: [] }), - headers: new Headers(), - }); - - const safeConfig = { ...baseConfig, safeMode: true }; - const result = await executeTool(safeConfig, 'list_sites_v1', {}, mockLogger); - - const parsed = JSON.parse(result.content[0].text); - expect(parsed).toHaveProperty('sites'); - }); - }); - - describe('executeTool - confirmation flow', () => { - it('should generate preview for destructive operation', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => ({ - dry_run: true, - site: { id: 1, name: 'Test' }, - would_delete: true, - }), - headers: new Headers(), - }); - - const result = await executeTool( - baseConfig, - 'delete_site_v1', - { site_id_or_domain: 1, confirm: true }, - mockLogger - ); - - expect(result.content[0].text).toContain('CONFIRMATION_REQUIRED'); - expect(result.content[0].text).toContain('preview'); - expect(result.content[0].text).toContain('user_confirmed'); - // Preview is a successful workflow step, not a failed call - expect(result.isError).toBeUndefined(); - }); - - it('should complete two-phase confirmation flow', async () => { - // Phase 1: Preview request - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => ({ dry_run: true, would_delete: true }), - headers: new Headers(), - }); - - const previewResult = await executeTool( - baseConfig, - 'delete_site_v1', - { site_id_or_domain: 1, confirm: true }, - mockLogger - ); - - expect(previewResult.content[0].text).toContain('CONFIRMATION_REQUIRED'); - - // Phase 2: Confirmed execution (need to clear logger mocks for fresh tracking) - vi.mocked(mockLogger.info).mockClear(); - - // Note: abilities are already cached from Phase 1, so no need to mock abilities fetch again - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => ({ success: true, deleted: true }), - headers: new Headers(), - }); - - const confirmResult = await executeTool( - baseConfig, - 'delete_site_v1', - { site_id_or_domain: 1, user_confirmed: true }, - mockLogger - ); - - const parsed = JSON.parse(confirmResult.content[0].text); - expect(parsed.success).toBe(true); - }); - - it('should reject user_confirmed without preview', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - // Skip preview, go straight to confirm - const result = await executeTool( - baseConfig, - 'delete_site_v1', - { site_id_or_domain: 999, user_confirmed: true }, - mockLogger - ); - - expect(result.content[0].text).toContain('PREVIEW_REQUIRED'); - expect(result.isError).toBe(true); - }); - - it('should reject conflicting parameters', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - const result = await executeTool( - baseConfig, - 'delete_site_v1', - { site_id_or_domain: 1, dry_run: true, user_confirmed: true }, - mockLogger - ); - - expect(result.content[0].text).toContain('CONFLICTING_PARAMETERS'); - expect(result.isError).toBe(true); - }); - - it('should allow explicit dry_run to bypass confirmation', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => ({ dry_run: true, preview: true }), - headers: new Headers(), - }); - - const result = await executeTool( - baseConfig, - 'delete_site_v1', - { site_id_or_domain: 1, dry_run: true }, - mockLogger - ); - - // Should return the dry_run result directly, not CONFIRMATION_REQUIRED - const parsed = JSON.parse(result.content[0].text); - expect(parsed.dry_run).toBe(true); - }); - }); - - describe('executeTool - error handling', () => { - it('should handle ability execution errors', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - mockFetch.mockResolvedValueOnce({ - ok: false, - status: 404, - statusText: 'Not Found', - text: async () => - JSON.stringify({ - code: 'site_not_found', - message: 'Site does not exist', - }), - }); - - const result = await executeTool( - baseConfig, - 'get_site_v1', - { site_id_or_domain: 999 }, - mockLogger - ); - - expect(result.content[0].text).toContain('error'); - expect(result.isError).toBe(true); - expect(mockLogger.error).toHaveBeenCalled(); - }); - - it('should validate input and reject invalid IDs', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - const result = await executeTool(baseConfig, 'get_site_v1', { site_id: -1 }, mockLogger); - - expect(result.content[0].text).toContain('error'); - expect(result.content[0].text).toContain('positive integer'); - expect(result.isError).toBe(true); - }); - - it('should handle tool not found', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - const result = await executeTool(baseConfig, 'nonexistent_tool_v1', {}, mockLogger); - - expect(result.content[0].text).toContain('error'); - expect(result.content[0].text).toContain('not found'); - expect(result.isError).toBe(true); - }); - }); - - describe('executeTool - logging', () => { - it('should log tool execution start and end', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => ({}), - headers: new Headers(), - }); - - await executeTool(baseConfig, 'list_sites_v1', {}, mockLogger); - - expect(mockLogger.debug).toHaveBeenCalledWith( - expect.stringContaining('started'), - expect.any(Object) - ); - - expect(mockLogger.info).toHaveBeenCalledWith( - expect.stringContaining('succeeded'), - expect.objectContaining({ durationMs: expect.any(Number) }) - ); - }); - - it('should log destructive operations', async () => { - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesFixture, - headers: new Headers(), - }); - - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => ({ preview: true }), - headers: new Headers(), - }); - - await executeTool( - baseConfig, - 'delete_site_v1', - { site_id_or_domain: 1, confirm: true }, - mockLogger - ); - - expect(mockLogger.info).toHaveBeenCalledWith( - expect.stringContaining('Destructive'), - expect.any(Object) - ); - }); - }); - - describe('non-primary namespace tools', () => { - it('resolves an {ns}__ tool name to the right ability URL', async () => { - const abilitiesPayload = [ - { - name: 'acme/do-thing-v1', - label: 'Acme Do Thing', - description: 'Third-party readonly ability', - category: 'acme-misc', - meta: { annotations: { readonly: true, destructive: false, idempotent: true } }, - }, - ]; - - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesPayload, - headers: new Headers(), - }); - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => ({ result: 'pong' }), - headers: new Headers(), - }); - - const config: Config = { - ...baseConfig, - abilityNamespaces: ['mainwp', 'acme'] as [string, ...string[]], - }; - - const result = await executeTool(config, 'acme__do_thing_v1', { input: 'ping' }, mockLogger); - - const executeCall = mockFetch.mock.calls[1]; - const url = executeCall[0] as string; - expect(url).toContain('/abilities/acme/do-thing-v1/run'); - expect(JSON.parse(result.content[0].text)).toEqual({ result: 'pong' }); - expect(result.isError).toBeUndefined(); - }); - - it('round-trips a hyphenated namespace through execute', async () => { - const abilitiesPayload = [ - { - name: 'acme-corp/do-thing-v1', - label: 'Acme Corp Do Thing', - description: 'Hyphenated-namespace ability', - category: 'acme-corp-misc', - meta: { annotations: { readonly: true, destructive: false, idempotent: true } }, - }, - ]; - - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => abilitiesPayload, - headers: new Headers(), - }); - mockFetch.mockResolvedValueOnce({ - ok: true, - json: async () => ({ ok: true }), - headers: new Headers(), - }); - - const config: Config = { - ...baseConfig, - abilityNamespaces: ['mainwp', 'acme-corp'] as [string, ...string[]], - }; - - const result = await executeTool(config, 'acme_corp__do_thing_v1', {}, mockLogger); - - const executeCall = mockFetch.mock.calls[1]; - const url = executeCall[0] as string; - expect(url).toContain('/abilities/acme-corp/do-thing-v1/run'); - expect(JSON.parse(result.content[0].text)).toEqual({ ok: true }); - expect(result.isError).toBeUndefined(); - }); - }); -}); diff --git a/tests/unit/security.test.ts b/tests/unit/security.test.ts deleted file mode 100644 index f818f23..0000000 --- a/tests/unit/security.test.ts +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Security Unit Tests - * - * Tests for input validation in security.ts. - */ - -import { describe, it, expect } from 'vitest'; -import { validateInput } from '../../src/security.js'; - -describe('validateInput', () => { - describe('plural ID fields (_ids)', () => { - it('accepts a valid array of numeric IDs', () => { - expect(() => validateInput({ site_ids: [1, 2, 3] })).not.toThrow(); - }); - - it('accepts a valid array of numeric string IDs', () => { - expect(() => validateInput({ site_ids: ['1', '2', '3'] })).not.toThrow(); - }); - - it('rejects non-array values', () => { - expect(() => validateInput({ site_ids: '123' })).toThrow('"site_ids" must be an array'); - expect(() => validateInput({ site_ids: 123 })).toThrow('"site_ids" must be an array'); - expect(() => validateInput({ site_ids: { id: 1 } })).toThrow('"site_ids" must be an array'); - }); - - it('rejects non-integer numeric strings', () => { - expect(() => validateInput({ site_ids: ['1.5'] })).toThrow('positive integer'); - }); - - it('rejects strings with trailing non-numeric characters', () => { - expect(() => validateInput({ site_ids: ['1abc'] })).toThrow('positive integer'); - }); - - it('rejects zero and negative IDs', () => { - expect(() => validateInput({ site_ids: [0] })).toThrow('positive integer'); - expect(() => validateInput({ site_ids: [-1] })).toThrow('positive integer'); - }); - }); -}); diff --git a/tsconfig.eslint.json b/tsconfig.eslint.json index 8eb7231..1aa5d26 100644 --- a/tsconfig.eslint.json +++ b/tsconfig.eslint.json @@ -4,6 +4,6 @@ "rootDir": ".", "noEmit": true }, - "include": ["src/**/*", "tests/**/*", "*.config.ts"], + "include": ["src/**/*", "tests/**/*", "scripts/**/*.ts", "*.config.ts"], "exclude": ["node_modules", "dist"] }