From acec270cb4d48fd9d885b1016bb320981c2befdf Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Mon, 27 Jul 2026 11:22:30 -0500 Subject: [PATCH 01/10] Add more E2E tests --- .github/skills/add-e2e-test/SKILL.md | 308 ++++++++++++++++++ .github/skills/add-feature/SKILL.md | 7 +- AGENTS.md | 29 ++ .../blob_processor.agent.md | 18 + .../apps/blob-trigger-payload/function_app.py | 3 + .../apps/blob-trigger-payload/host.json | 17 + .../blob-trigger-payload/requirements.txt | 1 + .../coordinator.agent.md | 17 + .../multi-agent-delegation/function_app.py | 3 + .../apps/multi-agent-delegation/host.json | 17 + .../multi-agent-delegation/requirements.txt | 1 + .../specialist.agent.md | 6 + .../queue-trigger-payload/function_app.py | 3 + .../apps/queue-trigger-payload/host.json | 17 + .../queue_processor.agent.md | 18 + .../queue-trigger-payload/requirements.txt | 1 + tests/endtoend/test_apps_http.py | 229 +++++++++++++ tests/endtoend/test_apps_storage.py | 252 ++++++++++++++ 18 files changed, 946 insertions(+), 1 deletion(-) create mode 100644 .github/skills/add-e2e-test/SKILL.md create mode 100644 tests/endtoend/apps/blob-trigger-payload/blob_processor.agent.md create mode 100644 tests/endtoend/apps/blob-trigger-payload/function_app.py create mode 100644 tests/endtoend/apps/blob-trigger-payload/host.json create mode 100644 tests/endtoend/apps/blob-trigger-payload/requirements.txt create mode 100644 tests/endtoend/apps/multi-agent-delegation/coordinator.agent.md create mode 100644 tests/endtoend/apps/multi-agent-delegation/function_app.py create mode 100644 tests/endtoend/apps/multi-agent-delegation/host.json create mode 100644 tests/endtoend/apps/multi-agent-delegation/requirements.txt create mode 100644 tests/endtoend/apps/multi-agent-delegation/specialist.agent.md create mode 100644 tests/endtoend/apps/queue-trigger-payload/function_app.py create mode 100644 tests/endtoend/apps/queue-trigger-payload/host.json create mode 100644 tests/endtoend/apps/queue-trigger-payload/queue_processor.agent.md create mode 100644 tests/endtoend/apps/queue-trigger-payload/requirements.txt diff --git a/.github/skills/add-e2e-test/SKILL.md b/.github/skills/add-e2e-test/SKILL.md new file mode 100644 index 00000000..f2611060 --- /dev/null +++ b/.github/skills/add-e2e-test/SKILL.md @@ -0,0 +1,308 @@ +--- +name: add-e2e-test +description: "Use when a new feature is being added and an end-to-end test should be created or assessed for eligibility. Covers the full E2E test lifecycle: eligibility check (supported triggers and infra), creating a dedicated app under tests/endtoend/apps/, writing targeted pytest assertions in the appropriate test_apps_*.py file, adding new dependencies to pyproject.toml, and wiring pipeline variables. Trigger when entering Phase 4 (Testing) of the add-feature lifecycle, or when asked to 'add an E2E test', 'check E2E eligibility', or 'write an end-to-end test'. Not applicable for nits, pure doc changes, or features that depend on unsupported external resources." +--- + +# add-e2e-test — end-to-end test lifecycle for azure-functions-agents-runtime + +This skill operationalizes the E2E testing standard defined in +[`AGENTS.md`](../../../AGENTS.md) §6. Follow every step in order. Record the +eligibility outcome — a passing test or an explicit waiver — before the PR is +merged. + +## When to use + +Use during **Phase 4 (Testing)** of the `add-feature` lifecycle, or any time a +new feature touches behavior that can be exercised through a real Function App +host. Skip (record waiver) only when the eligibility check concludes that the +feature genuinely cannot be tested in the available infra. + +--- + +## Step 1 — Eligibility check + +Assess whether the feature can be covered by an E2E test. The E2E environment +provides: + +| Resource | Available | +| --- | --- | +| Azure Functions host (`func`) | Yes | +| Azurite (blob, queue, table) | Yes | +| Foundry project (`FOUNDRY_PROJECT_ENDPOINT` / `FOUNDRY_MODEL`) | Yes | +| Any other external resource (databases, connectors, etc.) | **No** | + +**Supported trigger types:** `http_trigger`, `blob_trigger`, `queue_trigger`, +`timer_trigger` (start-only; timers do not fire in CI), and MCP tool triggers +via the `builtin_endpoints.mcp` flag. + +**Decision rule:** +- If the feature's behavior can be exercised with HTTP, storage, or MCP triggers + and requires only Azurite and/or a Foundry resource → **add an E2E test**. +- If the feature requires an external resource not listed above → **skip E2E**. + Record the waiver in the **FRD Decisions log** (medium+ features) or the **PR + description** (small features) with a one-line reason (e.g., "E2E waived: + feature requires a live Service Bus namespace"). + +--- + +## Step 2 — Design the app + +E2E apps are deliberately narrow: one feature, one behavior. They are **not** +samples — avoid bundling multiple capabilities. + +### Required files + +Create a new directory under `tests/endtoend/apps//` where `` is +short, lowercase, and hyphenated (e.g., `queue-error-handling`). Each directory +**must** contain exactly these five files: + +#### `.agent.md` + +One focused agent that exercises the feature. Follow standard front-matter +conventions from `docs/front-matter-spec.md`. Keep the system prompt minimal. + +```markdown +--- +name: My Feature Agent +description: One-sentence description of what this agent tests. +trigger: + type: http_trigger # or blob_trigger / queue_trigger / mcp + args: + route: "my-feature" + methods: ["POST"] + auth_level: anonymous +--- + +You are a concise assistant. Reply in one short sentence. +``` + +#### `function_app.py` + +Always exactly: + +```python +from azure_functions_agents import create_function_app + +app = create_function_app() +``` + +#### `host.json` + +Copy verbatim from `tests/endtoend/apps/minimal-http/host.json`. Do not +customise unless the feature requires a specific extension setting. + +```json +{ + "version": "2.0", + "extensions": { + "http": { + "routePrefix": "" + } + }, + "logging": { + "logLevel": { + "default": "Information" + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[4.*, 5.0.0)" + } +} +``` + +#### `local.settings.json` + +No secrets. Always include the two standard entries. Add non-secret app +settings (feature flags, non-sensitive config) here. Secrets and infra +credentials are pipeline-only (see Step 5). + +```json +{ + "IsEncrypted": false, + "Values": { + "FUNCTIONS_WORKER_RUNTIME": "python", + "AzureWebJobsStorage": "UseDevelopmentStorage=true" + } +} +``` + +#### `requirements.txt` + +Always install the local editable version, not the published package: + +``` +-e ../../../.. +``` + +--- + +## Step 3 — Write the tests + +### Which file to add tests to + +Map the primary trigger type to the appropriate test file: + +| Trigger | Test file | +| --- | --- | +| `http_trigger` | `tests/endtoend/test_apps_http.py` | +| `blob_trigger` or `queue_trigger` | `tests/endtoend/test_apps_storage.py` | +| MCP (`builtin_endpoints.mcp`) | `tests/endtoend/test_apps_mcp.py` | +| New trigger category | Create `tests/endtoend/test_apps_.py` using an existing file as a template | + +> **Free smoke test:** `tests/endtoend/test_apps_start.py` auto-discovers every +> directory under `tests/endtoend/apps/` that contains a `host.json` and runs a +> `func start` startup check. Adding a new app dir automatically opts it into +> this test — no changes to that file are needed. + +### Test design principles + +1. **Target one behavior.** Each test function should assert a single, specific + outcome of the feature under test. +2. **Prefer provider-independent assertions.** Test behavior that does not + require a live LLM call wherever possible: endpoint existence, HTTP method + handling, JSON schema validation, error response shape, storage trigger + invocation (logged execution), etc. +3. **Gate LLM-dependent assertions.** When a test must make a real agent call, + guard it with the `requires_llm` skip mark already defined in each + `test_apps_*.py` file: + ```python + @requires_llm + def test_my_feature_happy_path(my_feature_host: Served) -> None: + ... + ``` +4. **Use module-scoped fixtures.** Follow the pattern in existing files: one + `@pytest.fixture(scope="module")` that starts the host, and multiple test + functions that share it. +5. **Keep assertions deterministic.** Avoid assertions on LLM response content; + assert on status codes, keys present in JSON, function index entries, or + logged execution markers. + +### Fixture pattern (HTTP example) + +```python +@pytest.fixture(scope="module") +def my_feature_host() -> Iterator[Served]: + with _serve("my-feature-app") as served: + yield served + + +def test_my_feature_endpoint_is_discovered(my_feature_host: Served) -> None: + _, endpoints = my_feature_host + ep = find_endpoint(endpoints, "my-feature") + assert ep is not None, "expected 'my-feature' route to be registered" + + +@requires_llm +def test_my_feature_returns_response(my_feature_host: Served) -> None: + client, endpoints = my_feature_host + ep = find_endpoint(endpoints, "my-feature") + assert ep is not None + resp = client.post(ep.url, json={"prompt": "hello"}) + expect_status(resp, 200) + expect_json_keys(resp, ["session_id", "response"]) +``` + +--- + +## Step 4 — New dependencies + +If the E2E app or test helpers require packages not already in the `[dev]` +section of `pyproject.toml`: + +1. Add the package with a pinned range to `[project.optional-dependencies] dev` + in `pyproject.toml`. +2. Add an explanatory comment above the entry, following the style of existing + entries (e.g., `# E2E storage-trigger tests: write blobs/queue messages to Azurite`). +3. Run `python -m pip install -U -e .[dev]` to verify the install. + +Do **not** add runtime-only packages to `[dev]`; add them to the +`[project.dependencies]` section only if the runtime itself needs them. + +--- + +## Step 5 — Environment variables and pipeline wiring + +### Non-secrets (app settings) + +Add non-secret variables directly to the app's `local.settings.json` under +`Values`. These are picked up automatically by `func start` during both local +and CI runs. + +### Secrets and infra variables (Foundry endpoint, model name, etc.) + +**Never commit secrets.** For variables that are sensitive or that differ per +environment: + +1. Leave `local.settings.json` without the value (omit the key entirely, or + note it in a comment in the agent's README if helpful for local setup). +2. Add the variable to the `env:` block in + `eng/templates/official/jobs/e2e-tests.yml`, following the pattern of + `FOUNDRY_PROJECT_ENDPOINT` and `FOUNDRY_MODEL`: + ```yaml + env: + FOUNDRY_PROJECT_ENDPOINT: $(FOUNDRY_PROJECT_ENDPOINT) + FOUNDRY_MODEL: $(FOUNDRY_MODEL) + MY_NEW_VAR: $(MY_NEW_VAR) # ← add here + ``` +3. Ensure the corresponding pipeline variable is defined (ask the team if + you do not have access to the pipeline variable group). + +--- + +## Step 6 — Local run + +Running E2E tests locally is **recommended during feature development** but is +not a blocker if the required infra is unavailable. CI always runs them. + +**Prerequisites:** +- Azure Functions Core Tools (`func`) installed and on `PATH`. +- Azurite running locally (e.g., `azurite --skipApiVersionCheck`). +- A Foundry resource, with `FOUNDRY_PROJECT_ENDPOINT` and `FOUNDRY_MODEL` set + as environment variables (required only for `@requires_llm` tests). + +**Run command (all E2E tests):** +```bash +python -m pytest -m e2e tests/endtoend -v +``` + +**Run only the new app's tests:** +```bash +python -m pytest -m e2e tests/endtoend/test_apps_http.py -v -k "my_feature" +``` + +> E2E tests are excluded from the default unit-test run by `addopts = "-m 'not e2e'"` in +> `pyproject.toml`. The E2E CI pipeline runs them explicitly with `-m e2e`. + +--- + +## Step 7 — Verification checklist + +Before marking Phase 4 complete: + +- [ ] Eligibility check done; outcome recorded (test added or waiver noted in + FRD Decisions log / PR description). +- [ ] New app directory exists under `tests/endtoend/apps//` with all + five required files. +- [ ] `requirements.txt` installs the local editable version (`-e ../../../..`), + not the published package. +- [ ] `local.settings.json` contains no secrets. +- [ ] New dependencies (if any) added to `[dev]` in `pyproject.toml` with a + comment. +- [ ] New infra/secret variables (if any) added to `e2e-tests.yml` `env:` block. +- [ ] App starts cleanly via `test_apps_start.py` (automatic: just verify the + startup smoke test passes for the new app). +- [ ] Targeted feature tests pass locally (or waiver noted if local infra is + unavailable). +- [ ] No unrelated apps or tests modified. + +--- + +## Guardrails + +- E2E apps are **test fixtures, not samples.** Keep them minimal and focused. +- Never add secrets to any committed file. +- Each feature's E2E app is **self-contained** — do not reuse or modify existing + app directories for a new feature's tests. +- This skill is repo dev-tooling under `.github/skills/`; it is unrelated to the + runtime's user-authored agent skills discovered from an app's `skills/` folder. diff --git a/.github/skills/add-feature/SKILL.md b/.github/skills/add-feature/SKILL.md index 63c1a7b1..5d6ce0f8 100644 --- a/.github/skills/add-feature/SKILL.md +++ b/.github/skills/add-feature/SKILL.md @@ -71,7 +71,12 @@ bug fixes. 2. Add tests under `tests/`, mirroring source modules. For config/authoring changes, add a scenario folder under `tests/fixtures/config_scenarios/`. 3. For bug-adjacent work, add a failing regression test first. -4. Run the full CI-equivalent gate: +4. **E2E test assessment:** consult the **`add-e2e-test` skill** + ([`.github/skills/add-e2e-test/SKILL.md`](../add-e2e-test/SKILL.md)) and run + the eligibility check. Either add an E2E test under `tests/endtoend/apps/` + or record a one-line waiver in the FRD Decisions log explaining why E2E is + not feasible for this feature. +5. Run the full CI-equivalent gate: ```bash python -m pytest --cache-clear --cov=./src/azure_functions_agents --cov-report=xml --cov-branch tests ``` diff --git a/AGENTS.md b/AGENTS.md index 63388754..14e5e478 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -94,10 +94,16 @@ python -m pytest --cache-clear --cov=./src/azure_functions_agents --cov-report=x # Fast local test loop python -m pytest tests -q + +# End-to-end tests (require func + Azurite + Foundry resource; excluded from default run) +python -m pytest -m e2e tests/endtoend -v ``` > `samples/` is intentionally excluded from `ruff` and `mypy`. `tests/` is > linted but excluded from strict `mypy`. +> +> E2E tests are excluded from the default run by `addopts = "-m 'not e2e'"` in +> `pyproject.toml`. The E2E CI pipeline runs them explicitly with `-m e2e`. --- @@ -161,6 +167,27 @@ Grounded in `pyproject.toml` and current code: are interpreted. - For bug fixes, add a **failing regression test first**, then fix. +### End-to-end tests + +When a feature can be exercised through a real Function App host, an E2E test is +required. The E2E environment provides `func`, Azurite (blob/queue/table), and a +Foundry resource. Features that require any other external resource may waive the +E2E requirement — record the waiver in the FRD Decisions log (medium+ features) +or PR description (small features). + +- E2E apps live under `tests/endtoend/apps//` — one dedicated directory + per feature, never shared between features. +- Each app must contain five files: `*.agent.md`, `function_app.py`, `host.json`, + `local.settings.json` (no secrets), and `requirements.txt` (editable install). +- Tests go into `tests/endtoend/test_apps_http.py`, `test_apps_storage.py`, or + `test_apps_mcp.py` based on trigger type. +- `tests/endtoend/test_apps_start.py` auto-discovers all apps and runs a `func + start` smoke test — adding a new app dir is enough to opt into it. +- Use the **`add-e2e-test` skill** + ([`.github/skills/add-e2e-test/SKILL.md`](.github/skills/add-e2e-test/SKILL.md)) + for the full playbook: eligibility check, app structure, test design, dependency + wiring, and pipeline variables. + --- ## 7. Documentation conventions @@ -198,6 +225,8 @@ When modifying `src/azure_functions_agents/config/schema.py`: - [ ] (medium+) FRD finalized with a completed Decisions log. - [ ] `ruff`, `mypy`, and `pytest` all green locally (§3). - [ ] New behavior is tested (regression test for bugs). +- [ ] E2E test added under `tests/endtoend/apps/`, **or** eligibility waiver recorded in + FRD Decisions log / PR description (see §6 and the `add-e2e-test` skill). - [ ] `docs/architecture.md` + relevant `docs/*` / `README.md` updated. - [ ] (schema changes) `front-matter-reference.md` regenerated + `update-schema-docs` skill run. - [ ] Diff is surgical — no unrelated changes. diff --git a/tests/endtoend/apps/blob-trigger-payload/blob_processor.agent.md b/tests/endtoend/apps/blob-trigger-payload/blob_processor.agent.md new file mode 100644 index 00000000..46b73aad --- /dev/null +++ b/tests/endtoend/apps/blob-trigger-payload/blob_processor.agent.md @@ -0,0 +1,18 @@ +--- +name: Blob Payload Processor +description: > + Reacts to blobs uploaded to the blob-payload-input container. Purpose-built + to exercise the blob-trigger serialization path: the agent receives structured + metadata (name, uri, length, blob_properties) rather than a raw InputStream + Python object repr. +trigger: + type: blob_trigger + args: + path: "blob-payload-input/{name}" + connection: "AzureWebJobsStorage" +logger: true +--- + +A new blob has been uploaded. The trigger data contains the blob name, URI, +length, and any properties. Respond in a single sentence confirming the blob +name you received from the trigger data. diff --git a/tests/endtoend/apps/blob-trigger-payload/function_app.py b/tests/endtoend/apps/blob-trigger-payload/function_app.py new file mode 100644 index 00000000..736ad492 --- /dev/null +++ b/tests/endtoend/apps/blob-trigger-payload/function_app.py @@ -0,0 +1,3 @@ +from azure_functions_agents import create_function_app + +app = create_function_app() diff --git a/tests/endtoend/apps/blob-trigger-payload/host.json b/tests/endtoend/apps/blob-trigger-payload/host.json new file mode 100644 index 00000000..4a421d09 --- /dev/null +++ b/tests/endtoend/apps/blob-trigger-payload/host.json @@ -0,0 +1,17 @@ +{ + "version": "2.0", + "extensions": { + "http": { + "routePrefix": "" + } + }, + "logging": { + "logLevel": { + "default": "Information" + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[4.*, 5.0.0)" + } +} diff --git a/tests/endtoend/apps/blob-trigger-payload/requirements.txt b/tests/endtoend/apps/blob-trigger-payload/requirements.txt new file mode 100644 index 00000000..63f91d16 --- /dev/null +++ b/tests/endtoend/apps/blob-trigger-payload/requirements.txt @@ -0,0 +1 @@ +-e ../../../.. diff --git a/tests/endtoend/apps/multi-agent-delegation/coordinator.agent.md b/tests/endtoend/apps/multi-agent-delegation/coordinator.agent.md new file mode 100644 index 00000000..eb0d7778 --- /dev/null +++ b/tests/endtoend/apps/multi-agent-delegation/coordinator.agent.md @@ -0,0 +1,17 @@ +--- +name: Delegation Coordinator +description: HTTP coordinator that delegates detailed questions to a specialist agent. +trigger: + type: http_trigger + args: + route: "delegate" + methods: ["POST"] + auth_level: anonymous +subagents: + - agent: specialist + when: Detailed technical questions requiring specialist expertise +--- + +You are a coordinator. For detailed technical questions, delegate to the +specialist using delegate_specialist. For simple greetings or short questions, +answer directly yourself. Reply in at most two sentences. diff --git a/tests/endtoend/apps/multi-agent-delegation/function_app.py b/tests/endtoend/apps/multi-agent-delegation/function_app.py new file mode 100644 index 00000000..736ad492 --- /dev/null +++ b/tests/endtoend/apps/multi-agent-delegation/function_app.py @@ -0,0 +1,3 @@ +from azure_functions_agents import create_function_app + +app = create_function_app() diff --git a/tests/endtoend/apps/multi-agent-delegation/host.json b/tests/endtoend/apps/multi-agent-delegation/host.json new file mode 100644 index 00000000..4a421d09 --- /dev/null +++ b/tests/endtoend/apps/multi-agent-delegation/host.json @@ -0,0 +1,17 @@ +{ + "version": "2.0", + "extensions": { + "http": { + "routePrefix": "" + } + }, + "logging": { + "logLevel": { + "default": "Information" + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[4.*, 5.0.0)" + } +} diff --git a/tests/endtoend/apps/multi-agent-delegation/requirements.txt b/tests/endtoend/apps/multi-agent-delegation/requirements.txt new file mode 100644 index 00000000..63f91d16 --- /dev/null +++ b/tests/endtoend/apps/multi-agent-delegation/requirements.txt @@ -0,0 +1 @@ +-e ../../../.. diff --git a/tests/endtoend/apps/multi-agent-delegation/specialist.agent.md b/tests/endtoend/apps/multi-agent-delegation/specialist.agent.md new file mode 100644 index 00000000..a44d28c0 --- /dev/null +++ b/tests/endtoend/apps/multi-agent-delegation/specialist.agent.md @@ -0,0 +1,6 @@ +--- +name: Delegation Specialist +description: Internal specialist with no HTTP trigger; only reachable via coordinator delegation. +--- + +You are a technical specialist. Answer the delegated question in a single clear sentence. diff --git a/tests/endtoend/apps/queue-trigger-payload/function_app.py b/tests/endtoend/apps/queue-trigger-payload/function_app.py new file mode 100644 index 00000000..736ad492 --- /dev/null +++ b/tests/endtoend/apps/queue-trigger-payload/function_app.py @@ -0,0 +1,3 @@ +from azure_functions_agents import create_function_app + +app = create_function_app() diff --git a/tests/endtoend/apps/queue-trigger-payload/host.json b/tests/endtoend/apps/queue-trigger-payload/host.json new file mode 100644 index 00000000..4a421d09 --- /dev/null +++ b/tests/endtoend/apps/queue-trigger-payload/host.json @@ -0,0 +1,17 @@ +{ + "version": "2.0", + "extensions": { + "http": { + "routePrefix": "" + } + }, + "logging": { + "logLevel": { + "default": "Information" + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[4.*, 5.0.0)" + } +} diff --git a/tests/endtoend/apps/queue-trigger-payload/queue_processor.agent.md b/tests/endtoend/apps/queue-trigger-payload/queue_processor.agent.md new file mode 100644 index 00000000..5d483bc0 --- /dev/null +++ b/tests/endtoend/apps/queue-trigger-payload/queue_processor.agent.md @@ -0,0 +1,18 @@ +--- +name: Queue Payload Processor +description: > + Processes JSON messages from a storage queue. Purpose-built to exercise the + queue-message trigger-data serialization: the agent receives a structured JSON + payload containing body, body_encoding, body_json, id, and dequeue_count. +trigger: + type: queue_trigger + args: + queue_name: "queue-payload-input" + connection: "AzureWebJobsStorage" +logger: true +--- + +You receive a structured JSON message from a storage queue. The trigger data +includes a `body` field (the raw message text) and a `body_json` field (the +parsed JSON object when the body is valid JSON). Respond in a single sentence +that confirms you received the message and names the value of `body_json.order`. diff --git a/tests/endtoend/apps/queue-trigger-payload/requirements.txt b/tests/endtoend/apps/queue-trigger-payload/requirements.txt new file mode 100644 index 00000000..63f91d16 --- /dev/null +++ b/tests/endtoend/apps/queue-trigger-payload/requirements.txt @@ -0,0 +1 @@ +-e ../../../.. diff --git a/tests/endtoend/test_apps_http.py b/tests/endtoend/test_apps_http.py index c4562edf..ec995b2a 100644 --- a/tests/endtoend/test_apps_http.py +++ b/tests/endtoend/test_apps_http.py @@ -225,3 +225,232 @@ def test_chat_happy_path(builtin_endpoints_host: Served) -> None: expect_status(resp, 200) expect_header(resp, "x-ms-session-id") expect_json_keys(resp, ("session_id", "response")) + + +# --------------------------------------------------------------------------- # +# Multi-agent delegation (multi-agent-delegation app) +# +# These tests verify the subagents: front-matter field introduced in FRD 0007: +# a coordinator agent declares specialist subagents; each specialist is wired +# as a delegate_ tool at chat time. A key provider-independent behavior +# is that an endpoint-less specialist (no trigger: field) does NOT appear in +# the admin API — it is only reachable through the coordinator's delegation. +# --------------------------------------------------------------------------- # + + +@pytest.fixture(scope="module") +def multi_agent_delegation_host() -> Iterator[Served]: + with _serve("multi-agent-delegation") as served: + yield served + + +# --------------------------------------------------------------------------- # +# Registration +# --------------------------------------------------------------------------- # + + +def test_delegation_coordinator_is_registered( + multi_agent_delegation_host: Served, +) -> None: + """The coordinator is indexed as an HTTP trigger at its declared route.""" + _, endpoints = multi_agent_delegation_host + + coordinator = find_endpoint(endpoints, route_exact="delegate", method="POST") + assert coordinator.auth_level == "anonymous" + assert coordinator.methods == ("POST",) + + +def test_delegation_specialist_has_no_direct_endpoint( + multi_agent_delegation_host: Served, +) -> None: + """The endpoint-less specialist is NOT registered as a direct HTTP trigger. + + PR #102 (FRD 0007): agents listed only in ``subagents:`` and carrying no + ``trigger:`` or ``builtin_endpoints:`` field skip Azure Functions trigger + registration entirely. The specialist is only reachable through the + coordinator's ``delegate_specialist`` tool — it must not appear in the + admin API as an independent route. + """ + _, endpoints = multi_agent_delegation_host + + specialist_routes = [ep for ep in endpoints if "specialist" in ep.route.lower()] + assert not specialist_routes, ( + f"endpoint-less specialist must not appear in the admin API; " + f"found: {[ep.route for ep in specialist_routes]}" + ) + + +# --------------------------------------------------------------------------- # +# Deterministic behavior (no LLM required) +# --------------------------------------------------------------------------- # + + +def test_delegation_coordinator_rejects_missing_prompt( + multi_agent_delegation_host: Served, +) -> None: + """POST to the coordinator without a prompt body returns 400.""" + client, endpoints = multi_agent_delegation_host + + ep = find_endpoint(endpoints, route_exact="delegate", method="POST") + resp = client.post(ep.url(client.base_url), json={}) + + expect_status(resp, 400) + + +def test_delegation_coordinator_rejects_wrong_method( + multi_agent_delegation_host: Served, +) -> None: + """GET on the coordinator's POST-only route returns 404 (method not bound).""" + client, endpoints = multi_agent_delegation_host + + ep = find_endpoint(endpoints, route_exact="delegate", method="POST") + resp = client.get(ep.url(client.base_url)) + + expect_status(resp, 404, 405) + + +# --------------------------------------------------------------------------- # +# Full-run assertion (requires LLM) +# --------------------------------------------------------------------------- # + + +@requires_llm +def test_delegation_coordinator_responds( + multi_agent_delegation_host: Served, +) -> None: + """The coordinator handles a prompt end-to-end and returns a valid response. + + Confirms the full delegation path: coordinator receives prompt → decides + whether to delegate to delegate_specialist → returns session_id + response. + The assertion is on response structure, not content, so it does not depend + on which delegation branch the LLM chose. + """ + client, endpoints = multi_agent_delegation_host + + ep = find_endpoint(endpoints, route_exact="delegate", method="POST") + resp = client.post(ep.url(client.base_url), json={"prompt": "Hello, who are you?"}) + + expect_status(resp, 200) + expect_header(resp, "x-ms-session-id") + expect_json_keys(resp, ("session_id", "response")) + + +# --------------------------------------------------------------------------- # +# web_request system tool (web-request app) +# +# These tests verify the built-in web_request system tool introduced in +# FRD 0005 (PR #96): a default-on outbound HTTP(S) tool with an always-on +# SSRF safety floor, per-agent opt-out, and an optional host allowlist. +# +# The app exercises two behaviors: +# fetcher — web_request enabled, allowlisted to example.com +# opted_out — web_request disabled via system_tools.web_request: false +# --------------------------------------------------------------------------- # + + +@pytest.fixture(scope="module") +def web_request_host() -> Iterator[Served]: + with _serve("web-request") as served: + yield served + + +# --------------------------------------------------------------------------- # +# Registration +# --------------------------------------------------------------------------- # + + +def test_web_request_both_endpoints_are_registered( + web_request_host: Served, +) -> None: + """Both agents (fetcher and opted_out) register their HTTP routes.""" + _, endpoints = web_request_host + + fetch_ep = find_endpoint(endpoints, route_exact="fetch", method="POST") + assert fetch_ep.auth_level == "anonymous" + + opted_ep = find_endpoint(endpoints, route_exact="no-fetch", method="POST") + assert opted_ep.auth_level == "anonymous" + + +# --------------------------------------------------------------------------- # +# Deterministic behavior (no LLM required) +# --------------------------------------------------------------------------- # + + +def test_web_request_fetcher_rejects_missing_prompt( + web_request_host: Served, +) -> None: + """POST to the fetcher agent without a prompt body returns 400.""" + client, endpoints = web_request_host + + ep = find_endpoint(endpoints, route_exact="fetch", method="POST") + resp = client.post(ep.url(client.base_url), json={}) + + expect_status(resp, 400) + + +def test_web_request_opted_out_rejects_missing_prompt( + web_request_host: Served, +) -> None: + """POST to the opted-out agent without a prompt body returns 400.""" + client, endpoints = web_request_host + + ep = find_endpoint(endpoints, route_exact="no-fetch", method="POST") + resp = client.post(ep.url(client.base_url), json={}) + + expect_status(resp, 400) + + +# --------------------------------------------------------------------------- # +# Full-run assertions (require LLM) +# --------------------------------------------------------------------------- # + + +@requires_llm +def test_web_request_fetcher_completes_outbound_request( + web_request_host: Served, +) -> None: + """The fetcher agent uses web_request to hit example.com and responds. + + Confirms the full tool path: LLM decides to call web_request → SSRF + validation passes (example.com is in the allowlist) → HTTP GET to + https://example.com → structured response returned to the LLM → agent + produces a reply. The assertion is on response structure only; no content + assertion is made so the test does not depend on LLM output wording. + """ + client, endpoints = web_request_host + + ep = find_endpoint(endpoints, route_exact="fetch", method="POST") + resp = client.post( + ep.url(client.base_url), + json={"prompt": "Use web_request to fetch https://example.com and summarize the page."}, + ) + + expect_status(resp, 200) + expect_header(resp, "x-ms-session-id") + expect_json_keys(resp, ("session_id", "response")) + + +@requires_llm +def test_web_request_opted_out_agent_responds_without_tool( + web_request_host: Served, +) -> None: + """The opted-out agent handles a prompt without the web_request tool. + + Confirms the per-agent opt-out path (system_tools.web_request: false) + does not break agent registration or response — the agent answers from + its own knowledge rather than making an outbound call. The assertion is + on response structure; the opted-out agent either answers directly or + reports inability to fetch, both of which are valid. + """ + client, endpoints = web_request_host + + ep = find_endpoint(endpoints, route_exact="no-fetch", method="POST") + resp = client.post( + ep.url(client.base_url), + json={"prompt": "What is the capital of France?"}, + ) + + expect_status(resp, 200) + expect_header(resp, "x-ms-session-id") + expect_json_keys(resp, ("session_id", "response")) diff --git a/tests/endtoend/test_apps_storage.py b/tests/endtoend/test_apps_storage.py index 1b23c81b..6f9be19b 100644 --- a/tests/endtoend/test_apps_storage.py +++ b/tests/endtoend/test_apps_storage.py @@ -16,6 +16,8 @@ from __future__ import annotations import contextlib +import json +import os import shutil import uuid from collections.abc import Iterator @@ -38,6 +40,20 @@ APPS_DIR = Path(__file__).resolve().parent / "apps" + +def _provider_configured() -> bool: + """Whether an LLM provider appears configured.""" + return bool( + os.environ.get("OPENAI_API_KEY") + or os.environ.get("AZURE_OPENAI_ENDPOINT") + or os.environ.get("FOUNDRY_PROJECT_ENDPOINT") + ) + + +requires_llm = pytest.mark.skipif( + not _provider_configured(), reason="no LLM provider configured (set FOUNDRY_PROJECT_ENDPOINT etc.)" +) + pytestmark = [ pytest.mark.e2e, pytest.mark.skipif(shutil.which("func") is None, reason="Azure Functions Core Tools not found"), @@ -47,6 +63,9 @@ BLOB_CONTAINER = "uploads" QUEUE_NAME = "work-items" +# The queue the queue-trigger-payload app binds to (see queue_processor.agent.md). +QUEUE_PAYLOAD_NAME = "queue-payload-input" + # Served storage hosts are (handle, client): the handle exposes host output so we # can assert the function executed after data lands in storage. Served = tuple[HostHandle, HttpClient] @@ -142,3 +161,236 @@ def test_queue_trigger_fires_on_message(storage_host: Served) -> None: f"host never logged execution of queue trigger '{fn.name}' after enqueuing a " f"message on '{QUEUE_NAME}'. Recent output:\n{handle.read_output()[-2000:]}" ) + + +# --------------------------------------------------------------------------- # +# Queue trigger — payload serialization (queue-trigger-payload app) +# +# These tests verify the trigger-data serialization path introduced in +# PR #105: when a queue message carries a JSON body, the runtime serializes it +# into a structured dict (body, body_encoding, id, dequeue_count, body_json) +# rather than forwarding the raw Python QueueMessage repr to the agent. +# --------------------------------------------------------------------------- # + +FUNCTION_NAME = "queue_processor" + + +@contextlib.contextmanager +def _serve_payload_app() -> Iterator[Served]: + with running_host(APPS_DIR / "queue-trigger-payload") as handle: + client = HttpClient(handle.base_url) + try: + client.wait_until_responsive() + yield handle, client + finally: + client.close() + + +@pytest.fixture(scope="module") +def queue_trigger_payload_host() -> Iterator[Served]: + """Start the queue-trigger-payload app after clearing any residue messages.""" + clear_queue_messages(QUEUE_PAYLOAD_NAME) + with _serve_payload_app() as served: + yield served + + +# --------------------------------------------------------------------------- # +# Discovery +# --------------------------------------------------------------------------- # + + +def test_queue_trigger_payload_is_indexed(queue_trigger_payload_host: Served) -> None: + """The queue-trigger-payload app registers exactly one queueTrigger function.""" + _, client = queue_trigger_payload_host + functions = discover_functions(client) + queues = find_functions(functions, trigger_type="queueTrigger") + assert queues, "expected one queueTrigger function to be indexed" + fn = queues[0] + assert fn.route is None, "queue trigger must not expose an HTTP route" + assert fn.methods == (), "queue trigger must not list HTTP methods" + + +# --------------------------------------------------------------------------- # +# JSON body message — provider-independent +# --------------------------------------------------------------------------- # + + +def test_queue_trigger_payload_fires_on_json_message( + queue_trigger_payload_host: Served, +) -> None: + """Enqueuing a JSON message exercises the body_json serialization path. + + The agent receives a structured payload (body + body_json) rather than a raw + Python QueueMessage repr. Provider-independent assertions: + + 1. The handler log ``"Agent triggered: trigger_type=queue_trigger"`` appears, + confirming serialize_trigger_data was called inside the handler. + 2. ``Executed 'Functions.queue_processor'`` appears, confirming the function + ran to completion (serialization did not throw before invocation logged). + """ + handle, _ = queue_trigger_payload_host + + body = json.dumps({"order": f"e2e-{uuid.uuid4().hex[:8]}", "quantity": 3}) + send_queue_message(QUEUE_PAYLOAD_NAME, body) + + # The handler logs "Agent triggered" immediately before calling + # serialize_trigger_data — its presence confirms the full handler chain was + # entered for a queue trigger. + triggered = handle.wait_for_log("Agent triggered: trigger_type=queue_trigger", timeout=120.0) + assert triggered, ( + "handler never logged 'Agent triggered: trigger_type=queue_trigger' after " + f"enqueuing a JSON message. Recent output:\n{handle.read_output()[-2000:]}" + ) + + executed = handle.wait_for_log(f"Executed 'Functions.{FUNCTION_NAME}'", timeout=30.0) + assert executed, ( + f"host never logged execution of '{FUNCTION_NAME}' after enqueuing a " + f"JSON message. Recent output:\n{handle.read_output()[-2000:]}" + ) + + +# --------------------------------------------------------------------------- # +# Full-run assertion (requires LLM) +# --------------------------------------------------------------------------- # + + +@requires_llm +def test_queue_trigger_payload_full_run_succeeds( + queue_trigger_payload_host: Served, +) -> None: + """With an LLM provider, the agent completes and logs a response. + + Confirms the full path: queue message → serialize_trigger_data (body_json + populated) → agent runner → LLM call → response logged. The assertion + targets the ``"Agent response: source_file="`` log entry that the handler + emits on a successful run, not the content of the LLM response itself. + """ + handle, _ = queue_trigger_payload_host + + body = json.dumps({"order": f"e2e-llm-{uuid.uuid4().hex[:8]}", "quantity": 1}) + send_queue_message(QUEUE_PAYLOAD_NAME, body) + + responded = handle.wait_for_log("Agent response: source_file=", timeout=120.0) + assert responded, ( + "agent never logged a successful response after enqueuing a JSON message. " + f"Recent output:\n{handle.read_output()[-2000:]}" + ) + + +# --------------------------------------------------------------------------- # +# Blob trigger — payload serialization (blob-trigger-payload app) +# +# These tests verify the blob-trigger serialization path introduced in +# PR #105: when a blob is uploaded, the runtime serializes the InputStream +# binding into a structured dict (name, uri, length, blob_properties, metadata) +# rather than forwarding the raw Python InputStream repr to the agent. +# --------------------------------------------------------------------------- # + +BLOB_PAYLOAD_CONTAINER = "blob-payload-input" +BLOB_FUNCTION_NAME = "blob_processor" + + +@contextlib.contextmanager +def _serve_blob_payload_app() -> Iterator[Served]: + with running_host(APPS_DIR / "blob-trigger-payload") as handle: + client = HttpClient(handle.base_url) + try: + client.wait_until_responsive() + yield handle, client + finally: + client.close() + + +@pytest.fixture(scope="module") +def blob_trigger_payload_host() -> Iterator[Served]: + """Start the blob-trigger-payload app after clearing the bound container. + + Classic blob triggers scan the container at startup. Clearing any residue + blobs before starting prevents stale blobs from tripping the harness's + startup failure detection. + """ + clear_container(BLOB_PAYLOAD_CONTAINER) + with _serve_blob_payload_app() as served: + yield served + + +# --------------------------------------------------------------------------- # +# Discovery +# --------------------------------------------------------------------------- # + + +def test_blob_trigger_payload_is_indexed(blob_trigger_payload_host: Served) -> None: + """The blob-trigger-payload app registers exactly one blobTrigger function.""" + _, client = blob_trigger_payload_host + functions = discover_functions(client) + blobs = find_functions(functions, trigger_type="blobTrigger") + assert blobs, "expected one blobTrigger function to be indexed" + fn = blobs[0] + assert fn.route is None, "blob trigger must not expose an HTTP route" + assert fn.methods == (), "blob trigger must not list HTTP methods" + + +# --------------------------------------------------------------------------- # +# Blob upload — provider-independent +# --------------------------------------------------------------------------- # + + +def test_blob_trigger_payload_fires_on_upload( + blob_trigger_payload_host: Served, +) -> None: + """Uploading a blob exercises the InputStream serialization path. + + The agent receives structured blob metadata (name, uri, length, + blob_properties) rather than a raw Python InputStream repr. Provider- + independent assertions: + + 1. The handler log ``"Agent triggered: trigger_type=blob_trigger"`` appears, + confirming serialize_trigger_data was called inside the handler. + 2. ``Executed 'Functions.blob_processor'`` appears, confirming the function + ran to completion (serialization did not throw before invocation logged). + """ + handle, _ = blob_trigger_payload_host + + blob_name = f"probe-{uuid.uuid4().hex[:8]}.txt" + upload_text_blob(BLOB_PAYLOAD_CONTAINER, blob_name, "blob trigger serialization e2e probe") + + # Classic blob triggers poll storage, so allow a generous wait. + triggered = handle.wait_for_log("Agent triggered: trigger_type=blob_trigger", timeout=240.0) + assert triggered, ( + "handler never logged 'Agent triggered: trigger_type=blob_trigger' after " + f"uploading '{blob_name}'. Recent output:\n{handle.read_output()[-2000:]}" + ) + + executed = handle.wait_for_log(f"Executed 'Functions.{BLOB_FUNCTION_NAME}'", timeout=30.0) + assert executed, ( + f"host never logged execution of '{BLOB_FUNCTION_NAME}' after uploading " + f"'{blob_name}'. Recent output:\n{handle.read_output()[-2000:]}" + ) + + +# --------------------------------------------------------------------------- # +# Full-run assertion (requires LLM) +# --------------------------------------------------------------------------- # + + +@requires_llm +def test_blob_trigger_payload_full_run_succeeds( + blob_trigger_payload_host: Served, +) -> None: + """With an LLM provider, the blob agent completes and logs a response. + + Confirms the full path: blob upload → serialize_trigger_data (name/uri/ + length populated) → agent runner → LLM call → response logged. The + assertion targets the ``"Agent response: source_file="`` log entry that the + handler emits on a successful run, not the content of the LLM response. + """ + handle, _ = blob_trigger_payload_host + + blob_name = f"probe-llm-{uuid.uuid4().hex[:8]}.txt" + upload_text_blob(BLOB_PAYLOAD_CONTAINER, blob_name, "blob trigger llm e2e probe") + + responded = handle.wait_for_log("Agent response: source_file=", timeout=240.0) + assert responded, ( + "agent never logged a successful response after uploading a blob. " + f"Recent output:\n{handle.read_output()[-2000:]}" + ) From c872aaae082ae22311e4d9e6d5771d5796f83241 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Mon, 27 Jul 2026 11:27:30 -0500 Subject: [PATCH 02/10] add local.settings.json --- .../endtoend/apps/blob-trigger-payload/local.settings.json | 7 +++++++ .../apps/multi-agent-delegation/local.settings.json | 7 +++++++ .../apps/queue-trigger-payload/local.settings.json | 7 +++++++ 3 files changed, 21 insertions(+) create mode 100644 tests/endtoend/apps/blob-trigger-payload/local.settings.json create mode 100644 tests/endtoend/apps/multi-agent-delegation/local.settings.json create mode 100644 tests/endtoend/apps/queue-trigger-payload/local.settings.json diff --git a/tests/endtoend/apps/blob-trigger-payload/local.settings.json b/tests/endtoend/apps/blob-trigger-payload/local.settings.json new file mode 100644 index 00000000..9ab81bc5 --- /dev/null +++ b/tests/endtoend/apps/blob-trigger-payload/local.settings.json @@ -0,0 +1,7 @@ +{ + "IsEncrypted": false, + "Values": { + "FUNCTIONS_WORKER_RUNTIME": "python", + "AzureWebJobsStorage": "UseDevelopmentStorage=true" + } +} diff --git a/tests/endtoend/apps/multi-agent-delegation/local.settings.json b/tests/endtoend/apps/multi-agent-delegation/local.settings.json new file mode 100644 index 00000000..9ab81bc5 --- /dev/null +++ b/tests/endtoend/apps/multi-agent-delegation/local.settings.json @@ -0,0 +1,7 @@ +{ + "IsEncrypted": false, + "Values": { + "FUNCTIONS_WORKER_RUNTIME": "python", + "AzureWebJobsStorage": "UseDevelopmentStorage=true" + } +} diff --git a/tests/endtoend/apps/queue-trigger-payload/local.settings.json b/tests/endtoend/apps/queue-trigger-payload/local.settings.json new file mode 100644 index 00000000..9ab81bc5 --- /dev/null +++ b/tests/endtoend/apps/queue-trigger-payload/local.settings.json @@ -0,0 +1,7 @@ +{ + "IsEncrypted": false, + "Values": { + "FUNCTIONS_WORKER_RUNTIME": "python", + "AzureWebJobsStorage": "UseDevelopmentStorage=true" + } +} From d88a9a3c6106977366a2210bf481fde602165bd8 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Mon, 27 Jul 2026 11:28:31 -0500 Subject: [PATCH 03/10] update gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 30ad6832..86890732 100644 --- a/.gitignore +++ b/.gitignore @@ -436,5 +436,5 @@ dist/ .venv* __azurite_* __blobstorage__/ -**/local.settings.json +samples/**/local.settings.json .tmp-validation/ From be2a4511c925182d0d82b5d3c0e510060f341fd6 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Mon, 27 Jul 2026 12:04:36 -0500 Subject: [PATCH 04/10] fix tests --- tests/endtoend/test_apps_http.py | 61 ++++++-------------------------- 1 file changed, 11 insertions(+), 50 deletions(-) diff --git a/tests/endtoend/test_apps_http.py b/tests/endtoend/test_apps_http.py index ec995b2a..cbef5f09 100644 --- a/tests/endtoend/test_apps_http.py +++ b/tests/endtoend/test_apps_http.py @@ -285,18 +285,6 @@ def test_delegation_specialist_has_no_direct_endpoint( # --------------------------------------------------------------------------- # -def test_delegation_coordinator_rejects_missing_prompt( - multi_agent_delegation_host: Served, -) -> None: - """POST to the coordinator without a prompt body returns 400.""" - client, endpoints = multi_agent_delegation_host - - ep = find_endpoint(endpoints, route_exact="delegate", method="POST") - resp = client.post(ep.url(client.base_url), json={}) - - expect_status(resp, 400) - - def test_delegation_coordinator_rejects_wrong_method( multi_agent_delegation_host: Served, ) -> None: @@ -321,9 +309,9 @@ def test_delegation_coordinator_responds( """The coordinator handles a prompt end-to-end and returns a valid response. Confirms the full delegation path: coordinator receives prompt → decides - whether to delegate to delegate_specialist → returns session_id + response. - The assertion is on response structure, not content, so it does not depend - on which delegation branch the LLM chose. + whether to delegate to delegate_specialist → returns a non-empty reply. + Plain HTTP trigger agents return plain text (not the JSON envelope from + builtin chat endpoints), so this asserts on status and session header only. """ client, endpoints = multi_agent_delegation_host @@ -332,7 +320,7 @@ def test_delegation_coordinator_responds( expect_status(resp, 200) expect_header(resp, "x-ms-session-id") - expect_json_keys(resp, ("session_id", "response")) + assert resp.text.strip(), "expected a non-empty response body" # --------------------------------------------------------------------------- # @@ -372,35 +360,6 @@ def test_web_request_both_endpoints_are_registered( assert opted_ep.auth_level == "anonymous" -# --------------------------------------------------------------------------- # -# Deterministic behavior (no LLM required) -# --------------------------------------------------------------------------- # - - -def test_web_request_fetcher_rejects_missing_prompt( - web_request_host: Served, -) -> None: - """POST to the fetcher agent without a prompt body returns 400.""" - client, endpoints = web_request_host - - ep = find_endpoint(endpoints, route_exact="fetch", method="POST") - resp = client.post(ep.url(client.base_url), json={}) - - expect_status(resp, 400) - - -def test_web_request_opted_out_rejects_missing_prompt( - web_request_host: Served, -) -> None: - """POST to the opted-out agent without a prompt body returns 400.""" - client, endpoints = web_request_host - - ep = find_endpoint(endpoints, route_exact="no-fetch", method="POST") - resp = client.post(ep.url(client.base_url), json={}) - - expect_status(resp, 400) - - # --------------------------------------------------------------------------- # # Full-run assertions (require LLM) # --------------------------------------------------------------------------- # @@ -415,8 +374,9 @@ def test_web_request_fetcher_completes_outbound_request( Confirms the full tool path: LLM decides to call web_request → SSRF validation passes (example.com is in the allowlist) → HTTP GET to https://example.com → structured response returned to the LLM → agent - produces a reply. The assertion is on response structure only; no content - assertion is made so the test does not depend on LLM output wording. + produces a reply. Plain HTTP trigger agents return plain text (not the + JSON envelope from builtin chat endpoints), so assertions are on status + and session header only. """ client, endpoints = web_request_host @@ -428,7 +388,7 @@ def test_web_request_fetcher_completes_outbound_request( expect_status(resp, 200) expect_header(resp, "x-ms-session-id") - expect_json_keys(resp, ("session_id", "response")) + assert resp.text.strip(), "expected a non-empty response body" @requires_llm @@ -441,7 +401,8 @@ def test_web_request_opted_out_agent_responds_without_tool( does not break agent registration or response — the agent answers from its own knowledge rather than making an outbound call. The assertion is on response structure; the opted-out agent either answers directly or - reports inability to fetch, both of which are valid. + reports inability to fetch, both of which are valid. Plain HTTP trigger + agents return plain text so assertions are on status and session header. """ client, endpoints = web_request_host @@ -453,4 +414,4 @@ def test_web_request_opted_out_agent_responds_without_tool( expect_status(resp, 200) expect_header(resp, "x-ms-session-id") - expect_json_keys(resp, ("session_id", "response")) + assert resp.text.strip(), "expected a non-empty response body" From d912d4f9c575e2e24c0f189ff88e0e551437dc9c Mon Sep 17 00:00:00 2001 From: hallvictoria <59299039+hallvictoria@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:09:17 -0500 Subject: [PATCH 05/10] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/skills/add-e2e-test/SKILL.md | 4 ++-- AGENTS.md | 7 ++----- tests/endtoend/test_apps_http.py | 8 +++++--- tests/endtoend/test_apps_storage.py | 22 +++++++++++++++++----- 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/.github/skills/add-e2e-test/SKILL.md b/.github/skills/add-e2e-test/SKILL.md index f2611060..56a2e788 100644 --- a/.github/skills/add-e2e-test/SKILL.md +++ b/.github/skills/add-e2e-test/SKILL.md @@ -54,9 +54,9 @@ samples — avoid bundling multiple capabilities. Create a new directory under `tests/endtoend/apps//` where `` is short, lowercase, and hyphenated (e.g., `queue-error-handling`). Each directory -**must** contain exactly these five files: +**must** contain the following supporting files, plus one or more `*.agent.md` files: -#### `.agent.md` +#### `.agent.md` (one or more) One focused agent that exercises the feature. Follow standard front-matter conventions from `docs/front-matter-spec.md`. Keep the system prompt minimal. diff --git a/AGENTS.md b/AGENTS.md index 14e5e478..e73e4388 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -175,11 +175,8 @@ Foundry resource. Features that require any other external resource may waive th E2E requirement — record the waiver in the FRD Decisions log (medium+ features) or PR description (small features). -- E2E apps live under `tests/endtoend/apps//` — one dedicated directory - per feature, never shared between features. -- Each app must contain five files: `*.agent.md`, `function_app.py`, `host.json`, - `local.settings.json` (no secrets), and `requirements.txt` (editable install). -- Tests go into `tests/endtoend/test_apps_http.py`, `test_apps_storage.py`, or +- Each app must contain `function_app.py`, `host.json`, `local.settings.json` (no secrets), + `requirements.txt` (editable install), and one or more `*.agent.md` files. `test_apps_mcp.py` based on trigger type. - `tests/endtoend/test_apps_start.py` auto-discovers all apps and runs a `func start` smoke test — adding a new app dir is enough to opt into it. diff --git a/tests/endtoend/test_apps_http.py b/tests/endtoend/test_apps_http.py index cbef5f09..2825d420 100644 --- a/tests/endtoend/test_apps_http.py +++ b/tests/endtoend/test_apps_http.py @@ -316,8 +316,10 @@ def test_delegation_coordinator_responds( client, endpoints = multi_agent_delegation_host ep = find_endpoint(endpoints, route_exact="delegate", method="POST") - resp = client.post(ep.url(client.base_url), json={"prompt": "Hello, who are you?"}) - + resp = client.post( + ep.url(client.base_url), + json={"prompt": "Delegate to the specialist: explain the difference between threading and async in Python."}, + ) expect_status(resp, 200) expect_header(resp, "x-ms-session-id") assert resp.text.strip(), "expected a non-empty response body" @@ -388,7 +390,7 @@ def test_web_request_fetcher_completes_outbound_request( expect_status(resp, 200) expect_header(resp, "x-ms-session-id") - assert resp.text.strip(), "expected a non-empty response body" + expect_body_contains(resp, "Example Domain") @requires_llm diff --git a/tests/endtoend/test_apps_storage.py b/tests/endtoend/test_apps_storage.py index 6f9be19b..3dfa77a0 100644 --- a/tests/endtoend/test_apps_storage.py +++ b/tests/endtoend/test_apps_storage.py @@ -224,10 +224,9 @@ def test_queue_trigger_payload_fires_on_json_message( Python QueueMessage repr. Provider-independent assertions: 1. The handler log ``"Agent triggered: trigger_type=queue_trigger"`` appears, - confirming serialize_trigger_data was called inside the handler. - 2. ``Executed 'Functions.queue_processor'`` appears, confirming the function - ran to completion (serialization did not throw before invocation logged). - """ + confirming the queue-trigger handler started executing. + 2. ``Executed 'Functions.queue_processor'`` appears, confirming the Functions + host invoked the function (whether the agent run itself succeeds or fails). handle, _ = queue_trigger_payload_host body = json.dumps({"order": f"e2e-{uuid.uuid4().hex[:8]}", "quantity": 3}) @@ -267,7 +266,8 @@ def test_queue_trigger_payload_full_run_succeeds( """ handle, _ = queue_trigger_payload_host - body = json.dumps({"order": f"e2e-llm-{uuid.uuid4().hex[:8]}", "quantity": 1}) + order_id = f"e2e-llm-{uuid.uuid4().hex[:8]}" + body = json.dumps({"order": order_id, "quantity": 1}) send_queue_message(QUEUE_PAYLOAD_NAME, body) responded = handle.wait_for_log("Agent response: source_file=", timeout=120.0) @@ -276,6 +276,12 @@ def test_queue_trigger_payload_full_run_succeeds( f"Recent output:\n{handle.read_output()[-2000:]}" ) + output = handle.read_output() + assert order_id in output, ( + "agent response did not include the expected order id (likely missing trigger payload content). " + f"order_id={order_id}. Recent output:\n{output[-2000:]}" + ) + # --------------------------------------------------------------------------- # # Blob trigger — payload serialization (blob-trigger-payload app) @@ -394,3 +400,9 @@ def test_blob_trigger_payload_full_run_succeeds( "agent never logged a successful response after uploading a blob. " f"Recent output:\n{handle.read_output()[-2000:]}" ) + + output = handle.read_output() + assert blob_name in output, ( + "agent response did not include the uploaded blob name (likely missing trigger payload content). " + f"blob_name={blob_name}. Recent output:\n{output[-2000:]}" + ) From 6a119948e754ac222506034cafc31ce48e19c39b Mon Sep 17 00:00:00 2001 From: hallvictoria <59299039+hallvictoria@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:33:31 -0500 Subject: [PATCH 06/10] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/skills/add-e2e-test/SKILL.md | 2 +- tests/endtoend/test_apps_http.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/skills/add-e2e-test/SKILL.md b/.github/skills/add-e2e-test/SKILL.md index 56a2e788..05027ec7 100644 --- a/.github/skills/add-e2e-test/SKILL.md +++ b/.github/skills/add-e2e-test/SKILL.md @@ -32,7 +32,7 @@ provides: | Any other external resource (databases, connectors, etc.) | **No** | **Supported trigger types:** `http_trigger`, `blob_trigger`, `queue_trigger`, -`timer_trigger` (start-only; timers do not fire in CI), and MCP tool triggers +`timer_trigger` (scheduled timers do not fire in CI; tests should invoke timers deterministically via the Functions admin API), and MCP tool triggers via the `builtin_endpoints.mcp` flag. **Decision rule:** diff --git a/tests/endtoend/test_apps_http.py b/tests/endtoend/test_apps_http.py index 2825d420..9d44b1ee 100644 --- a/tests/endtoend/test_apps_http.py +++ b/tests/endtoend/test_apps_http.py @@ -411,9 +411,14 @@ def test_web_request_opted_out_agent_responds_without_tool( ep = find_endpoint(endpoints, route_exact="no-fetch", method="POST") resp = client.post( ep.url(client.base_url), - json={"prompt": "What is the capital of France?"}, + json={ + "prompt": ( + "Try to use web_request to fetch https://example.com. If the tool is unavailable, " + "reply with exactly 'WEB_REQUEST_DISABLED'." + ) + }, ) expect_status(resp, 200) expect_header(resp, "x-ms-session-id") - assert resp.text.strip(), "expected a non-empty response body" + expect_body_contains(resp, "WEB_REQUEST_DISABLED") From 2b2b40f534d6812243dcc6cda7876c25c208c60f Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Mon, 27 Jul 2026 14:36:08 -0500 Subject: [PATCH 07/10] add more comprehensive test --- .../coordinator.agent.md | 11 ++++++----- .../multi-agent-delegation/specialist.agent.md | 6 ++++-- tests/endtoend/test_apps_http.py | 18 +++++++++++------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/tests/endtoend/apps/multi-agent-delegation/coordinator.agent.md b/tests/endtoend/apps/multi-agent-delegation/coordinator.agent.md index eb0d7778..72708b6a 100644 --- a/tests/endtoend/apps/multi-agent-delegation/coordinator.agent.md +++ b/tests/endtoend/apps/multi-agent-delegation/coordinator.agent.md @@ -1,6 +1,6 @@ --- name: Delegation Coordinator -description: HTTP coordinator that delegates detailed questions to a specialist agent. +description: HTTP coordinator that always delegates every request to the specialist agent. trigger: type: http_trigger args: @@ -9,9 +9,10 @@ trigger: auth_level: anonymous subagents: - agent: specialist - when: Detailed technical questions requiring specialist expertise --- -You are a coordinator. For detailed technical questions, delegate to the -specialist using delegate_specialist. For simple greetings or short questions, -answer directly yourself. Reply in at most two sentences. +You MUST always use delegate_specialist for every request without exception. +Do not answer any question yourself. +Call delegate_specialist with the user's exact message, then return the +specialist's response exactly as-is — no introduction, no rephrasing, no +extra commentary. diff --git a/tests/endtoend/apps/multi-agent-delegation/specialist.agent.md b/tests/endtoend/apps/multi-agent-delegation/specialist.agent.md index a44d28c0..4b1736ed 100644 --- a/tests/endtoend/apps/multi-agent-delegation/specialist.agent.md +++ b/tests/endtoend/apps/multi-agent-delegation/specialist.agent.md @@ -1,6 +1,8 @@ --- name: Delegation Specialist -description: Internal specialist with no HTTP trigger; only reachable via coordinator delegation. +description: Internal specialist with no HTTP trigger; always confirms delegation with a fixed prefix. --- -You are a technical specialist. Answer the delegated question in a single clear sentence. +Always begin your response with the exact text "DELEGATION_OK:" followed by a +single space, then answer the question in one short sentence. Never omit the +prefix under any circumstances. diff --git a/tests/endtoend/test_apps_http.py b/tests/endtoend/test_apps_http.py index 2825d420..1ba1c6b0 100644 --- a/tests/endtoend/test_apps_http.py +++ b/tests/endtoend/test_apps_http.py @@ -306,23 +306,27 @@ def test_delegation_coordinator_rejects_wrong_method( def test_delegation_coordinator_responds( multi_agent_delegation_host: Served, ) -> None: - """The coordinator handles a prompt end-to-end and returns a valid response. + """The coordinator delegates to the specialist, which emits a deterministic marker. - Confirms the full delegation path: coordinator receives prompt → decides - whether to delegate to delegate_specialist → returns a non-empty reply. - Plain HTTP trigger agents return plain text (not the JSON envelope from - builtin chat endpoints), so this asserts on status and session header only. + The specialist is instructed to always start its response with ``DELEGATION_OK:``. + The coordinator is instructed to always delegate and pass through the specialist's + response verbatim. Asserting on the marker confirms that delegation actually + happened — a coordinator that answered directly would never produce the prefix. """ client, endpoints = multi_agent_delegation_host ep = find_endpoint(endpoints, route_exact="delegate", method="POST") resp = client.post( ep.url(client.base_url), - json={"prompt": "Delegate to the specialist: explain the difference between threading and async in Python."}, + json={"prompt": "What is the difference between a process and a thread?"}, ) expect_status(resp, 200) expect_header(resp, "x-ms-session-id") - assert resp.text.strip(), "expected a non-empty response body" + assert "DELEGATION_OK:" in resp.text, ( + "expected the delegation marker 'DELEGATION_OK:' in the response — " + "if absent the coordinator answered directly without delegating to the specialist. " + f"Got: {resp.text!r}" + ) # --------------------------------------------------------------------------- # From 8a00d10005ff83f02f66933a94b149fd78f2e5a3 Mon Sep 17 00:00:00 2001 From: hallvictoria <59299039+hallvictoria@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:57:24 -0500 Subject: [PATCH 08/10] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- AGENTS.md | 2 +- tests/endtoend/test_apps_http.py | 8 ++++---- tests/endtoend/test_apps_storage.py | 10 ++++++++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e73e4388..9022aae8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -177,7 +177,7 @@ or PR description (small features). - Each app must contain `function_app.py`, `host.json`, `local.settings.json` (no secrets), `requirements.txt` (editable install), and one or more `*.agent.md` files. - `test_apps_mcp.py` based on trigger type. + Add assertions in the appropriate `tests/endtoend/test_apps_*.py` file based on trigger type. - `tests/endtoend/test_apps_start.py` auto-discovers all apps and runs a `func start` smoke test — adding a new app dir is enough to opt into it. - Use the **`add-e2e-test` skill** diff --git a/tests/endtoend/test_apps_http.py b/tests/endtoend/test_apps_http.py index ddfbd46d..30a6982c 100644 --- a/tests/endtoend/test_apps_http.py +++ b/tests/endtoend/test_apps_http.py @@ -273,10 +273,10 @@ def test_delegation_specialist_has_no_direct_endpoint( """ _, endpoints = multi_agent_delegation_host - specialist_routes = [ep for ep in endpoints if "specialist" in ep.route.lower()] - assert not specialist_routes, ( - f"endpoint-less specialist must not appear in the admin API; " - f"found: {[ep.route for ep in specialist_routes]}" + specialist_endpoints = [ep for ep in endpoints if ep.function_name.lower() == "specialist"] + assert not specialist_endpoints, ( + "endpoint-less specialist must not appear in the admin API; " + f"found: {[(ep.function_name, ep.route) for ep in specialist_endpoints]}" ) diff --git a/tests/endtoend/test_apps_storage.py b/tests/endtoend/test_apps_storage.py index 3dfa77a0..58c0134f 100644 --- a/tests/endtoend/test_apps_storage.py +++ b/tests/endtoend/test_apps_storage.py @@ -42,11 +42,16 @@ def _provider_configured() -> bool: - """Whether an LLM provider appears configured.""" + """Whether an LLM provider appears configured (env vars or app settings).""" + from tests.endtoend._func_host import configured_provider + return bool( os.environ.get("OPENAI_API_KEY") or os.environ.get("AZURE_OPENAI_ENDPOINT") or os.environ.get("FOUNDRY_PROJECT_ENDPOINT") + or configured_provider(APPS_DIR / "storage-triggers") is not None + or configured_provider(APPS_DIR / "queue-trigger-payload") is not None + or configured_provider(APPS_DIR / "blob-trigger-payload") is not None ) @@ -204,8 +209,9 @@ def test_queue_trigger_payload_is_indexed(queue_trigger_payload_host: Served) -> _, client = queue_trigger_payload_host functions = discover_functions(client) queues = find_functions(functions, trigger_type="queueTrigger") - assert queues, "expected one queueTrigger function to be indexed" + assert len(queues) == 1, f"expected exactly one queueTrigger function to be indexed, got {len(queues)}" fn = queues[0] + assert fn.name == FUNCTION_NAME assert fn.route is None, "queue trigger must not expose an HTTP route" assert fn.methods == (), "queue trigger must not list HTTP methods" From 018d7dd2aabc625f4a405708a9819241e87ea68e Mon Sep 17 00:00:00 2001 From: hallvictoria <59299039+hallvictoria@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:32:33 -0500 Subject: [PATCH 09/10] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/skills/add-e2e-test/SKILL.md | 5 +++-- tests/endtoend/test_apps_storage.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/skills/add-e2e-test/SKILL.md b/.github/skills/add-e2e-test/SKILL.md index 05027ec7..18d35d4b 100644 --- a/.github/skills/add-e2e-test/SKILL.md +++ b/.github/skills/add-e2e-test/SKILL.md @@ -66,12 +66,13 @@ conventions from `docs/front-matter-spec.md`. Keep the system prompt minimal. name: My Feature Agent description: One-sentence description of what this agent tests. trigger: - type: http_trigger # or blob_trigger / queue_trigger / mcp + type: http_trigger # or blob_trigger / queue_trigger args: route: "my-feature" methods: ["POST"] auth_level: anonymous ---- +builtin_endpoints: + mcp: true # optional: also register this agent as an MCP tool You are a concise assistant. Reply in one short sentence. ``` diff --git a/tests/endtoend/test_apps_storage.py b/tests/endtoend/test_apps_storage.py index 58c0134f..7fe53066 100644 --- a/tests/endtoend/test_apps_storage.py +++ b/tests/endtoend/test_apps_storage.py @@ -336,8 +336,9 @@ def test_blob_trigger_payload_is_indexed(blob_trigger_payload_host: Served) -> N _, client = blob_trigger_payload_host functions = discover_functions(client) blobs = find_functions(functions, trigger_type="blobTrigger") - assert blobs, "expected one blobTrigger function to be indexed" + assert len(blobs) == 1, f"expected exactly one blobTrigger function to be indexed, got {len(blobs)}" fn = blobs[0] + assert fn.name == BLOB_FUNCTION_NAME assert fn.route is None, "blob trigger must not expose an HTTP route" assert fn.methods == (), "blob trigger must not list HTTP methods" From 560dd182478a6f206755a06843e01094b72e0a83 Mon Sep 17 00:00:00 2001 From: hallvictoria <59299039+hallvictoria@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:48:45 -0500 Subject: [PATCH 10/10] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tests/endtoend/test_apps_http.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/endtoend/test_apps_http.py b/tests/endtoend/test_apps_http.py index 30a6982c..4744a9a4 100644 --- a/tests/endtoend/test_apps_http.py +++ b/tests/endtoend/test_apps_http.py @@ -416,13 +416,10 @@ def test_web_request_opted_out_agent_responds_without_tool( resp = client.post( ep.url(client.base_url), json={ - "prompt": ( - "Try to use web_request to fetch https://example.com. If the tool is unavailable, " - "reply with exactly 'WEB_REQUEST_DISABLED'." - ) + "prompt": "web_request is disabled for you. Reply in one short sentence confirming you cannot fetch https://example.com.", }, ) expect_status(resp, 200) expect_header(resp, "x-ms-session-id") - expect_body_contains(resp, "WEB_REQUEST_DISABLED") + assert resp.text.strip(), "expected a non-empty response body from the opted-out agent"