diff --git a/e2e-tests/README.md b/e2e-tests/README.md index 3cdfaed5f..813f73f46 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -113,6 +113,23 @@ Feature lifecycle tests: describe what the test exercises end-to-end - `dev-lifecycle.test.ts` - `evals-lifecycle.test.ts` +- `ab-test-config-bundle.test.ts` — A/B test (config-bundle mode): create → run → pause → resume → promote, asserting + live execution state from AWS via `view ab-test` +- `ab-test-target-based.test.ts` — A/B test (target-based mode): two http-runtime gateway-targets on named runtime + endpoints, each scoped by its own online-eval → run → pause → resume → promote (control endpoint version-bumped to + treatment's) +- `httpgateway-all-targets.test.ts` — one `protocolType: None` (HTTP) gateway hosting every deployable target type + (http-runtime, mcp-server, lambda-function-arn, api-gateway, open-api-schema, smithy-model, web-search, passthrough), + deployed in a single stack. `passthrough` is gated, so its add/deploy run with `ENABLE_GATED_FEATURES=1`. Omits + `connector` (Bedrock FMKB, a private-beta CFN resource type). + +### Fixtures that provision external AWS resources + +Some gateway-target types reference AWS resources that `agentcore deploy` cannot create (an existing Lambda, a REST +API). `fixtures/gateway-targets/setup_target_prereqs.py` creates them idempotently (check-then-create, reused across +runs) and writes their identifiers to a per-run JSON file the test reads — mirroring `fixtures/import/`'s boto3 setup +pattern. If the IAM role lacks permission to create one (e.g. a restricted CI role without `lambda:*`/`apigateway:*`), +the fixture emits `null` for that identifier and the test skips the dependent target rather than failing the suite. ## Important Notes diff --git a/e2e-tests/ab-test-config-bundle.test.ts b/e2e-tests/ab-test-config-bundle.test.ts new file mode 100644 index 000000000..011afcdd3 --- /dev/null +++ b/e2e-tests/ab-test-config-bundle.test.ts @@ -0,0 +1,340 @@ +/** + * E2E test for A/B tests (config-bundle mode) across the AWS boundary. + * + * Flow: create project → add gateway → add config bundle (v1) → deploy → + * update bundle (v2) → deploy → add online-eval (Builtin evaluator) → deploy → + * run ab-test → view (poll RUNNING) → pause → view (PAUSED) → resume → + * view (RUNNING) → promote → archive + * + * A/B tests are fire-and-forget jobs, not project resources, so cleanup must + * `archive` the test explicitly — `remove all` does not touch it. + * + * Live-AWS behaviours this proves (per e2e-tests/README.md): pause / resume / + * promote return live execution state from AWS. `view ab-test --json` re-fetches + * server state; the live execution status (RUNNING/PAUSED/STOPPED) surfaces in + * the `lifecycleStatus` field (handler.refresh maps executionStatus → lifecycleStatus). + * + * Prerequisites: AWS credentials, npm, git, uv. + */ +import { parseJsonOutput, retry } from '../src/test-utils/index.js'; +import { + baseCanRun, + hasAws, + installCdkTarball, + runAgentCoreCLI, + teardownE2EProject, + writeAwsTargets, +} from './e2e-helper.js'; +import { randomUUID } from 'node:crypto'; +import { mkdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const canRun = baseCanRun && hasAws; + +describe.sequential('e2e: A/B test lifecycle (config-bundle mode)', () => { + let testDir: string; + let projectPath: string; + const suffix = String(Date.now()).slice(-8); + const agentName = `E2eAbt${suffix}`; + const gatewayName = 'abtgw'; + const bundleName = 'E2eAbtBundle'; + const onlineEvalName = 'E2eAbtEval'; + const abTestName = 'E2eAbtTest'; + + // Captured across the sequential steps. + let controlVersionId: string; + let abTestId: string; + + beforeAll(async () => { + if (!canRun) return; + + testDir = join(tmpdir(), `agentcore-e2e-ab-test-${randomUUID()}`); + await mkdir(testDir, { recursive: true }); + + const result = await runAgentCoreCLI( + [ + 'create', + '--name', + agentName, + '--language', + 'Python', + '--framework', + 'Strands', + '--model-provider', + 'Bedrock', + '--memory', + 'none', + '--json', + ], + testDir + ); + expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0); + projectPath = (parseJsonOutput(result.stdout) as { projectPath: string }).projectPath; + + await writeAwsTargets(projectPath); + installCdkTarball(projectPath); + }, 300000); + + afterAll(async () => { + // A/B tests are jobs, not project resources — archive explicitly before teardown. + if (abTestId && projectPath && hasAws) { + await runAgentCoreCLI(['archive', 'ab-test', '-i', abTestId, '--json'], projectPath); + } + if (projectPath && hasAws) { + await teardownE2EProject(projectPath, agentName, 'Bedrock'); + } + if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 }); + }, 600000); + + const run = (args: string[]) => runAgentCoreCLI(args, projectPath); + + const bundleComponents = (systemPrompt: string, temperature: number) => + JSON.stringify({ + [`{{runtime:${agentName}}}`]: { configuration: { systemPrompt, temperature } }, + }); + + // ── Gateway (required: AB tests resolve a deployed gateway ARN) ────────── + + it.skipIf(!canRun)( + 'adds a gateway', + async () => { + const result = await run(['add', 'gateway', '--name', gatewayName, '--protocol-type', 'None', '--json']); + expect(result.exitCode, `Add gateway failed: ${result.stdout}`).toBe(0); + expect((parseJsonOutput(result.stdout) as { success: boolean }).success).toBe(true); + }, + 60000 + ); + + // ── Config bundle v1 + deploy ──────────────────────────────────────────── + + it.skipIf(!canRun)( + 'adds config bundle (v1) and deploys', + async () => { + const add = await run([ + 'add', + 'config-bundle', + '--name', + bundleName, + '--description', + 'AB test bundle', + '--components', + bundleComponents('You are control: concise.', 0.5), + '--branch', + 'mainline', + '--commit-message', + 'v1 control', + '--json', + ]); + expect(add.exitCode, `Add config-bundle failed: ${add.stdout}`).toBe(0); + expect((parseJsonOutput(add.stdout) as { success: boolean }).success).toBe(true); + + const deploy = await run(['deploy', '--yes', '--json']); + if (deploy.exitCode !== 0) console.log('Deploy v1 stdout/stderr:', deploy.stdout, deploy.stderr); + expect(deploy.exitCode, 'Deploy v1 failed').toBe(0); + expect((parseJsonOutput(deploy.stdout) as { success: boolean }).success).toBe(true); + }, + 600000 + ); + + // ── Config bundle v2 (remove + re-add same name + redeploy = version bump) ─ + + it.skipIf(!canRun)( + 'updates config bundle to v2 (second version of the same bundle) and deploys', + async () => { + let result = await run(['remove', 'config-bundle', '--name', bundleName, '--json']); + expect(result.exitCode, `Remove config-bundle failed: ${result.stdout}`).toBe(0); + + result = await run([ + 'add', + 'config-bundle', + '--name', + bundleName, + '--description', + 'AB test bundle - treatment', + '--components', + bundleComponents('You are treatment: detailed and thorough.', 0.9), + '--branch', + 'mainline', + '--commit-message', + 'v2 treatment', + '--json', + ]); + expect(result.exitCode, `Re-add config-bundle failed: ${result.stdout}`).toBe(0); + + result = await run(['deploy', '--yes', '--json']); + expect(result.exitCode, `Redeploy failed: ${result.stdout}`).toBe(0); + }, + 600000 + ); + + it.skipIf(!canRun)( + 'config-bundle versions lists both versions (captures control = oldest)', + async () => { + const result = await run(['config-bundle', 'versions', '--name', bundleName, '--json']); + expect(result.exitCode, `cb versions failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { versions: { versionId: string }[] }; + expect(json.versions.length).toBeGreaterThanOrEqual(2); + // Versions are newest-first; oldest is the control (treatment uses LATEST). + controlVersionId = json.versions[json.versions.length - 1]!.versionId; + expect(controlVersionId).toBeTruthy(); + }, + 120000 + ); + + // ── Online-eval (Builtin evaluator — no custom evaluator resource needed) ── + + it.skipIf(!canRun)( + 'adds an online-eval config and deploys', + async () => { + const add = await run([ + 'add', + 'online-eval', + '--name', + onlineEvalName, + '--runtime', + agentName, + '--evaluator', + 'Builtin.Faithfulness', + '--sampling-rate', + '100', + '--json', + ]); + expect(add.exitCode, `Add online-eval failed: ${add.stdout}`).toBe(0); + const addJson = parseJsonOutput(add.stdout) as { success: boolean; configName: string }; + expect(addJson.success).toBe(true); + expect(addJson.configName).toBe(onlineEvalName); + + const deploy = await run(['deploy', '--yes', '--json']); + if (deploy.exitCode !== 0) console.log('Deploy eval stdout/stderr:', deploy.stdout, deploy.stderr); + expect(deploy.exitCode, 'Deploy online-eval failed').toBe(0); + expect((parseJsonOutput(deploy.stdout) as { success: boolean }).success).toBe(true); + }, + 600000 + ); + + // ── Create the A/B test ─────────────────────────────────────────────────── + + it.skipIf(!canRun)( + 'runs the A/B test (control = oldest version, treatment = LATEST)', + async () => { + expect(controlVersionId, 'Control version should have been captured').toBeTruthy(); + + // Auto-creates an IAM role and retries on AccessDenied while IAM propagates; + // retry the whole call to absorb propagation flakiness. + let runJson: { mode: string; variants: { name: string }[] } | undefined; + await retry( + async () => { + const result = await run([ + 'run', + 'ab-test', + '-n', + abTestName, + '-g', + gatewayName, + '--mode', + 'config-bundle', + '--control-bundle', + bundleName, + '--control-version', + controlVersionId, + '--treatment-bundle', + bundleName, + '--treatment-version', + 'LATEST', + '--online-eval', + onlineEvalName, + '--runtime', + agentName, + '--json', + ]); + + if (result.exitCode !== 0) console.log('run ab-test stdout/stderr:', result.stdout, result.stderr); + expect(result.exitCode, `run ab-test failed: ${result.stdout}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { + success: boolean; + id: string; + mode: string; + variants: { name: string }[]; + }; + expect(json.success).toBe(true); + expect(json.id).toBeTruthy(); + // Capture the id immediately so afterAll always archives the test, even if a + // later assertion fails. Done inside retry (before any throw) so an orphan is + // never left behind by a re-attempt. + abTestId = json.id; + runJson = json; + }, + 3, + 20000 + ); + // Deterministic checks live outside retry — a mismatch must not re-create the test. + expect(runJson!.mode).toBe('config-bundle'); + expect(runJson!.variants).toHaveLength(2); + }, + 300000 + ); + + // ── pause / resume / promote — live execution state from AWS ─────────────── + + const viewExecutionStatus = async (): Promise => { + const result = await run(['view', 'ab-test', abTestId, '--json']); + expect(result.exitCode, `view ab-test failed: ${result.stderr}`).toBe(0); + // Live execution status (RUNNING/PAUSED/STOPPED) surfaces in lifecycleStatus. + return (parseJsonOutput(result.stdout) as { lifecycleStatus: string }).lifecycleStatus; + }; + + it.skipIf(!canRun)( + 'view reports the test reaching RUNNING', + async () => { + expect(abTestId, 'AB test ID should have been captured').toBeTruthy(); + await retry( + async () => { + expect(await viewExecutionStatus()).toBe('RUNNING'); + }, + 12, + 10000 + ); + }, + 180000 + ); + + it.skipIf(!canRun)( + 'pause sets live execution state to PAUSED', + async () => { + const result = await run(['pause', 'ab-test', '-i', abTestId, '--json']); + expect(result.exitCode, `pause failed: ${result.stderr}`).toBe(0); + expect((parseJsonOutput(result.stdout) as { success: boolean; id: string }).success).toBe(true); + + await retry(async () => expect(await viewExecutionStatus()).toBe('PAUSED'), 6, 10000); + }, + 120000 + ); + + it.skipIf(!canRun)( + 'resume sets live execution state back to RUNNING', + async () => { + const result = await run(['resume', 'ab-test', '-i', abTestId, '--json']); + expect(result.exitCode, `resume failed: ${result.stderr}`).toBe(0); + expect((parseJsonOutput(result.stdout) as { success: boolean }).success).toBe(true); + + await retry(async () => expect(await viewExecutionStatus()).toBe('RUNNING'), 6, 10000); + }, + 120000 + ); + + it.skipIf(!canRun)( + 'promote stops the test and applies the winning variant to config', + async () => { + // promote waits for RUNNING (up to ~120s), stops the test, rewrites the bundle. + const result = await run(['promote', 'ab-test', '-i', abTestId, '--json']); + if (result.exitCode !== 0) console.log('promote stdout/stderr:', result.stdout, result.stderr); + expect(result.exitCode, `promote failed: ${result.stdout}`).toBe(0); + expect((parseJsonOutput(result.stdout) as { success: boolean; id: string }).success).toBe(true); + + await retry(async () => expect(await viewExecutionStatus()).toBe('STOPPED'), 6, 10000); + }, + 180000 + ); +}); diff --git a/e2e-tests/ab-test-target-based.test.ts b/e2e-tests/ab-test-target-based.test.ts new file mode 100644 index 000000000..403a29853 --- /dev/null +++ b/e2e-tests/ab-test-target-based.test.ts @@ -0,0 +1,324 @@ +/** + * E2E test for A/B tests (target-based mode) across the AWS boundary. + * + * Target-based mode compares two gateway-targets (rather than two config-bundle + * versions). Here both targets are http-runtime targets pointing at two named + * endpoints of the same agent runtime, each scoped by its own online-eval. + * + * Flow: create project → add gateway → add two runtime endpoints (control, + * treatment) → add two http-runtime targets → add two online-evals + * (one per endpoint) → deploy → run ab-test --mode target-based → + * view (RUNNING) → pause (PAUSED) → resume (RUNNING) → promote (STOPPED) → + * archive + * + * Promote in target-based mode (same runtime, both named endpoints) bumps the + * control endpoint's version to the treatment endpoint's — control keeps its + * identity. A/B tests are jobs, not project resources, so cleanup archives the + * test explicitly before teardown. + * + * Prerequisites: AWS credentials, npm, git, uv. + */ +import { parseJsonOutput, retry } from '../src/test-utils/index.js'; +import { + baseCanRun, + hasAws, + installCdkTarball, + runAgentCoreCLI, + teardownE2EProject, + writeAwsTargets, +} from './e2e-helper.js'; +import { randomUUID } from 'node:crypto'; +import { mkdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const canRun = baseCanRun && hasAws; + +describe.sequential('e2e: A/B test lifecycle (target-based mode)', () => { + let testDir: string; + let projectPath: string; + const suffix = String(Date.now()).slice(-8); + const agentName = `E2eAbtTb${suffix}`; + const gatewayName = 'abttbgw'; + const controlTarget = 'ctrlTarget'; + const treatmentTarget = 'treatTarget'; + const controlEndpoint = 'control'; + const treatmentEndpoint = 'treatment'; + const controlEval = 'E2eAbtTbCtrlEval'; + const treatmentEval = 'E2eAbtTbTreatEval'; + const abTestName = 'E2eAbtTbTest'; + + let abTestId: string; + + beforeAll(async () => { + if (!canRun) return; + + testDir = join(tmpdir(), `agentcore-e2e-ab-test-tb-${randomUUID()}`); + await mkdir(testDir, { recursive: true }); + + const result = await runAgentCoreCLI( + [ + 'create', + '--name', + agentName, + '--language', + 'Python', + '--framework', + 'Strands', + '--model-provider', + 'Bedrock', + '--memory', + 'none', + '--json', + ], + testDir + ); + expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0); + projectPath = (parseJsonOutput(result.stdout) as { projectPath: string }).projectPath; + + await writeAwsTargets(projectPath); + installCdkTarball(projectPath); + }, 300000); + + afterAll(async () => { + // A/B tests are jobs, not project resources — archive explicitly before teardown. + if (abTestId && projectPath && hasAws) { + await runAgentCoreCLI(['archive', 'ab-test', '-i', abTestId, '--json'], projectPath); + } + if (projectPath && hasAws) { + await teardownE2EProject(projectPath, agentName, 'Bedrock'); + } + if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 }); + }, 600000); + + const run = (args: string[]) => runAgentCoreCLI(args, projectPath); + + const assertSuccess = async (args: string[], label: string): Promise => { + const result = await run(args); + expect(result.exitCode, `${label} failed: ${result.stdout}\n${result.stderr}`).toBe(0); + expect((parseJsonOutput(result.stdout) as { success: boolean }).success, `${label} should succeed`).toBe(true); + }; + + // ── Gateway + two named endpoints on the agent runtime ─────────────────── + + it.skipIf(!canRun)('adds an HTTP gateway', () => + assertSuccess(['add', 'gateway', '--name', gatewayName, '--protocol-type', 'None', '--json'], 'add gateway') + ); + + it.skipIf(!canRun)('adds two runtime endpoints (control, treatment)', async () => { + await assertSuccess( + ['add', 'runtime-endpoint', '--runtime', agentName, '--endpoint', controlEndpoint, '--version', '1', '--json'], + 'add control endpoint' + ); + await assertSuccess( + ['add', 'runtime-endpoint', '--runtime', agentName, '--endpoint', treatmentEndpoint, '--version', '1', '--json'], + 'add treatment endpoint' + ); + }); + + // ── Two http-runtime targets, one per endpoint ─────────────────────────── + + it.skipIf(!canRun)('adds the control http-runtime target', () => + assertSuccess( + [ + 'add', + 'gateway-target', + '--gateway', + gatewayName, + '--name', + controlTarget, + '--type', + 'http-runtime', + '--runtime', + agentName, + '--runtime-endpoint', + controlEndpoint, + '--json', + ], + 'add control target' + ) + ); + + it.skipIf(!canRun)('adds the treatment http-runtime target', () => + assertSuccess( + [ + 'add', + 'gateway-target', + '--gateway', + gatewayName, + '--name', + treatmentTarget, + '--type', + 'http-runtime', + '--runtime', + agentName, + '--runtime-endpoint', + treatmentEndpoint, + '--json', + ], + 'add treatment target' + ) + ); + + // ── One online-eval per endpoint (target-based requires per-variant evals) ─ + + it.skipIf(!canRun)('adds two online-eval configs, one per endpoint', async () => { + await assertSuccess( + [ + 'add', + 'online-eval', + '--name', + controlEval, + '--runtime', + agentName, + '--endpoint', + controlEndpoint, + '--evaluator', + 'Builtin.Faithfulness', + '--sampling-rate', + '100', + '--json', + ], + 'add control online-eval' + ); + await assertSuccess( + [ + 'add', + 'online-eval', + '--name', + treatmentEval, + '--runtime', + agentName, + '--endpoint', + treatmentEndpoint, + '--evaluator', + 'Builtin.Faithfulness', + '--sampling-rate', + '100', + '--json', + ], + 'add treatment online-eval' + ); + }); + + // ── Deploy everything in one stack ─────────────────────────────────────── + + it.skipIf(!canRun)( + 'deploys the runtime, gateway, targets, and online-evals', + async () => { + const result = await run(['deploy', '--yes', '--json']); + if (result.exitCode !== 0) console.log('Deploy stdout/stderr:', result.stdout, result.stderr); + expect(result.exitCode, `Deploy failed: ${result.stdout}`).toBe(0); + expect((parseJsonOutput(result.stdout) as { success: boolean }).success).toBe(true); + }, + 900000 + ); + + // ── Create the target-based A/B test ───────────────────────────────────── + + it.skipIf(!canRun)( + 'runs the A/B test in target-based mode', + async () => { + let runJson: { mode: string; variants: { name: string; targetName?: string }[] } | undefined; + await retry( + async () => { + const result = await run([ + 'run', + 'ab-test', + '-n', + abTestName, + '-g', + gatewayName, + '--mode', + 'target-based', + '--control-target', + controlTarget, + '--treatment-target', + treatmentTarget, + '--control-online-eval', + controlEval, + '--treatment-online-eval', + treatmentEval, + '--runtime', + agentName, + '--json', + ]); + + if (result.exitCode !== 0) console.log('run ab-test stdout/stderr:', result.stdout, result.stderr); + expect(result.exitCode, `run ab-test failed: ${result.stdout}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { + success: boolean; + id: string; + mode: string; + variants: { name: string; targetName?: string }[]; + }; + expect(json.success).toBe(true); + expect(json.id).toBeTruthy(); + // Capture the id immediately so afterAll always archives the test, even if a + // later assertion fails. Done inside retry (before any throw) so an orphan is + // never left behind by a re-attempt. + abTestId = json.id; + runJson = json; + }, + 3, + 20000 + ); + // Deterministic checks live outside retry — a mismatch must not re-create the test. + expect(runJson!.mode).toBe('target-based'); + expect(runJson!.variants).toHaveLength(2); + }, + 300000 + ); + + // ── pause / resume / promote — live execution state from AWS ───────────── + + const viewExecutionStatus = async (): Promise => { + const result = await run(['view', 'ab-test', abTestId, '--json']); + expect(result.exitCode, `view ab-test failed: ${result.stderr}`).toBe(0); + // Live execution status (RUNNING/PAUSED/STOPPED) surfaces in lifecycleStatus. + return (parseJsonOutput(result.stdout) as { lifecycleStatus: string }).lifecycleStatus; + }; + + it.skipIf(!canRun)( + 'view reports the test reaching RUNNING', + async () => { + expect(abTestId, 'AB test ID should have been captured').toBeTruthy(); + await retry(async () => expect(await viewExecutionStatus()).toBe('RUNNING'), 12, 10000); + }, + 180000 + ); + + it.skipIf(!canRun)( + 'pause sets live execution state to PAUSED', + async () => { + await assertSuccess(['pause', 'ab-test', '-i', abTestId, '--json'], 'pause'); + await retry(async () => expect(await viewExecutionStatus()).toBe('PAUSED'), 6, 10000); + }, + 120000 + ); + + it.skipIf(!canRun)( + 'resume sets live execution state back to RUNNING', + async () => { + await assertSuccess(['resume', 'ab-test', '-i', abTestId, '--json'], 'resume'); + await retry(async () => expect(await viewExecutionStatus()).toBe('RUNNING'), 6, 10000); + }, + 120000 + ); + + it.skipIf(!canRun)( + 'promote stops the test and applies the winning target to config', + async () => { + // promote waits for RUNNING (up to ~120s), stops the test, then promotes the + // control endpoint to the treatment endpoint's version in agentcore.json. + const result = await run(['promote', 'ab-test', '-i', abTestId, '--json']); + if (result.exitCode !== 0) console.log('promote stdout/stderr:', result.stdout, result.stderr); + expect(result.exitCode, `promote failed: ${result.stdout}`).toBe(0); + expect((parseJsonOutput(result.stdout) as { success: boolean; id: string }).success).toBe(true); + + await retry(async () => expect(await viewExecutionStatus()).toBe('STOPPED'), 6, 10000); + }, + 180000 + ); +}); diff --git a/e2e-tests/fixtures/gateway-targets/.gitignore b/e2e-tests/fixtures/gateway-targets/.gitignore new file mode 100644 index 000000000..677a4df9e --- /dev/null +++ b/e2e-tests/fixtures/gateway-targets/.gitignore @@ -0,0 +1 @@ +gateway-target-prereqs-*.json diff --git a/e2e-tests/fixtures/gateway-targets/lambda-tools.json b/e2e-tests/fixtures/gateway-targets/lambda-tools.json new file mode 100644 index 000000000..976a66a82 --- /dev/null +++ b/e2e-tests/fixtures/gateway-targets/lambda-tools.json @@ -0,0 +1,13 @@ +[ + { + "name": "say_hello", + "description": "Returns a friendly greeting for the given name.", + "inputSchema": { + "type": "object", + "properties": { + "name": { "type": "string", "description": "Who to greet" } + }, + "required": ["name"] + } + } +] diff --git a/e2e-tests/fixtures/gateway-targets/openapi.json b/e2e-tests/fixtures/gateway-targets/openapi.json new file mode 100644 index 000000000..c1792f9c6 --- /dev/null +++ b/e2e-tests/fixtures/gateway-targets/openapi.json @@ -0,0 +1,38 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "E2E Greeting API", + "version": "1.0.0", + "description": "Minimal OpenAPI spec for the open-api-schema gateway-target e2e test." + }, + "servers": [{ "url": "https://api.example.com" }], + "paths": { + "/hello": { + "get": { + "operationId": "getHello", + "summary": "Return a greeting", + "parameters": [ + { + "name": "name", + "in": "query", + "required": false, + "schema": { "type": "string" } + } + ], + "responses": { + "200": { + "description": "A greeting message", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { "message": { "type": "string" } } + } + } + } + } + } + } + } + } +} diff --git a/e2e-tests/fixtures/gateway-targets/setup_target_prereqs.py b/e2e-tests/fixtures/gateway-targets/setup_target_prereqs.py new file mode 100644 index 000000000..4bf49aacf --- /dev/null +++ b/e2e-tests/fixtures/gateway-targets/setup_target_prereqs.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +"""Idempotently create the external AWS resources that some gateway targets need. + +Some gateway-target types reference AWS resources that must already exist before +`agentcore deploy` can wire them up: + + - lambda-function-arn → an existing Lambda function ARN + - api-gateway → an existing API Gateway REST API id + stage + +This script checks for each resource and creates it only if missing (so repeated +e2e runs reuse the same resources — no leaks, no per-run cost). It writes the +identifiers to gateway-target-prereqs-.json for the test to read. + +Run: uv run --with boto3 python3 setup_target_prereqs.py +Env: AWS_REGION, RESOURCE_SUFFIX (optional; defaults to a random hex) +""" +import io +import json +import os +import time +import uuid +import zipfile + +import boto3 +import botocore + +REGION = os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") or "us-east-1" +SUFFIX = os.environ.get("RESOURCE_SUFFIX") or uuid.uuid4().hex[:12] +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +OUT_FILE = os.path.join(SCRIPT_DIR, f"gateway-target-prereqs-{SUFFIX}.json") + +# Resource names are deterministic + account-scoped so runs reuse them (idempotent). +LAMBDA_ROLE_NAME = "e2e-gwtarget-lambda-role" +LAMBDA_FN_NAME = "e2e-gwtarget-fn" +# v2: the REST API method now carries an operationName (→ operationId in the +# OpenAPI export) which the AgentCore api-gateway target requires. +REST_API_NAME = "e2e-gwtarget-api-v2" +REST_API_STAGE = "prod" + + +def account_id(): + return boto3.client("sts", region_name=REGION).get_caller_identity()["Account"] + + +def ensure_lambda_role(): + """Create (or reuse) a basic Lambda execution role.""" + iam = boto3.client("iam") + arn = f"arn:aws:iam::{account_id()}:role/{LAMBDA_ROLE_NAME}" + try: + iam.get_role(RoleName=LAMBDA_ROLE_NAME) + print(f"Lambda role exists: {arn}") + return arn + except iam.exceptions.NoSuchEntityException: + pass + print(f"Creating Lambda role: {LAMBDA_ROLE_NAME}") + iam.create_role( + RoleName=LAMBDA_ROLE_NAME, + AssumeRolePolicyDocument=json.dumps({ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"Service": "lambda.amazonaws.com"}, + "Action": "sts:AssumeRole", + }], + }), + ) + iam.attach_role_policy( + RoleName=LAMBDA_ROLE_NAME, + PolicyArn="arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ) + print("Waiting 10s for role propagation...") + time.sleep(10) + return arn + + +def _lambda_zip(): + """A trivial handler — the gateway only needs a callable function to exist.""" + src = ( + "def handler(event, context):\n" + " name = (event or {}).get('name', 'world')\n" + " return {'message': f'hello {name}'}\n" + ) + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr("lambda_function.py", src) + return buf.getvalue() + + +def grant_agentcore_invoke(lam, arn): + """Resource-based policy so the AgentCore gateway service can invoke the function. + + The CDK grants lambda:InvokeFunction to the gateway role too, but that + identity-based policy can lose a propagation race against gateway-target + validation. A resource-based policy is an independent, account-scoped + authorization path. Idempotent (ignore an existing statement id).""" + try: + lam.add_permission( + FunctionName=LAMBDA_FN_NAME, + StatementId="agentcore-gateway-invoke", + Action="lambda:InvokeFunction", + Principal="bedrock-agentcore.amazonaws.com", + SourceAccount=account_id(), + ) + print("Added AgentCore invoke permission to Lambda") + except lam.exceptions.ResourceConflictException: + print("AgentCore invoke permission already present") + + +def ensure_lambda(): + lam = boto3.client("lambda", region_name=REGION) + try: + resp = lam.get_function(FunctionName=LAMBDA_FN_NAME) + arn = resp["Configuration"]["FunctionArn"] + print(f"Lambda exists: {arn}") + grant_agentcore_invoke(lam, arn) + return arn + except lam.exceptions.ResourceNotFoundException: + pass + role_arn = ensure_lambda_role() + print(f"Creating Lambda: {LAMBDA_FN_NAME}") + # New roles can briefly fail with InvalidParameterValueException during propagation. + arn = None + for attempt in range(5): + try: + resp = lam.create_function( + FunctionName=LAMBDA_FN_NAME, + Runtime="python3.12", + Role=role_arn, + Handler="lambda_function.handler", + Code={"ZipFile": _lambda_zip()}, + Timeout=10, + ) + arn = resp["FunctionArn"] + break + except lam.exceptions.ResourceConflictException: + # A parallel CI job (e.g. the preview/ga matrix) created it first — reuse it. + print(" Lambda created concurrently, reusing existing function") + arn = lam.get_function(FunctionName=LAMBDA_FN_NAME)["Configuration"]["FunctionArn"] + break + except lam.exceptions.ClientError as e: + if "cannot be assumed" in str(e) or "InvalidParameterValue" in str(e): + print(f" role not ready, retrying ({attempt + 1})...") + time.sleep(5) + continue + raise + if not arn: + raise RuntimeError("Lambda creation failed after retries") + grant_agentcore_invoke(lam, arn) + return arn + + +def _find_rest_apis(api): + """All REST API ids with our name, sorted (deterministic 'winner' = first).""" + return sorted( + item["id"] for item in api.get_rest_apis(limit=500).get("items", []) if item.get("name") == REST_API_NAME + ) + + +def ensure_rest_api(): + """Create (or reuse) a minimal REST API with one GET method deployed to a stage. + + API Gateway allows duplicate names, so parallel CI jobs can each create one. + We reap the race: keep the lowest-id API, delete the rest, so duplicates never + accumulate and every run deterministically resolves to the same API. + """ + api = boto3.client("apigateway", region_name=REGION) + existing = _find_rest_apis(api) + if existing: + winner, dupes = existing[0], existing[1:] + for dupe in dupes: + print(f"Deleting duplicate REST API: {dupe}") + try: + api.delete_rest_api(restApiId=dupe) + except api.exceptions.ClientError: + pass # best-effort; another job may be reaping it too + print(f"REST API exists: {winner}") + return winner, REST_API_STAGE + print(f"Creating REST API: {REST_API_NAME}") + rest_api_id = api.create_rest_api(name=REST_API_NAME, description="e2e gateway-target prereq")["id"] + root_id = next(r["id"] for r in api.get_resources(restApiId=rest_api_id)["items"] if r["path"] == "/") + res_id = api.create_resource(restApiId=rest_api_id, parentId=root_id, pathPart="hello")["id"] + # operationName → operationId in the OpenAPI export. AgentCore's api-gateway + # target rejects operations that have no operationId. + api.put_method( + restApiId=rest_api_id, + resourceId=res_id, + httpMethod="GET", + authorizationType="NONE", + operationName="getHello", + ) + api.put_integration( + restApiId=rest_api_id, + resourceId=res_id, + httpMethod="GET", + type="MOCK", + requestTemplates={"application/json": '{"statusCode": 200}'}, + ) + api.put_method_response( + restApiId=rest_api_id, resourceId=res_id, httpMethod="GET", statusCode="200", + ) + api.put_integration_response( + restApiId=rest_api_id, resourceId=res_id, httpMethod="GET", statusCode="200", + responseTemplates={"application/json": '{"message": "hello"}'}, + ) + api.create_deployment(restApiId=rest_api_id, stageName=REST_API_STAGE) + print(f"REST API created: {rest_api_id} (stage {REST_API_STAGE})") + + # Two jobs may have both reached the create path concurrently. Reconcile to the + # lowest id so every job converges on the same API and no duplicates survive. + all_ids = _find_rest_apis(api) + winner = all_ids[0] if all_ids else rest_api_id + for dupe in (i for i in all_ids if i != winner): + print(f"Deleting duplicate REST API created by a parallel job: {dupe}") + try: + api.delete_rest_api(restApiId=dupe) + except api.exceptions.ClientError: + pass + return winner, REST_API_STAGE + + +def _try(label, fn): + """Run a resource creator; on AccessDenied return None so the test can skip + the dependent target instead of failing the whole suite. Restricted CI roles + (e.g. e2e-github-actions) may lack lambda:*/apigateway:* permissions.""" + try: + return fn() + except botocore.exceptions.ClientError as e: + if e.response.get("Error", {}).get("Code") in ("AccessDeniedException", "AccessDenied", "UnauthorizedOperation"): + print(f"⚠️ Skipping {label}: not authorized ({e.response['Error']['Code']})") + return None + raise + + +def main(): + lambda_arn = _try("lambda", ensure_lambda) + api = _try("api-gateway", ensure_rest_api) + rest_api_id, stage = api if api else (None, None) + + out = { + "lambdaArn": lambda_arn, + "restApiId": rest_api_id, + "restApiStage": stage, + } + with open(OUT_FILE, "w") as f: + json.dump(out, f, indent=2) + print(f"\nWrote prereqs to {OUT_FILE}:") + print(json.dumps(out, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/e2e-tests/fixtures/gateway-targets/smithy.json b/e2e-tests/fixtures/gateway-targets/smithy.json new file mode 100644 index 000000000..38ccbd6a4 --- /dev/null +++ b/e2e-tests/fixtures/gateway-targets/smithy.json @@ -0,0 +1,41 @@ +{ + "smithy": "2.0", + "shapes": { + "com.example#Greeter": { + "type": "service", + "version": "2024-01-01", + "operations": [{ "target": "com.example#SayHello" }], + "traits": { + "aws.api#service": { "sdkId": "Greeter", "arnNamespace": "greeter" }, + "aws.auth#sigv4": { "name": "greeter" }, + "aws.protocols#restJson1": {}, + "smithy.api#documentation": "A minimal AWS service used by the smithy-model gateway-target e2e test." + } + }, + "com.example#SayHello": { + "type": "operation", + "input": { "target": "com.example#SayHelloInput" }, + "output": { "target": "com.example#SayHelloOutput" }, + "traits": { + "smithy.api#documentation": "Returns a greeting for the given name.", + "smithy.api#http": { "method": "GET", "uri": "/hello", "code": 200 }, + "smithy.api#readonly": {} + } + }, + "com.example#SayHelloInput": { + "type": "structure", + "members": { + "name": { + "target": "smithy.api#String", + "traits": { "smithy.api#httpQuery": "name" } + } + } + }, + "com.example#SayHelloOutput": { + "type": "structure", + "members": { + "message": { "target": "smithy.api#String" } + } + } + } +} diff --git a/e2e-tests/httpgateway-all-targets.test.ts b/e2e-tests/httpgateway-all-targets.test.ts new file mode 100644 index 000000000..9535eb3d4 --- /dev/null +++ b/e2e-tests/httpgateway-all-targets.test.ts @@ -0,0 +1,357 @@ +/** + * E2E test for an HTTP-runtime gateway hosting every deployable target type. + * + * A `protocolType: None` (HTTP) gateway is a superset — it can host MCP targets + * AND the HTTP-only target types — so one gateway carries all targets: + * + * http-runtime, mcp-server, lambda-function-arn, api-gateway, + * open-api-schema, smithy-model, web-search, passthrough + * + * Flow: create project → ensure external prereqs (Lambda, REST API via a boto3 + * fixture) → add gateway + credential → add every target → deploy ONE + * CloudFormation stack → assert the gateway and all targets are deployed → + * teardown. + * + * External resources that can't be created by `agentcore deploy` are provisioned + * idempotently by fixtures/gateway-targets/setup_target_prereqs.py (mirrors the + * import-resources.test.ts fixture pattern). + * + * `passthrough` is gated behind ENABLE_GATED_FEATURES, so the add/deploy steps + * run with that env var set. Omits `connector` (Bedrock FMKB), a private-beta + * CloudFormation resource type. + * + * Prerequisites: AWS credentials, npm, git, uv, python3 + boto3. + */ +import { hasAwsCredentials, hasCommand, parseJsonOutput, prereqs, spawnAndCollect } from '../src/test-utils/index.js'; +import { installCdkTarball, runAgentCoreCLI, writeAwsTargets } from './e2e-helper.js'; +import { deleteCredentialProvider } from './utils/credential-provider-cleanup.js'; +import { getLogger } from './utils/logger.js'; +import { BedrockAgentCoreControlClient } from '@aws-sdk/client-bedrock-agentcore-control'; +import { execSync } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { copyFile, mkdir, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const fixtureDir = join(__dirname, 'fixtures', 'gateway-targets'); + +const hasAws = hasAwsCredentials(); +const hasPython = + hasCommand('python3') && + (() => { + try { + execSync('uv run --with boto3 python3 -c "import boto3"', { stdio: 'ignore' }); + return true; + } catch { + return false; + } + })(); +const canRun = prereqs.npm && prereqs.git && prereqs.uv && hasAws && hasPython; + +interface Prereqs { + // null when the IAM role lacks permission to create the resource (restricted + // CI roles may lack lambda:*/apigateway:*); the dependent target is then skipped. + lambdaArn: string | null; + restApiId: string | null; + restApiStage: string | null; +} + +describe.sequential('e2e: HTTP gateway with all target types', () => { + const region = process.env.AWS_REGION ?? 'us-east-1'; + const suffix = Date.now().toString().slice(-8); + const agentName = `E2eGwAll${suffix}`; + const gatewayName = 'allgw'; + const credName = 'E2eGwCred'; + + let testDir: string; + let projectPath: string; + let prereqsData: Prereqs; + + const run = (args: string[]) => runAgentCoreCLI(args, projectPath); + // passthrough is gated; run add/deploy with ENABLE_GATED_FEATURES on (harmless for the others). + const runGated = (args: string[]) => spawnAndCollect('agentcore', args, projectPath, { ENABLE_GATED_FEATURES: '1' }); + + const assertAddTarget = async (args: string[], targetName: string): Promise => { + const result = await runGated(['add', 'gateway-target', '--gateway', gatewayName, '--json', ...args]); + expect(result.exitCode, `add target ${targetName} failed: ${result.stdout}\n${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean; toolName: string }; + expect(json.success, `add target ${targetName} should succeed`).toBe(true); + expect(json.toolName).toBe(targetName); + }; + + beforeAll(async () => { + if (!canRun) return; + + // 1. Provision external AWS resources (idempotent) and read their identifiers. + const setup = await spawnAndCollect( + 'uv', + ['run', '--with', 'boto3', 'python3', 'setup_target_prereqs.py'], + fixtureDir, + { + AWS_REGION: region, + RESOURCE_SUFFIX: suffix, + } + ); + if (setup.exitCode !== 0) { + throw new Error(`prereq setup failed (exit ${setup.exitCode}):\n${setup.stdout}\n${setup.stderr}`); + } + prereqsData = JSON.parse( + await readFile(join(fixtureDir, `gateway-target-prereqs-${suffix}.json`), 'utf-8') + ) as Prereqs; + + // 2. Create the CLI project. + testDir = join(tmpdir(), `agentcore-e2e-gw-all-${randomUUID()}`); + await mkdir(testDir, { recursive: true }); + const createResult = await runAgentCoreCLI( + [ + 'create', + '--name', + agentName, + '--language', + 'Python', + '--framework', + 'Strands', + '--model-provider', + 'Bedrock', + '--memory', + 'none', + '--json', + ], + testDir + ); + expect(createResult.exitCode, `Create failed: ${createResult.stderr}`).toBe(0); + projectPath = (parseJsonOutput(createResult.stdout) as { projectPath: string }).projectPath; + + await writeAwsTargets(projectPath); + installCdkTarball(projectPath); + + // 3. Copy schema fixtures into the project (add validates schema paths relative to project root). + for (const f of ['openapi.json', 'smithy.json', 'lambda-tools.json']) { + await copyFile(join(fixtureDir, f), join(projectPath, f)); + } + }, 600000); + + afterAll(async () => { + if (projectPath && hasAws) { + await runAgentCoreCLI(['remove', 'all', '--json'], projectPath); + const deploy = await runAgentCoreCLI(['deploy', '--yes', '--json'], projectPath); + if (deploy.exitCode !== 0) console.warn('Teardown deploy failed:', deploy.stderr); + // The api-key credential provider is created by a pre-deploy SDK call, not + // CloudFormation, so stack teardown does not reap it — delete it explicitly. + const client = new BedrockAgentCoreControlClient({ region }); + await deleteCredentialProvider(client, getLogger('teardown-gw-all'), credName); + } + if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 }); + }, 600000); + + // ── Gateway + supporting resources ────────────────────────────────────── + + it.skipIf(!canRun)( + 'adds an HTTP (protocolType None) gateway', + async () => { + const result = await run(['add', 'gateway', '--name', gatewayName, '--protocol-type', 'None', '--json']); + expect(result.exitCode, `add gateway failed: ${result.stdout}`).toBe(0); + expect((parseJsonOutput(result.stdout) as { success: boolean }).success).toBe(true); + }, + 60000 + ); + + it.skipIf(!canRun)( + 'adds an API-key credential for auth-requiring targets', + async () => { + const result = await run([ + 'add', + 'credential', + '--name', + credName, + '--type', + 'api-key', + '--api-key', + 'e2e-dummy-key', + '--json', + ]); + expect(result.exitCode, `add credential failed: ${result.stdout}`).toBe(0); + expect((parseJsonOutput(result.stdout) as { success: boolean }).success).toBe(true); + }, + 60000 + ); + + // ── One target of every deployable type ────────────────────────────────── + + it.skipIf(!canRun)('adds an http-runtime target', () => + assertAddTarget(['--name', 'tHttpRuntime', '--type', 'http-runtime', '--runtime', agentName], 'tHttpRuntime') + ); + + it.skipIf(!canRun)('adds an mcp-server target', () => + assertAddTarget( + ['--name', 'tMcpServer', '--type', 'mcp-server', '--endpoint', 'https://mcp.exa.ai/mcp'], + 'tMcpServer' + ) + ); + + // Lambda + API Gateway depend on resources the fixture may not be able to create + // under a restricted IAM role; skip the target when its prereq is absent. + it.skipIf(!canRun)('adds a lambda-function-arn target', async ctx => { + if (!prereqsData.lambdaArn) return ctx.skip(); + await assertAddTarget( + [ + '--name', + 'tLambda', + '--type', + 'lambda-function-arn', + '--lambda-arn', + prereqsData.lambdaArn, + '--tool-schema-file', + 'lambda-tools.json', + ], + 'tLambda' + ); + }); + + it.skipIf(!canRun)('adds an api-gateway target', async ctx => { + if (!prereqsData.restApiId || !prereqsData.restApiStage) return ctx.skip(); + await assertAddTarget( + [ + '--name', + 'tApiGw', + '--type', + 'api-gateway', + '--rest-api-id', + prereqsData.restApiId, + '--stage', + prereqsData.restApiStage, + ], + 'tApiGw' + ); + }); + + it.skipIf(!canRun)('adds an open-api-schema target (api-key outbound auth)', () => + assertAddTarget( + [ + '--name', + 'tOpenApi', + '--type', + 'open-api-schema', + '--schema', + 'openapi.json', + '--outbound-auth', + 'api-key', + '--credential-name', + credName, + ], + 'tOpenApi' + ) + ); + + it.skipIf(!canRun)('adds a smithy-model target', () => + assertAddTarget(['--name', 'tSmithy', '--type', 'smithy-model', '--schema', 'smithy.json'], 'tSmithy') + ); + + it.skipIf(!canRun)('adds a web-search target', () => + assertAddTarget(['--name', 'tWebSearch', '--type', 'web-search'], 'tWebSearch') + ); + + it.skipIf(!canRun)('adds a passthrough target (gated; gateway-iam-role auth)', () => + assertAddTarget( + [ + '--name', + 'tPassthrough', + '--type', + 'passthrough', + '--passthrough-endpoint', + 'https://example.com/mcp', + '--passthrough-protocol', + 'MCP', + '--outbound-auth', + 'gateway-iam-role', + '--signing-service', + 'execute-api', + '--signing-region', + region, + ], + 'tPassthrough' + ) + ); + + // ── Config sanity: every target landed in agentcore.json ───────────────── + + it.skipIf(!canRun)( + 'agentcore.json contains all added targets', + async () => { + const config = JSON.parse(await readFile(join(projectPath, 'agentcore', 'agentcore.json'), 'utf-8')) as { + agentCoreGateways: { name: string; targets: { name: string; targetType: string }[] }[]; + }; + const gw = config.agentCoreGateways.find(g => g.name === gatewayName); + expect(gw, `gateway ${gatewayName} should be in config`).toBeDefined(); + const types = new Set(gw!.targets.map(t => t.targetType)); + const expected = ['httpRuntime', 'mcpServer', 'openApiSchema', 'smithyModel', 'webSearch', 'passthrough']; + // Only expected when the fixture could provision their external prereqs. + if (prereqsData.lambdaArn) expected.push('lambdaFunctionArn'); + if (prereqsData.restApiId) expected.push('apiGateway'); + for (const t of expected) { + expect(types.has(t), `config should contain a ${t} target`).toBe(true); + } + }, + 30000 + ); + + // ── Deploy the whole stack and verify the gateway is live ──────────────── + + it.skipIf(!canRun)( + 'deploys the gateway and all targets in one stack', + async () => { + // No retry: a failed deploy can leave the CFN stack mid-rollback / + // REVIEW_IN_PROGRESS, and an immediate second deploy fails differently + // (masking the real error). One deploy, long timeout, surface the failure. + const result = await runGated(['deploy', '--yes', '--json']); + if (result.exitCode !== 0) { + console.log('Deploy stdout:', result.stdout); + console.log('Deploy stderr:', result.stderr); + } + expect(result.exitCode, `Deploy failed (stderr: ${result.stderr})`).toBe(0); + expect((parseJsonOutput(result.stdout) as { success: boolean }).success).toBe(true); + + // deployed-state.json should record a gateway with an id. + const state = JSON.parse( + await readFile(join(projectPath, 'agentcore', '.cli', 'deployed-state.json'), 'utf-8') + ) as { + targets: Record< + string, + { + resources?: { + gateways?: Record; + mcp?: { gateways?: Record }; + }; + } + >; + }; + const gateways = Object.values(state.targets).flatMap(t => [ + ...Object.values(t.resources?.gateways ?? {}), + ...Object.values(t.resources?.mcp?.gateways ?? {}), + ]); + expect(gateways.length, 'a gateway should be in deployed state').toBeGreaterThan(0); + expect(gateways[0]!.gatewayId, 'deployed gateway should have an id').toBeTruthy(); + }, + 900000 + ); + + it.skipIf(!canRun)( + 'status reports the gateway as deployed', + async () => { + const result = await run(['status', '--json']); + expect(result.exitCode, `status failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { + success: boolean; + resources: { resourceType: string; name: string; deploymentState: string }[]; + }; + expect(json.success).toBe(true); + const gw = json.resources.find(r => r.resourceType === 'gateway' && r.name === gatewayName); + expect(gw, `gateway ${gatewayName} should appear in status`).toBeDefined(); + expect(gw!.deploymentState).toBe('deployed'); + }, + 120000 + ); +});