From 70d11514121bc1d84bbd36348fd1437b88b7a4e5 Mon Sep 17 00:00:00 2001 From: Hweinstock Date: Wed, 10 Jun 2026 18:33:48 +0000 Subject: [PATCH 1/7] test(e2e): warn with skip reason for harness e2e tests --- e2e-tests/harness-e2e-helper.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/e2e-tests/harness-e2e-helper.ts b/e2e-tests/harness-e2e-helper.ts index c8ca0c456..21c1238ad 100644 --- a/e2e-tests/harness-e2e-helper.ts +++ b/e2e-tests/harness-e2e-helper.ts @@ -1,6 +1,7 @@ import { getHarness } from '../src/cli/aws/agentcore-harness.js'; import { hasAwsCredentials, parseJsonOutput, prereqs, retry, spawnAndCollect } from '../src/test-utils/index.js'; import { installCdkTarball, runAgentCoreCLI, teardownE2EProject, writeAwsTargets } from './e2e-helper.js'; +import { getLogger } from './utils/logger.js'; import { randomUUID } from 'node:crypto'; import { mkdir, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; @@ -26,6 +27,17 @@ export function createHarnessE2ESuite(cfg: HarnessE2EConfig) { const providerLabel = cfg.modelProvider === 'open_ai' ? 'OpenAI' : cfg.modelProvider === 'gemini' ? 'Gemini' : 'Bedrock'; + // Logged at collection time (not in beforeAll, which Vitest skips when every test in the + // suite is skipped) so the skip reason is always surfaced. + const logger = getLogger(`harness-${providerLabel.toLowerCase()}`); + if (!canRun) { + logger.warn( + `tests are skipped due to insufficient conditions. ` + + `npm=${prereqs.npm}, git=${prereqs.git}, hasAws=${hasAws}, ` + + `isPreviewBuild=${isPreviewBuild}, hasRequiredVar=${hasRequiredVar}` + ); + } + describe.sequential(`e2e: harness/${providerLabel} — create → deploy → invoke → teardown`, () => { let testDir: string; let projectPath: string; From 393bcace2d678d4996f4389e7c67f9c5e8dd1154 Mon Sep 17 00:00:00 2001 From: Hweinstock Date: Wed, 10 Jun 2026 18:34:01 +0000 Subject: [PATCH 2/7] ci(e2e): run full suite across preview and ga builds --- .github/workflows/e2e-tests-full.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e-tests-full.yml b/.github/workflows/e2e-tests-full.yml index 79cb44f9a..ba943f7eb 100644 --- a/.github/workflows/e2e-tests-full.yml +++ b/.github/workflows/e2e-tests-full.yml @@ -27,7 +27,7 @@ jobs: strategy: fail-fast: false matrix: - cdk-source: [npm, main] + cli-build: [preview, ga] shard: ['1/6', '2/6', '3/6', '4/6', '5/6', '6/6'] steps: - uses: actions/checkout@v6 @@ -57,9 +57,11 @@ jobs: E2E,${{ secrets.E2E_SECRET_ARN }} parse-json-secrets: true - run: npm ci - - run: npm run build + - name: Build CLI (${{ matrix.cli-build }}) + run: npm run build + env: + BUILD_PREVIEW: ${{ matrix.cli-build == 'preview' && '1' || '0' }} - name: Generate GitHub App Token - if: matrix.cdk-source == 'main' id: app-token uses: actions/create-github-app-token@v1 with: @@ -67,7 +69,6 @@ jobs: private-key: ${{ secrets.APP_PRIVATE_KEY }} owner: aws - name: Build CDK package from main - if: matrix.cdk-source == 'main' run: | git clone --depth 1 "https://x-access-token:${CDK_REPO_TOKEN}@github.com/${CDK_REPO}.git" /tmp/cdk-repo cd /tmp/cdk-repo @@ -80,7 +81,7 @@ jobs: CDK_REPO: ${{ secrets.CDK_REPO_NAME }} - name: Install CLI globally run: npm install -g "$(npm pack | tail -1)" - - name: Run E2E tests (${{ matrix.cdk-source }}, shard ${{ matrix.shard }}) + - name: Run E2E tests (${{ matrix.cli-build }}, shard ${{ matrix.shard }}) env: AWS_ACCOUNT_ID: ${{ steps.aws.outputs.account_id }} AWS_REGION: ${{ inputs.aws_region || 'us-east-1' }} @@ -95,6 +96,7 @@ jobs: CDP_API_KEY_ID: ${{ env.E2E_CDP_API_KEY_ID }} CDP_API_KEY_SECRET: ${{ env.E2E_CDP_API_KEY_SECRET }} CDP_WALLET_SECRET: ${{ env.E2E_CDP_WALLET_SECRET }} + BUILD_PREVIEW: ${{ matrix.cli-build == 'preview' && '1' || '0' }} run: npx vitest run --project e2e --shard=${{ matrix.shard }} browser-tests: From cce4e49d137d57e0ec3f79ead58ebac8f3a5db2a Mon Sep 17 00:00:00 2001 From: Hweinstock Date: Wed, 10 Jun 2026 18:34:07 +0000 Subject: [PATCH 3/7] docs(scripts): document BUILD_PREVIEW=1 for harness e2e tests --- scripts/run-e2e-local.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/run-e2e-local.sh b/scripts/run-e2e-local.sh index 0fcaee1cf..6962c1bd2 100755 --- a/scripts/run-e2e-local.sh +++ b/scripts/run-e2e-local.sh @@ -8,13 +8,18 @@ # # Optional env vars: # AWS_REGION — defaults to us-east-1 +# BUILD_PREVIEW — set to 1 to build a preview CLI and run the harness e2e tests +# (e2e-tests/harness-*.test.ts). Harness features are gated to preview +# builds; without this they self-skip. The var is read at build time +# (esbuild bakes __PREVIEW__=true) and at test runtime (skip gate). # # Usage: # export E2E_ROLE_ARN=arn:aws:iam:::role/ # export E2E_SECRET_ARN=arn:aws:secretsmanager:::secret: -# ./scripts/run-e2e-local.sh # runs strands-bedrock.test.ts (CI default) -# ./scripts/run-e2e-local.sh --all # runs the full e2e suite -# ./scripts/run-e2e-local.sh e2e-tests/foo.test.ts # runs a specific test file +# ./scripts/run-e2e-local.sh # runs strands-bedrock.test.ts (CI default) +# ./scripts/run-e2e-local.sh --all # runs the full e2e suite +# ./scripts/run-e2e-local.sh e2e-tests/foo.test.ts # runs a specific test file +# BUILD_PREVIEW=1 ./scripts/run-e2e-local.sh e2e-tests/harness-bedrock.test.ts # runs harness tests # # Prerequisites: aws CLI, node >=20.19, npm, git, uv, jq From ecea1612d0dd24c4f830620784c5b0bd24a00eeb Mon Sep 17 00:00:00 2001 From: Hweinstock Date: Wed, 10 Jun 2026 18:40:54 +0000 Subject: [PATCH 4/7] fix: avoid duplicate logs on errors --- e2e-tests/harness-e2e-helper.ts | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/e2e-tests/harness-e2e-helper.ts b/e2e-tests/harness-e2e-helper.ts index 21c1238ad..a5f68956a 100644 --- a/e2e-tests/harness-e2e-helper.ts +++ b/e2e-tests/harness-e2e-helper.ts @@ -27,8 +27,7 @@ export function createHarnessE2ESuite(cfg: HarnessE2EConfig) { const providerLabel = cfg.modelProvider === 'open_ai' ? 'OpenAI' : cfg.modelProvider === 'gemini' ? 'Gemini' : 'Bedrock'; - // Logged at collection time (not in beforeAll, which Vitest skips when every test in the - // suite is skipped) so the skip reason is always surfaced. + // note: this is created outside of beforeAll since beforeAll is skipped if all tests are skipped. const logger = getLogger(`harness-${providerLabel.toLowerCase()}`); if (!canRun) { logger.warn( @@ -98,12 +97,7 @@ export function createHarnessE2ESuite(cfg: HarnessE2EConfig) { async () => { const result = await runAgentCoreCLI(['deploy', '--yes', '--json'], projectPath); - if (result.exitCode !== 0) { - console.log('Deploy stdout:', result.stdout); - console.log('Deploy stderr:', result.stderr); - } - - expect(result.exitCode, `Deploy failed (stderr: ${result.stderr}, stdout: ${result.stdout})`).toBe(0); + expect(result.exitCode, `Deploy failed stderr=${result.stderr}, stdout=${result.stdout}`).toBe(0); const json = parseJsonOutput(result.stdout) as { success: boolean }; expect(json.success, 'Deploy should report success').toBe(true); @@ -127,12 +121,7 @@ export function createHarnessE2ESuite(cfg: HarnessE2EConfig) { projectPath ); - if (result.exitCode !== 0) { - console.log('Invoke stdout:', result.stdout); - console.log('Invoke stderr:', result.stderr); - } - - expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0); + expect(result.exitCode, `Invoke failed: stderr=${result.stderr}, stdout=${result.stdout}`).toBe(0); const json = parseJsonOutput(result.stdout) as { success: boolean }; expect(json.success, 'Invoke should report success').toBe(true); From a77e7e09f368deb43240ca62e671e749309e2583 Mon Sep 17 00:00:00 2001 From: Hweinstock Date: Wed, 10 Jun 2026 18:54:37 +0000 Subject: [PATCH 5/7] fix(ci): reduce sharding to 5 to avoid rate limiting --- .github/workflows/e2e-tests-full.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-tests-full.yml b/.github/workflows/e2e-tests-full.yml index ba943f7eb..0962bfce8 100644 --- a/.github/workflows/e2e-tests-full.yml +++ b/.github/workflows/e2e-tests-full.yml @@ -28,7 +28,7 @@ jobs: fail-fast: false matrix: cli-build: [preview, ga] - shard: ['1/6', '2/6', '3/6', '4/6', '5/6', '6/6'] + shard: ['1/5', '2/5', '3/5', '4/5', '5/5'] steps: - uses: actions/checkout@v6 with: From 8c8e72e68c829db1fcf14e6636dd7dd45b003dc4 Mon Sep 17 00:00:00 2001 From: Hweinstock Date: Wed, 10 Jun 2026 19:03:26 +0000 Subject: [PATCH 6/7] fix(e2e): avoid mis-labeled key arn --- e2e-tests/harness-gemini.test.ts | 2 +- e2e-tests/harness-openai.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-tests/harness-gemini.test.ts b/e2e-tests/harness-gemini.test.ts index 5db4a1a02..2b659929b 100644 --- a/e2e-tests/harness-gemini.test.ts +++ b/e2e-tests/harness-gemini.test.ts @@ -2,6 +2,6 @@ import { createHarnessE2ESuite } from './harness-e2e-helper.js'; createHarnessE2ESuite({ modelProvider: 'gemini', - apiKeyEnvVar: 'GEMINI_API_KEY_ARN', + apiKeyEnvVar: 'GEMINI_API_KEY', skipMemory: true, }); diff --git a/e2e-tests/harness-openai.test.ts b/e2e-tests/harness-openai.test.ts index 62343ac3f..95e1c18e2 100644 --- a/e2e-tests/harness-openai.test.ts +++ b/e2e-tests/harness-openai.test.ts @@ -1,3 +1,3 @@ import { createHarnessE2ESuite } from './harness-e2e-helper.js'; -createHarnessE2ESuite({ modelProvider: 'open_ai', apiKeyEnvVar: 'OPENAI_API_KEY_ARN', skipMemory: true }); +createHarnessE2ESuite({ modelProvider: 'open_ai', apiKeyEnvVar: 'OPENAI_API_KEY', skipMemory: true }); From cde2032c1b77f16936689560527fa22cecc171b8 Mon Sep 17 00:00:00 2001 From: Hweinstock Date: Wed, 10 Jun 2026 19:46:04 +0000 Subject: [PATCH 7/7] refactor(e2e): rename parameter to make it clear its an ARN --- e2e-tests/harness-e2e-helper.ts | 9 +++++---- e2e-tests/harness-gemini.test.ts | 2 +- e2e-tests/harness-openai.test.ts | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/e2e-tests/harness-e2e-helper.ts b/e2e-tests/harness-e2e-helper.ts index a5f68956a..9be2b42aa 100644 --- a/e2e-tests/harness-e2e-helper.ts +++ b/e2e-tests/harness-e2e-helper.ts @@ -15,13 +15,14 @@ const baseCanRun = prereqs.npm && prereqs.git && hasAws && isPreviewBuild; interface HarnessE2EConfig { modelProvider: 'bedrock' | 'open_ai' | 'gemini'; - apiKeyEnvVar?: string; + /** Env var holding the API key ARN — its value is passed as --api-key-arn. */ + apiKeyArnEnvVar?: string; skipMemory?: boolean; skipInvoke?: boolean; } export function createHarnessE2ESuite(cfg: HarnessE2EConfig) { - const hasRequiredVar = !cfg.apiKeyEnvVar || !!process.env[cfg.apiKeyEnvVar]; + const hasRequiredVar = !cfg.apiKeyArnEnvVar || !!process.env[cfg.apiKeyArnEnvVar]; const canRun = baseCanRun && hasRequiredVar; const providerLabel = @@ -62,8 +63,8 @@ export function createHarnessE2ESuite(cfg: HarnessE2EConfig) { '--skip-git', ]; - if (cfg.apiKeyEnvVar && process.env[cfg.apiKeyEnvVar]) { - createArgs.push('--api-key-arn', process.env[cfg.apiKeyEnvVar]!); + if (cfg.apiKeyArnEnvVar && process.env[cfg.apiKeyArnEnvVar]) { + createArgs.push('--api-key-arn', process.env[cfg.apiKeyArnEnvVar]!); } if (cfg.skipMemory) { diff --git a/e2e-tests/harness-gemini.test.ts b/e2e-tests/harness-gemini.test.ts index 2b659929b..81b0d5e7e 100644 --- a/e2e-tests/harness-gemini.test.ts +++ b/e2e-tests/harness-gemini.test.ts @@ -2,6 +2,6 @@ import { createHarnessE2ESuite } from './harness-e2e-helper.js'; createHarnessE2ESuite({ modelProvider: 'gemini', - apiKeyEnvVar: 'GEMINI_API_KEY', + apiKeyArnEnvVar: 'GEMINI_API_KEY_ARN', skipMemory: true, }); diff --git a/e2e-tests/harness-openai.test.ts b/e2e-tests/harness-openai.test.ts index 95e1c18e2..32ec14be7 100644 --- a/e2e-tests/harness-openai.test.ts +++ b/e2e-tests/harness-openai.test.ts @@ -1,3 +1,3 @@ import { createHarnessE2ESuite } from './harness-e2e-helper.js'; -createHarnessE2ESuite({ modelProvider: 'open_ai', apiKeyEnvVar: 'OPENAI_API_KEY', skipMemory: true }); +createHarnessE2ESuite({ modelProvider: 'open_ai', apiKeyArnEnvVar: 'OPENAI_API_KEY_ARN', skipMemory: true });