diff --git a/.github/workflows/e2e-tests-full.yml b/.github/workflows/e2e-tests-full.yml index 79cb44f9a..0962bfce8 100644 --- a/.github/workflows/e2e-tests-full.yml +++ b/.github/workflows/e2e-tests-full.yml @@ -27,8 +27,8 @@ jobs: strategy: fail-fast: false matrix: - cdk-source: [npm, main] - shard: ['1/6', '2/6', '3/6', '4/6', '5/6', '6/6'] + cli-build: [preview, ga] + shard: ['1/5', '2/5', '3/5', '4/5', '5/5'] steps: - uses: actions/checkout@v6 with: @@ -57,9 +57,11 @@ jobs: E2E,${{ secrets.E2E_SECRET_ARN }} parse-json-secrets: true - run: npm ci - - run: npm run build + - name: Build CLI (${{ matrix.cli-build }}) + run: npm run build + env: + BUILD_PREVIEW: ${{ matrix.cli-build == 'preview' && '1' || '0' }} - name: Generate GitHub App Token - if: matrix.cdk-source == 'main' id: app-token uses: actions/create-github-app-token@v1 with: @@ -67,7 +69,6 @@ jobs: private-key: ${{ secrets.APP_PRIVATE_KEY }} owner: aws - name: Build CDK package from main - if: matrix.cdk-source == 'main' run: | git clone --depth 1 "https://x-access-token:${CDK_REPO_TOKEN}@github.com/${CDK_REPO}.git" /tmp/cdk-repo cd /tmp/cdk-repo @@ -80,7 +81,7 @@ jobs: CDK_REPO: ${{ secrets.CDK_REPO_NAME }} - name: Install CLI globally run: npm install -g "$(npm pack | tail -1)" - - name: Run E2E tests (${{ matrix.cdk-source }}, shard ${{ matrix.shard }}) + - name: Run E2E tests (${{ matrix.cli-build }}, shard ${{ matrix.shard }}) env: AWS_ACCOUNT_ID: ${{ steps.aws.outputs.account_id }} AWS_REGION: ${{ inputs.aws_region || 'us-east-1' }} @@ -95,6 +96,7 @@ jobs: CDP_API_KEY_ID: ${{ env.E2E_CDP_API_KEY_ID }} CDP_API_KEY_SECRET: ${{ env.E2E_CDP_API_KEY_SECRET }} CDP_WALLET_SECRET: ${{ env.E2E_CDP_WALLET_SECRET }} + BUILD_PREVIEW: ${{ matrix.cli-build == 'preview' && '1' || '0' }} run: npx vitest run --project e2e --shard=${{ matrix.shard }} browser-tests: diff --git a/e2e-tests/harness-e2e-helper.ts b/e2e-tests/harness-e2e-helper.ts index c8ca0c456..9be2b42aa 100644 --- a/e2e-tests/harness-e2e-helper.ts +++ b/e2e-tests/harness-e2e-helper.ts @@ -1,6 +1,7 @@ import { getHarness } from '../src/cli/aws/agentcore-harness.js'; import { hasAwsCredentials, parseJsonOutput, prereqs, retry, spawnAndCollect } from '../src/test-utils/index.js'; import { installCdkTarball, runAgentCoreCLI, teardownE2EProject, writeAwsTargets } from './e2e-helper.js'; +import { getLogger } from './utils/logger.js'; import { randomUUID } from 'node:crypto'; import { mkdir, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; @@ -14,18 +15,29 @@ const baseCanRun = prereqs.npm && prereqs.git && hasAws && isPreviewBuild; interface HarnessE2EConfig { modelProvider: 'bedrock' | 'open_ai' | 'gemini'; - apiKeyEnvVar?: string; + /** Env var holding the API key ARN — its value is passed as --api-key-arn. */ + apiKeyArnEnvVar?: string; skipMemory?: boolean; skipInvoke?: boolean; } export function createHarnessE2ESuite(cfg: HarnessE2EConfig) { - const hasRequiredVar = !cfg.apiKeyEnvVar || !!process.env[cfg.apiKeyEnvVar]; + const hasRequiredVar = !cfg.apiKeyArnEnvVar || !!process.env[cfg.apiKeyArnEnvVar]; const canRun = baseCanRun && hasRequiredVar; const providerLabel = cfg.modelProvider === 'open_ai' ? 'OpenAI' : cfg.modelProvider === 'gemini' ? 'Gemini' : 'Bedrock'; + // note: this is created outside of beforeAll since beforeAll is skipped if all tests are skipped. + const logger = getLogger(`harness-${providerLabel.toLowerCase()}`); + if (!canRun) { + logger.warn( + `tests are skipped due to insufficient conditions. ` + + `npm=${prereqs.npm}, git=${prereqs.git}, hasAws=${hasAws}, ` + + `isPreviewBuild=${isPreviewBuild}, hasRequiredVar=${hasRequiredVar}` + ); + } + describe.sequential(`e2e: harness/${providerLabel} — create → deploy → invoke → teardown`, () => { let testDir: string; let projectPath: string; @@ -51,8 +63,8 @@ export function createHarnessE2ESuite(cfg: HarnessE2EConfig) { '--skip-git', ]; - if (cfg.apiKeyEnvVar && process.env[cfg.apiKeyEnvVar]) { - createArgs.push('--api-key-arn', process.env[cfg.apiKeyEnvVar]!); + if (cfg.apiKeyArnEnvVar && process.env[cfg.apiKeyArnEnvVar]) { + createArgs.push('--api-key-arn', process.env[cfg.apiKeyArnEnvVar]!); } if (cfg.skipMemory) { @@ -86,12 +98,7 @@ export function createHarnessE2ESuite(cfg: HarnessE2EConfig) { async () => { const result = await runAgentCoreCLI(['deploy', '--yes', '--json'], projectPath); - if (result.exitCode !== 0) { - console.log('Deploy stdout:', result.stdout); - console.log('Deploy stderr:', result.stderr); - } - - expect(result.exitCode, `Deploy failed (stderr: ${result.stderr}, stdout: ${result.stdout})`).toBe(0); + expect(result.exitCode, `Deploy failed stderr=${result.stderr}, stdout=${result.stdout}`).toBe(0); const json = parseJsonOutput(result.stdout) as { success: boolean }; expect(json.success, 'Deploy should report success').toBe(true); @@ -115,12 +122,7 @@ export function createHarnessE2ESuite(cfg: HarnessE2EConfig) { projectPath ); - if (result.exitCode !== 0) { - console.log('Invoke stdout:', result.stdout); - console.log('Invoke stderr:', result.stderr); - } - - expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0); + expect(result.exitCode, `Invoke failed: stderr=${result.stderr}, stdout=${result.stdout}`).toBe(0); const json = parseJsonOutput(result.stdout) as { success: boolean }; expect(json.success, 'Invoke should report success').toBe(true); diff --git a/e2e-tests/harness-gemini.test.ts b/e2e-tests/harness-gemini.test.ts index 5db4a1a02..81b0d5e7e 100644 --- a/e2e-tests/harness-gemini.test.ts +++ b/e2e-tests/harness-gemini.test.ts @@ -2,6 +2,6 @@ import { createHarnessE2ESuite } from './harness-e2e-helper.js'; createHarnessE2ESuite({ modelProvider: 'gemini', - apiKeyEnvVar: 'GEMINI_API_KEY_ARN', + apiKeyArnEnvVar: 'GEMINI_API_KEY_ARN', skipMemory: true, }); diff --git a/e2e-tests/harness-openai.test.ts b/e2e-tests/harness-openai.test.ts index 62343ac3f..32ec14be7 100644 --- a/e2e-tests/harness-openai.test.ts +++ b/e2e-tests/harness-openai.test.ts @@ -1,3 +1,3 @@ import { createHarnessE2ESuite } from './harness-e2e-helper.js'; -createHarnessE2ESuite({ modelProvider: 'open_ai', apiKeyEnvVar: 'OPENAI_API_KEY_ARN', skipMemory: true }); +createHarnessE2ESuite({ modelProvider: 'open_ai', apiKeyArnEnvVar: 'OPENAI_API_KEY_ARN', skipMemory: true }); diff --git a/scripts/run-e2e-local.sh b/scripts/run-e2e-local.sh index 0fcaee1cf..6962c1bd2 100755 --- a/scripts/run-e2e-local.sh +++ b/scripts/run-e2e-local.sh @@ -8,13 +8,18 @@ # # Optional env vars: # AWS_REGION — defaults to us-east-1 +# BUILD_PREVIEW — set to 1 to build a preview CLI and run the harness e2e tests +# (e2e-tests/harness-*.test.ts). Harness features are gated to preview +# builds; without this they self-skip. The var is read at build time +# (esbuild bakes __PREVIEW__=true) and at test runtime (skip gate). # # Usage: # export E2E_ROLE_ARN=arn:aws:iam:::role/ # export E2E_SECRET_ARN=arn:aws:secretsmanager:::secret: -# ./scripts/run-e2e-local.sh # runs strands-bedrock.test.ts (CI default) -# ./scripts/run-e2e-local.sh --all # runs the full e2e suite -# ./scripts/run-e2e-local.sh e2e-tests/foo.test.ts # runs a specific test file +# ./scripts/run-e2e-local.sh # runs strands-bedrock.test.ts (CI default) +# ./scripts/run-e2e-local.sh --all # runs the full e2e suite +# ./scripts/run-e2e-local.sh e2e-tests/foo.test.ts # runs a specific test file +# BUILD_PREVIEW=1 ./scripts/run-e2e-local.sh e2e-tests/harness-bedrock.test.ts # runs harness tests # # Prerequisites: aws CLI, node >=20.19, npm, git, uv, jq