diff --git a/e2e-tests/README.md b/e2e-tests/README.md index cdaa0066d..45090b613 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -120,5 +120,5 @@ Feature lifecycle tests: describe what the test exercises end-to-end - E2E tests create real AWS resources and **will incur costs** - Always include `teardownE2EProject()` in `afterAll` — never skip cleanup - Use unique agent names (timestamp suffix) to avoid conflicts with parallel runs -- Stale credential providers older than 30 minutes are cleaned up automatically in `beforeAll` via +- Stale credential providers older than 30 minutes are cleaned up automatically in the Vitest `globalSetup` hook via `cleanupStaleCredentialProviders()` diff --git a/e2e-tests/e2e-helper.ts b/e2e-tests/e2e-helper.ts index bd3730ef6..d4e9bf554 100644 --- a/e2e-tests/e2e-helper.ts +++ b/e2e-tests/e2e-helper.ts @@ -6,12 +6,9 @@ import { retry, spawnAndCollect, } from '../src/test-utils/index.js'; -import { - BedrockAgentCoreControlClient, - DeleteApiKeyCredentialProviderCommand, - GetAgentRuntimeCommand, - ListApiKeyCredentialProvidersCommand, -} from '@aws-sdk/client-bedrock-agentcore-control'; +import { deleteCredentialProvider } from './utils/credential-provider-cleanup.js'; +import { getLogger } from './utils/logger.js'; +import { BedrockAgentCoreControlClient, GetAgentRuntimeCommand } from '@aws-sdk/client-bedrock-agentcore-control'; import { execSync } from 'node:child_process'; import { randomUUID } from 'node:crypto'; import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'; @@ -71,8 +68,6 @@ export function createE2ESuite(cfg: E2EConfig) { beforeAll(async () => { if (!canRun) return; - await cleanupStaleCredentialProviders(); - testDir = join(tmpdir(), `agentcore-e2e-${randomUUID()}`); await mkdir(testDir, { recursive: true }); @@ -377,38 +372,6 @@ export function installCdkTarball(projectPath: string): void { } } -async function deleteCredentialProvider(client: BedrockAgentCoreControlClient, name: string): Promise { - try { - await client.send(new DeleteApiKeyCredentialProviderCommand({ name })); - console.log(`Deleted credential provider: ${name}`); - } catch (error) { - const code = (error as { name?: string }).name ?? 'Unknown'; - console.warn(`Failed to delete credential provider ${name}: [${code}]`); - } -} - -/** - * Delete stale E2e* credential providers older than the given max age. - * Runs in beforeAll to prevent accumulation from previous runs that - * crashed or timed out before their afterAll teardown could execute. - */ -export async function cleanupStaleCredentialProviders(maxAgeMs: number = 30 * 60 * 1000): Promise { - const region = process.env.AWS_REGION ?? 'us-east-1'; - const client = new BedrockAgentCoreControlClient({ region }); - const cutoff = new Date(Date.now() - maxAgeMs); - - let nextToken: string | undefined; - do { - const response = await client.send(new ListApiKeyCredentialProvidersCommand({ nextToken })); - const providers = response.credentialProviders ?? []; - const stale = providers.filter(p => p.name?.startsWith('E2e') && p.createdTime && p.createdTime < cutoff); - - await Promise.all(stale.map(p => deleteCredentialProvider(client, p.name!))); - - nextToken = response.nextToken; - } while (nextToken); -} - export async function teardownE2EProject(projectPath: string, agentName: string, modelProvider: string): Promise { await spawnAndCollect('agentcore', ['remove', 'all', '--json'], projectPath); const result = await spawnAndCollect('agentcore', ['deploy', '--yes', '--json'], projectPath); @@ -419,7 +382,7 @@ export async function teardownE2EProject(projectPath: string, agentName: string, if (modelProvider !== 'Bedrock' && agentName) { const region = process.env.AWS_REGION ?? 'us-east-1'; const client = new BedrockAgentCoreControlClient({ region }); - await deleteCredentialProvider(client, `${agentName}${modelProvider}`); + await deleteCredentialProvider(client, getLogger('teardown-e2e'), `${agentName}${modelProvider}`); } } diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts new file mode 100644 index 000000000..0f93f2857 --- /dev/null +++ b/e2e-tests/global-setup.ts @@ -0,0 +1,55 @@ +import { cleanupStaleCredentialProviders } from './utils/credential-provider-cleanup'; +import { getLogger } from './utils/logger'; +import { cleanUpOldStacks } from './utils/stack-cleanup'; +import { BedrockAgentCoreControlClient } from '@aws-sdk/client-bedrock-agentcore-control'; +import { CloudFormationClient } from '@aws-sdk/client-cloudformation'; +import type { TestProject } from 'vitest/node'; + +/** + * Global setup for the e2e test project. + * + * The returned function runs once after all e2e tests complete. + * + * @see https://vitest.dev/config/#globalsetup + */ +export default async function setup(_project: TestProject): Promise<() => void> { + const region = process.env.AWS_REGION ?? 'us-east-1'; + const logger = getLogger('global-setup'); + logger.info(`starting global setup in region: ${region}`); + logger.info(`cleaning up stale stacks...`); + + const startTime = Date.now(); + + const cfn = new CloudFormationClient({ region: region, maxAttempts: 10 }); + try { + await cleanUpOldStacks(cfn, logger.child('stack-cleanup')); + } catch (e) { + logger.error(String(e)); + logger.warn(`skipping the rest of stack cleanup due to fatal error`); + } finally { + cfn.destroy(); + } + + const stackCleanUpFinishTime = Date.now(); + logger.info(`done cleaning up stacks after ${(stackCleanUpFinishTime - startTime) / 1000} seconds`); + logger.info(`cleaning up stale credential providers...`); + const bedrockCPClient = new BedrockAgentCoreControlClient({ region: region, maxAttempts: 10 }); + + try { + await cleanupStaleCredentialProviders(bedrockCPClient, logger.child('credential-provider-cleanup'), { + minAgeMs: 30 * 60 * 1000, + prefix: 'E2e', + }); + } catch (e) { + logger.error(String(e)); + logger.warn(`failed to clean up all credential providers`); + } finally { + bedrockCPClient.destroy(); + } + + logger.info(`setup finished in ${(Date.now() - startTime) / 1000} seconds`); + + return function teardown(): void { + // one time cleanup runs here. + }; +} diff --git a/e2e-tests/harness-e2e-helper.ts b/e2e-tests/harness-e2e-helper.ts index 32617af5c..c8ca0c456 100644 --- a/e2e-tests/harness-e2e-helper.ts +++ b/e2e-tests/harness-e2e-helper.ts @@ -1,12 +1,6 @@ import { getHarness } from '../src/cli/aws/agentcore-harness.js'; import { hasAwsCredentials, parseJsonOutput, prereqs, retry, spawnAndCollect } from '../src/test-utils/index.js'; -import { - cleanupStaleCredentialProviders, - installCdkTarball, - runAgentCoreCLI, - teardownE2EProject, - writeAwsTargets, -} from './e2e-helper.js'; +import { installCdkTarball, runAgentCoreCLI, teardownE2EProject, writeAwsTargets } from './e2e-helper.js'; import { randomUUID } from 'node:crypto'; import { mkdir, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; @@ -41,8 +35,6 @@ export function createHarnessE2ESuite(cfg: HarnessE2EConfig) { beforeAll(async () => { if (!canRun) return; - await cleanupStaleCredentialProviders(); - testDir = join(tmpdir(), `agentcore-e2e-harness-${randomUUID()}`); await mkdir(testDir, { recursive: true }); diff --git a/e2e-tests/utils/credential-provider-cleanup.ts b/e2e-tests/utils/credential-provider-cleanup.ts new file mode 100644 index 000000000..a2f1ef42d --- /dev/null +++ b/e2e-tests/utils/credential-provider-cleanup.ts @@ -0,0 +1,42 @@ +import type { Logger } from './logger'; +import { + BedrockAgentCoreControlClient, + DeleteApiKeyCredentialProviderCommand, + ListApiKeyCredentialProvidersCommand, +} from '@aws-sdk/client-bedrock-agentcore-control'; + +export async function deleteCredentialProvider( + client: BedrockAgentCoreControlClient, + logger: Logger, + name: string +): Promise { + try { + await client.send(new DeleteApiKeyCredentialProviderCommand({ name })); + logger.info(`Deleted credential provider: ${name}`); + } catch (error) { + const err = error as Error; + logger.warn(`Failed to delete credential provider ${name}: ${err.name}:${err.message}`); + } +} + +export async function cleanupStaleCredentialProviders( + client: BedrockAgentCoreControlClient, + logger: Logger, + options: { + minAgeMs: number; + prefix: string; + } +): Promise { + const cutoff = new Date(Date.now() - options.minAgeMs); + + let nextToken: string | undefined; + do { + const response = await client.send(new ListApiKeyCredentialProvidersCommand({ nextToken })); + const providers = response.credentialProviders ?? []; + const stale = providers.filter(p => p.name?.startsWith(options.prefix) && p.createdTime && p.createdTime < cutoff); + + await Promise.all(stale.map(p => deleteCredentialProvider(client, logger, p.name!))); + + nextToken = response.nextToken; + } while (nextToken); +} diff --git a/e2e-tests/utils/logger.ts b/e2e-tests/utils/logger.ts new file mode 100644 index 000000000..842ec7f81 --- /dev/null +++ b/e2e-tests/utils/logger.ts @@ -0,0 +1,13 @@ +function prefixStr(msg: string, prefix?: string) { + return `${prefix ? `${prefix}:` : ''}${msg}`; +} + +export const getLogger = (prefix?: string) => ({ + debug: (msg: string) => console.debug(prefixStr(msg, `[${prefix}]`)), + info: (msg: string) => console.info(prefixStr(msg, `[${prefix}]`)), + warn: (msg: string) => console.warn(prefixStr(msg, `[${prefix}]`)), + error: (msg: string) => console.error(prefixStr(msg, `[${prefix}]`)), + child: (newPrefix: string) => getLogger(prefixStr(newPrefix, prefix)), +}); + +export type Logger = ReturnType; diff --git a/e2e-tests/utils/stack-cleanup.ts b/e2e-tests/utils/stack-cleanup.ts new file mode 100644 index 000000000..add0ea1d7 --- /dev/null +++ b/e2e-tests/utils/stack-cleanup.ts @@ -0,0 +1,97 @@ +import type { Logger } from './logger'; +import { + type CloudFormationClient, + DeleteStackCommand, + StackStatus, + type StackSummary, + paginateListStacks, + waitUntilStackDeleteComplete, +} from '@aws-sdk/client-cloudformation'; + +/** + * List every root stack whose name starts with given prefix and is older than given age, with an optional filter. + */ +async function listStacks( + cfn: CloudFormationClient, + logger: Logger, + options: { maxCount?: number; minStackAgeMs: number; statusFilter?: (status: StackStatus) => boolean; prefix: string } +): Promise { + const cutoff = new Date(Date.now() - options.minStackAgeMs); + logger.info(`listing stacks with cutoff=${cutoff.toISOString()}, prefix=${options.prefix}`); + + const stacks: StackSummary[] = []; + for await (const page of paginateListStacks( + { client: cfn }, + { + StackStatusFilter: Object.values(StackStatus).filter(options.statusFilter ?? (() => true)), + } + )) { + for (const summary of page.StackSummaries ?? []) { + if (options.maxCount !== undefined && stacks.length >= options.maxCount) return stacks; + if (!summary.StackName?.startsWith(options.prefix)) continue; + if (summary.ParentId) continue; // skip nested stacks. + if (!summary.CreationTime || summary.CreationTime > cutoff) continue; + stacks.push(summary); + } + } + return stacks; +} + +/** + * Delete a single stack and block until CloudFormation confirms it is gone. + * Skip cleanups that fail. + */ +async function deleteStackAndVerify(cfn: CloudFormationClient, logger: Logger, stackName: string): Promise { + await cfn.send(new DeleteStackCommand({ StackName: stackName })); + logger.info(`deleting stack with name ${stackName}`); + const startTime = Date.now(); + try { + const result = await waitUntilStackDeleteComplete( + { client: cfn, maxWaitTime: 60 * 3, minDelay: 15 }, + { StackName: stackName } + ); + + logger.info(`finished deleting stack in ${(Date.now() - startTime) / 1000} seconds`); + + if (String(result.state) === 'SUCCESS') { + return true; + } + } catch (e) { + const err = e as Error; + logger.error(`failed to delete stack with name ${stackName} after ${(Date.now() - startTime) / 1000} seconds`); + logger.error(`skipping stack after error: ${err.name}:${err.message}`); + } + + // DELETE_FAILED, timed out, or otherwise did not reach DELETE_COMPLETE. + return false; +} + +export async function cleanUpOldStacks( + client: CloudFormationClient, + logger: Logger, + options?: { maxStacksDeleted?: number; retries?: number } +) { + const stacks = await listStacks(client, logger, { + statusFilter: s => + ![StackStatus.DELETE_COMPLETE, StackStatus.DELETE_IN_PROGRESS].includes(s as never) && + !s.toString().endsWith('_IN_PROGRESS'), + prefix: 'AgentCore-E2e', + minStackAgeMs: 3 * 60 * 60 * 1000, + maxCount: options?.maxStacksDeleted, + }); + logger.info(`found ${stacks.length} stacks`); + if (stacks.length === 0) { + logger.info(`no stacks found!`); + } else { + const names = stacks.map(s => s.StackName!); + + logger.info(`deleting ${names.length} stacks with names=${names.join(',')}`); + const results = await Promise.allSettled(names.map(name => deleteStackAndVerify(client, logger, name))); + const passed = results.filter(p => p.status === 'fulfilled' && p.value); + logger.info(`deleted ${passed.length} of ${names.length} remaining stacks`); + + if (options?.retries !== undefined && options.retries > 0 && passed.length !== names.length) { + await cleanUpOldStacks(client, logger, { ...options, retries: options.retries - 1 }); + } + } +} diff --git a/vitest.config.ts b/vitest.config.ts index 6b1e61e9c..d1e678033 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -67,6 +67,7 @@ export default defineConfig({ include: ['e2e-tests/**/*.test.ts'], testTimeout: 600000, hookTimeout: 600000, + globalSetup: ['./e2e-tests/global-setup.ts'], }, }, {