feat(ci): clean up stale stacks with global vitest setup hook

e2e-tests/README.md

@@ -120,5 +120,5 @@ Feature lifecycle tests: describe what the test exercises end-to-end
 - E2E tests create real AWS resources and **will incur costs**
 - Always include `teardownE2EProject()` in `afterAll` — never skip cleanup
 - Use unique agent names (timestamp suffix) to avoid conflicts with parallel runs
-- Stale credential providers older than 30 minutes are cleaned up automatically in `beforeAll` via
+- Stale credential providers older than 30 minutes are cleaned up automatically in the Vitest `globalSetup` hook via
   `cleanupStaleCredentialProviders()`

e2e-tests/e2e-helper.ts

@@ -6,12 +6,9 @@ import {
   retry,
   spawnAndCollect,
 } from '../src/test-utils/index.js';
-import {
-  BedrockAgentCoreControlClient,
-  DeleteApiKeyCredentialProviderCommand,
-  GetAgentRuntimeCommand,
-  ListApiKeyCredentialProvidersCommand,
-} from '@aws-sdk/client-bedrock-agentcore-control';
+import { deleteCredentialProvider } from './utils/credential-provider-cleanup.js';
+import { getLogger } from './utils/logger.js';
+import { BedrockAgentCoreControlClient, GetAgentRuntimeCommand } from '@aws-sdk/client-bedrock-agentcore-control';
 import { execSync } from 'node:child_process';
 import { randomUUID } from 'node:crypto';
 import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
@@ -71,8 +68,6 @@ export function createE2ESuite(cfg: E2EConfig) {
     beforeAll(async () => {
       if (!canRun) return;
 
-      await cleanupStaleCredentialProviders();
-
       testDir = join(tmpdir(), `agentcore-e2e-${randomUUID()}`);
       await mkdir(testDir, { recursive: true });
 
@@ -377,38 +372,6 @@ export function installCdkTarball(projectPath: string): void {
   }
 }
 
-async function deleteCredentialProvider(client: BedrockAgentCoreControlClient, name: string): Promise<void> {
-  try {
-    await client.send(new DeleteApiKeyCredentialProviderCommand({ name }));
-    console.log(`Deleted credential provider: ${name}`);
-  } catch (error) {
-    const code = (error as { name?: string }).name ?? 'Unknown';
-    console.warn(`Failed to delete credential provider ${name}: [${code}]`);
-  }
-}
-
-/**
- * Delete stale E2e* credential providers older than the given max age.
- * Runs in beforeAll to prevent accumulation from previous runs that
- * crashed or timed out before their afterAll teardown could execute.
- */
-export async function cleanupStaleCredentialProviders(maxAgeMs: number = 30 * 60 * 1000): Promise<void> {
-  const region = process.env.AWS_REGION ?? 'us-east-1';
-  const client = new BedrockAgentCoreControlClient({ region });
-  const cutoff = new Date(Date.now() - maxAgeMs);
-
-  let nextToken: string | undefined;
-  do {
-    const response = await client.send(new ListApiKeyCredentialProvidersCommand({ nextToken }));
-    const providers = response.credentialProviders ?? [];
-    const stale = providers.filter(p => p.name?.startsWith('E2e') && p.createdTime && p.createdTime < cutoff);
-
-    await Promise.all(stale.map(p => deleteCredentialProvider(client, p.name!)));
-
-    nextToken = response.nextToken;
-  } while (nextToken);
-}
-
 export async function teardownE2EProject(projectPath: string, agentName: string, modelProvider: string): Promise<void> {
   await spawnAndCollect('agentcore', ['remove', 'all', '--json'], projectPath);
   const result = await spawnAndCollect('agentcore', ['deploy', '--yes', '--json'], projectPath);
@@ -419,7 +382,7 @@ export async function teardownE2EProject(projectPath: string, agentName: string,
   if (modelProvider !== 'Bedrock' && agentName) {
     const region = process.env.AWS_REGION ?? 'us-east-1';
     const client = new BedrockAgentCoreControlClient({ region });
-    await deleteCredentialProvider(client, `${agentName}${modelProvider}`);
+    await deleteCredentialProvider(client, getLogger('teardown-e2e'), `${agentName}${modelProvider}`);
   }
 }
 

e2e-tests/global-setup.ts

@@ -0,0 +1,55 @@
+import { cleanupStaleCredentialProviders } from './utils/credential-provider-cleanup';
+import { getLogger } from './utils/logger';
+import { cleanUpOldStacks } from './utils/stack-cleanup';
+import { BedrockAgentCoreControlClient } from '@aws-sdk/client-bedrock-agentcore-control';
+import { CloudFormationClient } from '@aws-sdk/client-cloudformation';
+import type { TestProject } from 'vitest/node';
+
+/**
+ * Global setup for the e2e test project.
+ *
+ * The returned function runs once after all e2e tests complete.
+ *
+ * @see https://vitest.dev/config/#globalsetup
+ */
+export default async function setup(_project: TestProject): Promise<() => void> {
+  const region = process.env.AWS_REGION ?? 'us-east-1';
+  const logger = getLogger('global-setup');
+  logger.info(`starting global setup in region: ${region}`);
+  logger.info(`cleaning up stale stacks...`);
+
+  const startTime = Date.now();
+
+  const cfn = new CloudFormationClient({ region: region, maxAttempts: 10 });
+  try {
+    await cleanUpOldStacks(cfn, logger.child('stack-cleanup'));
+  } catch (e) {
+    logger.error(String(e));
+    logger.warn(`skipping the rest of stack cleanup due to fatal error`);
+  } finally {
+    cfn.destroy();
+  }
+
+  const stackCleanUpFinishTime = Date.now();
+  logger.info(`done cleaning up stacks after ${(stackCleanUpFinishTime - startTime) / 1000} seconds`);
+  logger.info(`cleaning up stale credential providers...`);
+  const bedrockCPClient = new BedrockAgentCoreControlClient({ region: region, maxAttempts: 10 });
+
+  try {
+    await cleanupStaleCredentialProviders(bedrockCPClient, logger.child('credential-provider-cleanup'), {
+      minAgeMs: 30 * 60 * 1000,
+      prefix: 'E2e',
+    });
+  } catch (e) {
+    logger.error(String(e));
+    logger.warn(`failed to clean up all credential providers`);
+  } finally {
+    bedrockCPClient.destroy();
+  }
+
+  logger.info(`setup finished in ${(Date.now() - startTime) / 1000} seconds`);
+
+  return function teardown(): void {
+    // one time cleanup runs here.
+  };
+}

e2e-tests/harness-e2e-helper.ts

@@ -1,12 +1,6 @@
 import { getHarness } from '../src/cli/aws/agentcore-harness.js';
 import { hasAwsCredentials, parseJsonOutput, prereqs, retry, spawnAndCollect } from '../src/test-utils/index.js';
-import {
-  cleanupStaleCredentialProviders,
-  installCdkTarball,
-  runAgentCoreCLI,
-  teardownE2EProject,
-  writeAwsTargets,
-} from './e2e-helper.js';
+import { installCdkTarball, runAgentCoreCLI, teardownE2EProject, writeAwsTargets } from './e2e-helper.js';
 import { randomUUID } from 'node:crypto';
 import { mkdir, readFile, rm } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
@@ -41,8 +35,6 @@ export function createHarnessE2ESuite(cfg: HarnessE2EConfig) {
     beforeAll(async () => {
       if (!canRun) return;
 
-      await cleanupStaleCredentialProviders();
-
       testDir = join(tmpdir(), `agentcore-e2e-harness-${randomUUID()}`);
       await mkdir(testDir, { recursive: true });
 

e2e-tests/utils/credential-provider-cleanup.ts

@@ -0,0 +1,42 @@
+import type { Logger } from './logger';
+import {
+  BedrockAgentCoreControlClient,
+  DeleteApiKeyCredentialProviderCommand,
+  ListApiKeyCredentialProvidersCommand,
+} from '@aws-sdk/client-bedrock-agentcore-control';
+
+export async function deleteCredentialProvider(
+  client: BedrockAgentCoreControlClient,
+  logger: Logger,
+  name: string
+): Promise<void> {
+  try {
+    await client.send(new DeleteApiKeyCredentialProviderCommand({ name }));
+    logger.info(`Deleted credential provider: ${name}`);
+  } catch (error) {
+    const err = error as Error;
+    logger.warn(`Failed to delete credential provider ${name}: ${err.name}:${err.message}`);
+  }
+}
+
+export async function cleanupStaleCredentialProviders(
+  client: BedrockAgentCoreControlClient,
+  logger: Logger,
+  options: {
+    minAgeMs: number;
+    prefix: string;
+  }
+): Promise<void> {
+  const cutoff = new Date(Date.now() - options.minAgeMs);
+
+  let nextToken: string | undefined;
+  do {
+    const response = await client.send(new ListApiKeyCredentialProvidersCommand({ nextToken }));
+    const providers = response.credentialProviders ?? [];
+    const stale = providers.filter(p => p.name?.startsWith(options.prefix) && p.createdTime && p.createdTime < cutoff);
+
+    await Promise.all(stale.map(p => deleteCredentialProvider(client, logger, p.name!)));
+
+    nextToken = response.nextToken;
+  } while (nextToken);
+}

e2e-tests/utils/logger.ts

@@ -0,0 +1,13 @@
+function prefixStr(msg: string, prefix?: string) {
+  return `${prefix ? `${prefix}:` : ''}${msg}`;
+}
+
+export const getLogger = (prefix?: string) => ({
+  debug: (msg: string) => console.debug(prefixStr(msg, `[${prefix}]`)),
+  info: (msg: string) => console.info(prefixStr(msg, `[${prefix}]`)),
+  warn: (msg: string) => console.warn(prefixStr(msg, `[${prefix}]`)),
+  error: (msg: string) => console.error(prefixStr(msg, `[${prefix}]`)),
+  child: (newPrefix: string) => getLogger(prefixStr(newPrefix, prefix)),
+});
+
+export type Logger = ReturnType<typeof getLogger>;

e2e-tests/utils/stack-cleanup.ts

@@ -0,0 +1,97 @@
+import type { Logger } from './logger';
+import {
+  type CloudFormationClient,
+  DeleteStackCommand,
+  StackStatus,
+  type StackSummary,
+  paginateListStacks,
+  waitUntilStackDeleteComplete,
+} from '@aws-sdk/client-cloudformation';
+
+/**
+ * List every root stack whose name starts with given prefix and is older than given age, with an optional filter.
+ */
+async function listStacks(
+  cfn: CloudFormationClient,
+  logger: Logger,
+  options: { maxCount?: number; minStackAgeMs: number; statusFilter?: (status: StackStatus) => boolean; prefix: string }
+): Promise<StackSummary[]> {
+  const cutoff = new Date(Date.now() - options.minStackAgeMs);
+  logger.info(`listing stacks with cutoff=${cutoff.toISOString()}, prefix=${options.prefix}`);
+
+  const stacks: StackSummary[] = [];
+  for await (const page of paginateListStacks(
+    { client: cfn },
+    {
+      StackStatusFilter: Object.values(StackStatus).filter(options.statusFilter ?? (() => true)),
+    }
+  )) {
+    for (const summary of page.StackSummaries ?? []) {
+      if (options.maxCount !== undefined && stacks.length >= options.maxCount) return stacks;
+      if (!summary.StackName?.startsWith(options.prefix)) continue;
+      if (summary.ParentId) continue; // skip nested stacks.
+      if (!summary.CreationTime || summary.CreationTime > cutoff) continue;
+      stacks.push(summary);
+    }
+  }
+  return stacks;
+}
+
+/**
+ * Delete a single stack and block until CloudFormation confirms it is gone.
+ * Skip cleanups that fail.
+ */
+async function deleteStackAndVerify(cfn: CloudFormationClient, logger: Logger, stackName: string): Promise<boolean> {
+  await cfn.send(new DeleteStackCommand({ StackName: stackName }));
+  logger.info(`deleting stack with name ${stackName}`);
+  const startTime = Date.now();
+  try {
+    const result = await waitUntilStackDeleteComplete(
+      { client: cfn, maxWaitTime: 60 * 3, minDelay: 15 },
+      { StackName: stackName }
+    );
+
+    logger.info(`finished deleting stack in ${(Date.now() - startTime) / 1000} seconds`);
+
+    if (String(result.state) === 'SUCCESS') {
+      return true;
+    }
+  } catch (e) {
+    const err = e as Error;
+    logger.error(`failed to delete stack with name ${stackName} after ${(Date.now() - startTime) / 1000} seconds`);
+    logger.error(`skipping stack after error: ${err.name}:${err.message}`);
+  }
+
+  // DELETE_FAILED, timed out, or otherwise did not reach DELETE_COMPLETE.
+  return false;
+}
+
+export async function cleanUpOldStacks(
+  client: CloudFormationClient,
+  logger: Logger,
+  options?: { maxStacksDeleted?: number; retries?: number }
+) {
+  const stacks = await listStacks(client, logger, {
+    statusFilter: s =>
+      ![StackStatus.DELETE_COMPLETE, StackStatus.DELETE_IN_PROGRESS].includes(s as never) &&
+      !s.toString().endsWith('_IN_PROGRESS'),
+    prefix: 'AgentCore-E2e',
+    minStackAgeMs: 3 * 60 * 60 * 1000,
+    maxCount: options?.maxStacksDeleted,
+  });
+  logger.info(`found ${stacks.length} stacks`);
+  if (stacks.length === 0) {
+    logger.info(`no stacks found!`);
+  } else {
+    const names = stacks.map(s => s.StackName!);
+
+    logger.info(`deleting ${names.length} stacks with names=${names.join(',')}`);
+    const results = await Promise.allSettled(names.map(name => deleteStackAndVerify(client, logger, name)));
+    const passed = results.filter(p => p.status === 'fulfilled' && p.value);
+    logger.info(`deleted ${passed.length} of ${names.length} remaining stacks`);
+
+    if (options?.retries !== undefined && options.retries > 0 && passed.length !== names.length) {
+      await cleanUpOldStacks(client, logger, { ...options, retries: options.retries - 1 });
+    }
+  }
+}

vitest.config.ts

@@ -67,6 +67,7 @@ export default defineConfig({
           include: ['e2e-tests/**/*.test.ts'],
           testTimeout: 600000,
           hookTimeout: 600000,
+          globalSetup: ['./e2e-tests/global-setup.ts'],
         },
       },
       {