fix(deploy): handle PYTHON_3_14 region mismatch and recover from REVIEW_IN_PROGRESS stacks

src/cli/__tests__/errors.test.ts

@@ -2,8 +2,10 @@ import {
   AgentAlreadyExistsError,
   getErrorMessage,
   isChangesetInProgressError,
+  isEarlyValidationError,
   isExpiredTokenError,
   isNoCredentialsError,
+  isReviewInProgressError,
   isStackInProgressError,
 } from '../errors.js';
 import { describe, expect, it } from 'vitest';
@@ -204,4 +206,57 @@ describe('errors', () => {
       expect(isChangesetInProgressError({})).toBe(false);
     });
   });
+
+  describe('isEarlyValidationError', () => {
+    it('detects AWS::EarlyValidation::PropertyValidation messages', () => {
+      expect(
+        isEarlyValidationError(
+          new Error('The following hook(s)/validation failed: [AWS::EarlyValidation::PropertyValidation]')
+        )
+      ).toBe(true);
+      expect(isEarlyValidationError(new Error('EarlyValidation::PropertyValidation rejected runtime'))).toBe(true);
+    });
+
+    it('detects other AWS::EarlyValidation:: hook subtypes', () => {
+      // Future-proofing: any hook under the EarlyValidation namespace should match.
+      expect(isEarlyValidationError(new Error('AWS::EarlyValidation::SomeOtherCheck failed for resource'))).toBe(true);
+    });
+
+    it('detects messages that only carry the hook(s)/validation framing', () => {
+      // When CFN logs the hook name with different separators, we still recognize it.
+      expect(
+        isEarlyValidationError(
+          new Error('The following hook(s)/validation failed: [AWS-EarlyValidation-PropertyValidation]')
+        )
+      ).toBe(true);
+    });
+
+    it('returns false for unrelated errors', () => {
+      expect(isEarlyValidationError(new Error('CREATE_FAILED'))).toBe(false);
+      expect(isEarlyValidationError(new Error('Stack not found'))).toBe(false);
+      expect(isEarlyValidationError(null)).toBe(false);
+    });
+  });
+
+  describe('isReviewInProgressError', () => {
+    it('detects REVIEW_IN_PROGRESS errors', () => {
+      expect(isReviewInProgressError(new Error('Stack "MyStack" is currently in REVIEW_IN_PROGRESS state.'))).toBe(
+        true
+      );
+      expect(isReviewInProgressError(new Error('Stack "MyStack" is in REVIEW_IN_PROGRESS state'))).toBe(true);
+      expect(isReviewInProgressError(new Error('REVIEW_IN_PROGRESS state detected'))).toBe(true);
+    });
+
+    it('does not match bare REVIEW_IN_PROGRESS token in passing references', () => {
+      // e.g. an event-stream history message mentioning prior statuses
+      expect(
+        isReviewInProgressError(new Error('History: REVIEW_IN_PROGRESS -> CREATE_IN_PROGRESS -> CREATE_COMPLETE'))
+      ).toBe(false);
+    });
+
+    it('returns false for unrelated errors', () => {
+      expect(isReviewInProgressError(new Error('UPDATE_IN_PROGRESS'))).toBe(false);
+      expect(isReviewInProgressError(null)).toBe(false);
+    });
+  });
 });

src/cli/cloudformation/__tests__/stack-cleanup.test.ts

@@ -0,0 +1,292 @@
+import { recoverReviewInProgressStack } from '../stack-cleanup.js';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const { mockSend, DescribeStacksCommand, ListChangeSetsCommand, DeleteStackCommand } = vi.hoisted(() => ({
+  mockSend: vi.fn(),
+  DescribeStacksCommand: class {
+    constructor(public input: unknown) {}
+  },
+  ListChangeSetsCommand: class {
+    constructor(public input: unknown) {}
+  },
+  DeleteStackCommand: class {
+    constructor(public input: unknown) {}
+  },
+}));
+
+vi.mock('@aws-sdk/client-cloudformation', () => ({
+  CloudFormationClient: class {
+    send = mockSend;
+  },
+  DescribeStacksCommand,
+  ListChangeSetsCommand,
+  DeleteStackCommand,
+}));
+
+vi.mock('../../aws', () => ({
+  getCredentialProvider: vi.fn().mockReturnValue({}),
+}));
+
+describe('recoverReviewInProgressStack', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  function makeClient() {
+    return { send: mockSend } as unknown as Parameters<typeof recoverReviewInProgressStack>[2] extends infer O
+      ? O extends { client?: infer C }
+        ? C
+        : never
+      : never;
+  }
+
+  it('deletes stack when REVIEW_IN_PROGRESS and all change sets failed', async () => {
+    // describe
+    mockSend.mockResolvedValueOnce({ Stacks: [{ StackStatus: 'REVIEW_IN_PROGRESS' }] });
+    // list change sets - one FAILED
+    mockSend.mockResolvedValueOnce({ Summaries: [{ Status: 'FAILED', ExecutionStatus: 'UNAVAILABLE' }] });
+    // delete stack
+    mockSend.mockResolvedValueOnce({});
+    // poll - stack no longer exists
+    const validationErr = new Error('Stack does not exist');
+    validationErr.name = 'ValidationError';
+    mockSend.mockRejectedValueOnce(validationErr);
+
+    const result = await recoverReviewInProgressStack('us-east-1', 'MyStack', {
+      pollIntervalMs: 1,
+      timeoutMs: 1000,
+      client: makeClient(),
+    });
+
+    expect(result.deleted).toBe(true);
+    expect(result.changeSetCount).toBe(1);
+    expect(result.allChangeSetsNonExecuted).toBe(true);
+
+    // Verify DeleteStackCommand was sent
+    const deleteCalls = mockSend.mock.calls.filter(c => c[0] instanceof DeleteStackCommand);
+    expect(deleteCalls.length).toBe(1);
+  });
+
+  it('throws when stack is not in REVIEW_IN_PROGRESS', async () => {
+    mockSend.mockResolvedValueOnce({ Stacks: [{ StackStatus: 'CREATE_COMPLETE' }] });
+    await expect(recoverReviewInProgressStack('us-east-1', 'MyStack', { client: makeClient() })).rejects.toThrow(
+      /expected status REVIEW_IN_PROGRESS/
+    );
+  });
+
+  it('throws when stack does not exist at all', async () => {
+    mockSend.mockResolvedValueOnce({ Stacks: [] });
+    await expect(recoverReviewInProgressStack('us-east-1', 'MyStack', { client: makeClient() })).rejects.toThrow(
+      /not found/
+    );
+  });
+
+  it('refuses to delete when a change set has been executed', async () => {
+    mockSend.mockResolvedValueOnce({ Stacks: [{ StackStatus: 'REVIEW_IN_PROGRESS' }] });
+    mockSend.mockResolvedValueOnce({
+      Summaries: [{ Status: 'CREATE_COMPLETE', ExecutionStatus: 'EXECUTE_COMPLETE' }],
+    });
+
+    await expect(recoverReviewInProgressStack('us-east-1', 'MyStack', { client: makeClient() })).rejects.toThrow(
+      /at least one change set has been executed/
+    );
+
+    // Ensure delete was not called
+    const deleteCalls = mockSend.mock.calls.filter(c => c[0] instanceof DeleteStackCommand);
+    expect(deleteCalls.length).toBe(0);
+  });
+
+  it('handles paginated change set lists', async () => {
+    mockSend.mockResolvedValueOnce({ Stacks: [{ StackStatus: 'REVIEW_IN_PROGRESS' }] });
+    mockSend.mockResolvedValueOnce({
+      Summaries: [{ Status: 'FAILED' }],
+      NextToken: 'page-2',
+    });
+    mockSend.mockResolvedValueOnce({
+      Summaries: [{ Status: 'OBSOLETE' }],
+    });
+    mockSend.mockResolvedValueOnce({}); // delete
+    const validationErr = new Error('Stack does not exist');
+    validationErr.name = 'ValidationError';
+    mockSend.mockRejectedValueOnce(validationErr); // poll
+
+    const result = await recoverReviewInProgressStack('us-east-1', 'MyStack', {
+      pollIntervalMs: 1,
+      timeoutMs: 1000,
+      client: makeClient(),
+    });
+    expect(result.deleted).toBe(true);
+    expect(result.changeSetCount).toBe(2);
+  });
+
+  it('handles DELETE_FAILED during poll', async () => {
+    mockSend.mockResolvedValueOnce({ Stacks: [{ StackStatus: 'REVIEW_IN_PROGRESS' }] });
+    mockSend.mockResolvedValueOnce({ Summaries: [{ Status: 'FAILED' }] });
+    mockSend.mockResolvedValueOnce({}); // delete
+    mockSend.mockResolvedValueOnce({
+      Stacks: [{ StackStatus: 'DELETE_FAILED', StackStatusReason: 'permission denied' }],
+    });
+
+    await expect(
+      recoverReviewInProgressStack('us-east-1', 'MyStack', {
+        pollIntervalMs: 1,
+        timeoutMs: 1000,
+        client: makeClient(),
+      })
+    ).rejects.toThrow(/Failed to delete stack/);
+  });
+
+  it('proceeds with deletion when ListChangeSets returns no summaries (with warning)', async () => {
+    // CloudFormation may auto-purge old change sets; REVIEW_IN_PROGRESS itself
+    // is sufficient evidence that the stack contains no resources.
+    mockSend.mockResolvedValueOnce({ Stacks: [{ StackStatus: 'REVIEW_IN_PROGRESS' }] });
+    mockSend.mockResolvedValueOnce({ Summaries: [] });
+    mockSend.mockResolvedValueOnce({}); // delete
+    const validationErr = new Error('Stack does not exist');
+    validationErr.name = 'ValidationError';
+    mockSend.mockRejectedValueOnce(validationErr);
+
+    const warnings: string[] = [];
+    const result = await recoverReviewInProgressStack('us-east-1', 'MyStack', {
+      pollIntervalMs: 1,
+      timeoutMs: 1000,
+      client: makeClient(),
+      onWarning: msg => warnings.push(msg),
+    });
+    expect(result.deleted).toBe(true);
+    expect(result.changeSetCount).toBe(0);
+    expect(warnings.length).toBe(1);
+    expect(warnings[0]).toContain('no change sets');
+    const deleteCalls = mockSend.mock.calls.filter(c => c[0] instanceof DeleteStackCommand);
+    expect(deleteCalls.length).toBe(1);
+  });
+
+  it('treats Status=CREATE_COMPLETE + ExecutionStatus=AVAILABLE as recoverable', async () => {
+    // This is the typical shape of the change set that *put* the stack into
+    // REVIEW_IN_PROGRESS in the first place — created successfully but never
+    // executed (e.g. the user is recovering from a prior early-validation
+    // failure flow).
+    mockSend.mockResolvedValueOnce({ Stacks: [{ StackStatus: 'REVIEW_IN_PROGRESS' }] });
+    mockSend.mockResolvedValueOnce({
+      Summaries: [{ Status: 'CREATE_COMPLETE', ExecutionStatus: 'AVAILABLE' }],
+    });
+    mockSend.mockResolvedValueOnce({}); // delete
+    const validationErr = new Error('Stack does not exist');
+    validationErr.name = 'ValidationError';
+    mockSend.mockRejectedValueOnce(validationErr);
+
+    const result = await recoverReviewInProgressStack('us-east-1', 'MyStack', {
+      pollIntervalMs: 1,
+      timeoutMs: 1000,
+      client: makeClient(),
+    });
+    expect(result.deleted).toBe(true);
+    expect(result.allChangeSetsNonExecuted).toBe(true);
+  });
+
+  it('treats ExecutionStatus=EXECUTE_FAILED as recoverable', async () => {
+    mockSend.mockResolvedValueOnce({ Stacks: [{ StackStatus: 'REVIEW_IN_PROGRESS' }] });
+    mockSend.mockResolvedValueOnce({
+      Summaries: [{ Status: 'CREATE_COMPLETE', ExecutionStatus: 'EXECUTE_FAILED' }],
+    });
+    mockSend.mockResolvedValueOnce({}); // delete
+    const validationErr = new Error('Stack does not exist');
+    validationErr.name = 'ValidationError';
+    mockSend.mockRejectedValueOnce(validationErr);
+
+    const result = await recoverReviewInProgressStack('us-east-1', 'MyStack', {
+      pollIntervalMs: 1,
+      timeoutMs: 1000,
+      client: makeClient(),
+    });
+    expect(result.deleted).toBe(true);
+  });
+
+  it('throws a Timed out error when the deadline elapses without delete completing', async () => {
+    mockSend.mockResolvedValueOnce({ Stacks: [{ StackStatus: 'REVIEW_IN_PROGRESS' }] });
+    mockSend.mockResolvedValueOnce({ Summaries: [{ Status: 'FAILED' }] });
+    mockSend.mockResolvedValueOnce({}); // delete
+    // Subsequent describes always return DELETE_IN_PROGRESS
+    mockSend.mockResolvedValue({ Stacks: [{ StackStatus: 'DELETE_IN_PROGRESS' }] });
+
+    await expect(
+      recoverReviewInProgressStack('us-east-1', 'MyStack', {
+        pollIntervalMs: 1,
+        timeoutMs: 5,
+        client: makeClient(),
+      })
+    ).rejects.toThrow(/Timed out waiting for stack/);
+  });
+
+  it('re-throws non-ValidationError errors raised by DescribeStacks during polling', async () => {
+    mockSend.mockResolvedValueOnce({ Stacks: [{ StackStatus: 'REVIEW_IN_PROGRESS' }] });
+    mockSend.mockResolvedValueOnce({ Summaries: [{ Status: 'FAILED' }] });
+    mockSend.mockResolvedValueOnce({}); // delete
+    const accessDenied = new Error('Access denied');
+    accessDenied.name = 'AccessDeniedException';
+    mockSend.mockRejectedValueOnce(accessDenied);
+
+    await expect(
+      recoverReviewInProgressStack('us-east-1', 'MyStack', {
+        pollIntervalMs: 1,
+        timeoutMs: 1000,
+        client: makeClient(),
+      })
+    ).rejects.toThrow(/Access denied/);
+  });
+
+  it('treats ValidationException (SDK v3 surface) as stack-deleted during polling', async () => {
+    mockSend.mockResolvedValueOnce({ Stacks: [{ StackStatus: 'REVIEW_IN_PROGRESS' }] });
+    mockSend.mockResolvedValueOnce({ Summaries: [{ Status: 'FAILED' }] });
+    mockSend.mockResolvedValueOnce({}); // delete
+    const validationEx = new Error('Stack with id MyStack does not exist');
+    validationEx.name = 'ValidationException';
+    mockSend.mockRejectedValueOnce(validationEx);
+
+    const result = await recoverReviewInProgressStack('us-east-1', 'MyStack', {
+      pollIntervalMs: 1,
+      timeoutMs: 1000,
+      client: makeClient(),
+    });
+    expect(result.deleted).toBe(true);
+  });
+
+  it('treats arbitrary "does not exist" errors as stack-deleted during polling', async () => {
+    mockSend.mockResolvedValueOnce({ Stacks: [{ StackStatus: 'REVIEW_IN_PROGRESS' }] });
+    mockSend.mockResolvedValueOnce({ Summaries: [{ Status: 'FAILED' }] });
+    mockSend.mockResolvedValueOnce({}); // delete
+    const wrappedErr = new Error('Stack with id MyStack does not exist');
+    wrappedErr.name = 'CloudFormationServiceException';
+    mockSend.mockRejectedValueOnce(wrappedErr);
+
+    const result = await recoverReviewInProgressStack('us-east-1', 'MyStack', {
+      pollIntervalMs: 1,
+      timeoutMs: 1000,
+      client: makeClient(),
+    });
+    expect(result.deleted).toBe(true);
+  });
+
+  it('invokes onProgress callback for each poll iteration', async () => {
+    mockSend.mockResolvedValueOnce({ Stacks: [{ StackStatus: 'REVIEW_IN_PROGRESS' }] });
+    mockSend.mockResolvedValueOnce({ Summaries: [{ Status: 'FAILED' }] });
+    mockSend.mockResolvedValueOnce({}); // delete
+    // First poll returns DELETE_IN_PROGRESS, second poll returns "not found"
+    mockSend.mockResolvedValueOnce({ Stacks: [{ StackStatus: 'DELETE_IN_PROGRESS' }] });
+    const validationErr = new Error('Stack does not exist');
+    validationErr.name = 'ValidationError';
+    mockSend.mockRejectedValueOnce(validationErr);
+
+    const heartbeats: { stackStatus: string; elapsedMs: number }[] = [];
+    const result = await recoverReviewInProgressStack('us-east-1', 'MyStack', {
+      pollIntervalMs: 1,
+      timeoutMs: 1000,
+      client: makeClient(),
+      onProgress: info => heartbeats.push(info),
+    });
+    expect(result.deleted).toBe(true);
+    expect(heartbeats.length).toBeGreaterThanOrEqual(2);
+    expect(heartbeats[0]!.stackStatus).toBe('DELETE_IN_PROGRESS');
+    expect(heartbeats[heartbeats.length - 1]!.stackStatus).toBe('DELETE_COMPLETE');
+  });
+});

src/cli/cloudformation/__tests__/stack-status.test.ts

@@ -50,7 +50,6 @@ describe('checkStackStatus', () => {
       'DELETE_IN_PROGRESS',
       'ROLLBACK_IN_PROGRESS',
       'UPDATE_ROLLBACK_IN_PROGRESS',
-      'REVIEW_IN_PROGRESS',
     ];
 
     for (const status of inProgressStatuses) {
@@ -59,9 +58,20 @@ describe('checkStackStatus', () => {
       expect(result.canDeploy, `${status} should block deploy`).toBe(false);
       expect(result.exists).toBe(true);
       expect(result.message).toContain(status);
+      expect(result.isRecoverableReview).toBeFalsy();
     }
   });
 
+  it('returns canDeploy false and isRecoverableReview true for REVIEW_IN_PROGRESS', async () => {
+    mockSend.mockResolvedValue({ Stacks: [{ StackStatus: 'REVIEW_IN_PROGRESS' }] });
+    const result = await checkStackStatus('us-east-1', 'MyStack');
+    expect(result.canDeploy).toBe(false);
+    expect(result.exists).toBe(true);
+    expect(result.status).toBe('REVIEW_IN_PROGRESS');
+    expect(result.isRecoverableReview).toBe(true);
+    expect(result.message).toContain('--recover');
+  });
+
   it('returns canDeploy false for failed status', async () => {
     const failedStatuses = ['CREATE_FAILED', 'ROLLBACK_FAILED', 'DELETE_FAILED', 'UPDATE_ROLLBACK_FAILED'];
 

src/cli/cloudformation/index.ts

@@ -1,5 +1,6 @@
 export * from './bootstrap';
 export * from './outputs';
+export * from './stack-cleanup';
 export * from './stack-discovery';
 export * from './stack-status';
 export * from './types';