From 7bd647707fdf38bd0963c1ee87991c7e4b23efe8 Mon Sep 17 00:00:00 2001
From: Eric Zhang <chong.zhang@pingcap.com>
Date: Thu, 30 Jul 2026 17:41:45 +0800
Subject: [PATCH 1/9] feat(ci): add production deployment workflows

---
 .dockerignore                                 |  23 ++
 .github/workflows/ci.yml                      |  56 +++
 .github/workflows/deploy-production.yml       | 350 ++++++++++++++++++
 .github/workflows/rollback-production.yml     | 238 ++++++++++++
 .gitignore                                    |   1 +
 README.md                                     |   8 +
 ...eep-analysis-duplicate-ops.service.spec.ts |  17 +-
 apps/api/src/user-profile.service.spec.ts     |   4 +-
 docs/production-deployment.md                 | 164 ++++++++
 infra/terraform/ecr.tf                        |   4 +-
 infra/terraform/ecs_services.tf               |   2 +-
 infra/terraform/ecs_tasks.tf                  |  16 +-
 infra/terraform/main.tf                       |   4 +-
 infra/terraform/outputs.tf                    |  20 +
 infra/terraform/terraform.tfvars.example      |   4 +
 infra/terraform/variables.tf                  |  30 ++
 16 files changed, 928 insertions(+), 13 deletions(-)
 create mode 100644 .dockerignore
 create mode 100644 .github/workflows/ci.yml
 create mode 100644 .github/workflows/deploy-production.yml
 create mode 100644 .github/workflows/rollback-production.yml
 create mode 100644 docs/production-deployment.md

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..5b5246d
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,23 @@
+# Source-control and CI metadata
+.git
+.github
+
+# Local dependencies, caches, logs, and build output
+**/node_modules
+.pnpm-store
+**/dist
+coverage
+logs
+*.log
+*.tsbuildinfo
+.DS_Store
+
+# Local environment and credential files
+.env
+.env.*
+.sentryclirc
+
+# Terraform configuration is not part of the application image. Excluding the
+# whole directory also keeps local variables, state, plans, and backups out of
+# the Docker build context.
+infra/terraform
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..f5170d3
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,56 @@
+name: CI
+
+on:
+  pull_request:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+env:
+  TERRAFORM_VERSION: 1.15.4
+  TF_WORKING_DIR: infra/terraform
+
+jobs:
+  verify:
+    name: Verify
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+
+      - name: Set up pnpm
+        uses: pnpm/action-setup@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version-file: .nvmrc
+          cache: pnpm
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Verify application
+        run: pnpm verify
+
+      - name: Set up Terraform
+        uses: hashicorp/setup-terraform@v4
+        with:
+          terraform_version: ${{ env.TERRAFORM_VERSION }}
+          terraform_wrapper: false
+
+      - name: Check Terraform formatting
+        run: terraform fmt -check -recursive "$TF_WORKING_DIR"
+
+      - name: Validate Terraform
+        run: |
+          terraform -chdir="$TF_WORKING_DIR" init -backend=false -input=false
+          terraform -chdir="$TF_WORKING_DIR" validate
diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml
new file mode 100644
index 0000000..3b2d213
--- /dev/null
+++ b/.github/workflows/deploy-production.yml
@@ -0,0 +1,350 @@
+name: Deploy production
+run-name: Deploy ${{ github.sha }} to production
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: production
+  cancel-in-progress: false
+
+env:
+  AWS_REGION: ap-southeast-1
+  ECR_REGISTRY: 401696231252.dkr.ecr.ap-southeast-1.amazonaws.com
+  API_REPOSITORY: mem9-node-prod-api
+  WORKER_REPOSITORY: mem9-node-prod-worker
+  TF_STATE_BUCKET: mem9-node-prod-terraform-state
+  TF_STATE_KMS_KEY_ARN: arn:aws:kms:ap-southeast-1:401696231252:key/4cd4c7a1-c1a6-47ad-9205-548678b84d86
+  TERRAFORM_VERSION: 1.15.4
+  TF_WORKING_DIR: infra/terraform
+  TF_IN_AUTOMATION: "true"
+  TF_INPUT: "false"
+
+jobs:
+  deploy:
+    name: Build and deploy current main
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    permissions:
+      actions: write
+      contents: read
+      id-token: write
+
+    steps:
+      - name: Check out the release commit
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.sha }}
+
+      - name: Verify release source
+        run: |
+          test "$GITHUB_REF" = "refs/heads/main"
+          test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
+
+      - name: Configure AWS image credentials
+        uses: aws-actions/configure-aws-credentials@v6
+        with:
+          role-to-assume: arn:aws:iam::401696231252:role/mem9-node-prod-github-prepare
+          role-session-name: mem9-node-production-images
+          aws-region: ${{ env.AWS_REGION }}
+          mask-aws-account-id: true
+
+      - name: Log in to Amazon ECR
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Build and push API image
+        run: |
+          if aws ecr describe-images \
+            --repository-name "$API_REPOSITORY" \
+            --image-ids "imageTag=$GITHUB_SHA" \
+            --region "$AWS_REGION" > /dev/null 2>&1; then
+            echo "API image $GITHUB_SHA already exists; reusing it."
+          else
+            docker build \
+              --platform linux/amd64 \
+              --file apps/api/Dockerfile \
+              --tag "$ECR_REGISTRY/$API_REPOSITORY:$GITHUB_SHA" \
+              .
+            docker push "$ECR_REGISTRY/$API_REPOSITORY:$GITHUB_SHA"
+          fi
+
+      - name: Build and push worker image
+        run: |
+          if aws ecr describe-images \
+            --repository-name "$WORKER_REPOSITORY" \
+            --image-ids "imageTag=$GITHUB_SHA" \
+            --region "$AWS_REGION" > /dev/null 2>&1; then
+            echo "Worker image $GITHUB_SHA already exists; reusing it."
+          else
+            docker build \
+              --platform linux/amd64 \
+              --file apps/worker/Dockerfile \
+              --tag "$ECR_REGISTRY/$WORKER_REPOSITORY:$GITHUB_SHA" \
+              .
+            docker push "$ECR_REGISTRY/$WORKER_REPOSITORY:$GITHUB_SHA"
+          fi
+
+      - name: Lock release images
+        run: |
+          API_IMAGE="$ECR_REGISTRY/$API_REPOSITORY:$GITHUB_SHA"
+          WORKER_IMAGE="$ECR_REGISTRY/$WORKER_REPOSITORY:$GITHUB_SHA"
+          API_DIGEST=$(aws ecr describe-images \
+            --repository-name "$API_REPOSITORY" \
+            --image-ids "imageTag=$GITHUB_SHA" \
+            --query 'imageDetails[0].imageDigest' \
+            --output text \
+            --region "$AWS_REGION")
+          WORKER_DIGEST=$(aws ecr describe-images \
+            --repository-name "$WORKER_REPOSITORY" \
+            --image-ids "imageTag=$GITHUB_SHA" \
+            --query 'imageDetails[0].imageDigest' \
+            --output text \
+            --region "$AWS_REGION")
+
+          test -n "$API_DIGEST" && test "$API_DIGEST" != "None"
+          test -n "$WORKER_DIGEST" && test "$WORKER_DIGEST" != "None"
+
+          {
+            echo "API_IMAGE=$API_IMAGE"
+            echo "WORKER_IMAGE=$WORKER_IMAGE"
+            echo "API_DIGEST=$API_DIGEST"
+            echo "WORKER_DIGEST=$WORKER_DIGEST"
+          } >> "$GITHUB_ENV"
+
+      - name: Configure AWS deployment credentials
+        uses: aws-actions/configure-aws-credentials@v6
+        with:
+          role-to-assume: arn:aws:iam::401696231252:role/mem9-node-prod-github-apply
+          role-session-name: mem9-node-production-deploy
+          aws-region: ${{ env.AWS_REGION }}
+          mask-aws-account-id: true
+
+      - name: Read current ECS release
+        run: |
+          API_PREVIOUS_TASK_DEFINITION=$(aws ecs describe-services \
+            --cluster mem9-node-prod-cluster \
+            --services mem9-node-prod-api \
+            --query 'services[0].taskDefinition' \
+            --output text \
+            --region "$AWS_REGION")
+          WORKER_PREVIOUS_TASK_DEFINITION=$(aws ecs describe-services \
+            --cluster mem9-node-prod-cluster \
+            --services mem9-node-prod-worker \
+            --query 'services[0].taskDefinition' \
+            --output text \
+            --region "$AWS_REGION")
+
+          test -n "$API_PREVIOUS_TASK_DEFINITION" && test "$API_PREVIOUS_TASK_DEFINITION" != "None"
+          test -n "$WORKER_PREVIOUS_TASK_DEFINITION" && test "$WORKER_PREVIOUS_TASK_DEFINITION" != "None"
+
+          {
+            echo "API_PREVIOUS_TASK_DEFINITION=$API_PREVIOUS_TASK_DEFINITION"
+            echo "WORKER_PREVIOUS_TASK_DEFINITION=$WORKER_PREVIOUS_TASK_DEFINITION"
+          } >> "$GITHUB_ENV"
+
+      - name: Set up Terraform
+        uses: hashicorp/setup-terraform@v4
+        with:
+          terraform_version: ${{ env.TERRAFORM_VERSION }}
+          terraform_wrapper: false
+
+      - name: Write production variables
+        env:
+          TF_VARS: ${{ secrets.TF_VARS }}
+        run: |
+          test -n "$TF_VARS"
+          printf '%s\n' "$TF_VARS" > "$TF_WORKING_DIR/production.auto.tfvars"
+
+      - name: Initialize Terraform
+        run: |
+          terraform -chdir="$TF_WORKING_DIR" init \
+            -input=false \
+            -backend-config="bucket=$TF_STATE_BUCKET" \
+            -backend-config="key=mem9-node/production/terraform.tfstate" \
+            -backend-config="region=$AWS_REGION" \
+            -backend-config="encrypt=true" \
+            -backend-config="kms_key_id=$TF_STATE_KMS_KEY_ARN" \
+            -backend-config="use_lockfile=true"
+
+      - name: Plan production release
+        run: |
+          terraform -chdir="$TF_WORKING_DIR" plan \
+            -input=false \
+            -var="release_id=$GITHUB_SHA" \
+            -var="api_image=$API_IMAGE" \
+            -var="worker_image=$WORKER_IMAGE" \
+            -out=tfplan
+
+      - name: Summarize Terraform plan
+        run: |
+          PLAN_JSON=$(mktemp)
+          terraform -chdir="$TF_WORKING_DIR" show -json tfplan > "$PLAN_JSON"
+
+          ADD_COUNT=$(jq '[.resource_changes[]? | select(.change.actions | index("create"))] | length' "$PLAN_JSON")
+          CHANGE_COUNT=$(jq '[.resource_changes[]? | select(.change.actions | index("update"))] | length' "$PLAN_JSON")
+          DESTROY_COUNT=$(jq '[.resource_changes[]? | select(.change.actions | index("delete"))] | length' "$PLAN_JSON")
+
+          {
+            echo "## Production plan"
+            echo
+            echo "- Release: \`$GITHUB_SHA\`"
+            echo "- API image: \`$API_IMAGE\`"
+            echo "- Worker image: \`$WORKER_IMAGE\`"
+            echo "- Terraform: **$ADD_COUNT add**, **$CHANGE_COUNT change**, **$DESTROY_COUNT destroy**"
+            echo "- Database migration: not run"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Record pre-deployment restore point
+        run: |
+          jq -n \
+            --arg release "$GITHUB_SHA" \
+            --arg previousApiTaskDefinition "$API_PREVIOUS_TASK_DEFINITION" \
+            --arg previousWorkerTaskDefinition "$WORKER_PREVIOUS_TASK_DEFINITION" \
+            --arg workflowActor "$GITHUB_ACTOR" \
+            --arg repository "$GITHUB_REPOSITORY" \
+            --arg runId "$GITHUB_RUN_ID" \
+            --arg preparedAt "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \
+            '{
+              release: $release,
+              previousApiTaskDefinition: $previousApiTaskDefinition,
+              previousWorkerTaskDefinition: $previousWorkerTaskDefinition,
+              workflowActor: $workflowActor,
+              repository: $repository,
+              runId: $runId,
+              preparedAt: $preparedAt,
+              status: "prepared"
+            }' > restore-point.json
+
+      - name: Upload pre-deployment restore point
+        id: upload-restore-point
+        uses: actions/upload-artifact@v7
+        with:
+          name: production-restore-point-${{ github.sha }}-${{ github.run_id }}
+          path: restore-point.json
+          if-no-files-found: error
+          retention-days: 90
+
+      - name: Apply production release
+        run: terraform -chdir="$TF_WORKING_DIR" apply -input=false -auto-approve tfplan
+
+      - name: Wait for ECS services
+        run: |
+          ECS_CLUSTER=$(terraform -chdir="$TF_WORKING_DIR" output -raw ecs_cluster_name)
+          API_SERVICE=$(terraform -chdir="$TF_WORKING_DIR" output -raw api_service_name)
+          WORKER_SERVICE=$(terraform -chdir="$TF_WORKING_DIR" output -raw worker_service_name)
+
+          aws ecs wait services-stable \
+            --cluster "$ECS_CLUSTER" \
+            --services "$API_SERVICE" "$WORKER_SERVICE" \
+            --region "$AWS_REGION"
+
+      - name: Record successful release
+        run: |
+          API_TASK_DEFINITION=$(terraform -chdir="$TF_WORKING_DIR" output -raw api_task_definition_arn)
+          WORKER_TASK_DEFINITION=$(terraform -chdir="$TF_WORKING_DIR" output -raw worker_task_definition_arn)
+
+          jq -n \
+            --arg release "$GITHUB_SHA" \
+            --arg apiImage "$API_IMAGE" \
+            --arg apiDigest "$API_DIGEST" \
+            --arg workerImage "$WORKER_IMAGE" \
+            --arg workerDigest "$WORKER_DIGEST" \
+            --arg apiTaskDefinition "$API_TASK_DEFINITION" \
+            --arg workerTaskDefinition "$WORKER_TASK_DEFINITION" \
+            --arg previousApiTaskDefinition "$API_PREVIOUS_TASK_DEFINITION" \
+            --arg previousWorkerTaskDefinition "$WORKER_PREVIOUS_TASK_DEFINITION" \
+            --arg workflowActor "$GITHUB_ACTOR" \
+            --arg repository "$GITHUB_REPOSITORY" \
+            --arg runId "$GITHUB_RUN_ID" \
+            --arg deployedAt "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \
+            '{
+              release: $release,
+              apiImage: $apiImage,
+              apiDigest: $apiDigest,
+              workerImage: $workerImage,
+              workerDigest: $workerDigest,
+              apiTaskDefinition: $apiTaskDefinition,
+              workerTaskDefinition: $workerTaskDefinition,
+              previousApiTaskDefinition: $previousApiTaskDefinition,
+              previousWorkerTaskDefinition: $previousWorkerTaskDefinition,
+              workflowActor: $workflowActor,
+              repository: $repository,
+              runId: $runId,
+              deployedAt: $deployedAt,
+              status: "deployed"
+            }' > release.json
+
+          {
+            echo
+            echo "## Production deployed"
+            echo
+            echo "- API task definition: \`$API_TASK_DEFINITION\`"
+            echo "- Worker task definition: \`$WORKER_TASK_DEFINITION\`"
+            echo "- ECS services: stable"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload release record
+        uses: actions/upload-artifact@v7
+        with:
+          name: production-release-${{ github.sha }}-${{ github.run_id }}
+          path: release.json
+          if-no-files-found: error
+          retention-days: 90
+
+      - name: Delete pre-deployment restore point
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RESTORE_POINT_ARTIFACT_ID: ${{ steps.upload-restore-point.outputs.artifact-id }}
+        run: |
+          test -n "$RESTORE_POINT_ARTIFACT_ID"
+          gh api --method DELETE \
+            "/repos/$GITHUB_REPOSITORY/actions/artifacts/$RESTORE_POINT_ARTIFACT_ID"
+
+      - name: Keep the latest 30 task definition revisions
+        env:
+          TASK_DEFINITION_RETENTION: "30"
+        run: |
+          for family in mem9-node-prod-api mem9-node-prod-worker; do
+            aws ecs list-task-definitions \
+              --family-prefix "$family" \
+              --status ACTIVE \
+              --sort DESC \
+              --region "$AWS_REGION" \
+              --query 'taskDefinitionArns[]' \
+              --output text \
+              | tr '\t' '\n' \
+              | grep "/$family:" \
+              | tail -n "+$((TASK_DEFINITION_RETENTION + 1))" \
+              | while read -r task_definition; do
+                  if [ -n "$task_definition" ]; then
+                    aws ecs deregister-task-definition \
+                      --task-definition "$task_definition" \
+                      --region "$AWS_REGION" > /dev/null
+                  fi
+                done
+          done
+
+      - name: Keep the latest 30 release records
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh api --paginate "/repos/$GITHUB_REPOSITORY/actions/artifacts?per_page=100" \
+            --jq '.artifacts[]
+              | select(.expired == false)
+              | select(.name | startswith("production-release-"))
+              | [.created_at, (.id | tostring)]
+              | @tsv' \
+            | sort -r \
+            | tail -n +31 \
+            | cut -f2 \
+            | while read -r artifact_id; do
+                if [ -n "$artifact_id" ]; then
+                  gh api --method DELETE "/repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact_id"
+                fi
+              done
diff --git a/.github/workflows/rollback-production.yml b/.github/workflows/rollback-production.yml
new file mode 100644
index 0000000..e8363fb
--- /dev/null
+++ b/.github/workflows/rollback-production.yml
@@ -0,0 +1,238 @@
+name: Roll back production
+run-name: Roll back production to its previous release
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: production
+  cancel-in-progress: false
+
+env:
+  AWS_REGION: ap-southeast-1
+  ECS_CLUSTER: mem9-node-prod-cluster
+  API_SERVICE: mem9-node-prod-api
+  WORKER_SERVICE: mem9-node-prod-worker
+
+jobs:
+  rollback:
+    name: Roll back to previous successful release
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      actions: write
+      contents: read
+      id-token: write
+
+    steps:
+      - name: Configure AWS deployment credentials
+        uses: aws-actions/configure-aws-credentials@v6
+        with:
+          role-to-assume: arn:aws:iam::401696231252:role/mem9-node-prod-github-apply
+          role-session-name: mem9-node-production-rollback
+          aws-region: ${{ env.AWS_REGION }}
+          mask-aws-account-id: true
+
+      - name: Read current ECS release
+        run: |
+          CURRENT_API_TASK_DEFINITION=$(aws ecs describe-services \
+            --cluster "$ECS_CLUSTER" \
+            --services "$API_SERVICE" \
+            --query 'services[0].taskDefinition' \
+            --output text \
+            --region "$AWS_REGION")
+          CURRENT_WORKER_TASK_DEFINITION=$(aws ecs describe-services \
+            --cluster "$ECS_CLUSTER" \
+            --services "$WORKER_SERVICE" \
+            --query 'services[0].taskDefinition' \
+            --output text \
+            --region "$AWS_REGION")
+
+          test -n "$CURRENT_API_TASK_DEFINITION" && test "$CURRENT_API_TASK_DEFINITION" != "None"
+          test -n "$CURRENT_WORKER_TASK_DEFINITION" && test "$CURRENT_WORKER_TASK_DEFINITION" != "None"
+
+          {
+            echo "CURRENT_API_TASK_DEFINITION=$CURRENT_API_TASK_DEFINITION"
+            echo "CURRENT_WORKER_TASK_DEFINITION=$CURRENT_WORKER_TASK_DEFINITION"
+          } >> "$GITHUB_ENV"
+
+      - name: Find the previous release pair
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          RECORD_DIR="$RUNNER_TEMP/production-releases"
+          mkdir -p "$RECORD_DIR"
+
+          gh api --paginate "/repos/$GITHUB_REPOSITORY/actions/artifacts?per_page=100" \
+            --jq '.artifacts[]
+              | select(.expired == false)
+              | select(.name | startswith("production-release-"))
+              | [.created_at, (.id | tostring)]
+              | @tsv' \
+            | sort -r > "$RECORD_DIR/releases.tsv"
+
+          gh api --paginate "/repos/$GITHUB_REPOSITORY/actions/artifacts?per_page=100" \
+            --jq '.artifacts[]
+              | select(.expired == false)
+              | select(.name | startswith("production-restore-point-"))
+              | [.created_at, (.id | tostring)]
+              | @tsv' \
+            | sort -r > "$RECORD_DIR/restore-points.tsv"
+
+          TARGET_API_TASK_DEFINITION=""
+          TARGET_WORKER_TASK_DEFINITION=""
+          SOURCE_RELEASE=""
+          FALLBACK_API_TASK_DEFINITION=""
+          FALLBACK_WORKER_TASK_DEFINITION=""
+          FALLBACK_RELEASE=""
+
+          while IFS=$'\t' read -r _ artifact_id; do
+            test -n "$artifact_id" || continue
+
+            archive="$RECORD_DIR/$artifact_id.zip"
+            record="$RECORD_DIR/$artifact_id.json"
+            gh api "/repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact_id/zip" > "$archive"
+            unzip -p "$archive" release.json > "$record"
+
+            recorded_api=$(jq -r '.apiTaskDefinition // empty' "$record")
+            recorded_worker=$(jq -r '.workerTaskDefinition // empty' "$record")
+
+            if [ -z "$FALLBACK_API_TASK_DEFINITION" ] && [ -n "$recorded_api" ] && [ -n "$recorded_worker" ]; then
+              FALLBACK_API_TASK_DEFINITION="$recorded_api"
+              FALLBACK_WORKER_TASK_DEFINITION="$recorded_worker"
+              FALLBACK_RELEASE=$(jq -r '.release // empty' "$record")
+            fi
+
+            if [ "$recorded_api" = "$CURRENT_API_TASK_DEFINITION" ] && \
+               [ "$recorded_worker" = "$CURRENT_WORKER_TASK_DEFINITION" ]; then
+              previous_api=$(jq -r '.previousApiTaskDefinition // empty' "$record")
+              previous_worker=$(jq -r '.previousWorkerTaskDefinition // empty' "$record")
+
+              if [ -n "$previous_api" ] && [ -n "$previous_worker" ] && \
+                 [ "$previous_api" != "$CURRENT_API_TASK_DEFINITION" ] && \
+                 [ "$previous_worker" != "$CURRENT_WORKER_TASK_DEFINITION" ]; then
+                TARGET_API_TASK_DEFINITION="$previous_api"
+                TARGET_WORKER_TASK_DEFINITION="$previous_worker"
+                SOURCE_RELEASE=$(jq -r '.release // empty' "$record")
+                break
+              fi
+            fi
+          done < "$RECORD_DIR/releases.tsv"
+
+          if [ -z "$TARGET_API_TASK_DEFINITION" ] && \
+             [ -n "$FALLBACK_API_TASK_DEFINITION" ] && \
+             [ -n "$FALLBACK_WORKER_TASK_DEFINITION" ]; then
+            TARGET_API_TASK_DEFINITION="$FALLBACK_API_TASK_DEFINITION"
+            TARGET_WORKER_TASK_DEFINITION="$FALLBACK_WORKER_TASK_DEFINITION"
+            SOURCE_RELEASE="unrecorded-current-to-$FALLBACK_RELEASE"
+          fi
+
+          if [ -z "$TARGET_API_TASK_DEFINITION" ] || [ -z "$TARGET_WORKER_TASK_DEFINITION" ]; then
+            while IFS=$'\t' read -r _ artifact_id; do
+              test -n "$artifact_id" || continue
+
+              archive="$RECORD_DIR/restore-$artifact_id.zip"
+              record="$RECORD_DIR/restore-$artifact_id.json"
+              gh api "/repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact_id/zip" > "$archive"
+              unzip -p "$archive" restore-point.json > "$record"
+
+              previous_api=$(jq -r '.previousApiTaskDefinition // empty' "$record")
+              previous_worker=$(jq -r '.previousWorkerTaskDefinition // empty' "$record")
+
+              if [ -n "$previous_api" ] && [ -n "$previous_worker" ] && \
+                 { [ "$previous_api" != "$CURRENT_API_TASK_DEFINITION" ] || \
+                   [ "$previous_worker" != "$CURRENT_WORKER_TASK_DEFINITION" ]; }; then
+                TARGET_API_TASK_DEFINITION="$previous_api"
+                TARGET_WORKER_TASK_DEFINITION="$previous_worker"
+                SOURCE_RELEASE="failed-$(jq -r '.release // "unknown"' "$record")"
+                break
+              fi
+            done < "$RECORD_DIR/restore-points.tsv"
+          fi
+
+          test -n "$TARGET_API_TASK_DEFINITION"
+          test -n "$TARGET_WORKER_TASK_DEFINITION"
+          test "$TARGET_API_TASK_DEFINITION" != "$CURRENT_API_TASK_DEFINITION" || \
+            test "$TARGET_WORKER_TASK_DEFINITION" != "$CURRENT_WORKER_TASK_DEFINITION"
+
+          {
+            echo "TARGET_API_TASK_DEFINITION=$TARGET_API_TASK_DEFINITION"
+            echo "TARGET_WORKER_TASK_DEFINITION=$TARGET_WORKER_TASK_DEFINITION"
+            echo "SOURCE_RELEASE=$SOURCE_RELEASE"
+          } >> "$GITHUB_ENV"
+
+      - name: Verify target task definitions
+        run: |
+          test "$(aws ecs describe-task-definition \
+            --task-definition "$TARGET_API_TASK_DEFINITION" \
+            --query 'taskDefinition.status' \
+            --output text \
+            --region "$AWS_REGION")" = "ACTIVE"
+          test "$(aws ecs describe-task-definition \
+            --task-definition "$TARGET_WORKER_TASK_DEFINITION" \
+            --query 'taskDefinition.status' \
+            --output text \
+            --region "$AWS_REGION")" = "ACTIVE"
+
+      - name: Roll back ECS services
+        run: |
+          aws ecs update-service \
+            --cluster "$ECS_CLUSTER" \
+            --service "$API_SERVICE" \
+            --task-definition "$TARGET_API_TASK_DEFINITION" \
+            --region "$AWS_REGION" > /dev/null
+          aws ecs update-service \
+            --cluster "$ECS_CLUSTER" \
+            --service "$WORKER_SERVICE" \
+            --task-definition "$TARGET_WORKER_TASK_DEFINITION" \
+            --region "$AWS_REGION" > /dev/null
+
+      - name: Wait for ECS services
+        run: |
+          aws ecs wait services-stable \
+            --cluster "$ECS_CLUSTER" \
+            --services "$API_SERVICE" "$WORKER_SERVICE" \
+            --region "$AWS_REGION"
+
+      - name: Record rollback
+        run: |
+          jq -n \
+            --arg sourceRelease "$SOURCE_RELEASE" \
+            --arg fromApiTaskDefinition "$CURRENT_API_TASK_DEFINITION" \
+            --arg fromWorkerTaskDefinition "$CURRENT_WORKER_TASK_DEFINITION" \
+            --arg toApiTaskDefinition "$TARGET_API_TASK_DEFINITION" \
+            --arg toWorkerTaskDefinition "$TARGET_WORKER_TASK_DEFINITION" \
+            --arg workflowActor "$GITHUB_ACTOR" \
+            --arg rolledBackAt "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \
+            '{
+              sourceRelease: $sourceRelease,
+              fromApiTaskDefinition: $fromApiTaskDefinition,
+              fromWorkerTaskDefinition: $fromWorkerTaskDefinition,
+              toApiTaskDefinition: $toApiTaskDefinition,
+              toWorkerTaskDefinition: $toWorkerTaskDefinition,
+              workflowActor: $workflowActor,
+              rolledBackAt: $rolledBackAt,
+              status: "rolled-back"
+            }' > rollback.json
+
+          {
+            echo "## Production rolled back"
+            echo
+            echo "- From API: \`$CURRENT_API_TASK_DEFINITION\`"
+            echo "- To API: \`$TARGET_API_TASK_DEFINITION\`"
+            echo "- From worker: \`$CURRENT_WORKER_TASK_DEFINITION\`"
+            echo "- To worker: \`$TARGET_WORKER_TASK_DEFINITION\`"
+            echo "- ECS services: stable"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload rollback record
+        uses: actions/upload-artifact@v7
+        with:
+          name: production-rollback-${{ github.run_id }}
+          path: rollback.json
+          if-no-files-found: error
+          retention-days: 90
diff --git a/.gitignore b/.gitignore
index 21d66fd..c58f6db 100644
--- a/.gitignore
+++ b/.gitignore
@@ -49,6 +49,7 @@ web_modules/
 
 # Optional npm cache directory
 .npm
+.pnpm-store/
 
 # Optional eslint cache
 .eslintcache
diff --git a/README.md b/README.md
index 338f23d..d3ab224 100644
--- a/README.md
+++ b/README.md
@@ -48,6 +48,14 @@ Worker health: `http://127.0.0.1:3001/health/live`
 - Build and upload Sentry sourcemaps: `pnpm build:with-sourcemaps`
 - Verify: `pnpm verify`
 
+## Production Deployment
+
+Pull requests run application and Terraform checks. Production is released
+manually from the current `main` commit; one workflow builds immutable API and
+worker images, applies Terraform, and waits for ECS. Rollback automatically
+selects the previous recorded task-definition pair. See [Production
+deployment](docs/production-deployment.md) for setup and operating details.
+
 ## API Flow
 
 ### Create a job
diff --git a/apps/api/src/deep-analysis-duplicate-ops.service.spec.ts b/apps/api/src/deep-analysis-duplicate-ops.service.spec.ts
index b0a4318..b362916 100644
--- a/apps/api/src/deep-analysis-duplicate-ops.service.spec.ts
+++ b/apps/api/src/deep-analysis-duplicate-ops.service.spec.ts
@@ -60,11 +60,18 @@ describe('deep analysis duplicate ops service', () => {
   });
 
   it('delegates duplicate deletion to the mem9 source client', async () => {
+    let resolveDeletionStarted!: () => void;
+    const deletionStarted = new Promise<void>((resolve) => {
+      resolveDeletionStarted = resolve;
+    });
     const source = {
-      deleteMemories: jest.fn(async () => ({
-        deletedMemoryIds: ['mem_2'],
-        failedMemoryIds: ['mem_3'],
-      })),
+      deleteMemories: jest.fn(async () => {
+        resolveDeletionStarted();
+        return {
+          deletedMemoryIds: ['mem_2'],
+          failedMemoryIds: ['mem_3'],
+        };
+      }),
     };
     const service = new DeepAnalysisDuplicateOpsService(
       {
@@ -114,7 +121,7 @@ describe('deep analysis duplicate ops service', () => {
     );
 
     const result = await service.deleteDuplicateMemories(createContext(), 'dar_1');
-    await new Promise<void>((resolve) => setTimeout(resolve, 0));
+    await deletionStarted;
 
     expect(result.duplicateCleanup.status).toBe('QUEUED');
     expect(result.duplicateCleanup.totalCount).toBe(2);
diff --git a/apps/api/src/user-profile.service.spec.ts b/apps/api/src/user-profile.service.spec.ts
index bc5270c..c80acb3 100644
--- a/apps/api/src/user-profile.service.spec.ts
+++ b/apps/api/src/user-profile.service.spec.ts
@@ -259,7 +259,7 @@ describe('user profile service', () => {
     const result = await service.getProfile(createContext());
 
     expect(result.summary.text).toBe('你是一位目标驱动的长期成长型用户，持续推进英语学习、健康管理');
-    expect(result.summary.message).toBe('当前可用画像信息较少，已根据现有 facts、insights 和 pinned 生成初步总结，但画像可能不稳定。');
+    expect(result.summary.message).toBe('当前可用记忆信息较少，已根据现有记忆生成初步总结，但画像可能不稳定。');
     expect(result.summary.evidence).toHaveLength(1);
   });
 
@@ -274,7 +274,7 @@ describe('user profile service', () => {
     const result = await service.getProfile(createContext());
 
     expect(result.summary.text).toContain('整体画像：一个偏理性、重视稳定成长的人');
-    expect(result.summary.message).toBe('当前可用画像信息较少，已根据现有 facts、insights 和 pinned 生成初步总结，但画像可能不稳定。');
+    expect(result.summary.message).toBe('当前可用记忆信息较少，已根据现有记忆生成初步总结，但画像可能不稳定。');
     expect(result.summary.evidence).toHaveLength(1);
   });
 
diff --git a/docs/production-deployment.md b/docs/production-deployment.md
new file mode 100644
index 0000000..948b8fb
--- /dev/null
+++ b/docs/production-deployment.md
@@ -0,0 +1,164 @@
+# Production deployment
+
+Production is released manually from the current `main` commit. Image builds,
+Terraform plan/apply, and ECS deployment happen in one workflow run, so there
+is no separately selected image SHA or saved intermediate Terraform plan.
+
+## Workflows
+
+### Pull-request checks
+
+`.github/workflows/ci.yml` runs for pull requests targeting `main`:
+
+1. Install dependencies from the lockfile.
+2. Run `pnpm verify` (typecheck, unit tests, and application build).
+3. Check Terraform formatting.
+4. Initialize Terraform without a backend and validate the configuration.
+
+The repository does not currently define a benchmark command. Add it to this
+workflow after the project has a stable benchmark script and failure threshold.
+
+### Production release
+
+`.github/workflows/deploy-production.yml` has no inputs. Run it from the `main`
+branch in GitHub Actions. The workflow:
+
+1. Locks the run to the `main` commit identified by `github.sha`.
+2. Assumes the image role through GitHub OIDC.
+3. Builds and pushes API and worker images tagged with that full SHA. A rerun
+   reuses an existing immutable image and builds only a missing image.
+4. Reads both ECR image digests.
+5. Assumes the deployment role through GitHub OIDC.
+6. Reads the currently deployed API and worker task definitions for rollback.
+7. Initializes the KMS-encrypted S3 Terraform backend.
+8. Creates a local `tfplan` in the same runner.
+9. Uploads a temporary restore point containing the task-definition pair that
+   is currently online, then immediately applies the plan.
+10. Waits for both ECS services to become stable.
+11. Uploads a release record containing the SHA, image pair, task-definition
+    pair, and previous task-definition pair.
+12. Deletes the temporary restore point after the successful release record is
+    safely uploaded. Failed runs keep their restore point for recovery.
+
+Terraform owns the ECS task definitions and the ECS services' task-definition
+references. Normal deployment must not call `aws ecs update-service` directly.
+
+The workflow does not use a GitHub Environment. The **Run workflow** action is
+the production release decision. The job-level branch check and both AWS role
+trust policies must restrict production access to `refs/heads/main`.
+
+### Production rollback
+
+`.github/workflows/rollback-production.yml` also has no inputs. It finds the
+release record matching the task-definition pair currently used by ECS and
+switches both services to the previous pair recorded by that release.
+
+If the current deployment failed before it could write a success record, the
+workflow falls back to the newest recorded successful task-definition pair. If
+there has never been a successful CI release, it uses the newest failed run's
+pre-deployment restore point. It refuses to continue if it cannot find a usable
+record or if either target task definition is inactive.
+
+Rollback changes the ECS service references directly and intentionally leaves
+Terraform state untouched. The next production release refreshes the state and
+deploys its new task-definition pair. API and worker updates are coordinated by
+one workflow but are not an atomic AWS transaction.
+
+## Terraform state
+
+Terraform state and its lock file use the existing S3 backend:
+
+```text
+s3://mem9-node-prod-terraform-state/mem9-node/production/terraform.tfstate
+s3://mem9-node-prod-terraform-state/mem9-node/production/terraform.tfstate.tflock
+```
+
+The bucket must keep:
+
+- versioning;
+- default SSE-KMS encryption;
+- Block Public Access;
+- an HTTPS-only bucket policy;
+- state and lock-file permissions scoped to the two objects above.
+
+The bucket no longer stores `tfplan` or release-candidate files. Existing
+`mem9-node/production/plans/` lifecycle rules and IAM permissions are harmless
+but can be removed after the new workflow has been verified.
+
+### Migrate an existing local state
+
+Run this once from the machine holding the real production
+`terraform.tfstate`. Never run production Terraform against a new empty state.
+
+```bash
+terraform -chdir=infra/terraform init -migrate-state \
+  -backend-config="bucket=mem9-node-prod-terraform-state" \
+  -backend-config="key=mem9-node/production/terraform.tfstate" \
+  -backend-config="region=ap-southeast-1" \
+  -backend-config="encrypt=true" \
+  -backend-config="kms_key_id=<TF_STATE_KMS_KEY_ARN>" \
+  -backend-config="use_lockfile=true"
+```
+
+Confirm the remote state before removing the local backup.
+
+## GitHub configuration
+
+Create one repository-level GitHub Actions secret named `TF_VARS`. Its value is
+the complete production `terraform.tfvars` content. The workflow writes it to
+the ephemeral runner without printing it and removes the runner after the job.
+
+Do not add these release-specific values to `TF_VARS`:
+
+```text
+release_id
+api_image
+worker_image
+```
+
+The workflow supplies those three values from the release SHA and its two ECR
+images. Command-line `-var` values take precedence if an older local file still
+contains them, but removing them from `TF_VARS` keeps ownership unambiguous.
+
+Protect `main` and require the `Verify` status check before merge. Block force
+pushes and branch deletion. A benchmark is not a required check until a real
+benchmark command exists in the repository.
+
+## AWS OIDC roles
+
+Keep the two existing roles but restrict both trust policies to this repository
+and the `main` branch subject:
+
+```text
+repo:mem9-ai/mem9-node:ref:refs/heads/main
+```
+
+- `mem9-node-prod-github-prepare`: ECR login, image lookup, and image push for
+  the API and worker repositories only.
+- `mem9-node-prod-github-apply`: Terraform state/KMS access, infrastructure
+  plan/apply permissions, ECS read/update permissions, and task-definition
+  retention permissions.
+
+The apply role's old Environment-based subject must be changed before the new
+deployment and rollback workflows can assume it. The unused GitHub
+`production` Environment may remain or be deleted after verification.
+
+## Retention
+
+- ECR images use immutable full-SHA tags.
+- ECS keeps the latest 30 active API revisions and 30 active worker revisions.
+- GitHub keeps the latest 30 successful release-record artifacts, subject to
+  the repository's artifact retention limit.
+- A failed deployment keeps its temporary pre-deployment restore point. A
+  successful deployment deletes that temporary record after uploading the
+  permanent release record.
+- Rollback records are stored separately and do not participate in selecting a
+  rollback target.
+
+Do not introduce a count-based ECR lifecycle rule until it can preserve every
+image referenced by the retained release records.
+
+## Database migrations
+
+These workflows do not run database migrations. Schema changes require a
+separately reviewed, backward-compatible migration process.
diff --git a/infra/terraform/ecr.tf b/infra/terraform/ecr.tf
index da4b755..58fbd26 100644
--- a/infra/terraform/ecr.tf
+++ b/infra/terraform/ecr.tf
@@ -1,6 +1,6 @@
 resource "aws_ecr_repository" "api" {
   name                 = "${var.name_prefix}-api"
-  image_tag_mutability = "MUTABLE"
+  image_tag_mutability = "IMMUTABLE"
 
   image_scanning_configuration {
     scan_on_push = true
@@ -9,7 +9,7 @@ resource "aws_ecr_repository" "api" {
 
 resource "aws_ecr_repository" "worker" {
   name                 = "${var.name_prefix}-worker"
-  image_tag_mutability = "MUTABLE"
+  image_tag_mutability = "IMMUTABLE"
 
   image_scanning_configuration {
     scan_on_push = true
diff --git a/infra/terraform/ecs_services.tf b/infra/terraform/ecs_services.tf
index 21555a8..442f397 100644
--- a/infra/terraform/ecs_services.tf
+++ b/infra/terraform/ecs_services.tf
@@ -2,7 +2,7 @@ resource "aws_ecs_service" "api" {
   name            = "${var.name_prefix}-api"
   cluster         = aws_ecs_cluster.this.id
   task_definition = aws_ecs_task_definition.api.arn
-  desired_count   = 4
+  desired_count   = 6
   launch_type     = "FARGATE"
 
   network_configuration {
diff --git a/infra/terraform/ecs_tasks.tf b/infra/terraform/ecs_tasks.tf
index 28707ba..04e84f6 100644
--- a/infra/terraform/ecs_tasks.tf
+++ b/infra/terraform/ecs_tasks.tf
@@ -4,13 +4,14 @@ resource "aws_ecs_task_definition" "api" {
   network_mode             = "awsvpc"
   cpu                      = "512"
   memory                   = "1024"
+  skip_destroy             = true
   execution_role_arn       = aws_iam_role.task_execution.arn
   task_role_arn            = aws_iam_role.task_runtime.arn
 
   container_definitions = jsonencode([
     {
       name      = "api"
-      image     = "${aws_ecr_repository.api.repository_url}:latest"
+      image     = var.api_image
       essential = true
 
       portMappings = [
@@ -86,6 +87,11 @@ resource "aws_ecs_task_definition" "api" {
       }
     }
   ])
+
+  tags = {
+    Release = var.release_id
+    Service = "api"
+  }
 }
 
 resource "aws_ecs_task_definition" "worker" {
@@ -94,13 +100,14 @@ resource "aws_ecs_task_definition" "worker" {
   network_mode             = "awsvpc"
   cpu                      = "512"
   memory                   = "1024"
+  skip_destroy             = true
   execution_role_arn       = aws_iam_role.task_execution.arn
   task_role_arn            = aws_iam_role.task_runtime.arn
 
   container_definitions = jsonencode([
     {
       name      = "worker"
-      image     = "${aws_ecr_repository.worker.repository_url}:latest"
+      image     = var.worker_image
       essential = true
 
       portMappings = [
@@ -172,4 +179,9 @@ resource "aws_ecs_task_definition" "worker" {
       }
     }
   ])
+
+  tags = {
+    Release = var.release_id
+    Service = "worker"
+  }
 }
diff --git a/infra/terraform/main.tf b/infra/terraform/main.tf
index a5e57cc..cfcad9d 100644
--- a/infra/terraform/main.tf
+++ b/infra/terraform/main.tf
@@ -1,5 +1,7 @@
 terraform {
-  required_version = ">= 1.7.0"
+  required_version = ">= 1.10.0, < 2.0.0"
+
+  backend "s3" {}
 
   required_providers {
     aws = {
diff --git a/infra/terraform/outputs.tf b/infra/terraform/outputs.tf
index ea473e6..c53bb66 100644
--- a/infra/terraform/outputs.tf
+++ b/infra/terraform/outputs.tf
@@ -17,3 +17,23 @@ output "analysis_llm_queue_url" {
 output "api_load_balancer_dns_name" {
   value = aws_lb.api.dns_name
 }
+
+output "api_task_definition_arn" {
+  value = aws_ecs_task_definition.api.arn
+}
+
+output "worker_task_definition_arn" {
+  value = aws_ecs_task_definition.worker.arn
+}
+
+output "ecs_cluster_name" {
+  value = aws_ecs_cluster.this.name
+}
+
+output "api_service_name" {
+  value = aws_ecs_service.api.name
+}
+
+output "worker_service_name" {
+  value = aws_ecs_service.worker.name
+}
diff --git a/infra/terraform/terraform.tfvars.example b/infra/terraform/terraform.tfvars.example
index ae91d85..ac060fe 100644
--- a/infra/terraform/terraform.tfvars.example
+++ b/infra/terraform/terraform.tfvars.example
@@ -2,6 +2,10 @@ aws_region = "ap-southeast-1"
 
 name_prefix = "mem9-node-prod"
 
+release_id   = "0123456789abcdef0123456789abcdef01234567"
+api_image    = "401696231252.dkr.ecr.ap-southeast-1.amazonaws.com/mem9-node-prod-api:0123456789abcdef0123456789abcdef01234567"
+worker_image = "401696231252.dkr.ecr.ap-southeast-1.amazonaws.com/mem9-node-prod-worker:0123456789abcdef0123456789abcdef01234567"
+
 api_alb_idle_timeout_seconds = 120
 
 vpc_id = "vpc-xxxxxxxxxxxxxxxxx"
diff --git a/infra/terraform/variables.tf b/infra/terraform/variables.tf
index f4a817d..bc7370d 100644
--- a/infra/terraform/variables.tf
+++ b/infra/terraform/variables.tf
@@ -8,6 +8,36 @@ variable "name_prefix" {
   default = "mem9-analysis"
 }
 
+variable "release_id" {
+  description = "Git commit SHA that groups the API and worker task definitions into one release."
+  type        = string
+
+  validation {
+    condition     = can(regex("^[0-9a-f]{40}$", var.release_id))
+    error_message = "release_id must be a full 40-character lowercase Git commit SHA."
+  }
+}
+
+variable "api_image" {
+  description = "Immutable API ECR image URI tagged with the release Git commit SHA."
+  type        = string
+
+  validation {
+    condition     = can(regex(":[0-9a-f]{40}$", var.api_image))
+    error_message = "api_image must end with a full 40-character lowercase Git commit SHA tag."
+  }
+}
+
+variable "worker_image" {
+  description = "Immutable worker ECR image URI tagged with the release Git commit SHA."
+  type        = string
+
+  validation {
+    condition     = can(regex(":[0-9a-f]{40}$", var.worker_image))
+    error_message = "worker_image must end with a full 40-character lowercase Git commit SHA tag."
+  }
+}
+
 variable "api_alb_idle_timeout_seconds" {
   type    = number
   default = 120

From 909db7abd7acec5acbcd27851a4fcb3a9203d154 Mon Sep 17 00:00:00 2001
From: Eric Zhang <chong.zhang@pingcap.com>
Date: Thu, 30 Jul 2026 18:11:45 +0800
Subject: [PATCH 2/9] fix(ci): make production workflow prepare-only

---
 .github/workflows/deploy-production.yml   | 209 +++----------------
 .github/workflows/rollback-production.yml | 238 ----------------------
 README.md                                 |   8 +-
 docs/production-deployment.md             | 142 ++++++-------
 4 files changed, 91 insertions(+), 506 deletions(-)
 delete mode 100644 .github/workflows/rollback-production.yml

diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml
index 3b2d213..8eeb36e 100644
--- a/.github/workflows/deploy-production.yml
+++ b/.github/workflows/deploy-production.yml
@@ -1,5 +1,5 @@
-name: Deploy production
-run-name: Deploy ${{ github.sha }} to production
+name: Prepare production release
+run-name: Prepare ${{ github.sha }} for production
 
 on:
   workflow_dispatch:
@@ -8,7 +8,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: production
+  group: production-prepare
   cancel-in-progress: false
 
 env:
@@ -24,13 +24,12 @@ env:
   TF_INPUT: "false"
 
 jobs:
-  deploy:
-    name: Build and deploy current main
+  prepare:
+    name: Build images and plan current main
     if: github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
     timeout-minutes: 60
     permissions:
-      actions: write
       contents: read
       id-token: write
 
@@ -115,37 +114,23 @@ jobs:
             echo "WORKER_DIGEST=$WORKER_DIGEST"
           } >> "$GITHUB_ENV"
 
-      - name: Configure AWS deployment credentials
+      - name: Verify release image contents
+        run: |
+          docker pull "$API_IMAGE"
+          docker pull "$WORKER_IMAGE"
+          docker run --rm --entrypoint test "$API_IMAGE" \
+            -f /app/apps/api/dist/apps/api/src/main.js
+          docker run --rm --entrypoint test "$WORKER_IMAGE" \
+            -f /app/apps/worker/dist/apps/worker/src/main.js
+
+      - name: Configure AWS Terraform credentials
         uses: aws-actions/configure-aws-credentials@v6
         with:
           role-to-assume: arn:aws:iam::401696231252:role/mem9-node-prod-github-apply
-          role-session-name: mem9-node-production-deploy
+          role-session-name: mem9-node-production-plan
           aws-region: ${{ env.AWS_REGION }}
           mask-aws-account-id: true
 
-      - name: Read current ECS release
-        run: |
-          API_PREVIOUS_TASK_DEFINITION=$(aws ecs describe-services \
-            --cluster mem9-node-prod-cluster \
-            --services mem9-node-prod-api \
-            --query 'services[0].taskDefinition' \
-            --output text \
-            --region "$AWS_REGION")
-          WORKER_PREVIOUS_TASK_DEFINITION=$(aws ecs describe-services \
-            --cluster mem9-node-prod-cluster \
-            --services mem9-node-prod-worker \
-            --query 'services[0].taskDefinition' \
-            --output text \
-            --region "$AWS_REGION")
-
-          test -n "$API_PREVIOUS_TASK_DEFINITION" && test "$API_PREVIOUS_TASK_DEFINITION" != "None"
-          test -n "$WORKER_PREVIOUS_TASK_DEFINITION" && test "$WORKER_PREVIOUS_TASK_DEFINITION" != "None"
-
-          {
-            echo "API_PREVIOUS_TASK_DEFINITION=$API_PREVIOUS_TASK_DEFINITION"
-            echo "WORKER_PREVIOUS_TASK_DEFINITION=$WORKER_PREVIOUS_TASK_DEFINITION"
-          } >> "$GITHUB_ENV"
-
       - name: Set up Terraform
         uses: hashicorp/setup-terraform@v4
         with:
@@ -179,7 +164,7 @@ jobs:
             -var="worker_image=$WORKER_IMAGE" \
             -out=tfplan
 
-      - name: Summarize Terraform plan
+      - name: Summarize prepared release
         run: |
           PLAN_JSON=$(mktemp)
           terraform -chdir="$TF_WORKING_DIR" show -json tfplan > "$PLAN_JSON"
@@ -189,162 +174,14 @@ jobs:
           DESTROY_COUNT=$(jq '[.resource_changes[]? | select(.change.actions | index("delete"))] | length' "$PLAN_JSON")
 
           {
-            echo "## Production plan"
+            echo "## Production preparation"
             echo
             echo "- Release: \`$GITHUB_SHA\`"
-            echo "- API image: \`$API_IMAGE\`"
-            echo "- Worker image: \`$WORKER_IMAGE\`"
+            echo "- API image: \`$API_IMAGE@$API_DIGEST\`"
+            echo "- Worker image: \`$WORKER_IMAGE@$WORKER_DIGEST\`"
+            echo "- Image contents: verified"
             echo "- Terraform: **$ADD_COUNT add**, **$CHANGE_COUNT change**, **$DESTROY_COUNT destroy**"
+            echo "- Terraform apply: not run"
+            echo "- ECS services: unchanged"
             echo "- Database migration: not run"
           } >> "$GITHUB_STEP_SUMMARY"
-
-      - name: Record pre-deployment restore point
-        run: |
-          jq -n \
-            --arg release "$GITHUB_SHA" \
-            --arg previousApiTaskDefinition "$API_PREVIOUS_TASK_DEFINITION" \
-            --arg previousWorkerTaskDefinition "$WORKER_PREVIOUS_TASK_DEFINITION" \
-            --arg workflowActor "$GITHUB_ACTOR" \
-            --arg repository "$GITHUB_REPOSITORY" \
-            --arg runId "$GITHUB_RUN_ID" \
-            --arg preparedAt "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \
-            '{
-              release: $release,
-              previousApiTaskDefinition: $previousApiTaskDefinition,
-              previousWorkerTaskDefinition: $previousWorkerTaskDefinition,
-              workflowActor: $workflowActor,
-              repository: $repository,
-              runId: $runId,
-              preparedAt: $preparedAt,
-              status: "prepared"
-            }' > restore-point.json
-
-      - name: Upload pre-deployment restore point
-        id: upload-restore-point
-        uses: actions/upload-artifact@v7
-        with:
-          name: production-restore-point-${{ github.sha }}-${{ github.run_id }}
-          path: restore-point.json
-          if-no-files-found: error
-          retention-days: 90
-
-      - name: Apply production release
-        run: terraform -chdir="$TF_WORKING_DIR" apply -input=false -auto-approve tfplan
-
-      - name: Wait for ECS services
-        run: |
-          ECS_CLUSTER=$(terraform -chdir="$TF_WORKING_DIR" output -raw ecs_cluster_name)
-          API_SERVICE=$(terraform -chdir="$TF_WORKING_DIR" output -raw api_service_name)
-          WORKER_SERVICE=$(terraform -chdir="$TF_WORKING_DIR" output -raw worker_service_name)
-
-          aws ecs wait services-stable \
-            --cluster "$ECS_CLUSTER" \
-            --services "$API_SERVICE" "$WORKER_SERVICE" \
-            --region "$AWS_REGION"
-
-      - name: Record successful release
-        run: |
-          API_TASK_DEFINITION=$(terraform -chdir="$TF_WORKING_DIR" output -raw api_task_definition_arn)
-          WORKER_TASK_DEFINITION=$(terraform -chdir="$TF_WORKING_DIR" output -raw worker_task_definition_arn)
-
-          jq -n \
-            --arg release "$GITHUB_SHA" \
-            --arg apiImage "$API_IMAGE" \
-            --arg apiDigest "$API_DIGEST" \
-            --arg workerImage "$WORKER_IMAGE" \
-            --arg workerDigest "$WORKER_DIGEST" \
-            --arg apiTaskDefinition "$API_TASK_DEFINITION" \
-            --arg workerTaskDefinition "$WORKER_TASK_DEFINITION" \
-            --arg previousApiTaskDefinition "$API_PREVIOUS_TASK_DEFINITION" \
-            --arg previousWorkerTaskDefinition "$WORKER_PREVIOUS_TASK_DEFINITION" \
-            --arg workflowActor "$GITHUB_ACTOR" \
-            --arg repository "$GITHUB_REPOSITORY" \
-            --arg runId "$GITHUB_RUN_ID" \
-            --arg deployedAt "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \
-            '{
-              release: $release,
-              apiImage: $apiImage,
-              apiDigest: $apiDigest,
-              workerImage: $workerImage,
-              workerDigest: $workerDigest,
-              apiTaskDefinition: $apiTaskDefinition,
-              workerTaskDefinition: $workerTaskDefinition,
-              previousApiTaskDefinition: $previousApiTaskDefinition,
-              previousWorkerTaskDefinition: $previousWorkerTaskDefinition,
-              workflowActor: $workflowActor,
-              repository: $repository,
-              runId: $runId,
-              deployedAt: $deployedAt,
-              status: "deployed"
-            }' > release.json
-
-          {
-            echo
-            echo "## Production deployed"
-            echo
-            echo "- API task definition: \`$API_TASK_DEFINITION\`"
-            echo "- Worker task definition: \`$WORKER_TASK_DEFINITION\`"
-            echo "- ECS services: stable"
-          } >> "$GITHUB_STEP_SUMMARY"
-
-      - name: Upload release record
-        uses: actions/upload-artifact@v7
-        with:
-          name: production-release-${{ github.sha }}-${{ github.run_id }}
-          path: release.json
-          if-no-files-found: error
-          retention-days: 90
-
-      - name: Delete pre-deployment restore point
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ github.token }}
-          RESTORE_POINT_ARTIFACT_ID: ${{ steps.upload-restore-point.outputs.artifact-id }}
-        run: |
-          test -n "$RESTORE_POINT_ARTIFACT_ID"
-          gh api --method DELETE \
-            "/repos/$GITHUB_REPOSITORY/actions/artifacts/$RESTORE_POINT_ARTIFACT_ID"
-
-      - name: Keep the latest 30 task definition revisions
-        env:
-          TASK_DEFINITION_RETENTION: "30"
-        run: |
-          for family in mem9-node-prod-api mem9-node-prod-worker; do
-            aws ecs list-task-definitions \
-              --family-prefix "$family" \
-              --status ACTIVE \
-              --sort DESC \
-              --region "$AWS_REGION" \
-              --query 'taskDefinitionArns[]' \
-              --output text \
-              | tr '\t' '\n' \
-              | grep "/$family:" \
-              | tail -n "+$((TASK_DEFINITION_RETENTION + 1))" \
-              | while read -r task_definition; do
-                  if [ -n "$task_definition" ]; then
-                    aws ecs deregister-task-definition \
-                      --task-definition "$task_definition" \
-                      --region "$AWS_REGION" > /dev/null
-                  fi
-                done
-          done
-
-      - name: Keep the latest 30 release records
-        continue-on-error: true
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          gh api --paginate "/repos/$GITHUB_REPOSITORY/actions/artifacts?per_page=100" \
-            --jq '.artifacts[]
-              | select(.expired == false)
-              | select(.name | startswith("production-release-"))
-              | [.created_at, (.id | tostring)]
-              | @tsv' \
-            | sort -r \
-            | tail -n +31 \
-            | cut -f2 \
-            | while read -r artifact_id; do
-                if [ -n "$artifact_id" ]; then
-                  gh api --method DELETE "/repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact_id"
-                fi
-              done
diff --git a/.github/workflows/rollback-production.yml b/.github/workflows/rollback-production.yml
deleted file mode 100644
index e8363fb..0000000
--- a/.github/workflows/rollback-production.yml
+++ /dev/null
@@ -1,238 +0,0 @@
-name: Roll back production
-run-name: Roll back production to its previous release
-
-on:
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-concurrency:
-  group: production
-  cancel-in-progress: false
-
-env:
-  AWS_REGION: ap-southeast-1
-  ECS_CLUSTER: mem9-node-prod-cluster
-  API_SERVICE: mem9-node-prod-api
-  WORKER_SERVICE: mem9-node-prod-worker
-
-jobs:
-  rollback:
-    name: Roll back to previous successful release
-    if: github.ref == 'refs/heads/main'
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    permissions:
-      actions: write
-      contents: read
-      id-token: write
-
-    steps:
-      - name: Configure AWS deployment credentials
-        uses: aws-actions/configure-aws-credentials@v6
-        with:
-          role-to-assume: arn:aws:iam::401696231252:role/mem9-node-prod-github-apply
-          role-session-name: mem9-node-production-rollback
-          aws-region: ${{ env.AWS_REGION }}
-          mask-aws-account-id: true
-
-      - name: Read current ECS release
-        run: |
-          CURRENT_API_TASK_DEFINITION=$(aws ecs describe-services \
-            --cluster "$ECS_CLUSTER" \
-            --services "$API_SERVICE" \
-            --query 'services[0].taskDefinition' \
-            --output text \
-            --region "$AWS_REGION")
-          CURRENT_WORKER_TASK_DEFINITION=$(aws ecs describe-services \
-            --cluster "$ECS_CLUSTER" \
-            --services "$WORKER_SERVICE" \
-            --query 'services[0].taskDefinition' \
-            --output text \
-            --region "$AWS_REGION")
-
-          test -n "$CURRENT_API_TASK_DEFINITION" && test "$CURRENT_API_TASK_DEFINITION" != "None"
-          test -n "$CURRENT_WORKER_TASK_DEFINITION" && test "$CURRENT_WORKER_TASK_DEFINITION" != "None"
-
-          {
-            echo "CURRENT_API_TASK_DEFINITION=$CURRENT_API_TASK_DEFINITION"
-            echo "CURRENT_WORKER_TASK_DEFINITION=$CURRENT_WORKER_TASK_DEFINITION"
-          } >> "$GITHUB_ENV"
-
-      - name: Find the previous release pair
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          RECORD_DIR="$RUNNER_TEMP/production-releases"
-          mkdir -p "$RECORD_DIR"
-
-          gh api --paginate "/repos/$GITHUB_REPOSITORY/actions/artifacts?per_page=100" \
-            --jq '.artifacts[]
-              | select(.expired == false)
-              | select(.name | startswith("production-release-"))
-              | [.created_at, (.id | tostring)]
-              | @tsv' \
-            | sort -r > "$RECORD_DIR/releases.tsv"
-
-          gh api --paginate "/repos/$GITHUB_REPOSITORY/actions/artifacts?per_page=100" \
-            --jq '.artifacts[]
-              | select(.expired == false)
-              | select(.name | startswith("production-restore-point-"))
-              | [.created_at, (.id | tostring)]
-              | @tsv' \
-            | sort -r > "$RECORD_DIR/restore-points.tsv"
-
-          TARGET_API_TASK_DEFINITION=""
-          TARGET_WORKER_TASK_DEFINITION=""
-          SOURCE_RELEASE=""
-          FALLBACK_API_TASK_DEFINITION=""
-          FALLBACK_WORKER_TASK_DEFINITION=""
-          FALLBACK_RELEASE=""
-
-          while IFS=$'\t' read -r _ artifact_id; do
-            test -n "$artifact_id" || continue
-
-            archive="$RECORD_DIR/$artifact_id.zip"
-            record="$RECORD_DIR/$artifact_id.json"
-            gh api "/repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact_id/zip" > "$archive"
-            unzip -p "$archive" release.json > "$record"
-
-            recorded_api=$(jq -r '.apiTaskDefinition // empty' "$record")
-            recorded_worker=$(jq -r '.workerTaskDefinition // empty' "$record")
-
-            if [ -z "$FALLBACK_API_TASK_DEFINITION" ] && [ -n "$recorded_api" ] && [ -n "$recorded_worker" ]; then
-              FALLBACK_API_TASK_DEFINITION="$recorded_api"
-              FALLBACK_WORKER_TASK_DEFINITION="$recorded_worker"
-              FALLBACK_RELEASE=$(jq -r '.release // empty' "$record")
-            fi
-
-            if [ "$recorded_api" = "$CURRENT_API_TASK_DEFINITION" ] && \
-               [ "$recorded_worker" = "$CURRENT_WORKER_TASK_DEFINITION" ]; then
-              previous_api=$(jq -r '.previousApiTaskDefinition // empty' "$record")
-              previous_worker=$(jq -r '.previousWorkerTaskDefinition // empty' "$record")
-
-              if [ -n "$previous_api" ] && [ -n "$previous_worker" ] && \
-                 [ "$previous_api" != "$CURRENT_API_TASK_DEFINITION" ] && \
-                 [ "$previous_worker" != "$CURRENT_WORKER_TASK_DEFINITION" ]; then
-                TARGET_API_TASK_DEFINITION="$previous_api"
-                TARGET_WORKER_TASK_DEFINITION="$previous_worker"
-                SOURCE_RELEASE=$(jq -r '.release // empty' "$record")
-                break
-              fi
-            fi
-          done < "$RECORD_DIR/releases.tsv"
-
-          if [ -z "$TARGET_API_TASK_DEFINITION" ] && \
-             [ -n "$FALLBACK_API_TASK_DEFINITION" ] && \
-             [ -n "$FALLBACK_WORKER_TASK_DEFINITION" ]; then
-            TARGET_API_TASK_DEFINITION="$FALLBACK_API_TASK_DEFINITION"
-            TARGET_WORKER_TASK_DEFINITION="$FALLBACK_WORKER_TASK_DEFINITION"
-            SOURCE_RELEASE="unrecorded-current-to-$FALLBACK_RELEASE"
-          fi
-
-          if [ -z "$TARGET_API_TASK_DEFINITION" ] || [ -z "$TARGET_WORKER_TASK_DEFINITION" ]; then
-            while IFS=$'\t' read -r _ artifact_id; do
-              test -n "$artifact_id" || continue
-
-              archive="$RECORD_DIR/restore-$artifact_id.zip"
-              record="$RECORD_DIR/restore-$artifact_id.json"
-              gh api "/repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact_id/zip" > "$archive"
-              unzip -p "$archive" restore-point.json > "$record"
-
-              previous_api=$(jq -r '.previousApiTaskDefinition // empty' "$record")
-              previous_worker=$(jq -r '.previousWorkerTaskDefinition // empty' "$record")
-
-              if [ -n "$previous_api" ] && [ -n "$previous_worker" ] && \
-                 { [ "$previous_api" != "$CURRENT_API_TASK_DEFINITION" ] || \
-                   [ "$previous_worker" != "$CURRENT_WORKER_TASK_DEFINITION" ]; }; then
-                TARGET_API_TASK_DEFINITION="$previous_api"
-                TARGET_WORKER_TASK_DEFINITION="$previous_worker"
-                SOURCE_RELEASE="failed-$(jq -r '.release // "unknown"' "$record")"
-                break
-              fi
-            done < "$RECORD_DIR/restore-points.tsv"
-          fi
-
-          test -n "$TARGET_API_TASK_DEFINITION"
-          test -n "$TARGET_WORKER_TASK_DEFINITION"
-          test "$TARGET_API_TASK_DEFINITION" != "$CURRENT_API_TASK_DEFINITION" || \
-            test "$TARGET_WORKER_TASK_DEFINITION" != "$CURRENT_WORKER_TASK_DEFINITION"
-
-          {
-            echo "TARGET_API_TASK_DEFINITION=$TARGET_API_TASK_DEFINITION"
-            echo "TARGET_WORKER_TASK_DEFINITION=$TARGET_WORKER_TASK_DEFINITION"
-            echo "SOURCE_RELEASE=$SOURCE_RELEASE"
-          } >> "$GITHUB_ENV"
-
-      - name: Verify target task definitions
-        run: |
-          test "$(aws ecs describe-task-definition \
-            --task-definition "$TARGET_API_TASK_DEFINITION" \
-            --query 'taskDefinition.status' \
-            --output text \
-            --region "$AWS_REGION")" = "ACTIVE"
-          test "$(aws ecs describe-task-definition \
-            --task-definition "$TARGET_WORKER_TASK_DEFINITION" \
-            --query 'taskDefinition.status' \
-            --output text \
-            --region "$AWS_REGION")" = "ACTIVE"
-
-      - name: Roll back ECS services
-        run: |
-          aws ecs update-service \
-            --cluster "$ECS_CLUSTER" \
-            --service "$API_SERVICE" \
-            --task-definition "$TARGET_API_TASK_DEFINITION" \
-            --region "$AWS_REGION" > /dev/null
-          aws ecs update-service \
-            --cluster "$ECS_CLUSTER" \
-            --service "$WORKER_SERVICE" \
-            --task-definition "$TARGET_WORKER_TASK_DEFINITION" \
-            --region "$AWS_REGION" > /dev/null
-
-      - name: Wait for ECS services
-        run: |
-          aws ecs wait services-stable \
-            --cluster "$ECS_CLUSTER" \
-            --services "$API_SERVICE" "$WORKER_SERVICE" \
-            --region "$AWS_REGION"
-
-      - name: Record rollback
-        run: |
-          jq -n \
-            --arg sourceRelease "$SOURCE_RELEASE" \
-            --arg fromApiTaskDefinition "$CURRENT_API_TASK_DEFINITION" \
-            --arg fromWorkerTaskDefinition "$CURRENT_WORKER_TASK_DEFINITION" \
-            --arg toApiTaskDefinition "$TARGET_API_TASK_DEFINITION" \
-            --arg toWorkerTaskDefinition "$TARGET_WORKER_TASK_DEFINITION" \
-            --arg workflowActor "$GITHUB_ACTOR" \
-            --arg rolledBackAt "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \
-            '{
-              sourceRelease: $sourceRelease,
-              fromApiTaskDefinition: $fromApiTaskDefinition,
-              fromWorkerTaskDefinition: $fromWorkerTaskDefinition,
-              toApiTaskDefinition: $toApiTaskDefinition,
-              toWorkerTaskDefinition: $toWorkerTaskDefinition,
-              workflowActor: $workflowActor,
-              rolledBackAt: $rolledBackAt,
-              status: "rolled-back"
-            }' > rollback.json
-
-          {
-            echo "## Production rolled back"
-            echo
-            echo "- From API: \`$CURRENT_API_TASK_DEFINITION\`"
-            echo "- To API: \`$TARGET_API_TASK_DEFINITION\`"
-            echo "- From worker: \`$CURRENT_WORKER_TASK_DEFINITION\`"
-            echo "- To worker: \`$TARGET_WORKER_TASK_DEFINITION\`"
-            echo "- ECS services: stable"
-          } >> "$GITHUB_STEP_SUMMARY"
-
-      - name: Upload rollback record
-        uses: actions/upload-artifact@v7
-        with:
-          name: production-rollback-${{ github.run_id }}
-          path: rollback.json
-          if-no-files-found: error
-          retention-days: 90
diff --git a/README.md b/README.md
index d3ab224..99e4b87 100644
--- a/README.md
+++ b/README.md
@@ -50,10 +50,10 @@ Worker health: `http://127.0.0.1:3001/health/live`
 
 ## Production Deployment
 
-Pull requests run application and Terraform checks. Production is released
-manually from the current `main` commit; one workflow builds immutable API and
-worker images, applies Terraform, and waits for ECS. Rollback automatically
-selects the previous recorded task-definition pair. See [Production
+Pull requests run application and Terraform checks. The current production
+workflow is intentionally prepare-only: it builds SHA-tagged API and worker
+images, verifies their contents, and creates a production Terraform plan. It
+does not apply Terraform or change ECS. See [Production
 deployment](docs/production-deployment.md) for setup and operating details.
 
 ## API Flow
diff --git a/docs/production-deployment.md b/docs/production-deployment.md
index 948b8fb..d74a676 100644
--- a/docs/production-deployment.md
+++ b/docs/production-deployment.md
@@ -1,12 +1,10 @@
-# Production deployment
+# Production release preparation
 
-Production is released manually from the current `main` commit. Image builds,
-Terraform plan/apply, and ECS deployment happen in one workflow run, so there
-is no separately selected image SHA or saved intermediate Terraform plan.
+The first rollout phase prepares and inspects production materials without
+changing the running service. There is currently no active CI deployment or
+rollback workflow.
 
-## Workflows
-
-### Pull-request checks
+## Pull-request checks
 
 `.github/workflows/ci.yml` runs for pull requests targeting `main`:
 
@@ -15,54 +13,47 @@ is no separately selected image SHA or saved intermediate Terraform plan.
 3. Check Terraform formatting.
 4. Initialize Terraform without a backend and validate the configuration.
 
-The repository does not currently define a benchmark command. Add it to this
-workflow after the project has a stable benchmark script and failure threshold.
+The repository does not currently define a benchmark command. Add one only
+after the project has a stable benchmark script and failure threshold.
 
-### Production release
+## Prepare a production release
 
-`.github/workflows/deploy-production.yml` has no inputs. Run it from the `main`
-branch in GitHub Actions. The workflow:
+After this workflow has been merged, open GitHub Actions, select **Prepare
+production release**, and run it from `main`. It:
 
-1. Locks the run to the `main` commit identified by `github.sha`.
+1. Locks the run to the current `main` commit identified by `github.sha`.
 2. Assumes the image role through GitHub OIDC.
-3. Builds and pushes API and worker images tagged with that full SHA. A rerun
-   reuses an existing immutable image and builds only a missing image.
-4. Reads both ECR image digests.
-5. Assumes the deployment role through GitHub OIDC.
-6. Reads the currently deployed API and worker task definitions for rollback.
-7. Initializes the KMS-encrypted S3 Terraform backend.
-8. Creates a local `tfplan` in the same runner.
-9. Uploads a temporary restore point containing the task-definition pair that
-   is currently online, then immediately applies the plan.
-10. Waits for both ECS services to become stable.
-11. Uploads a release record containing the SHA, image pair, task-definition
-    pair, and previous task-definition pair.
-12. Deletes the temporary restore point after the successful release record is
-    safely uploaded. Failed runs keep their restore point for recovery.
-
-Terraform owns the ECS task definitions and the ECS services' task-definition
-references. Normal deployment must not call `aws ecs update-service` directly.
-
-The workflow does not use a GitHub Environment. The **Run workflow** action is
-the production release decision. The job-level branch check and both AWS role
-trust policies must restrict production access to `refs/heads/main`.
-
-### Production rollback
-
-`.github/workflows/rollback-production.yml` also has no inputs. It finds the
-release record matching the task-definition pair currently used by ECS and
-switches both services to the previous pair recorded by that release.
-
-If the current deployment failed before it could write a success record, the
-workflow falls back to the newest recorded successful task-definition pair. If
-there has never been a successful CI release, it uses the newest failed run's
-pre-deployment restore point. It refuses to continue if it cannot find a usable
-record or if either target task definition is inactive.
-
-Rollback changes the ECS service references directly and intentionally leaves
-Terraform state untouched. The next production release refreshes the state and
-deploys its new task-definition pair. API and worker updates are coordinated by
-one workflow but are not an atomic AWS transaction.
+3. Builds and pushes API and worker images tagged with the full commit SHA.
+4. Resolves both ECR image digests and verifies the built entrypoint files.
+5. Assumes the Terraform role through GitHub OIDC.
+6. Initializes the KMS-encrypted S3 Terraform backend.
+7. Creates a local, ephemeral production `tfplan`.
+8. Writes the image identities and Terraform change counts to Job Summary.
+
+The workflow writes SHA-tagged images to ECR. It does **not**:
+
+- run `terraform apply`;
+- register or activate new ECS task-definition revisions;
+- update either ECS service;
+- run database migrations;
+- save the `tfplan` after the runner is deleted.
+
+Rerunning the same commit reuses an existing image and rebuilds only a missing
+one. Normal operation must not overwrite a SHA tag.
+
+## What to review
+
+Before enabling real deployment, run the preparation workflow and confirm:
+
+- both images exist under the expected SHA and have valid digests;
+- both container entrypoint files pass the image-content check;
+- the Terraform change counts are expected;
+- the detailed Terraform log contains no unexpected resource changes;
+- OIDC, the remote backend, and `TF_VARS` all work in GitHub Actions.
+
+A later, separately reviewed change can add Terraform apply, ECS stability
+waiting, release records, retention, and rollback. That keeps the first
+production-affecting run out of this initial CI change.
 
 ## Terraform state
 
@@ -81,9 +72,9 @@ The bucket must keep:
 - an HTTPS-only bucket policy;
 - state and lock-file permissions scoped to the two objects above.
 
-The bucket no longer stores `tfplan` or release-candidate files. Existing
+The bucket does not store `tfplan`. Existing
 `mem9-node/production/plans/` lifecycle rules and IAM permissions are harmless
-but can be removed after the new workflow has been verified.
+but can be removed after this workflow has been verified.
 
 ### Migrate an existing local state
 
@@ -106,7 +97,8 @@ Confirm the remote state before removing the local backup.
 
 Create one repository-level GitHub Actions secret named `TF_VARS`. Its value is
 the complete production `terraform.tfvars` content. The workflow writes it to
-the ephemeral runner without printing it and removes the runner after the job.
+the ephemeral runner without printing it; GitHub deletes the runner after the
+job.
 
 Do not add these release-specific values to `TF_VARS`:
 
@@ -116,9 +108,7 @@ api_image
 worker_image
 ```
 
-The workflow supplies those three values from the release SHA and its two ECR
-images. Command-line `-var` values take precedence if an older local file still
-contains them, but removing them from `TF_VARS` keeps ownership unambiguous.
+The workflow supplies those values from the release SHA and its two images.
 
 Protect `main` and require the `Verify` status check before merge. Block force
 pushes and branch deletion. A benchmark is not a required check until a real
@@ -126,8 +116,8 @@ benchmark command exists in the repository.
 
 ## AWS OIDC roles
 
-Keep the two existing roles but restrict both trust policies to this repository
-and the `main` branch subject:
+Both trust policies must restrict production access to this repository and the
+`main` branch subject:
 
 ```text
 repo:mem9-ai/mem9-node:ref:refs/heads/main
@@ -135,30 +125,26 @@ repo:mem9-ai/mem9-node:ref:refs/heads/main
 
 - `mem9-node-prod-github-prepare`: ECR login, image lookup, and image push for
   the API and worker repositories only.
-- `mem9-node-prod-github-apply`: Terraform state/KMS access, infrastructure
-  plan/apply permissions, ECS read/update permissions, and task-definition
-  retention permissions.
+- `mem9-node-prod-github-apply`: currently used for Terraform state access and
+  read-only planning. Its broader apply permissions are reserved for the later
+  deployment phase.
 
-The apply role's old Environment-based subject must be changed before the new
-deployment and rollback workflows can assume it. The unused GitHub
-`production` Environment may remain or be deleted after verification.
+The preparation workflow has no GitHub Environment approval because it does
+not change the running service. The **Run workflow** action only starts material
+preparation.
 
-## Retention
+## Retention and rollback
 
-- ECR images use immutable full-SHA tags.
-- ECS keeps the latest 30 active API revisions and 30 active worker revisions.
-- GitHub keeps the latest 30 successful release-record artifacts, subject to
-  the repository's artifact retention limit.
-- A failed deployment keeps its temporary pre-deployment restore point. A
-  successful deployment deletes that temporary record after uploading the
-  permanent release record.
-- Rollback records are stored separately and do not participate in selecting a
-  rollback target.
+The current workflow does not clean up ECR images, task-definition revisions,
+or release records. Keep the prepared SHA images during validation. Add bounded
+retention together with real deployment and release records so cleanup cannot
+delete an image needed for rollback.
 
-Do not introduce a count-based ECR lifecycle rule until it can preserve every
-image referenced by the retained release records.
+There is no active rollback workflow until CI owns at least one successful
+production release record. Existing manual rollback procedures remain the
+fallback in the meantime.
 
 ## Database migrations
 
-These workflows do not run database migrations. Schema changes require a
+The workflow does not run database migrations. Schema changes require a
 separately reviewed, backward-compatible migration process.

From 8af2a902bc71836e09e848feb5e8eca0739e3af6 Mon Sep 17 00:00:00 2001
From: Eric Zhang <chong.zhang@pingcap.com>
Date: Thu, 30 Jul 2026 18:25:28 +0800
Subject: [PATCH 3/9] fix(ci): preserve disabled production release path

---
 .github/workflows/deploy-production.yml   | 220 +++++++++++++++++---
 .github/workflows/rollback-production.yml | 239 ++++++++++++++++++++++
 README.md                                 |   9 +-
 docs/production-deployment.md             | 148 ++++++++------
 4 files changed, 525 insertions(+), 91 deletions(-)
 create mode 100644 .github/workflows/rollback-production.yml

diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml
index 8eeb36e..88333f2 100644
--- a/.github/workflows/deploy-production.yml
+++ b/.github/workflows/deploy-production.yml
@@ -1,5 +1,5 @@
-name: Prepare production release
-run-name: Prepare ${{ github.sha }} for production
+name: Deploy production
+run-name: Deploy ${{ github.sha }} to production
 
 on:
   workflow_dispatch:
@@ -8,7 +8,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: production-prepare
+  group: production
   cancel-in-progress: false
 
 env:
@@ -24,12 +24,13 @@ env:
   TF_INPUT: "false"
 
 jobs:
-  prepare:
-    name: Build images and plan current main
+  deploy:
+    name: Build and deploy current main
     if: github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
     timeout-minutes: 60
     permissions:
+      actions: write
       contents: read
       id-token: write
 
@@ -114,23 +115,37 @@ jobs:
             echo "WORKER_DIGEST=$WORKER_DIGEST"
           } >> "$GITHUB_ENV"
 
-      - name: Verify release image contents
-        run: |
-          docker pull "$API_IMAGE"
-          docker pull "$WORKER_IMAGE"
-          docker run --rm --entrypoint test "$API_IMAGE" \
-            -f /app/apps/api/dist/apps/api/src/main.js
-          docker run --rm --entrypoint test "$WORKER_IMAGE" \
-            -f /app/apps/worker/dist/apps/worker/src/main.js
-
-      - name: Configure AWS Terraform credentials
+      - name: Configure AWS deployment credentials
         uses: aws-actions/configure-aws-credentials@v6
         with:
           role-to-assume: arn:aws:iam::401696231252:role/mem9-node-prod-github-apply
-          role-session-name: mem9-node-production-plan
+          role-session-name: mem9-node-production-deploy
           aws-region: ${{ env.AWS_REGION }}
           mask-aws-account-id: true
 
+      - name: Read current ECS release
+        run: |
+          API_PREVIOUS_TASK_DEFINITION=$(aws ecs describe-services \
+            --cluster mem9-node-prod-cluster \
+            --services mem9-node-prod-api \
+            --query 'services[0].taskDefinition' \
+            --output text \
+            --region "$AWS_REGION")
+          WORKER_PREVIOUS_TASK_DEFINITION=$(aws ecs describe-services \
+            --cluster mem9-node-prod-cluster \
+            --services mem9-node-prod-worker \
+            --query 'services[0].taskDefinition' \
+            --output text \
+            --region "$AWS_REGION")
+
+          test -n "$API_PREVIOUS_TASK_DEFINITION" && test "$API_PREVIOUS_TASK_DEFINITION" != "None"
+          test -n "$WORKER_PREVIOUS_TASK_DEFINITION" && test "$WORKER_PREVIOUS_TASK_DEFINITION" != "None"
+
+          {
+            echo "API_PREVIOUS_TASK_DEFINITION=$API_PREVIOUS_TASK_DEFINITION"
+            echo "WORKER_PREVIOUS_TASK_DEFINITION=$WORKER_PREVIOUS_TASK_DEFINITION"
+          } >> "$GITHUB_ENV"
+
       - name: Set up Terraform
         uses: hashicorp/setup-terraform@v4
         with:
@@ -164,7 +179,7 @@ jobs:
             -var="worker_image=$WORKER_IMAGE" \
             -out=tfplan
 
-      - name: Summarize prepared release
+      - name: Summarize Terraform plan
         run: |
           PLAN_JSON=$(mktemp)
           terraform -chdir="$TF_WORKING_DIR" show -json tfplan > "$PLAN_JSON"
@@ -174,14 +189,173 @@ jobs:
           DESTROY_COUNT=$(jq '[.resource_changes[]? | select(.change.actions | index("delete"))] | length' "$PLAN_JSON")
 
           {
-            echo "## Production preparation"
+            echo "## Production plan"
             echo
             echo "- Release: \`$GITHUB_SHA\`"
-            echo "- API image: \`$API_IMAGE@$API_DIGEST\`"
-            echo "- Worker image: \`$WORKER_IMAGE@$WORKER_DIGEST\`"
-            echo "- Image contents: verified"
+            echo "- API image: \`$API_IMAGE\`"
+            echo "- Worker image: \`$WORKER_IMAGE\`"
             echo "- Terraform: **$ADD_COUNT add**, **$CHANGE_COUNT change**, **$DESTROY_COUNT destroy**"
-            echo "- Terraform apply: not run"
-            echo "- ECS services: unchanged"
             echo "- Database migration: not run"
           } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Record pre-deployment restore point
+        # Temporary guard: keep the complete release path for validation, but
+        # do not run any step after the Terraform plan until it is approved.
+        if: ${{ false }}
+        run: |
+          jq -n \
+            --arg release "$GITHUB_SHA" \
+            --arg previousApiTaskDefinition "$API_PREVIOUS_TASK_DEFINITION" \
+            --arg previousWorkerTaskDefinition "$WORKER_PREVIOUS_TASK_DEFINITION" \
+            --arg workflowActor "$GITHUB_ACTOR" \
+            --arg repository "$GITHUB_REPOSITORY" \
+            --arg runId "$GITHUB_RUN_ID" \
+            --arg preparedAt "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \
+            '{
+              release: $release,
+              previousApiTaskDefinition: $previousApiTaskDefinition,
+              previousWorkerTaskDefinition: $previousWorkerTaskDefinition,
+              workflowActor: $workflowActor,
+              repository: $repository,
+              runId: $runId,
+              preparedAt: $preparedAt,
+              status: "prepared"
+            }' > restore-point.json
+
+      - name: Upload pre-deployment restore point
+        if: ${{ false }}
+        id: upload-restore-point
+        uses: actions/upload-artifact@v7
+        with:
+          name: production-restore-point-${{ github.sha }}-${{ github.run_id }}
+          path: restore-point.json
+          if-no-files-found: error
+          retention-days: 90
+
+      - name: Apply production release
+        if: ${{ false }}
+        run: terraform -chdir="$TF_WORKING_DIR" apply -input=false -auto-approve tfplan
+
+      - name: Wait for ECS services
+        if: ${{ false }}
+        run: |
+          ECS_CLUSTER=$(terraform -chdir="$TF_WORKING_DIR" output -raw ecs_cluster_name)
+          API_SERVICE=$(terraform -chdir="$TF_WORKING_DIR" output -raw api_service_name)
+          WORKER_SERVICE=$(terraform -chdir="$TF_WORKING_DIR" output -raw worker_service_name)
+
+          aws ecs wait services-stable \
+            --cluster "$ECS_CLUSTER" \
+            --services "$API_SERVICE" "$WORKER_SERVICE" \
+            --region "$AWS_REGION"
+
+      - name: Record successful release
+        if: ${{ false }}
+        run: |
+          API_TASK_DEFINITION=$(terraform -chdir="$TF_WORKING_DIR" output -raw api_task_definition_arn)
+          WORKER_TASK_DEFINITION=$(terraform -chdir="$TF_WORKING_DIR" output -raw worker_task_definition_arn)
+
+          jq -n \
+            --arg release "$GITHUB_SHA" \
+            --arg apiImage "$API_IMAGE" \
+            --arg apiDigest "$API_DIGEST" \
+            --arg workerImage "$WORKER_IMAGE" \
+            --arg workerDigest "$WORKER_DIGEST" \
+            --arg apiTaskDefinition "$API_TASK_DEFINITION" \
+            --arg workerTaskDefinition "$WORKER_TASK_DEFINITION" \
+            --arg previousApiTaskDefinition "$API_PREVIOUS_TASK_DEFINITION" \
+            --arg previousWorkerTaskDefinition "$WORKER_PREVIOUS_TASK_DEFINITION" \
+            --arg workflowActor "$GITHUB_ACTOR" \
+            --arg repository "$GITHUB_REPOSITORY" \
+            --arg runId "$GITHUB_RUN_ID" \
+            --arg deployedAt "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \
+            '{
+              release: $release,
+              apiImage: $apiImage,
+              apiDigest: $apiDigest,
+              workerImage: $workerImage,
+              workerDigest: $workerDigest,
+              apiTaskDefinition: $apiTaskDefinition,
+              workerTaskDefinition: $workerTaskDefinition,
+              previousApiTaskDefinition: $previousApiTaskDefinition,
+              previousWorkerTaskDefinition: $previousWorkerTaskDefinition,
+              workflowActor: $workflowActor,
+              repository: $repository,
+              runId: $runId,
+              deployedAt: $deployedAt,
+              status: "deployed"
+            }' > release.json
+
+          {
+            echo
+            echo "## Production deployed"
+            echo
+            echo "- API task definition: \`$API_TASK_DEFINITION\`"
+            echo "- Worker task definition: \`$WORKER_TASK_DEFINITION\`"
+            echo "- ECS services: stable"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload release record
+        if: ${{ false }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: production-release-${{ github.sha }}-${{ github.run_id }}
+          path: release.json
+          if-no-files-found: error
+          retention-days: 90
+
+      - name: Delete pre-deployment restore point
+        if: ${{ false }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RESTORE_POINT_ARTIFACT_ID: ${{ steps.upload-restore-point.outputs.artifact-id }}
+        run: |
+          test -n "$RESTORE_POINT_ARTIFACT_ID"
+          gh api --method DELETE \
+            "/repos/$GITHUB_REPOSITORY/actions/artifacts/$RESTORE_POINT_ARTIFACT_ID"
+
+      - name: Keep the latest 30 task definition revisions
+        if: ${{ false }}
+        env:
+          TASK_DEFINITION_RETENTION: "30"
+        run: |
+          for family in mem9-node-prod-api mem9-node-prod-worker; do
+            aws ecs list-task-definitions \
+              --family-prefix "$family" \
+              --status ACTIVE \
+              --sort DESC \
+              --region "$AWS_REGION" \
+              --query 'taskDefinitionArns[]' \
+              --output text \
+              | tr '\t' '\n' \
+              | grep "/$family:" \
+              | tail -n "+$((TASK_DEFINITION_RETENTION + 1))" \
+              | while read -r task_definition; do
+                  if [ -n "$task_definition" ]; then
+                    aws ecs deregister-task-definition \
+                      --task-definition "$task_definition" \
+                      --region "$AWS_REGION" > /dev/null
+                  fi
+                done
+          done
+
+      - name: Keep the latest 30 release records
+        if: ${{ false }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh api --paginate "/repos/$GITHUB_REPOSITORY/actions/artifacts?per_page=100" \
+            --jq '.artifacts[]
+              | select(.expired == false)
+              | select(.name | startswith("production-release-"))
+              | [.created_at, (.id | tostring)]
+              | @tsv' \
+            | sort -r \
+            | tail -n +31 \
+            | cut -f2 \
+            | while read -r artifact_id; do
+                if [ -n "$artifact_id" ]; then
+                  gh api --method DELETE "/repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact_id"
+                fi
+              done
diff --git a/.github/workflows/rollback-production.yml b/.github/workflows/rollback-production.yml
new file mode 100644
index 0000000..1f1d451
--- /dev/null
+++ b/.github/workflows/rollback-production.yml
@@ -0,0 +1,239 @@
+name: Roll back production
+run-name: Roll back production to its previous release
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: production
+  cancel-in-progress: false
+
+env:
+  AWS_REGION: ap-southeast-1
+  ECS_CLUSTER: mem9-node-prod-cluster
+  API_SERVICE: mem9-node-prod-api
+  WORKER_SERVICE: mem9-node-prod-worker
+
+jobs:
+  rollback:
+    name: Roll back to previous successful release
+    # Temporary guard: rollback becomes usable after CI owns a verified release.
+    if: github.ref == 'refs/heads/main' && false
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      actions: write
+      contents: read
+      id-token: write
+
+    steps:
+      - name: Configure AWS deployment credentials
+        uses: aws-actions/configure-aws-credentials@v6
+        with:
+          role-to-assume: arn:aws:iam::401696231252:role/mem9-node-prod-github-apply
+          role-session-name: mem9-node-production-rollback
+          aws-region: ${{ env.AWS_REGION }}
+          mask-aws-account-id: true
+
+      - name: Read current ECS release
+        run: |
+          CURRENT_API_TASK_DEFINITION=$(aws ecs describe-services \
+            --cluster "$ECS_CLUSTER" \
+            --services "$API_SERVICE" \
+            --query 'services[0].taskDefinition' \
+            --output text \
+            --region "$AWS_REGION")
+          CURRENT_WORKER_TASK_DEFINITION=$(aws ecs describe-services \
+            --cluster "$ECS_CLUSTER" \
+            --services "$WORKER_SERVICE" \
+            --query 'services[0].taskDefinition' \
+            --output text \
+            --region "$AWS_REGION")
+
+          test -n "$CURRENT_API_TASK_DEFINITION" && test "$CURRENT_API_TASK_DEFINITION" != "None"
+          test -n "$CURRENT_WORKER_TASK_DEFINITION" && test "$CURRENT_WORKER_TASK_DEFINITION" != "None"
+
+          {
+            echo "CURRENT_API_TASK_DEFINITION=$CURRENT_API_TASK_DEFINITION"
+            echo "CURRENT_WORKER_TASK_DEFINITION=$CURRENT_WORKER_TASK_DEFINITION"
+          } >> "$GITHUB_ENV"
+
+      - name: Find the previous release pair
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          RECORD_DIR="$RUNNER_TEMP/production-releases"
+          mkdir -p "$RECORD_DIR"
+
+          gh api --paginate "/repos/$GITHUB_REPOSITORY/actions/artifacts?per_page=100" \
+            --jq '.artifacts[]
+              | select(.expired == false)
+              | select(.name | startswith("production-release-"))
+              | [.created_at, (.id | tostring)]
+              | @tsv' \
+            | sort -r > "$RECORD_DIR/releases.tsv"
+
+          gh api --paginate "/repos/$GITHUB_REPOSITORY/actions/artifacts?per_page=100" \
+            --jq '.artifacts[]
+              | select(.expired == false)
+              | select(.name | startswith("production-restore-point-"))
+              | [.created_at, (.id | tostring)]
+              | @tsv' \
+            | sort -r > "$RECORD_DIR/restore-points.tsv"
+
+          TARGET_API_TASK_DEFINITION=""
+          TARGET_WORKER_TASK_DEFINITION=""
+          SOURCE_RELEASE=""
+          FALLBACK_API_TASK_DEFINITION=""
+          FALLBACK_WORKER_TASK_DEFINITION=""
+          FALLBACK_RELEASE=""
+
+          while IFS=$'\t' read -r _ artifact_id; do
+            test -n "$artifact_id" || continue
+
+            archive="$RECORD_DIR/$artifact_id.zip"
+            record="$RECORD_DIR/$artifact_id.json"
+            gh api "/repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact_id/zip" > "$archive"
+            unzip -p "$archive" release.json > "$record"
+
+            recorded_api=$(jq -r '.apiTaskDefinition // empty' "$record")
+            recorded_worker=$(jq -r '.workerTaskDefinition // empty' "$record")
+
+            if [ -z "$FALLBACK_API_TASK_DEFINITION" ] && [ -n "$recorded_api" ] && [ -n "$recorded_worker" ]; then
+              FALLBACK_API_TASK_DEFINITION="$recorded_api"
+              FALLBACK_WORKER_TASK_DEFINITION="$recorded_worker"
+              FALLBACK_RELEASE=$(jq -r '.release // empty' "$record")
+            fi
+
+            if [ "$recorded_api" = "$CURRENT_API_TASK_DEFINITION" ] && \
+               [ "$recorded_worker" = "$CURRENT_WORKER_TASK_DEFINITION" ]; then
+              previous_api=$(jq -r '.previousApiTaskDefinition // empty' "$record")
+              previous_worker=$(jq -r '.previousWorkerTaskDefinition // empty' "$record")
+
+              if [ -n "$previous_api" ] && [ -n "$previous_worker" ] && \
+                 [ "$previous_api" != "$CURRENT_API_TASK_DEFINITION" ] && \
+                 [ "$previous_worker" != "$CURRENT_WORKER_TASK_DEFINITION" ]; then
+                TARGET_API_TASK_DEFINITION="$previous_api"
+                TARGET_WORKER_TASK_DEFINITION="$previous_worker"
+                SOURCE_RELEASE=$(jq -r '.release // empty' "$record")
+                break
+              fi
+            fi
+          done < "$RECORD_DIR/releases.tsv"
+
+          if [ -z "$TARGET_API_TASK_DEFINITION" ] && \
+             [ -n "$FALLBACK_API_TASK_DEFINITION" ] && \
+             [ -n "$FALLBACK_WORKER_TASK_DEFINITION" ]; then
+            TARGET_API_TASK_DEFINITION="$FALLBACK_API_TASK_DEFINITION"
+            TARGET_WORKER_TASK_DEFINITION="$FALLBACK_WORKER_TASK_DEFINITION"
+            SOURCE_RELEASE="unrecorded-current-to-$FALLBACK_RELEASE"
+          fi
+
+          if [ -z "$TARGET_API_TASK_DEFINITION" ] || [ -z "$TARGET_WORKER_TASK_DEFINITION" ]; then
+            while IFS=$'\t' read -r _ artifact_id; do
+              test -n "$artifact_id" || continue
+
+              archive="$RECORD_DIR/restore-$artifact_id.zip"
+              record="$RECORD_DIR/restore-$artifact_id.json"
+              gh api "/repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact_id/zip" > "$archive"
+              unzip -p "$archive" restore-point.json > "$record"
+
+              previous_api=$(jq -r '.previousApiTaskDefinition // empty' "$record")
+              previous_worker=$(jq -r '.previousWorkerTaskDefinition // empty' "$record")
+
+              if [ -n "$previous_api" ] && [ -n "$previous_worker" ] && \
+                 { [ "$previous_api" != "$CURRENT_API_TASK_DEFINITION" ] || \
+                   [ "$previous_worker" != "$CURRENT_WORKER_TASK_DEFINITION" ]; }; then
+                TARGET_API_TASK_DEFINITION="$previous_api"
+                TARGET_WORKER_TASK_DEFINITION="$previous_worker"
+                SOURCE_RELEASE="failed-$(jq -r '.release // "unknown"' "$record")"
+                break
+              fi
+            done < "$RECORD_DIR/restore-points.tsv"
+          fi
+
+          test -n "$TARGET_API_TASK_DEFINITION"
+          test -n "$TARGET_WORKER_TASK_DEFINITION"
+          test "$TARGET_API_TASK_DEFINITION" != "$CURRENT_API_TASK_DEFINITION" || \
+            test "$TARGET_WORKER_TASK_DEFINITION" != "$CURRENT_WORKER_TASK_DEFINITION"
+
+          {
+            echo "TARGET_API_TASK_DEFINITION=$TARGET_API_TASK_DEFINITION"
+            echo "TARGET_WORKER_TASK_DEFINITION=$TARGET_WORKER_TASK_DEFINITION"
+            echo "SOURCE_RELEASE=$SOURCE_RELEASE"
+          } >> "$GITHUB_ENV"
+
+      - name: Verify target task definitions
+        run: |
+          test "$(aws ecs describe-task-definition \
+            --task-definition "$TARGET_API_TASK_DEFINITION" \
+            --query 'taskDefinition.status' \
+            --output text \
+            --region "$AWS_REGION")" = "ACTIVE"
+          test "$(aws ecs describe-task-definition \
+            --task-definition "$TARGET_WORKER_TASK_DEFINITION" \
+            --query 'taskDefinition.status' \
+            --output text \
+            --region "$AWS_REGION")" = "ACTIVE"
+
+      - name: Roll back ECS services
+        run: |
+          aws ecs update-service \
+            --cluster "$ECS_CLUSTER" \
+            --service "$API_SERVICE" \
+            --task-definition "$TARGET_API_TASK_DEFINITION" \
+            --region "$AWS_REGION" > /dev/null
+          aws ecs update-service \
+            --cluster "$ECS_CLUSTER" \
+            --service "$WORKER_SERVICE" \
+            --task-definition "$TARGET_WORKER_TASK_DEFINITION" \
+            --region "$AWS_REGION" > /dev/null
+
+      - name: Wait for ECS services
+        run: |
+          aws ecs wait services-stable \
+            --cluster "$ECS_CLUSTER" \
+            --services "$API_SERVICE" "$WORKER_SERVICE" \
+            --region "$AWS_REGION"
+
+      - name: Record rollback
+        run: |
+          jq -n \
+            --arg sourceRelease "$SOURCE_RELEASE" \
+            --arg fromApiTaskDefinition "$CURRENT_API_TASK_DEFINITION" \
+            --arg fromWorkerTaskDefinition "$CURRENT_WORKER_TASK_DEFINITION" \
+            --arg toApiTaskDefinition "$TARGET_API_TASK_DEFINITION" \
+            --arg toWorkerTaskDefinition "$TARGET_WORKER_TASK_DEFINITION" \
+            --arg workflowActor "$GITHUB_ACTOR" \
+            --arg rolledBackAt "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \
+            '{
+              sourceRelease: $sourceRelease,
+              fromApiTaskDefinition: $fromApiTaskDefinition,
+              fromWorkerTaskDefinition: $fromWorkerTaskDefinition,
+              toApiTaskDefinition: $toApiTaskDefinition,
+              toWorkerTaskDefinition: $toWorkerTaskDefinition,
+              workflowActor: $workflowActor,
+              rolledBackAt: $rolledBackAt,
+              status: "rolled-back"
+            }' > rollback.json
+
+          {
+            echo "## Production rolled back"
+            echo
+            echo "- From API: \`$CURRENT_API_TASK_DEFINITION\`"
+            echo "- To API: \`$TARGET_API_TASK_DEFINITION\`"
+            echo "- From worker: \`$CURRENT_WORKER_TASK_DEFINITION\`"
+            echo "- To worker: \`$TARGET_WORKER_TASK_DEFINITION\`"
+            echo "- ECS services: stable"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload rollback record
+        uses: actions/upload-artifact@v7
+        with:
+          name: production-rollback-${{ github.run_id }}
+          path: rollback.json
+          if-no-files-found: error
+          retention-days: 90
diff --git a/README.md b/README.md
index 99e4b87..66252a7 100644
--- a/README.md
+++ b/README.md
@@ -50,10 +50,11 @@ Worker health: `http://127.0.0.1:3001/health/live`
 
 ## Production Deployment
 
-Pull requests run application and Terraform checks. The current production
-workflow is intentionally prepare-only: it builds SHA-tagged API and worker
-images, verifies their contents, and creates a production Terraform plan. It
-does not apply Terraform or change ECS. See [Production
+Pull requests run application and Terraform checks. Production is released
+manually from the current `main` commit; one workflow builds immutable API and
+worker images, applies Terraform, and waits for ECS. Rollback automatically
+selects the previous recorded task-definition pair. Apply and rollback are
+temporarily disabled while the workflow is validated. See [Production
 deployment](docs/production-deployment.md) for setup and operating details.
 
 ## API Flow
diff --git a/docs/production-deployment.md b/docs/production-deployment.md
index d74a676..9bc40f0 100644
--- a/docs/production-deployment.md
+++ b/docs/production-deployment.md
@@ -1,10 +1,18 @@
-# Production release preparation
+# Production deployment
 
-The first rollout phase prepares and inspects production materials without
-changing the running service. There is currently no active CI deployment or
-rollback workflow.
+Production is released manually from the current `main` commit. Image builds,
+Terraform plan/apply, and ECS deployment happen in one workflow run, so there
+is no separately selected image SHA or saved intermediate Terraform plan.
 
-## Pull-request checks
+> **Temporary validation guard:** the workflow currently stops after the
+> Terraform plan summary. Every step from the restore point through apply,
+> ECS update, release recording, and retention has `if: false`; the rollback
+> job is disabled the same way. Remove those guards only after the prepared
+> images and Terraform plan have been verified.
+
+## Workflows
+
+### Pull-request checks
 
 `.github/workflows/ci.yml` runs for pull requests targeting `main`:
 
@@ -13,47 +21,54 @@ rollback workflow.
 3. Check Terraform formatting.
 4. Initialize Terraform without a backend and validate the configuration.
 
-The repository does not currently define a benchmark command. Add one only
-after the project has a stable benchmark script and failure threshold.
+The repository does not currently define a benchmark command. Add it to this
+workflow after the project has a stable benchmark script and failure threshold.
 
-## Prepare a production release
+### Production release
 
-After this workflow has been merged, open GitHub Actions, select **Prepare
-production release**, and run it from `main`. It:
+`.github/workflows/deploy-production.yml` has no inputs. Run it from the `main`
+branch in GitHub Actions. The workflow:
 
-1. Locks the run to the current `main` commit identified by `github.sha`.
+1. Locks the run to the `main` commit identified by `github.sha`.
 2. Assumes the image role through GitHub OIDC.
-3. Builds and pushes API and worker images tagged with the full commit SHA.
-4. Resolves both ECR image digests and verifies the built entrypoint files.
-5. Assumes the Terraform role through GitHub OIDC.
-6. Initializes the KMS-encrypted S3 Terraform backend.
-7. Creates a local, ephemeral production `tfplan`.
-8. Writes the image identities and Terraform change counts to Job Summary.
-
-The workflow writes SHA-tagged images to ECR. It does **not**:
-
-- run `terraform apply`;
-- register or activate new ECS task-definition revisions;
-- update either ECS service;
-- run database migrations;
-- save the `tfplan` after the runner is deleted.
-
-Rerunning the same commit reuses an existing image and rebuilds only a missing
-one. Normal operation must not overwrite a SHA tag.
-
-## What to review
-
-Before enabling real deployment, run the preparation workflow and confirm:
-
-- both images exist under the expected SHA and have valid digests;
-- both container entrypoint files pass the image-content check;
-- the Terraform change counts are expected;
-- the detailed Terraform log contains no unexpected resource changes;
-- OIDC, the remote backend, and `TF_VARS` all work in GitHub Actions.
-
-A later, separately reviewed change can add Terraform apply, ECS stability
-waiting, release records, retention, and rollback. That keeps the first
-production-affecting run out of this initial CI change.
+3. Builds and pushes API and worker images tagged with that full SHA. A rerun
+   reuses an existing immutable image and builds only a missing image.
+4. Reads both ECR image digests.
+5. Assumes the deployment role through GitHub OIDC.
+6. Reads the currently deployed API and worker task definitions for rollback.
+7. Initializes the KMS-encrypted S3 Terraform backend.
+8. Creates a local `tfplan` in the same runner.
+9. Uploads a temporary restore point containing the task-definition pair that
+   is currently online, then immediately applies the plan.
+10. Waits for both ECS services to become stable.
+11. Uploads a release record containing the SHA, image pair, task-definition
+    pair, and previous task-definition pair.
+12. Deletes the temporary restore point after the successful release record is
+    safely uploaded. Failed runs keep their restore point for recovery.
+
+Terraform owns the ECS task definitions and the ECS services' task-definition
+references. Normal deployment must not call `aws ecs update-service` directly.
+
+The workflow does not use a GitHub Environment. The **Run workflow** action is
+the production release decision. The job-level branch check and both AWS role
+trust policies must restrict production access to `refs/heads/main`.
+
+### Production rollback
+
+`.github/workflows/rollback-production.yml` also has no inputs. It finds the
+release record matching the task-definition pair currently used by ECS and
+switches both services to the previous pair recorded by that release.
+
+If the current deployment failed before it could write a success record, the
+workflow falls back to the newest recorded successful task-definition pair. If
+there has never been a successful CI release, it uses the newest failed run's
+pre-deployment restore point. It refuses to continue if it cannot find a usable
+record or if either target task definition is inactive.
+
+Rollback changes the ECS service references directly and intentionally leaves
+Terraform state untouched. The next production release refreshes the state and
+deploys its new task-definition pair. API and worker updates are coordinated by
+one workflow but are not an atomic AWS transaction.
 
 ## Terraform state
 
@@ -72,9 +87,9 @@ The bucket must keep:
 - an HTTPS-only bucket policy;
 - state and lock-file permissions scoped to the two objects above.
 
-The bucket does not store `tfplan`. Existing
+The bucket no longer stores `tfplan` or release-candidate files. Existing
 `mem9-node/production/plans/` lifecycle rules and IAM permissions are harmless
-but can be removed after this workflow has been verified.
+but can be removed after the new workflow has been verified.
 
 ### Migrate an existing local state
 
@@ -97,8 +112,7 @@ Confirm the remote state before removing the local backup.
 
 Create one repository-level GitHub Actions secret named `TF_VARS`. Its value is
 the complete production `terraform.tfvars` content. The workflow writes it to
-the ephemeral runner without printing it; GitHub deletes the runner after the
-job.
+the ephemeral runner without printing it and removes the runner after the job.
 
 Do not add these release-specific values to `TF_VARS`:
 
@@ -108,7 +122,9 @@ api_image
 worker_image
 ```
 
-The workflow supplies those values from the release SHA and its two images.
+The workflow supplies those three values from the release SHA and its two ECR
+images. Command-line `-var` values take precedence if an older local file still
+contains them, but removing them from `TF_VARS` keeps ownership unambiguous.
 
 Protect `main` and require the `Verify` status check before merge. Block force
 pushes and branch deletion. A benchmark is not a required check until a real
@@ -116,8 +132,8 @@ benchmark command exists in the repository.
 
 ## AWS OIDC roles
 
-Both trust policies must restrict production access to this repository and the
-`main` branch subject:
+Keep the two existing roles but restrict both trust policies to this repository
+and the `main` branch subject:
 
 ```text
 repo:mem9-ai/mem9-node:ref:refs/heads/main
@@ -125,26 +141,30 @@ repo:mem9-ai/mem9-node:ref:refs/heads/main
 
 - `mem9-node-prod-github-prepare`: ECR login, image lookup, and image push for
   the API and worker repositories only.
-- `mem9-node-prod-github-apply`: currently used for Terraform state access and
-  read-only planning. Its broader apply permissions are reserved for the later
-  deployment phase.
+- `mem9-node-prod-github-apply`: Terraform state/KMS access, infrastructure
+  plan/apply permissions, ECS read/update permissions, and task-definition
+  retention permissions.
 
-The preparation workflow has no GitHub Environment approval because it does
-not change the running service. The **Run workflow** action only starts material
-preparation.
+The apply role's old Environment-based subject must be changed before the new
+deployment and rollback workflows can assume it. The unused GitHub
+`production` Environment may remain or be deleted after verification.
 
-## Retention and rollback
+## Retention
 
-The current workflow does not clean up ECR images, task-definition revisions,
-or release records. Keep the prepared SHA images during validation. Add bounded
-retention together with real deployment and release records so cleanup cannot
-delete an image needed for rollback.
+- ECR images use immutable full-SHA tags.
+- ECS keeps the latest 30 active API revisions and 30 active worker revisions.
+- GitHub keeps the latest 30 successful release-record artifacts, subject to
+  the repository's artifact retention limit.
+- A failed deployment keeps its temporary pre-deployment restore point. A
+  successful deployment deletes that temporary record after uploading the
+  permanent release record.
+- Rollback records are stored separately and do not participate in selecting a
+  rollback target.
 
-There is no active rollback workflow until CI owns at least one successful
-production release record. Existing manual rollback procedures remain the
-fallback in the meantime.
+Do not introduce a count-based ECR lifecycle rule until it can preserve every
+image referenced by the retained release records.
 
 ## Database migrations
 
-The workflow does not run database migrations. Schema changes require a
+These workflows do not run database migrations. Schema changes require a
 separately reviewed, backward-compatible migration process.

From 4283f3584766511a00a37ebacc50cc01d9f746f0 Mon Sep 17 00:00:00 2001
From: Eric Zhang <chong.zhang@pingcap.com>
Date: Mon, 3 Aug 2026 18:19:57 +0800
Subject: [PATCH 4/9] test(ci): run production preparation from feat/ci

---
 .github/workflows/deploy-production.yml | 9 +++++++--
 docs/production-deployment.md           | 4 ++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml
index 88333f2..f06d281 100644
--- a/.github/workflows/deploy-production.yml
+++ b/.github/workflows/deploy-production.yml
@@ -3,6 +3,10 @@ run-name: Deploy ${{ github.sha }} to production
 
 on:
   workflow_dispatch:
+  # Temporary PR validation trigger. Remove before merging.
+  push:
+    branches:
+      - feat/ci
 
 permissions:
   contents: read
@@ -26,7 +30,7 @@ env:
 jobs:
   deploy:
     name: Build and deploy current main
-    if: github.ref == 'refs/heads/main'
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/feat/ci'
     runs-on: ubuntu-latest
     timeout-minutes: 60
     permissions:
@@ -42,7 +46,8 @@ jobs:
 
       - name: Verify release source
         run: |
-          test "$GITHUB_REF" = "refs/heads/main"
+          test "$GITHUB_REF" = "refs/heads/main" || \
+            test "$GITHUB_REF" = "refs/heads/feat/ci"
           test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
 
       - name: Configure AWS image credentials
diff --git a/docs/production-deployment.md b/docs/production-deployment.md
index 9bc40f0..0ea671a 100644
--- a/docs/production-deployment.md
+++ b/docs/production-deployment.md
@@ -10,6 +10,10 @@ is no separately selected image SHA or saved intermediate Terraform plan.
 > job is disabled the same way. Remove those guards only after the prepared
 > images and Terraform plan have been verified.
 
+> The `feat/ci` push trigger and branch allowance are temporary so this PR can
+> run before merge. Remove both, together with the matching temporary AWS OIDC
+> branch trust, before merging to `main`.
+
 ## Workflows
 
 ### Pull-request checks

From 15a49560a5a34e8554e70b5f9f9a8870f546013d Mon Sep 17 00:00:00 2001
From: Eric Zhang <chong.zhang@pingcap.com>
Date: Mon, 3 Aug 2026 18:38:29 +0800
Subject: [PATCH 5/9] test(ci): remove temporary branch validation access

---
 .github/workflows/deploy-production.yml |  9 ++-------
 docs/production-deployment.md           | 17 ++++++++++-------
 2 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml
index f06d281..88333f2 100644
--- a/.github/workflows/deploy-production.yml
+++ b/.github/workflows/deploy-production.yml
@@ -3,10 +3,6 @@ run-name: Deploy ${{ github.sha }} to production
 
 on:
   workflow_dispatch:
-  # Temporary PR validation trigger. Remove before merging.
-  push:
-    branches:
-      - feat/ci
 
 permissions:
   contents: read
@@ -30,7 +26,7 @@ env:
 jobs:
   deploy:
     name: Build and deploy current main
-    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/feat/ci'
+    if: github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
     timeout-minutes: 60
     permissions:
@@ -46,8 +42,7 @@ jobs:
 
       - name: Verify release source
         run: |
-          test "$GITHUB_REF" = "refs/heads/main" || \
-            test "$GITHUB_REF" = "refs/heads/feat/ci"
+          test "$GITHUB_REF" = "refs/heads/main"
           test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
 
       - name: Configure AWS image credentials
diff --git a/docs/production-deployment.md b/docs/production-deployment.md
index 0ea671a..015c746 100644
--- a/docs/production-deployment.md
+++ b/docs/production-deployment.md
@@ -10,10 +10,6 @@ is no separately selected image SHA or saved intermediate Terraform plan.
 > job is disabled the same way. Remove those guards only after the prepared
 > images and Terraform plan have been verified.
 
-> The `feat/ci` push trigger and branch allowance are temporary so this PR can
-> run before merge. Remove both, together with the matching temporary AWS OIDC
-> branch trust, before merging to `main`.
-
 ## Workflows
 
 ### Pull-request checks
@@ -149,9 +145,16 @@ repo:mem9-ai/mem9-node:ref:refs/heads/main
   plan/apply permissions, ECS read/update permissions, and task-definition
   retention permissions.
 
-The apply role's old Environment-based subject must be changed before the new
-deployment and rollback workflows can assume it. The unused GitHub
-`production` Environment may remain or be deleted after verification.
+Terraform's S3 bucket refresh also requires these two read-only actions on the
+application payload bucket:
+
+```text
+s3:GetAccelerateConfiguration
+s3:GetReplicationConfiguration
+```
+
+Both role trust policies currently use the `main` branch subject shown above.
+The unused GitHub `production` Environment may remain or be deleted.
 
 ## Retention
 

From ce7510ac6ae7342eb8217f4ccbf75f863726319d Mon Sep 17 00:00:00 2001
From: Eric Zhang <chong.zhang@pingcap.com>
Date: Mon, 3 Aug 2026 20:08:16 +0800
Subject: [PATCH 6/9] fix(terraform): preserve ECS cluster tags

---
 infra/terraform/main.tf | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/infra/terraform/main.tf b/infra/terraform/main.tf
index cfcad9d..8053b18 100644
--- a/infra/terraform/main.tf
+++ b/infra/terraform/main.tf
@@ -77,6 +77,12 @@ resource "aws_sqs_queue" "analysis_llm" {
 
 resource "aws_ecs_cluster" "this" {
   name = "${var.name_prefix}-cluster"
+
+  tags = {
+    component   = "node"
+    environment = "prod"
+    servicetype = "mem9"
+  }
 }
 
 resource "aws_cloudwatch_log_group" "api" {

From d6deb91e0460a245fac64ad295fcf540591952a1 Mon Sep 17 00:00:00 2001
From: Eric Zhang <chong.zhang@pingcap.com>
Date: Mon, 3 Aug 2026 20:11:39 +0800
Subject: [PATCH 7/9] test(ci): verify cluster tag preservation

---
 .github/workflows/deploy-production.yml | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml
index 88333f2..f06d281 100644
--- a/.github/workflows/deploy-production.yml
+++ b/.github/workflows/deploy-production.yml
@@ -3,6 +3,10 @@ run-name: Deploy ${{ github.sha }} to production
 
 on:
   workflow_dispatch:
+  # Temporary PR validation trigger. Remove before merging.
+  push:
+    branches:
+      - feat/ci
 
 permissions:
   contents: read
@@ -26,7 +30,7 @@ env:
 jobs:
   deploy:
     name: Build and deploy current main
-    if: github.ref == 'refs/heads/main'
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/feat/ci'
     runs-on: ubuntu-latest
     timeout-minutes: 60
     permissions:
@@ -42,7 +46,8 @@ jobs:
 
       - name: Verify release source
         run: |
-          test "$GITHUB_REF" = "refs/heads/main"
+          test "$GITHUB_REF" = "refs/heads/main" || \
+            test "$GITHUB_REF" = "refs/heads/feat/ci"
           test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
 
       - name: Configure AWS image credentials

From 2fbe5920a0e607531ad66734cddb5c4390a94a9f Mon Sep 17 00:00:00 2001
From: Eric Zhang <chong.zhang@pingcap.com>
Date: Mon, 3 Aug 2026 20:15:51 +0800
Subject: [PATCH 8/9] test(ci): remove temporary tag verification trigger

---
 .github/workflows/deploy-production.yml | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml
index f06d281..88333f2 100644
--- a/.github/workflows/deploy-production.yml
+++ b/.github/workflows/deploy-production.yml
@@ -3,10 +3,6 @@ run-name: Deploy ${{ github.sha }} to production
 
 on:
   workflow_dispatch:
-  # Temporary PR validation trigger. Remove before merging.
-  push:
-    branches:
-      - feat/ci
 
 permissions:
   contents: read
@@ -30,7 +26,7 @@ env:
 jobs:
   deploy:
     name: Build and deploy current main
-    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/feat/ci'
+    if: github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
     timeout-minutes: 60
     permissions:
@@ -46,8 +42,7 @@ jobs:
 
       - name: Verify release source
         run: |
-          test "$GITHUB_REF" = "refs/heads/main" || \
-            test "$GITHUB_REF" = "refs/heads/feat/ci"
+          test "$GITHUB_REF" = "refs/heads/main"
           test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
 
       - name: Configure AWS image credentials

From 20548b4ada92e22d1bebb300fd0dda5fc40b7b0d Mon Sep 17 00:00:00 2001
From: Eric Zhang <chong.zhang@pingcap.com>
Date: Tue, 4 Aug 2026 17:22:57 +0800
Subject: [PATCH 9/9] feat(ci): enable production deploy and rollback

---
 .github/workflows/deploy-production.yml   | 11 -----------
 .github/workflows/rollback-production.yml |  3 +--
 README.md                                 |  3 +--
 docs/production-deployment.md             |  6 ------
 4 files changed, 2 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml
index 88333f2..3b2d213 100644
--- a/.github/workflows/deploy-production.yml
+++ b/.github/workflows/deploy-production.yml
@@ -199,9 +199,6 @@ jobs:
           } >> "$GITHUB_STEP_SUMMARY"
 
       - name: Record pre-deployment restore point
-        # Temporary guard: keep the complete release path for validation, but
-        # do not run any step after the Terraform plan until it is approved.
-        if: ${{ false }}
         run: |
           jq -n \
             --arg release "$GITHUB_SHA" \
@@ -223,7 +220,6 @@ jobs:
             }' > restore-point.json
 
       - name: Upload pre-deployment restore point
-        if: ${{ false }}
         id: upload-restore-point
         uses: actions/upload-artifact@v7
         with:
@@ -233,11 +229,9 @@ jobs:
           retention-days: 90
 
       - name: Apply production release
-        if: ${{ false }}
         run: terraform -chdir="$TF_WORKING_DIR" apply -input=false -auto-approve tfplan
 
       - name: Wait for ECS services
-        if: ${{ false }}
         run: |
           ECS_CLUSTER=$(terraform -chdir="$TF_WORKING_DIR" output -raw ecs_cluster_name)
           API_SERVICE=$(terraform -chdir="$TF_WORKING_DIR" output -raw api_service_name)
@@ -249,7 +243,6 @@ jobs:
             --region "$AWS_REGION"
 
       - name: Record successful release
-        if: ${{ false }}
         run: |
           API_TASK_DEFINITION=$(terraform -chdir="$TF_WORKING_DIR" output -raw api_task_definition_arn)
           WORKER_TASK_DEFINITION=$(terraform -chdir="$TF_WORKING_DIR" output -raw worker_task_definition_arn)
@@ -295,7 +288,6 @@ jobs:
           } >> "$GITHUB_STEP_SUMMARY"
 
       - name: Upload release record
-        if: ${{ false }}
         uses: actions/upload-artifact@v7
         with:
           name: production-release-${{ github.sha }}-${{ github.run_id }}
@@ -304,7 +296,6 @@ jobs:
           retention-days: 90
 
       - name: Delete pre-deployment restore point
-        if: ${{ false }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ github.token }}
@@ -315,7 +306,6 @@ jobs:
             "/repos/$GITHUB_REPOSITORY/actions/artifacts/$RESTORE_POINT_ARTIFACT_ID"
 
       - name: Keep the latest 30 task definition revisions
-        if: ${{ false }}
         env:
           TASK_DEFINITION_RETENTION: "30"
         run: |
@@ -340,7 +330,6 @@ jobs:
           done
 
       - name: Keep the latest 30 release records
-        if: ${{ false }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ github.token }}
diff --git a/.github/workflows/rollback-production.yml b/.github/workflows/rollback-production.yml
index 1f1d451..e8363fb 100644
--- a/.github/workflows/rollback-production.yml
+++ b/.github/workflows/rollback-production.yml
@@ -20,8 +20,7 @@ env:
 jobs:
   rollback:
     name: Roll back to previous successful release
-    # Temporary guard: rollback becomes usable after CI owns a verified release.
-    if: github.ref == 'refs/heads/main' && false
+    if: github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
     timeout-minutes: 30
     permissions:
diff --git a/README.md b/README.md
index 66252a7..d3ab224 100644
--- a/README.md
+++ b/README.md
@@ -53,8 +53,7 @@ Worker health: `http://127.0.0.1:3001/health/live`
 Pull requests run application and Terraform checks. Production is released
 manually from the current `main` commit; one workflow builds immutable API and
 worker images, applies Terraform, and waits for ECS. Rollback automatically
-selects the previous recorded task-definition pair. Apply and rollback are
-temporarily disabled while the workflow is validated. See [Production
+selects the previous recorded task-definition pair. See [Production
 deployment](docs/production-deployment.md) for setup and operating details.
 
 ## API Flow
diff --git a/docs/production-deployment.md b/docs/production-deployment.md
index 015c746..847e121 100644
--- a/docs/production-deployment.md
+++ b/docs/production-deployment.md
@@ -4,12 +4,6 @@ Production is released manually from the current `main` commit. Image builds,
 Terraform plan/apply, and ECS deployment happen in one workflow run, so there
 is no separately selected image SHA or saved intermediate Terraform plan.
 
-> **Temporary validation guard:** the workflow currently stops after the
-> Terraform plan summary. Every step from the restore point through apply,
-> ECS update, release recording, and retention has `if: false`; the rollback
-> job is disabled the same way. Remove those guards only after the prepared
-> images and Terraform plan have been verified.
-
 ## Workflows
 
 ### Pull-request checks