diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5b5246d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,23 @@ +# Source-control and CI metadata +.git +.github + +# Local dependencies, caches, logs, and build output +**/node_modules +.pnpm-store +**/dist +coverage +logs +*.log +*.tsbuildinfo +.DS_Store + +# Local environment and credential files +.env +.env.* +.sentryclirc + +# Terraform configuration is not part of the application image. Excluding the +# whole directory also keeps local variables, state, plans, and backups out of +# the Docker build context. +infra/terraform diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..f5170d3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,56 @@ +name: CI + +on: + pull_request: + branches: + - main + +permissions: + contents: read + +concurrency: + group: ci-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + TERRAFORM_VERSION: 1.15.4 + TF_WORKING_DIR: infra/terraform + +jobs: + verify: + name: Verify + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Set up pnpm + uses: pnpm/action-setup@v4 + + - name: Set up Node.js + uses: actions/setup-node@v6 + with: + node-version-file: .nvmrc + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Verify application + run: pnpm verify + + - name: Set up Terraform + uses: hashicorp/setup-terraform@v4 + with: + terraform_version: ${{ env.TERRAFORM_VERSION }} + terraform_wrapper: false + + - name: Check Terraform formatting + run: terraform fmt -check -recursive "$TF_WORKING_DIR" + + - name: Validate Terraform + run: | + terraform -chdir="$TF_WORKING_DIR" init -backend=false -input=false + terraform -chdir="$TF_WORKING_DIR" validate diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml new file mode 100644 index 0000000..3b2d213 --- /dev/null +++ b/.github/workflows/deploy-production.yml @@ -0,0 +1,350 @@ +name: Deploy production +run-name: Deploy ${{ github.sha }} to production + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: production + cancel-in-progress: false + +env: + AWS_REGION: ap-southeast-1 + ECR_REGISTRY: 401696231252.dkr.ecr.ap-southeast-1.amazonaws.com + API_REPOSITORY: mem9-node-prod-api + WORKER_REPOSITORY: mem9-node-prod-worker + TF_STATE_BUCKET: mem9-node-prod-terraform-state + TF_STATE_KMS_KEY_ARN: arn:aws:kms:ap-southeast-1:401696231252:key/4cd4c7a1-c1a6-47ad-9205-548678b84d86 + TERRAFORM_VERSION: 1.15.4 + TF_WORKING_DIR: infra/terraform + TF_IN_AUTOMATION: "true" + TF_INPUT: "false" + +jobs: + deploy: + name: Build and deploy current main + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + actions: write + contents: read + id-token: write + + steps: + - name: Check out the release commit + uses: actions/checkout@v6 + with: + ref: ${{ github.sha }} + + - name: Verify release source + run: | + test "$GITHUB_REF" = "refs/heads/main" + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + + - name: Configure AWS image credentials + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: arn:aws:iam::401696231252:role/mem9-node-prod-github-prepare + role-session-name: mem9-node-production-images + aws-region: ${{ env.AWS_REGION }} + mask-aws-account-id: true + + - name: Log in to Amazon ECR + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build and push API image + run: | + if aws ecr describe-images \ + --repository-name "$API_REPOSITORY" \ + --image-ids "imageTag=$GITHUB_SHA" \ + --region "$AWS_REGION" > /dev/null 2>&1; then + echo "API image $GITHUB_SHA already exists; reusing it." + else + docker build \ + --platform linux/amd64 \ + --file apps/api/Dockerfile \ + --tag "$ECR_REGISTRY/$API_REPOSITORY:$GITHUB_SHA" \ + . + docker push "$ECR_REGISTRY/$API_REPOSITORY:$GITHUB_SHA" + fi + + - name: Build and push worker image + run: | + if aws ecr describe-images \ + --repository-name "$WORKER_REPOSITORY" \ + --image-ids "imageTag=$GITHUB_SHA" \ + --region "$AWS_REGION" > /dev/null 2>&1; then + echo "Worker image $GITHUB_SHA already exists; reusing it." + else + docker build \ + --platform linux/amd64 \ + --file apps/worker/Dockerfile \ + --tag "$ECR_REGISTRY/$WORKER_REPOSITORY:$GITHUB_SHA" \ + . + docker push "$ECR_REGISTRY/$WORKER_REPOSITORY:$GITHUB_SHA" + fi + + - name: Lock release images + run: | + API_IMAGE="$ECR_REGISTRY/$API_REPOSITORY:$GITHUB_SHA" + WORKER_IMAGE="$ECR_REGISTRY/$WORKER_REPOSITORY:$GITHUB_SHA" + API_DIGEST=$(aws ecr describe-images \ + --repository-name "$API_REPOSITORY" \ + --image-ids "imageTag=$GITHUB_SHA" \ + --query 'imageDetails[0].imageDigest' \ + --output text \ + --region "$AWS_REGION") + WORKER_DIGEST=$(aws ecr describe-images \ + --repository-name "$WORKER_REPOSITORY" \ + --image-ids "imageTag=$GITHUB_SHA" \ + --query 'imageDetails[0].imageDigest' \ + --output text \ + --region "$AWS_REGION") + + test -n "$API_DIGEST" && test "$API_DIGEST" != "None" + test -n "$WORKER_DIGEST" && test "$WORKER_DIGEST" != "None" + + { + echo "API_IMAGE=$API_IMAGE" + echo "WORKER_IMAGE=$WORKER_IMAGE" + echo "API_DIGEST=$API_DIGEST" + echo "WORKER_DIGEST=$WORKER_DIGEST" + } >> "$GITHUB_ENV" + + - name: Configure AWS deployment credentials + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: arn:aws:iam::401696231252:role/mem9-node-prod-github-apply + role-session-name: mem9-node-production-deploy + aws-region: ${{ env.AWS_REGION }} + mask-aws-account-id: true + + - name: Read current ECS release + run: | + API_PREVIOUS_TASK_DEFINITION=$(aws ecs describe-services \ + --cluster mem9-node-prod-cluster \ + --services mem9-node-prod-api \ + --query 'services[0].taskDefinition' \ + --output text \ + --region "$AWS_REGION") + WORKER_PREVIOUS_TASK_DEFINITION=$(aws ecs describe-services \ + --cluster mem9-node-prod-cluster \ + --services mem9-node-prod-worker \ + --query 'services[0].taskDefinition' \ + --output text \ + --region "$AWS_REGION") + + test -n "$API_PREVIOUS_TASK_DEFINITION" && test "$API_PREVIOUS_TASK_DEFINITION" != "None" + test -n "$WORKER_PREVIOUS_TASK_DEFINITION" && test "$WORKER_PREVIOUS_TASK_DEFINITION" != "None" + + { + echo "API_PREVIOUS_TASK_DEFINITION=$API_PREVIOUS_TASK_DEFINITION" + echo "WORKER_PREVIOUS_TASK_DEFINITION=$WORKER_PREVIOUS_TASK_DEFINITION" + } >> "$GITHUB_ENV" + + - name: Set up Terraform + uses: hashicorp/setup-terraform@v4 + with: + terraform_version: ${{ env.TERRAFORM_VERSION }} + terraform_wrapper: false + + - name: Write production variables + env: + TF_VARS: ${{ secrets.TF_VARS }} + run: | + test -n "$TF_VARS" + printf '%s\n' "$TF_VARS" > "$TF_WORKING_DIR/production.auto.tfvars" + + - name: Initialize Terraform + run: | + terraform -chdir="$TF_WORKING_DIR" init \ + -input=false \ + -backend-config="bucket=$TF_STATE_BUCKET" \ + -backend-config="key=mem9-node/production/terraform.tfstate" \ + -backend-config="region=$AWS_REGION" \ + -backend-config="encrypt=true" \ + -backend-config="kms_key_id=$TF_STATE_KMS_KEY_ARN" \ + -backend-config="use_lockfile=true" + + - name: Plan production release + run: | + terraform -chdir="$TF_WORKING_DIR" plan \ + -input=false \ + -var="release_id=$GITHUB_SHA" \ + -var="api_image=$API_IMAGE" \ + -var="worker_image=$WORKER_IMAGE" \ + -out=tfplan + + - name: Summarize Terraform plan + run: | + PLAN_JSON=$(mktemp) + terraform -chdir="$TF_WORKING_DIR" show -json tfplan > "$PLAN_JSON" + + ADD_COUNT=$(jq '[.resource_changes[]? | select(.change.actions | index("create"))] | length' "$PLAN_JSON") + CHANGE_COUNT=$(jq '[.resource_changes[]? | select(.change.actions | index("update"))] | length' "$PLAN_JSON") + DESTROY_COUNT=$(jq '[.resource_changes[]? | select(.change.actions | index("delete"))] | length' "$PLAN_JSON") + + { + echo "## Production plan" + echo + echo "- Release: \`$GITHUB_SHA\`" + echo "- API image: \`$API_IMAGE\`" + echo "- Worker image: \`$WORKER_IMAGE\`" + echo "- Terraform: **$ADD_COUNT add**, **$CHANGE_COUNT change**, **$DESTROY_COUNT destroy**" + echo "- Database migration: not run" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Record pre-deployment restore point + run: | + jq -n \ + --arg release "$GITHUB_SHA" \ + --arg previousApiTaskDefinition "$API_PREVIOUS_TASK_DEFINITION" \ + --arg previousWorkerTaskDefinition "$WORKER_PREVIOUS_TASK_DEFINITION" \ + --arg workflowActor "$GITHUB_ACTOR" \ + --arg repository "$GITHUB_REPOSITORY" \ + --arg runId "$GITHUB_RUN_ID" \ + --arg preparedAt "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \ + '{ + release: $release, + previousApiTaskDefinition: $previousApiTaskDefinition, + previousWorkerTaskDefinition: $previousWorkerTaskDefinition, + workflowActor: $workflowActor, + repository: $repository, + runId: $runId, + preparedAt: $preparedAt, + status: "prepared" + }' > restore-point.json + + - name: Upload pre-deployment restore point + id: upload-restore-point + uses: actions/upload-artifact@v7 + with: + name: production-restore-point-${{ github.sha }}-${{ github.run_id }} + path: restore-point.json + if-no-files-found: error + retention-days: 90 + + - name: Apply production release + run: terraform -chdir="$TF_WORKING_DIR" apply -input=false -auto-approve tfplan + + - name: Wait for ECS services + run: | + ECS_CLUSTER=$(terraform -chdir="$TF_WORKING_DIR" output -raw ecs_cluster_name) + API_SERVICE=$(terraform -chdir="$TF_WORKING_DIR" output -raw api_service_name) + WORKER_SERVICE=$(terraform -chdir="$TF_WORKING_DIR" output -raw worker_service_name) + + aws ecs wait services-stable \ + --cluster "$ECS_CLUSTER" \ + --services "$API_SERVICE" "$WORKER_SERVICE" \ + --region "$AWS_REGION" + + - name: Record successful release + run: | + API_TASK_DEFINITION=$(terraform -chdir="$TF_WORKING_DIR" output -raw api_task_definition_arn) + WORKER_TASK_DEFINITION=$(terraform -chdir="$TF_WORKING_DIR" output -raw worker_task_definition_arn) + + jq -n \ + --arg release "$GITHUB_SHA" \ + --arg apiImage "$API_IMAGE" \ + --arg apiDigest "$API_DIGEST" \ + --arg workerImage "$WORKER_IMAGE" \ + --arg workerDigest "$WORKER_DIGEST" \ + --arg apiTaskDefinition "$API_TASK_DEFINITION" \ + --arg workerTaskDefinition "$WORKER_TASK_DEFINITION" \ + --arg previousApiTaskDefinition "$API_PREVIOUS_TASK_DEFINITION" \ + --arg previousWorkerTaskDefinition "$WORKER_PREVIOUS_TASK_DEFINITION" \ + --arg workflowActor "$GITHUB_ACTOR" \ + --arg repository "$GITHUB_REPOSITORY" \ + --arg runId "$GITHUB_RUN_ID" \ + --arg deployedAt "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \ + '{ + release: $release, + apiImage: $apiImage, + apiDigest: $apiDigest, + workerImage: $workerImage, + workerDigest: $workerDigest, + apiTaskDefinition: $apiTaskDefinition, + workerTaskDefinition: $workerTaskDefinition, + previousApiTaskDefinition: $previousApiTaskDefinition, + previousWorkerTaskDefinition: $previousWorkerTaskDefinition, + workflowActor: $workflowActor, + repository: $repository, + runId: $runId, + deployedAt: $deployedAt, + status: "deployed" + }' > release.json + + { + echo + echo "## Production deployed" + echo + echo "- API task definition: \`$API_TASK_DEFINITION\`" + echo "- Worker task definition: \`$WORKER_TASK_DEFINITION\`" + echo "- ECS services: stable" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload release record + uses: actions/upload-artifact@v7 + with: + name: production-release-${{ github.sha }}-${{ github.run_id }} + path: release.json + if-no-files-found: error + retention-days: 90 + + - name: Delete pre-deployment restore point + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + RESTORE_POINT_ARTIFACT_ID: ${{ steps.upload-restore-point.outputs.artifact-id }} + run: | + test -n "$RESTORE_POINT_ARTIFACT_ID" + gh api --method DELETE \ + "/repos/$GITHUB_REPOSITORY/actions/artifacts/$RESTORE_POINT_ARTIFACT_ID" + + - name: Keep the latest 30 task definition revisions + env: + TASK_DEFINITION_RETENTION: "30" + run: | + for family in mem9-node-prod-api mem9-node-prod-worker; do + aws ecs list-task-definitions \ + --family-prefix "$family" \ + --status ACTIVE \ + --sort DESC \ + --region "$AWS_REGION" \ + --query 'taskDefinitionArns[]' \ + --output text \ + | tr '\t' '\n' \ + | grep "/$family:" \ + | tail -n "+$((TASK_DEFINITION_RETENTION + 1))" \ + | while read -r task_definition; do + if [ -n "$task_definition" ]; then + aws ecs deregister-task-definition \ + --task-definition "$task_definition" \ + --region "$AWS_REGION" > /dev/null + fi + done + done + + - name: Keep the latest 30 release records + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: | + gh api --paginate "/repos/$GITHUB_REPOSITORY/actions/artifacts?per_page=100" \ + --jq '.artifacts[] + | select(.expired == false) + | select(.name | startswith("production-release-")) + | [.created_at, (.id | tostring)] + | @tsv' \ + | sort -r \ + | tail -n +31 \ + | cut -f2 \ + | while read -r artifact_id; do + if [ -n "$artifact_id" ]; then + gh api --method DELETE "/repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact_id" + fi + done diff --git a/.github/workflows/rollback-production.yml b/.github/workflows/rollback-production.yml new file mode 100644 index 0000000..e8363fb --- /dev/null +++ b/.github/workflows/rollback-production.yml @@ -0,0 +1,238 @@ +name: Roll back production +run-name: Roll back production to its previous release + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: production + cancel-in-progress: false + +env: + AWS_REGION: ap-southeast-1 + ECS_CLUSTER: mem9-node-prod-cluster + API_SERVICE: mem9-node-prod-api + WORKER_SERVICE: mem9-node-prod-worker + +jobs: + rollback: + name: Roll back to previous successful release + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + actions: write + contents: read + id-token: write + + steps: + - name: Configure AWS deployment credentials + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: arn:aws:iam::401696231252:role/mem9-node-prod-github-apply + role-session-name: mem9-node-production-rollback + aws-region: ${{ env.AWS_REGION }} + mask-aws-account-id: true + + - name: Read current ECS release + run: | + CURRENT_API_TASK_DEFINITION=$(aws ecs describe-services \ + --cluster "$ECS_CLUSTER" \ + --services "$API_SERVICE" \ + --query 'services[0].taskDefinition' \ + --output text \ + --region "$AWS_REGION") + CURRENT_WORKER_TASK_DEFINITION=$(aws ecs describe-services \ + --cluster "$ECS_CLUSTER" \ + --services "$WORKER_SERVICE" \ + --query 'services[0].taskDefinition' \ + --output text \ + --region "$AWS_REGION") + + test -n "$CURRENT_API_TASK_DEFINITION" && test "$CURRENT_API_TASK_DEFINITION" != "None" + test -n "$CURRENT_WORKER_TASK_DEFINITION" && test "$CURRENT_WORKER_TASK_DEFINITION" != "None" + + { + echo "CURRENT_API_TASK_DEFINITION=$CURRENT_API_TASK_DEFINITION" + echo "CURRENT_WORKER_TASK_DEFINITION=$CURRENT_WORKER_TASK_DEFINITION" + } >> "$GITHUB_ENV" + + - name: Find the previous release pair + env: + GH_TOKEN: ${{ github.token }} + run: | + RECORD_DIR="$RUNNER_TEMP/production-releases" + mkdir -p "$RECORD_DIR" + + gh api --paginate "/repos/$GITHUB_REPOSITORY/actions/artifacts?per_page=100" \ + --jq '.artifacts[] + | select(.expired == false) + | select(.name | startswith("production-release-")) + | [.created_at, (.id | tostring)] + | @tsv' \ + | sort -r > "$RECORD_DIR/releases.tsv" + + gh api --paginate "/repos/$GITHUB_REPOSITORY/actions/artifacts?per_page=100" \ + --jq '.artifacts[] + | select(.expired == false) + | select(.name | startswith("production-restore-point-")) + | [.created_at, (.id | tostring)] + | @tsv' \ + | sort -r > "$RECORD_DIR/restore-points.tsv" + + TARGET_API_TASK_DEFINITION="" + TARGET_WORKER_TASK_DEFINITION="" + SOURCE_RELEASE="" + FALLBACK_API_TASK_DEFINITION="" + FALLBACK_WORKER_TASK_DEFINITION="" + FALLBACK_RELEASE="" + + while IFS=$'\t' read -r _ artifact_id; do + test -n "$artifact_id" || continue + + archive="$RECORD_DIR/$artifact_id.zip" + record="$RECORD_DIR/$artifact_id.json" + gh api "/repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact_id/zip" > "$archive" + unzip -p "$archive" release.json > "$record" + + recorded_api=$(jq -r '.apiTaskDefinition // empty' "$record") + recorded_worker=$(jq -r '.workerTaskDefinition // empty' "$record") + + if [ -z "$FALLBACK_API_TASK_DEFINITION" ] && [ -n "$recorded_api" ] && [ -n "$recorded_worker" ]; then + FALLBACK_API_TASK_DEFINITION="$recorded_api" + FALLBACK_WORKER_TASK_DEFINITION="$recorded_worker" + FALLBACK_RELEASE=$(jq -r '.release // empty' "$record") + fi + + if [ "$recorded_api" = "$CURRENT_API_TASK_DEFINITION" ] && \ + [ "$recorded_worker" = "$CURRENT_WORKER_TASK_DEFINITION" ]; then + previous_api=$(jq -r '.previousApiTaskDefinition // empty' "$record") + previous_worker=$(jq -r '.previousWorkerTaskDefinition // empty' "$record") + + if [ -n "$previous_api" ] && [ -n "$previous_worker" ] && \ + [ "$previous_api" != "$CURRENT_API_TASK_DEFINITION" ] && \ + [ "$previous_worker" != "$CURRENT_WORKER_TASK_DEFINITION" ]; then + TARGET_API_TASK_DEFINITION="$previous_api" + TARGET_WORKER_TASK_DEFINITION="$previous_worker" + SOURCE_RELEASE=$(jq -r '.release // empty' "$record") + break + fi + fi + done < "$RECORD_DIR/releases.tsv" + + if [ -z "$TARGET_API_TASK_DEFINITION" ] && \ + [ -n "$FALLBACK_API_TASK_DEFINITION" ] && \ + [ -n "$FALLBACK_WORKER_TASK_DEFINITION" ]; then + TARGET_API_TASK_DEFINITION="$FALLBACK_API_TASK_DEFINITION" + TARGET_WORKER_TASK_DEFINITION="$FALLBACK_WORKER_TASK_DEFINITION" + SOURCE_RELEASE="unrecorded-current-to-$FALLBACK_RELEASE" + fi + + if [ -z "$TARGET_API_TASK_DEFINITION" ] || [ -z "$TARGET_WORKER_TASK_DEFINITION" ]; then + while IFS=$'\t' read -r _ artifact_id; do + test -n "$artifact_id" || continue + + archive="$RECORD_DIR/restore-$artifact_id.zip" + record="$RECORD_DIR/restore-$artifact_id.json" + gh api "/repos/$GITHUB_REPOSITORY/actions/artifacts/$artifact_id/zip" > "$archive" + unzip -p "$archive" restore-point.json > "$record" + + previous_api=$(jq -r '.previousApiTaskDefinition // empty' "$record") + previous_worker=$(jq -r '.previousWorkerTaskDefinition // empty' "$record") + + if [ -n "$previous_api" ] && [ -n "$previous_worker" ] && \ + { [ "$previous_api" != "$CURRENT_API_TASK_DEFINITION" ] || \ + [ "$previous_worker" != "$CURRENT_WORKER_TASK_DEFINITION" ]; }; then + TARGET_API_TASK_DEFINITION="$previous_api" + TARGET_WORKER_TASK_DEFINITION="$previous_worker" + SOURCE_RELEASE="failed-$(jq -r '.release // "unknown"' "$record")" + break + fi + done < "$RECORD_DIR/restore-points.tsv" + fi + + test -n "$TARGET_API_TASK_DEFINITION" + test -n "$TARGET_WORKER_TASK_DEFINITION" + test "$TARGET_API_TASK_DEFINITION" != "$CURRENT_API_TASK_DEFINITION" || \ + test "$TARGET_WORKER_TASK_DEFINITION" != "$CURRENT_WORKER_TASK_DEFINITION" + + { + echo "TARGET_API_TASK_DEFINITION=$TARGET_API_TASK_DEFINITION" + echo "TARGET_WORKER_TASK_DEFINITION=$TARGET_WORKER_TASK_DEFINITION" + echo "SOURCE_RELEASE=$SOURCE_RELEASE" + } >> "$GITHUB_ENV" + + - name: Verify target task definitions + run: | + test "$(aws ecs describe-task-definition \ + --task-definition "$TARGET_API_TASK_DEFINITION" \ + --query 'taskDefinition.status' \ + --output text \ + --region "$AWS_REGION")" = "ACTIVE" + test "$(aws ecs describe-task-definition \ + --task-definition "$TARGET_WORKER_TASK_DEFINITION" \ + --query 'taskDefinition.status' \ + --output text \ + --region "$AWS_REGION")" = "ACTIVE" + + - name: Roll back ECS services + run: | + aws ecs update-service \ + --cluster "$ECS_CLUSTER" \ + --service "$API_SERVICE" \ + --task-definition "$TARGET_API_TASK_DEFINITION" \ + --region "$AWS_REGION" > /dev/null + aws ecs update-service \ + --cluster "$ECS_CLUSTER" \ + --service "$WORKER_SERVICE" \ + --task-definition "$TARGET_WORKER_TASK_DEFINITION" \ + --region "$AWS_REGION" > /dev/null + + - name: Wait for ECS services + run: | + aws ecs wait services-stable \ + --cluster "$ECS_CLUSTER" \ + --services "$API_SERVICE" "$WORKER_SERVICE" \ + --region "$AWS_REGION" + + - name: Record rollback + run: | + jq -n \ + --arg sourceRelease "$SOURCE_RELEASE" \ + --arg fromApiTaskDefinition "$CURRENT_API_TASK_DEFINITION" \ + --arg fromWorkerTaskDefinition "$CURRENT_WORKER_TASK_DEFINITION" \ + --arg toApiTaskDefinition "$TARGET_API_TASK_DEFINITION" \ + --arg toWorkerTaskDefinition "$TARGET_WORKER_TASK_DEFINITION" \ + --arg workflowActor "$GITHUB_ACTOR" \ + --arg rolledBackAt "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" \ + '{ + sourceRelease: $sourceRelease, + fromApiTaskDefinition: $fromApiTaskDefinition, + fromWorkerTaskDefinition: $fromWorkerTaskDefinition, + toApiTaskDefinition: $toApiTaskDefinition, + toWorkerTaskDefinition: $toWorkerTaskDefinition, + workflowActor: $workflowActor, + rolledBackAt: $rolledBackAt, + status: "rolled-back" + }' > rollback.json + + { + echo "## Production rolled back" + echo + echo "- From API: \`$CURRENT_API_TASK_DEFINITION\`" + echo "- To API: \`$TARGET_API_TASK_DEFINITION\`" + echo "- From worker: \`$CURRENT_WORKER_TASK_DEFINITION\`" + echo "- To worker: \`$TARGET_WORKER_TASK_DEFINITION\`" + echo "- ECS services: stable" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload rollback record + uses: actions/upload-artifact@v7 + with: + name: production-rollback-${{ github.run_id }} + path: rollback.json + if-no-files-found: error + retention-days: 90 diff --git a/.gitignore b/.gitignore index 21d66fd..c58f6db 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,7 @@ web_modules/ # Optional npm cache directory .npm +.pnpm-store/ # Optional eslint cache .eslintcache diff --git a/README.md b/README.md index 338f23d..d3ab224 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,14 @@ Worker health: `http://127.0.0.1:3001/health/live` - Build and upload Sentry sourcemaps: `pnpm build:with-sourcemaps` - Verify: `pnpm verify` +## Production Deployment + +Pull requests run application and Terraform checks. Production is released +manually from the current `main` commit; one workflow builds immutable API and +worker images, applies Terraform, and waits for ECS. Rollback automatically +selects the previous recorded task-definition pair. See [Production +deployment](docs/production-deployment.md) for setup and operating details. + ## API Flow ### Create a job diff --git a/apps/api/src/deep-analysis-duplicate-ops.service.spec.ts b/apps/api/src/deep-analysis-duplicate-ops.service.spec.ts index b0a4318..b362916 100644 --- a/apps/api/src/deep-analysis-duplicate-ops.service.spec.ts +++ b/apps/api/src/deep-analysis-duplicate-ops.service.spec.ts @@ -60,11 +60,18 @@ describe('deep analysis duplicate ops service', () => { }); it('delegates duplicate deletion to the mem9 source client', async () => { + let resolveDeletionStarted!: () => void; + const deletionStarted = new Promise((resolve) => { + resolveDeletionStarted = resolve; + }); const source = { - deleteMemories: jest.fn(async () => ({ - deletedMemoryIds: ['mem_2'], - failedMemoryIds: ['mem_3'], - })), + deleteMemories: jest.fn(async () => { + resolveDeletionStarted(); + return { + deletedMemoryIds: ['mem_2'], + failedMemoryIds: ['mem_3'], + }; + }), }; const service = new DeepAnalysisDuplicateOpsService( { @@ -114,7 +121,7 @@ describe('deep analysis duplicate ops service', () => { ); const result = await service.deleteDuplicateMemories(createContext(), 'dar_1'); - await new Promise((resolve) => setTimeout(resolve, 0)); + await deletionStarted; expect(result.duplicateCleanup.status).toBe('QUEUED'); expect(result.duplicateCleanup.totalCount).toBe(2); diff --git a/apps/api/src/user-profile.service.spec.ts b/apps/api/src/user-profile.service.spec.ts index bc5270c..c80acb3 100644 --- a/apps/api/src/user-profile.service.spec.ts +++ b/apps/api/src/user-profile.service.spec.ts @@ -259,7 +259,7 @@ describe('user profile service', () => { const result = await service.getProfile(createContext()); expect(result.summary.text).toBe('你是一位目标驱动的长期成长型用户,持续推进英语学习、健康管理'); - expect(result.summary.message).toBe('当前可用画像信息较少,已根据现有 facts、insights 和 pinned 生成初步总结,但画像可能不稳定。'); + expect(result.summary.message).toBe('当前可用记忆信息较少,已根据现有记忆生成初步总结,但画像可能不稳定。'); expect(result.summary.evidence).toHaveLength(1); }); @@ -274,7 +274,7 @@ describe('user profile service', () => { const result = await service.getProfile(createContext()); expect(result.summary.text).toContain('整体画像:一个偏理性、重视稳定成长的人'); - expect(result.summary.message).toBe('当前可用画像信息较少,已根据现有 facts、insights 和 pinned 生成初步总结,但画像可能不稳定。'); + expect(result.summary.message).toBe('当前可用记忆信息较少,已根据现有记忆生成初步总结,但画像可能不稳定。'); expect(result.summary.evidence).toHaveLength(1); }); diff --git a/docs/production-deployment.md b/docs/production-deployment.md new file mode 100644 index 0000000..847e121 --- /dev/null +++ b/docs/production-deployment.md @@ -0,0 +1,171 @@ +# Production deployment + +Production is released manually from the current `main` commit. Image builds, +Terraform plan/apply, and ECS deployment happen in one workflow run, so there +is no separately selected image SHA or saved intermediate Terraform plan. + +## Workflows + +### Pull-request checks + +`.github/workflows/ci.yml` runs for pull requests targeting `main`: + +1. Install dependencies from the lockfile. +2. Run `pnpm verify` (typecheck, unit tests, and application build). +3. Check Terraform formatting. +4. Initialize Terraform without a backend and validate the configuration. + +The repository does not currently define a benchmark command. Add it to this +workflow after the project has a stable benchmark script and failure threshold. + +### Production release + +`.github/workflows/deploy-production.yml` has no inputs. Run it from the `main` +branch in GitHub Actions. The workflow: + +1. Locks the run to the `main` commit identified by `github.sha`. +2. Assumes the image role through GitHub OIDC. +3. Builds and pushes API and worker images tagged with that full SHA. A rerun + reuses an existing immutable image and builds only a missing image. +4. Reads both ECR image digests. +5. Assumes the deployment role through GitHub OIDC. +6. Reads the currently deployed API and worker task definitions for rollback. +7. Initializes the KMS-encrypted S3 Terraform backend. +8. Creates a local `tfplan` in the same runner. +9. Uploads a temporary restore point containing the task-definition pair that + is currently online, then immediately applies the plan. +10. Waits for both ECS services to become stable. +11. Uploads a release record containing the SHA, image pair, task-definition + pair, and previous task-definition pair. +12. Deletes the temporary restore point after the successful release record is + safely uploaded. Failed runs keep their restore point for recovery. + +Terraform owns the ECS task definitions and the ECS services' task-definition +references. Normal deployment must not call `aws ecs update-service` directly. + +The workflow does not use a GitHub Environment. The **Run workflow** action is +the production release decision. The job-level branch check and both AWS role +trust policies must restrict production access to `refs/heads/main`. + +### Production rollback + +`.github/workflows/rollback-production.yml` also has no inputs. It finds the +release record matching the task-definition pair currently used by ECS and +switches both services to the previous pair recorded by that release. + +If the current deployment failed before it could write a success record, the +workflow falls back to the newest recorded successful task-definition pair. If +there has never been a successful CI release, it uses the newest failed run's +pre-deployment restore point. It refuses to continue if it cannot find a usable +record or if either target task definition is inactive. + +Rollback changes the ECS service references directly and intentionally leaves +Terraform state untouched. The next production release refreshes the state and +deploys its new task-definition pair. API and worker updates are coordinated by +one workflow but are not an atomic AWS transaction. + +## Terraform state + +Terraform state and its lock file use the existing S3 backend: + +```text +s3://mem9-node-prod-terraform-state/mem9-node/production/terraform.tfstate +s3://mem9-node-prod-terraform-state/mem9-node/production/terraform.tfstate.tflock +``` + +The bucket must keep: + +- versioning; +- default SSE-KMS encryption; +- Block Public Access; +- an HTTPS-only bucket policy; +- state and lock-file permissions scoped to the two objects above. + +The bucket no longer stores `tfplan` or release-candidate files. Existing +`mem9-node/production/plans/` lifecycle rules and IAM permissions are harmless +but can be removed after the new workflow has been verified. + +### Migrate an existing local state + +Run this once from the machine holding the real production +`terraform.tfstate`. Never run production Terraform against a new empty state. + +```bash +terraform -chdir=infra/terraform init -migrate-state \ + -backend-config="bucket=mem9-node-prod-terraform-state" \ + -backend-config="key=mem9-node/production/terraform.tfstate" \ + -backend-config="region=ap-southeast-1" \ + -backend-config="encrypt=true" \ + -backend-config="kms_key_id=" \ + -backend-config="use_lockfile=true" +``` + +Confirm the remote state before removing the local backup. + +## GitHub configuration + +Create one repository-level GitHub Actions secret named `TF_VARS`. Its value is +the complete production `terraform.tfvars` content. The workflow writes it to +the ephemeral runner without printing it and removes the runner after the job. + +Do not add these release-specific values to `TF_VARS`: + +```text +release_id +api_image +worker_image +``` + +The workflow supplies those three values from the release SHA and its two ECR +images. Command-line `-var` values take precedence if an older local file still +contains them, but removing them from `TF_VARS` keeps ownership unambiguous. + +Protect `main` and require the `Verify` status check before merge. Block force +pushes and branch deletion. A benchmark is not a required check until a real +benchmark command exists in the repository. + +## AWS OIDC roles + +Keep the two existing roles but restrict both trust policies to this repository +and the `main` branch subject: + +```text +repo:mem9-ai/mem9-node:ref:refs/heads/main +``` + +- `mem9-node-prod-github-prepare`: ECR login, image lookup, and image push for + the API and worker repositories only. +- `mem9-node-prod-github-apply`: Terraform state/KMS access, infrastructure + plan/apply permissions, ECS read/update permissions, and task-definition + retention permissions. + +Terraform's S3 bucket refresh also requires these two read-only actions on the +application payload bucket: + +```text +s3:GetAccelerateConfiguration +s3:GetReplicationConfiguration +``` + +Both role trust policies currently use the `main` branch subject shown above. +The unused GitHub `production` Environment may remain or be deleted. + +## Retention + +- ECR images use immutable full-SHA tags. +- ECS keeps the latest 30 active API revisions and 30 active worker revisions. +- GitHub keeps the latest 30 successful release-record artifacts, subject to + the repository's artifact retention limit. +- A failed deployment keeps its temporary pre-deployment restore point. A + successful deployment deletes that temporary record after uploading the + permanent release record. +- Rollback records are stored separately and do not participate in selecting a + rollback target. + +Do not introduce a count-based ECR lifecycle rule until it can preserve every +image referenced by the retained release records. + +## Database migrations + +These workflows do not run database migrations. Schema changes require a +separately reviewed, backward-compatible migration process. diff --git a/infra/terraform/ecr.tf b/infra/terraform/ecr.tf index da4b755..58fbd26 100644 --- a/infra/terraform/ecr.tf +++ b/infra/terraform/ecr.tf @@ -1,6 +1,6 @@ resource "aws_ecr_repository" "api" { name = "${var.name_prefix}-api" - image_tag_mutability = "MUTABLE" + image_tag_mutability = "IMMUTABLE" image_scanning_configuration { scan_on_push = true @@ -9,7 +9,7 @@ resource "aws_ecr_repository" "api" { resource "aws_ecr_repository" "worker" { name = "${var.name_prefix}-worker" - image_tag_mutability = "MUTABLE" + image_tag_mutability = "IMMUTABLE" image_scanning_configuration { scan_on_push = true diff --git a/infra/terraform/ecs_services.tf b/infra/terraform/ecs_services.tf index 21555a8..442f397 100644 --- a/infra/terraform/ecs_services.tf +++ b/infra/terraform/ecs_services.tf @@ -2,7 +2,7 @@ resource "aws_ecs_service" "api" { name = "${var.name_prefix}-api" cluster = aws_ecs_cluster.this.id task_definition = aws_ecs_task_definition.api.arn - desired_count = 4 + desired_count = 6 launch_type = "FARGATE" network_configuration { diff --git a/infra/terraform/ecs_tasks.tf b/infra/terraform/ecs_tasks.tf index 28707ba..04e84f6 100644 --- a/infra/terraform/ecs_tasks.tf +++ b/infra/terraform/ecs_tasks.tf @@ -4,13 +4,14 @@ resource "aws_ecs_task_definition" "api" { network_mode = "awsvpc" cpu = "512" memory = "1024" + skip_destroy = true execution_role_arn = aws_iam_role.task_execution.arn task_role_arn = aws_iam_role.task_runtime.arn container_definitions = jsonencode([ { name = "api" - image = "${aws_ecr_repository.api.repository_url}:latest" + image = var.api_image essential = true portMappings = [ @@ -86,6 +87,11 @@ resource "aws_ecs_task_definition" "api" { } } ]) + + tags = { + Release = var.release_id + Service = "api" + } } resource "aws_ecs_task_definition" "worker" { @@ -94,13 +100,14 @@ resource "aws_ecs_task_definition" "worker" { network_mode = "awsvpc" cpu = "512" memory = "1024" + skip_destroy = true execution_role_arn = aws_iam_role.task_execution.arn task_role_arn = aws_iam_role.task_runtime.arn container_definitions = jsonencode([ { name = "worker" - image = "${aws_ecr_repository.worker.repository_url}:latest" + image = var.worker_image essential = true portMappings = [ @@ -172,4 +179,9 @@ resource "aws_ecs_task_definition" "worker" { } } ]) + + tags = { + Release = var.release_id + Service = "worker" + } } diff --git a/infra/terraform/main.tf b/infra/terraform/main.tf index a5e57cc..8053b18 100644 --- a/infra/terraform/main.tf +++ b/infra/terraform/main.tf @@ -1,5 +1,7 @@ terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.10.0, < 2.0.0" + + backend "s3" {} required_providers { aws = { @@ -75,6 +77,12 @@ resource "aws_sqs_queue" "analysis_llm" { resource "aws_ecs_cluster" "this" { name = "${var.name_prefix}-cluster" + + tags = { + component = "node" + environment = "prod" + servicetype = "mem9" + } } resource "aws_cloudwatch_log_group" "api" { diff --git a/infra/terraform/outputs.tf b/infra/terraform/outputs.tf index ea473e6..c53bb66 100644 --- a/infra/terraform/outputs.tf +++ b/infra/terraform/outputs.tf @@ -17,3 +17,23 @@ output "analysis_llm_queue_url" { output "api_load_balancer_dns_name" { value = aws_lb.api.dns_name } + +output "api_task_definition_arn" { + value = aws_ecs_task_definition.api.arn +} + +output "worker_task_definition_arn" { + value = aws_ecs_task_definition.worker.arn +} + +output "ecs_cluster_name" { + value = aws_ecs_cluster.this.name +} + +output "api_service_name" { + value = aws_ecs_service.api.name +} + +output "worker_service_name" { + value = aws_ecs_service.worker.name +} diff --git a/infra/terraform/terraform.tfvars.example b/infra/terraform/terraform.tfvars.example index ae91d85..ac060fe 100644 --- a/infra/terraform/terraform.tfvars.example +++ b/infra/terraform/terraform.tfvars.example @@ -2,6 +2,10 @@ aws_region = "ap-southeast-1" name_prefix = "mem9-node-prod" +release_id = "0123456789abcdef0123456789abcdef01234567" +api_image = "401696231252.dkr.ecr.ap-southeast-1.amazonaws.com/mem9-node-prod-api:0123456789abcdef0123456789abcdef01234567" +worker_image = "401696231252.dkr.ecr.ap-southeast-1.amazonaws.com/mem9-node-prod-worker:0123456789abcdef0123456789abcdef01234567" + api_alb_idle_timeout_seconds = 120 vpc_id = "vpc-xxxxxxxxxxxxxxxxx" diff --git a/infra/terraform/variables.tf b/infra/terraform/variables.tf index f4a817d..bc7370d 100644 --- a/infra/terraform/variables.tf +++ b/infra/terraform/variables.tf @@ -8,6 +8,36 @@ variable "name_prefix" { default = "mem9-analysis" } +variable "release_id" { + description = "Git commit SHA that groups the API and worker task definitions into one release." + type = string + + validation { + condition = can(regex("^[0-9a-f]{40}$", var.release_id)) + error_message = "release_id must be a full 40-character lowercase Git commit SHA." + } +} + +variable "api_image" { + description = "Immutable API ECR image URI tagged with the release Git commit SHA." + type = string + + validation { + condition = can(regex(":[0-9a-f]{40}$", var.api_image)) + error_message = "api_image must end with a full 40-character lowercase Git commit SHA tag." + } +} + +variable "worker_image" { + description = "Immutable worker ECR image URI tagged with the release Git commit SHA." + type = string + + validation { + condition = can(regex(":[0-9a-f]{40}$", var.worker_image)) + error_message = "worker_image must end with a full 40-character lowercase Git commit SHA tag." + } +} + variable "api_alb_idle_timeout_seconds" { type = number default = 120