feat: auto-roll back deloyments

.github/workflows/deploy_dev.yml → .github/workflows/deploy_dev_full.yml

@@ -1,4 +1,4 @@
-name: Dev Deploy
+name: Dev Deploy Infra and Lambda
 
 on:
   workflow_dispatch:

.github/workflows/deploy_dev_lambda_only.yml

@@ -0,0 +1,49 @@
+name: Dev Deploy Lambda Only
+# This allows us to deploy only lambda changes to dev without touching infra.
+
+on:
+  workflow_dispatch:
+
+permissions:
+  id-token: write
+  contents: write
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      lambdas_dirs: ${{ steps.lambdas_dirs.outputs.just_outputs }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get lambdas Directories
+        id: lambdas_dirs
+        uses: chrispsheehan/just-aws-oidc-action@0.3.0
+        with:
+          just_action: lambda-get-directories
+
+  build:
+    uses: ./.github/workflows/build.yml
+    needs: 
+      - setup
+    with:
+      environment: dev
+      version: ${{ github.sha }}
+      matrix: ${{ needs.setup.outputs.lambdas_dirs }}
+
+  get_build:
+    needs: build
+    uses: ./.github/workflows/build_get.yml
+    with:
+      environment: dev
+      version: ${{ github.sha }}
+
+  deploy:
+    uses: ./.github/workflows/deploy.yml
+    needs:
+      - get_build
+    with:
+      environment: dev
+      lambda_version: ${{ needs.get_build.outputs.lambda_version }}
+      lambda_bucket: ${{ needs.get_build.outputs.lambda_bucket }}
+      matrix: ${{ needs.get_build.outputs.lambda_version_files }}

.github/workflows/deploy_prod.yml → .github/workflows/deploy_prod_full.yml

@@ -1,4 +1,4 @@
-name: Prod Deploy
+name: Prod Deploy Infra and Lambda
 
 on:
   workflow_dispatch:

.github/workflows/deploy_prod_lambda_only.yml

@@ -0,0 +1,28 @@
+name: Prod Deploy Lambda Only
+# This allows us to deploy only lambda changes to prod found in the version s3 folder (only).
+
+on:
+  workflow_dispatch:
+
+permissions:
+  id-token: write
+  contents: write
+
+jobs:
+  get_build:
+    uses: ./.github/workflows/build_get.yml
+    with:
+      environment: ci
+      version: 0.5.1
+
+  deploy:
+    uses: ./.github/workflows/deploy.yml
+    needs: 
+      - get_build
+    with:
+      environment: prod
+      lambda_version: ${{ needs.get_build.outputs.lambda_version }}
+      lambda_bucket: ${{ needs.get_build.outputs.lambda_bucket }}
+      matrix: ${{ needs.get_build.outputs.lambda_version_files }}
+      # we can also define a lambda-only deployment here if needed, as below
+      # matrix: '["api"]'

README.md

@@ -36,7 +36,8 @@ module "lambda_example" {
 - we can handle an initial lag while lambda warms up/boots
 ```hcl
 provisioned_config = {
-    fixed = 0
+    fixed                = 0
+    reserved_concurrency = 2 # only allow 2 concurrent executions THIS ALSO SERVES AS A LIMIT TO AVOID THROTTLING
 }
 ```
 
@@ -45,7 +46,8 @@ provisioned_config = {
 - we never want lag due to warm up and can predict traffic
 ```hcl
 provisioned_config = {
-    fixed = 1
+    fixed                = 10
+    reserved_concurrency = 50
 }
 ```
 
@@ -65,12 +67,19 @@ provisioned_config = {
     }
 }
 ```
+- before scaling the lambda alias will match the minmum value
+![a](docs/lambda-config-before.png)
+- when the trigger percent is exceeded the lambda moves into `In progress (1/2)` state as an additional provisioned lambda is added.
+![a](docs/lamba-scaling-up.png)
+- after scaling the lambda alias will show an additional provisioned lambda
+![a](docs/lambda-config-after.png)
+
 
 ## 🚦 types of lambda deploy
 
 ```hcl
 module "lambda_example" {
-  source = "../lambda"
+  source = "../_shared/lambda"
   ...
   deployment_config = var.your_deployment_config
 }
@@ -109,3 +118,41 @@ deployment_config = {
     percentage       = 10
     interval_minutes = 1
 }
+```
+
+## 🔥↩️ deployment roll-back
+
+- use cloudwatch metrics and alarms to automatically roll-back a deployment
+- create a [cloudwatch_metric_alarm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_metric_alarm) resource and pass in as per below
+
+```hcl
+module "lambda_example" {
+  source = "../_shared/lambda"
+  ...
+  codedeploy_alarm_names = [
+    local.api_5xx_alarm_name
+  ]
+}
+```
+- if the alarm triggers during a deployment you will see the below in the CI
+
+```
+📦 Running: lambda-deploy
+🚀 Started deployment: d-40UUQH3DF
+Attempt 1: Deployment status is InProgress
+Attempt 2: Deployment status is InProgress
+Attempt 3: Deployment status is InProgress
+Attempt 4: Deployment status is InProgress
+Attempt 5: Deployment status is Stopped
+❌ Deployment d-40UUQH3DF failed or was stopped.
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+|                                                                                                                    GetDeployment                                                                                                                    |
++--------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+|  ErrorCode   |  ALARM_ACTIVE                                                                                                                                                                                                                        |
+|  ErrorMessage|  One or more alarms have been activated according to the Amazon CloudWatch metrics you selected, and the affected deployments have been stopped. Activated alarms: <dev-aws-serverless-github-deploy-api-api-v2-5xx-rate-critical>   |
+|  Status      |  Stopped                                                                                                                                                                                                                             |
++--------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+error: Recipe `lambda-deploy` failed with exit code 1
+Error: Process completed with exit code 1.
+
+```

infra/live/dev/aws/api/terragrunt.hcl

@@ -2,6 +2,12 @@ include {
   path = find_in_parent_folders("root.hcl")
 }
 
+inputs = {
+  api_5xx_alarm_threshold           = 20.0
+  api_5xx_alarm_evaluation_periods  = 1
+  api_5xx_alarm_datapoints_to_alarm = 1
+}
+
 terraform {
   source = "../../../../modules//aws//api"
 }

infra/live/prod/aws/api/terragrunt.hcl

@@ -2,6 +2,12 @@ include {
   path = find_in_parent_folders("root.hcl")
 }
 
+inputs = {
+  api_5xx_alarm_threshold           = 5.0
+  api_5xx_alarm_evaluation_periods  = 1
+  api_5xx_alarm_datapoints_to_alarm = 1
+}
+
 terraform {
   source = "../../../../modules//aws//api"
 }

infra/modules/aws/_shared/lambda/locals.tf

@@ -4,7 +4,7 @@ locals {
   compute_platform = "Lambda"
 
   lambda_bootstrap_zip_key = "bootstrap/bootstrap-lambda.zip"
-  lambda_name              = "${var.environment}-${var.project_name}-${var.lambda_name}"
+  lambda_name              = var.lambda_name
 
   deploy_all_at_once_type = "AllAtOnce"
   deploy_canary_type      = "TimeBasedCanary"
@@ -15,11 +15,18 @@ locals {
     canary      = local.deploy_canary_type
     linear      = local.deploy_linear_type
   }
+  deploy_strategy = local.deploy_config_type_map[var.deployment_config.strategy]
   deploy_config = {
-    type    = local.deploy_config_type_map[var.deployment_config.strategy]
+    type    = local.deploy_strategy
     percent = var.deployment_config.percentage
     minutes = var.deployment_config.interval_minutes
   }
+  deploy_config_suffix = lower((
+    var.deployment_config.strategy == "all_at_once"
+    ? local.deploy_strategy
+    : "${local.deploy_strategy}-${local.deploy_config.percent}-${local.deploy_config.minutes}"
+  ))
+  deployment_config_name = "${local.lambda_name}-deploy-${local.deploy_config_suffix}"
 
   fixed_mode      = try(var.provisioned_config.fixed != null, true) && try(var.provisioned_config.fixed > 0, false)
   auto_scale_mode = try(var.provisioned_config.auto_scale != null, false)

infra/modules/aws/_shared/lambda/main.tf

@@ -4,7 +4,7 @@ resource "aws_iam_role" "iam_for_lambda" {
 }
 
 resource "aws_iam_policy" "lambda_cloudwatch_logs" {
-  name   = "${var.project_name}-${var.environment}-lambda-cloudwatch-logs"
+  name   = "${local.lambda_name}-logs"
   policy = data.aws_iam_policy_document.lambda_cloudwatch_logs.json
 }
 
@@ -108,7 +108,7 @@ resource "aws_iam_role_policy" "cd_lambda" {
 }
 
 resource "aws_codedeploy_deployment_config" "lambda_config" {
-  deployment_config_name = "${local.lambda_name}-deploy-config"
+  deployment_config_name = local.deployment_config_name
   compute_platform       = local.compute_platform
 
   traffic_routing_config {
@@ -133,21 +133,35 @@ resource "aws_codedeploy_deployment_config" "lambda_config" {
 }
 
 resource "aws_codedeploy_deployment_group" "dg" {
+  depends_on = [aws_codedeploy_deployment_config.lambda_config] # to prevent DeploymentConfigInUseException
+
   app_name              = aws_codedeploy_app.app.name
-  deployment_group_name = "${local.lambda_name}-dg"
+  deployment_group_name = "${local.deployment_config_name}-dg"
   service_role_arn      = aws_iam_role.code_deploy_role.arn
 
   deployment_style {
     deployment_type   = "BLUE_GREEN"
     deployment_option = "WITH_TRAFFIC_CONTROL"
   }
 
-  deployment_config_name = aws_codedeploy_deployment_config.lambda_config.deployment_config_name
+  deployment_config_name = local.deployment_config_name
 
   auto_rollback_configuration {
     enabled = true
     events  = ["DEPLOYMENT_FAILURE", "DEPLOYMENT_STOP_ON_ALARM"]
   }
+
+  dynamic "alarm_configuration" {
+    for_each = length(var.codedeploy_alarm_names) > 0 ? [1] : []
+    content {
+      enabled = true
+      alarms  = var.codedeploy_alarm_names
+    }
+  }
+
+  lifecycle {
+    create_before_destroy = true # to prevent DeploymentConfigInUseException
+  }
 }
 
 resource "aws_appautoscaling_target" "pc_target" {

infra/modules/aws/_shared/lambda/variables.tf

@@ -42,6 +42,12 @@ variable "additional_policy_arns" {
   default     = []
 }
 
+variable "codedeploy_alarm_names" {
+  description = "Optional list of CloudWatch alarm names that trigger CodeDeploy rollback"
+  type        = list(string)
+  default     = []
+}
+
 variable "deployment_config" {
   description = "Traffic shifting: all_at_once | canary | linear"
   type = object({

infra/modules/aws/api/local.tf

@@ -0,0 +1,5 @@
+locals {
+  lambda_name           = "${var.environment}-${var.project_name}-api"
+  apigw_http_5xx_metric = "5xx"
+  api_5xx_alarm_name    = "${local.lambda_name}-api-v2-5xx-rate-critical"
+}