Yet More Datadog Setup

ops/services/20-microservices/ecs.tf

@@ -40,7 +40,7 @@ resource "aws_ecs_task_definition" "ecs_task" {
     }
   }, {
     name = "datadog-agent"
-    image = "public.ecr.aws/datadog/agent:7.80.3"
+    image = "public.ecr.aws/datadog/agent:7.81.0-fips"
     essential = false
 
     readonlyRootFilesystem = true

ops/services/30-monitors/config/defaults.yml

@@ -10,7 +10,7 @@ enabled:
   sqs: false
   sns: false
   lambda: false
-  s3: true
+  s3: false
   rds: true
 ecs:
   notify_no_data: true
@@ -29,5 +29,6 @@ s3:
   no_data_timeframe_minutes: 10
   timeframe: "last_5m"
 rds:
+  notify_no_data: true
   no_data_timeframe_minutes: 10
   timeframe: "last_10m"

ops/services/30-monitors/config/test.yml

@@ -5,3 +5,5 @@ notifications:
   victorops: false
 ecs:
   notify_no_data: false # test shuts down overnight
+rds:
+  notify_no_data: false # test shuts down overnight

ops/services/30-monitors/main.tf

@@ -5,7 +5,7 @@ module "platform" {
   app                 = local.app
   env                 = local.env
   service             = local.service
-  root_module = "https://github.com/CMSgov/bluebutton-web-server/tree/main/ops/services/${basename(abspath(path.module))}"
+  root_module         = "https://github.com/CMSgov/bluebutton-web-server/tree/main/ops/services/${basename(abspath(path.module))}"
   ssm_hierarchy_roots = ["bb"]
 }
 
@@ -22,8 +22,8 @@ data "aws_secretsmanager_secret_version" "datadog_cicd_application_key" {
 }
 
 locals {
-  env         = terraform.workspace
-  service     = "monitors"
+  env     = terraform.workspace
+  service = "monitors"
 
   default_tags = module.platform.default_tags
 
@@ -49,9 +49,105 @@ locals {
 }
 
 module "common_datadog_monitors" {
-  source = "github.com/CMSgov/cdap/terraform/modules/datadog_monitors?ref=6ded520857376f46bb317dca898e5df6a9ecc93b"
+  source = "github.com/CMSgov/cdap/terraform/modules/datadog_monitors?ref=0aba1af484320d0d121d804c05f36cf1a4d978c9"
 
-  app            = local.app
-  env            = local.env
-  monitor_config = local.monitor_config
+  app             = local.app
+  env             = local.env
+  monitor_config  = local.monitor_config
+  custom_monitors = local.custom_monitors
+}
+
+locals {
+  custom_monitors = [
+    {
+      name    = "[${upper(local.env)}] [${local.app}] ALB — Target Response Time High"
+      type    = "metric alert"
+      message = "ALB target has high average response time."
+      query   = "avg(last_1h):avg:aws.applicationelb.target_response_time.average{application:${local.app}, environment:${local.env}} > ${local.env == "test" ? 1 : 0.35}"
+
+      thresholds = {
+        critical = local.env == "test" ? 1 : 0.35
+        warning  = local.env == "test" ? 0.75 : 0.25
+      }
+
+      notify_no_data           = local.env != "test"
+      no_data_timeframe_minute = 60
+
+      # TODO are we sure about this?
+      require_full_window = false
+    },
+    {
+      # TODO need PR from CDAP for this to work
+      create = local.env != "test"
+
+      name    = "[${upper(local.env)}] [${local.app}] ALB — Request Rate Low"
+      type    = "metric alert"
+      message = "Request rate / throughput is low."
+      query   = "avg(last_1h):sum:aws.applicationelb.request_count{application:${local.app}, environment:${local.env}}.as_rate() < ${local.env == "prod" ? 0.1 : 0.001}"
+
+      thresholds = {
+        critical = local.env == "prod" ? 0.1 : 0.001
+        warning  = local.env == "prod" ? 0.5 : 0.005
+      }
+
+      notify_no_data           = true
+      no_data_timeframe_minute = 60
+
+      require_full_window = false
+    },
+    {
+      # TODO need PR from CDAP for this to work
+      create = local.env != "test"
+
+      name    = "[${upper(local.env)}] [${local.app}] ALB — Maximum Unhealthy Hosts"
+      type    = "metric alert"
+      message = "Maximum number of unhealthy hosts passed threshold."
+      query   = "max(last_1h):sum:aws.applicationelb.un_healthy_host_count.maximum{application:${local.app}, environment:${local.env}} >= 1"
+
+      thresholds = {
+        # TODO should this be 2 or something?
+        critical = 1
+        warning  = 0.5
+      }
+
+      notify_no_data           = true
+      no_data_timeframe_minute = 60
+
+      require_full_window = false
+    },
+    {
+      name    = "[${upper(local.env)}] [${local.app}] ALB — Target 5xx Count High"
+      type    = "metric alert"
+      message = "High number of 5xx response codes generated by the target."
+      query   = "sum(last_1h):sum:aws.applicationelb.httpcode_target_5xx{application:${local.app}, environment:${local.env}}.as_count() > 1000"
+
+      thresholds = {
+        critical = 1000
+        warning  = 100
+      }
+
+      # TODO when editing this, why does the on_missing_data appear as "Show No data" even though that's not the default for counts?
+      notify_no_data           = local.env != "test" # TODO should probably be false
+      no_data_timeframe_minute = 60
+
+      require_full_window = false
+    },
+    {
+      name    = "[${upper(local.env)}] [${local.app}] APM — Error Rate High"
+      type    = "metric alert"
+      message = "Service ${local.app} has high error rate."
+      # TODO what evaluation window?
+      query = "sum(last_1h):sum:trace.django.request.errors{env:${local.env},service:${local.app},span.kind:server}.as_count() / sum:trace.django.request.hits{env:${local.env},service:${local.app},span.kind:server}.as_count() > 0.02"
+
+      thresholds = {
+        critical = 0.02
+        warning  = 0.01
+      }
+
+      notify_no_data           = local.env != "test"
+      no_data_timeframe_minute = 60
+
+      require_full_window = false
+    },
+  ]
 }

ops/services/40-dashboards/main.tf

@@ -5,7 +5,7 @@ module "platform" {
   app                 = local.app
   env                 = local.env
   service             = local.service
-  root_module = "https://github.com/CMSgov/bluebutton-web-server/tree/main/ops/services/${basename(abspath(path.module))}"
+  root_module         = "https://github.com/CMSgov/bluebutton-web-server/tree/main/ops/services/${basename(abspath(path.module))}"
   ssm_hierarchy_roots = ["bb"]
 }
 
@@ -22,16 +22,16 @@ data "aws_secretsmanager_secret_version" "datadog_cicd_application_key" {
 }
 
 locals {
-  env         = terraform.workspace
-  service     = "dashboards"
+  env     = terraform.workspace
+  service = "dashboards"
 
   default_tags = module.platform.default_tags
 
   create_dashboards = local.env == "prod"
 }
 
 module "datadog_dashboard" {
-  source      = "github.com/CMSgov/cdap/terraform/modules/datadog_dashboard?ref=6ded520857376f46bb317dca898e5df6a9ecc93b"
+  source = "github.com/CMSgov/cdap/terraform/modules/datadog_dashboard?ref=0aba1af484320d0d121d804c05f36cf1a4d978c9"
 
   app         = local.app
   runbook_url = "https://github.com/CMSgov/bluebutton-web-server/blob/master/ops/services/RUNBOOK.md"
@@ -43,7 +43,7 @@ module "datadog_dashboard" {
     sns    = false
     sqs    = false
     aurora = true
-    s3     = true
+    s3     = false
     apm    = true
   }
 
@@ -58,5 +58,7 @@ module "datadog_dashboard" {
     apm    = "1h"
   }
 
+  apm_primary_operation = "django.request"
+
   count = local.create_dashboards ? 1 : 0
 }