From 5624e89ef188aee641d8411ea06a4365b473b44d Mon Sep 17 00:00:00 2001
From: Anna Montare <267455234+annamontare-nava@users.noreply.github.com>
Date: Tue, 7 Jul 2026 12:06:23 -0600
Subject: [PATCH 01/14] notify on missing data for RDS

since these resources are now tagged correctly
---
 ops/services/30-monitors/config/defaults.yml | 1 +
 ops/services/30-monitors/config/test.yml     | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/ops/services/30-monitors/config/defaults.yml b/ops/services/30-monitors/config/defaults.yml
index 3ea0f9b82..1e9305817 100644
--- a/ops/services/30-monitors/config/defaults.yml
+++ b/ops/services/30-monitors/config/defaults.yml
@@ -29,5 +29,6 @@ s3:
   no_data_timeframe_minutes: 10
   timeframe: "last_5m"
 rds:
+  notify_no_data: true
   no_data_timeframe_minutes: 10
   timeframe: "last_10m"
diff --git a/ops/services/30-monitors/config/test.yml b/ops/services/30-monitors/config/test.yml
index b44bbc656..09e90ed20 100644
--- a/ops/services/30-monitors/config/test.yml
+++ b/ops/services/30-monitors/config/test.yml
@@ -5,3 +5,5 @@ notifications:
   victorops: false
 ecs:
   notify_no_data: false # test shuts down overnight
+rds:
+  notify_no_data: false # test shuts down overnight

From af8e7938802f4e092c81a20691aa852992fdd95a Mon Sep 17 00:00:00 2001
From: Anna Montare <267455234+annamontare-nava@users.noreply.github.com>
Date: Wed, 8 Jul 2026 11:44:44 -0600
Subject: [PATCH 02/14] switch to fips-compliant version of datadog agent, and
 newer version

---
 ops/services/20-microservices/ecs.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ops/services/20-microservices/ecs.tf b/ops/services/20-microservices/ecs.tf
index ffce8f911..9da96987b 100644
--- a/ops/services/20-microservices/ecs.tf
+++ b/ops/services/20-microservices/ecs.tf
@@ -40,7 +40,7 @@ resource "aws_ecs_task_definition" "ecs_task" {
     }
   }, {
     name = "datadog-agent"
-    image = "public.ecr.aws/datadog/agent:7.80.3"
+    image = "public.ecr.aws/datadog/agent:7.81.0-fips"
     essential = false
 
     readonlyRootFilesystem = true

From 5531f410bd6062255540678c40ce9986f5d97b97 Mon Sep 17 00:00:00 2001
From: Anna Montare <267455234+annamontare-nava@users.noreply.github.com>
Date: Wed, 8 Jul 2026 14:34:26 -0600
Subject: [PATCH 03/14] add first custom monitor!

---
 ops/services/30-monitors/main.tf | 29 ++++++++++++++++++++++++++---
 1 file changed, 26 insertions(+), 3 deletions(-)

diff --git a/ops/services/30-monitors/main.tf b/ops/services/30-monitors/main.tf
index 7ba2c518a..d50be59d8 100644
--- a/ops/services/30-monitors/main.tf
+++ b/ops/services/30-monitors/main.tf
@@ -51,7 +51,30 @@ locals {
 module "common_datadog_monitors" {
   source = "github.com/CMSgov/cdap/terraform/modules/datadog_monitors?ref=6ded520857376f46bb317dca898e5df6a9ecc93b"
 
-  app            = local.app
-  env            = local.env
-  monitor_config = local.monitor_config
+  app             = local.app
+  env             = local.env
+  monitor_config  = local.monitor_config
+  custom_monitors = local.custom_monitors
+}
+
+locals {
+  custom_monitors = [
+    {
+      name    = "[${upper(local.env)}] [${local.app}] ALB — Target Response Time High"
+      type    = "metric alert"
+      message = "ALB target has high average response time."
+      query   = "avg(last_1h):avg:aws.applicationelb.target_response_time.average{application:${local.app}, environment:${local.env}} > 0.35"
+
+      thresholds = {
+        critical = 0.35
+        warning  = 0.25
+      }
+
+      notify_no_data           = local.env != "test"
+      no_data_timeframe_minute = 60
+
+      # TODO the CDAP module doesn't actually do anything with this, open a PR
+      require_full_window = false
+    },
+  ]
 }

From 73802b7edae8e7000de18f2235872daec9e10fa3 Mon Sep 17 00:00:00 2001
From: Anna Montare <267455234+annamontare-nava@users.noreply.github.com>
Date: Wed, 8 Jul 2026 14:35:58 -0600
Subject: [PATCH 04/14] tofu formatting

---
 ops/services/30-monitors/main.tf | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ops/services/30-monitors/main.tf b/ops/services/30-monitors/main.tf
index d50be59d8..ceb9e5563 100644
--- a/ops/services/30-monitors/main.tf
+++ b/ops/services/30-monitors/main.tf
@@ -5,7 +5,7 @@ module "platform" {
   app                 = local.app
   env                 = local.env
   service             = local.service
-  root_module = "https://github.com/CMSgov/bluebutton-web-server/tree/main/ops/services/${basename(abspath(path.module))}"
+  root_module         = "https://github.com/CMSgov/bluebutton-web-server/tree/main/ops/services/${basename(abspath(path.module))}"
   ssm_hierarchy_roots = ["bb"]
 }
 
@@ -22,8 +22,8 @@ data "aws_secretsmanager_secret_version" "datadog_cicd_application_key" {
 }
 
 locals {
-  env         = terraform.workspace
-  service     = "monitors"
+  env     = terraform.workspace
+  service = "monitors"
 
   default_tags = module.platform.default_tags
 

From 7fb6d2cd70b7ad67848ff3fb6bc4d3a869040587 Mon Sep 17 00:00:00 2001
From: Anna Montare <267455234+annamontare-nava@users.noreply.github.com>
Date: Thu, 9 Jul 2026 13:21:05 -0600
Subject: [PATCH 05/14] update cdap modules version

Changes in their repo allow setting apm_primary_operation and fix custom
monitors not reading an argument
---
 ops/services/30-monitors/main.tf   | 4 ++--
 ops/services/40-dashboards/main.tf | 4 +++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/ops/services/30-monitors/main.tf b/ops/services/30-monitors/main.tf
index ceb9e5563..cadfb58f7 100644
--- a/ops/services/30-monitors/main.tf
+++ b/ops/services/30-monitors/main.tf
@@ -49,7 +49,7 @@ locals {
 }
 
 module "common_datadog_monitors" {
-  source = "github.com/CMSgov/cdap/terraform/modules/datadog_monitors?ref=6ded520857376f46bb317dca898e5df6a9ecc93b"
+  source = "github.com/CMSgov/cdap/terraform/modules/datadog_monitors?ref=0aba1af484320d0d121d804c05f36cf1a4d978c9"
 
   app             = local.app
   env             = local.env
@@ -73,7 +73,7 @@ locals {
       notify_no_data           = local.env != "test"
       no_data_timeframe_minute = 60
 
-      # TODO the CDAP module doesn't actually do anything with this, open a PR
+      # TODO are we sure about this?
       require_full_window = false
     },
   ]
diff --git a/ops/services/40-dashboards/main.tf b/ops/services/40-dashboards/main.tf
index 4b3eed228..4d51dcb52 100644
--- a/ops/services/40-dashboards/main.tf
+++ b/ops/services/40-dashboards/main.tf
@@ -31,7 +31,7 @@ locals {
 }
 
 module "datadog_dashboard" {
-  source      = "github.com/CMSgov/cdap/terraform/modules/datadog_dashboard?ref=6ded520857376f46bb317dca898e5df6a9ecc93b"
+  source      = "github.com/CMSgov/cdap/terraform/modules/datadog_dashboard?ref=0aba1af484320d0d121d804c05f36cf1a4d978c9"
 
   app         = local.app
   runbook_url = "https://github.com/CMSgov/bluebutton-web-server/blob/master/ops/services/RUNBOOK.md"
@@ -58,5 +58,7 @@ module "datadog_dashboard" {
     apm    = "1h"
   }
 
+  apm_primary_operation = "django.request"
+
   count = local.create_dashboards ? 1 : 0
 }

From 7ae08d68430d1084d215ca70ea83f1fe2e7c62ed Mon Sep 17 00:00:00 2001
From: Anna Montare <267455234+annamontare-nava@users.noreply.github.com>
Date: Thu, 9 Jul 2026 16:06:45 -0600
Subject: [PATCH 06/14] add monitor for apm error rate

---
 ops/services/30-monitors/main.tf | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/ops/services/30-monitors/main.tf b/ops/services/30-monitors/main.tf
index cadfb58f7..93eaaf2de 100644
--- a/ops/services/30-monitors/main.tf
+++ b/ops/services/30-monitors/main.tf
@@ -76,5 +76,22 @@ locals {
       # TODO are we sure about this?
       require_full_window = false
     },
+    {
+      name    = "[${upper(local.env)}] [${local.app}] APM — Error Rate High"
+      type    = "metric alert"
+      message = "Service ${local.app} has high error rate."
+      # TODO what evaluation window?
+      query   = "sum(last_1h):sum:trace.django.request.errors{env:${local.env},service:${local.app},span.kind:server}.as_count() / sum:trace.django.request.hits{env:${local.env},service:${local.app},span.kind:server}.as_count() > 0.02"
+
+      thresholds = {
+        critical = 0.02
+        warning  = 0.01
+      }
+
+      notify_no_data           = local.env != "test"
+      no_data_timeframe_minute = 60
+
+      require_full_window = false
+    },
   ]
 }

From ade3663dce8cf604519a61373ad966f3fb35f43b Mon Sep 17 00:00:00 2001
From: Anna Montare <267455234+annamontare-nava@users.noreply.github.com>
Date: Fri, 10 Jul 2026 16:06:18 -0600
Subject: [PATCH 07/14] add a low request rate / throughput monitor

---
 ops/services/30-monitors/main.tf | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/ops/services/30-monitors/main.tf b/ops/services/30-monitors/main.tf
index 93eaaf2de..8cfe034f3 100644
--- a/ops/services/30-monitors/main.tf
+++ b/ops/services/30-monitors/main.tf
@@ -76,6 +76,25 @@ locals {
       # TODO are we sure about this?
       require_full_window = false
     },
+    {
+      # TODO need PR from CDAP for this to work
+      create = local.env != "test"
+
+      name    = "[${upper(local.env)}] [${local.app}] ALB — Request Rate Low"
+      type    = "metric alert"
+      message = "Request rate / throughput is low."
+      query   = "avg(last_1h):sum:aws.applicationelb.request_count{application:${local.app}, environment:${local.env}}.as_rate() < ${local.env == "prod" ? 0.1 : 0.001}"
+
+      thresholds = {
+        critical = local.env == "prod" ? 0.1 : 0.001
+        warning  = local.env == "prod" ? 0.5 : 0.005
+      }
+
+      notify_no_data           = true
+      no_data_timeframe_minute = 60
+
+      require_full_window = false
+    },
     {
       name    = "[${upper(local.env)}] [${local.app}] APM — Error Rate High"
       type    = "metric alert"

From 948805756b75876a7accd4b86d54808e3129f152 Mon Sep 17 00:00:00 2001
From: Anna Montare <267455234+annamontare-nava@users.noreply.github.com>
Date: Fri, 10 Jul 2026 16:07:34 -0600
Subject: [PATCH 08/14] tofu fmt

---
 ops/services/30-monitors/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ops/services/30-monitors/main.tf b/ops/services/30-monitors/main.tf
index 8cfe034f3..391d5e2f2 100644
--- a/ops/services/30-monitors/main.tf
+++ b/ops/services/30-monitors/main.tf
@@ -100,7 +100,7 @@ locals {
       type    = "metric alert"
       message = "Service ${local.app} has high error rate."
       # TODO what evaluation window?
-      query   = "sum(last_1h):sum:trace.django.request.errors{env:${local.env},service:${local.app},span.kind:server}.as_count() / sum:trace.django.request.hits{env:${local.env},service:${local.app},span.kind:server}.as_count() > 0.02"
+      query = "sum(last_1h):sum:trace.django.request.errors{env:${local.env},service:${local.app},span.kind:server}.as_count() / sum:trace.django.request.hits{env:${local.env},service:${local.app},span.kind:server}.as_count() > 0.02"
 
       thresholds = {
         critical = 0.02

From 5c7698d90de6921fe609b002b3c7c89e76211bcb Mon Sep 17 00:00:00 2001
From: Anna Montare <267455234+annamontare-nava@users.noreply.github.com>
Date: Fri, 10 Jul 2026 16:10:28 -0600
Subject: [PATCH 09/14] add an unhealthy hosts monitor

---
 ops/services/30-monitors/main.tf | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/ops/services/30-monitors/main.tf b/ops/services/30-monitors/main.tf
index 391d5e2f2..05d28a9b2 100644
--- a/ops/services/30-monitors/main.tf
+++ b/ops/services/30-monitors/main.tf
@@ -95,6 +95,26 @@ locals {
 
       require_full_window = false
     },
+    {
+      # TODO need PR from CDAP for this to work
+      create = local.env != "test"
+
+      name    = "[${upper(local.env)}] [${local.app}] ALB — Maximum Unhealthy Hosts"
+      type    = "metric alert"
+      message = "Maximum number of unhealthy hosts passed threshold."
+      query   = "max(last_1h):sum:aws.applicationelb.un_healthy_host_count.maximum{application:${local.app} ,environment:${local.env}} >= 1"
+
+      thresholds = {
+        # TODO should this be 2 or something?
+        critical = 1
+        warning  = 0.5
+      }
+
+      notify_no_data           = true
+      no_data_timeframe_minute = 60
+
+      require_full_window = false
+    },
     {
       name    = "[${upper(local.env)}] [${local.app}] APM — Error Rate High"
       type    = "metric alert"

From 781522ab4d39662a00a55e3f057e31b1891915f6 Mon Sep 17 00:00:00 2001
From: Anna Montare <267455234+annamontare-nava@users.noreply.github.com>
Date: Fri, 10 Jul 2026 18:08:22 -0600
Subject: [PATCH 10/14] fix query formatting

---
 ops/services/30-monitors/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ops/services/30-monitors/main.tf b/ops/services/30-monitors/main.tf
index 05d28a9b2..fa6fd3c6c 100644
--- a/ops/services/30-monitors/main.tf
+++ b/ops/services/30-monitors/main.tf
@@ -102,7 +102,7 @@ locals {
       name    = "[${upper(local.env)}] [${local.app}] ALB — Maximum Unhealthy Hosts"
       type    = "metric alert"
       message = "Maximum number of unhealthy hosts passed threshold."
-      query   = "max(last_1h):sum:aws.applicationelb.un_healthy_host_count.maximum{application:${local.app} ,environment:${local.env}} >= 1"
+      query   = "max(last_1h):sum:aws.applicationelb.un_healthy_host_count.maximum{application:${local.app}, environment:${local.env}} >= 1"
 
       thresholds = {
         # TODO should this be 2 or something?

From 530987fce7fe02e53fb7d1d718f99cdea4aee3a1 Mon Sep 17 00:00:00 2001
From: Anna Montare <267455234+annamontare-nava@users.noreply.github.com>
Date: Fri, 10 Jul 2026 18:10:05 -0600
Subject: [PATCH 11/14] add target 5xx error threshold monitor

---
 ops/services/30-monitors/main.tf | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/ops/services/30-monitors/main.tf b/ops/services/30-monitors/main.tf
index fa6fd3c6c..83333c61a 100644
--- a/ops/services/30-monitors/main.tf
+++ b/ops/services/30-monitors/main.tf
@@ -115,6 +115,23 @@ locals {
 
       require_full_window = false
     },
+    {
+      name    = "[${upper(local.env)}] [${local.app}] ALB — Target 5xx Count High"
+      type    = "metric alert"
+      message = "High number of 5xx response codes generated by the target."
+      query   = "sum(last_1h):sum:aws.applicationelb.httpcode_target_5xx{application:${local.app}, environment:${local.env}}.as_count() > 1000"
+
+      thresholds = {
+        critical = 1000
+        warning  = 100
+      }
+
+      # TODO when editing this, why does the on_missing_data appear as "Show No data" even though that's not the default for counts?
+      notify_no_data           = local.env != "test" # TODO should probably be false
+      no_data_timeframe_minute = 60
+
+      require_full_window = false
+    },
     {
       name    = "[${upper(local.env)}] [${local.app}] APM — Error Rate High"
       type    = "metric alert"

From b4f5c979f09a8e54f642903db3bca283230484e1 Mon Sep 17 00:00:00 2001
From: Anna Montare <267455234+annamontare-nava@users.noreply.github.com>
Date: Mon, 13 Jul 2026 11:59:24 -0600
Subject: [PATCH 12/14] disable s3 monitors and dashboard section

---
 ops/services/30-monitors/config/defaults.yml | 2 +-
 ops/services/40-dashboards/main.tf           | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ops/services/30-monitors/config/defaults.yml b/ops/services/30-monitors/config/defaults.yml
index 1e9305817..6eacf383c 100644
--- a/ops/services/30-monitors/config/defaults.yml
+++ b/ops/services/30-monitors/config/defaults.yml
@@ -10,7 +10,7 @@ enabled:
   sqs: false
   sns: false
   lambda: false
-  s3: true
+  s3: false
   rds: true
 ecs:
   notify_no_data: true
diff --git a/ops/services/40-dashboards/main.tf b/ops/services/40-dashboards/main.tf
index 4d51dcb52..dddd41170 100644
--- a/ops/services/40-dashboards/main.tf
+++ b/ops/services/40-dashboards/main.tf
@@ -43,7 +43,7 @@ module "datadog_dashboard" {
     sns    = false
     sqs    = false
     aurora = true
-    s3     = true
+    s3     = false
     apm    = true
   }
 

From bdab20577766035c8aa94931b97de957140427a1 Mon Sep 17 00:00:00 2001
From: Anna Montare <267455234+annamontare-nava@users.noreply.github.com>
Date: Mon, 13 Jul 2026 12:01:10 -0600
Subject: [PATCH 13/14] tofu fmt

---
 ops/services/40-dashboards/main.tf | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ops/services/40-dashboards/main.tf b/ops/services/40-dashboards/main.tf
index dddd41170..98e6b7bdc 100644
--- a/ops/services/40-dashboards/main.tf
+++ b/ops/services/40-dashboards/main.tf
@@ -5,7 +5,7 @@ module "platform" {
   app                 = local.app
   env                 = local.env
   service             = local.service
-  root_module = "https://github.com/CMSgov/bluebutton-web-server/tree/main/ops/services/${basename(abspath(path.module))}"
+  root_module         = "https://github.com/CMSgov/bluebutton-web-server/tree/main/ops/services/${basename(abspath(path.module))}"
   ssm_hierarchy_roots = ["bb"]
 }
 
@@ -22,8 +22,8 @@ data "aws_secretsmanager_secret_version" "datadog_cicd_application_key" {
 }
 
 locals {
-  env         = terraform.workspace
-  service     = "dashboards"
+  env     = terraform.workspace
+  service = "dashboards"
 
   default_tags = module.platform.default_tags
 
@@ -31,7 +31,7 @@ locals {
 }
 
 module "datadog_dashboard" {
-  source      = "github.com/CMSgov/cdap/terraform/modules/datadog_dashboard?ref=0aba1af484320d0d121d804c05f36cf1a4d978c9"
+  source = "github.com/CMSgov/cdap/terraform/modules/datadog_dashboard?ref=0aba1af484320d0d121d804c05f36cf1a4d978c9"
 
   app         = local.app
   runbook_url = "https://github.com/CMSgov/bluebutton-web-server/blob/master/ops/services/RUNBOOK.md"

From 919a5f7cc46ceacc041f98dc0974221e84629065 Mon Sep 17 00:00:00 2001
From: Anna Montare <267455234+annamontare-nava@users.noreply.github.com>
Date: Wed, 15 Jul 2026 12:42:46 -0600
Subject: [PATCH 14/14] set different thresholds on test for ALB target
 response time monitor

This one is pretty noisy so far due to the very low volume of requests
on test.
---
 ops/services/30-monitors/main.tf | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ops/services/30-monitors/main.tf b/ops/services/30-monitors/main.tf
index 83333c61a..f6284a858 100644
--- a/ops/services/30-monitors/main.tf
+++ b/ops/services/30-monitors/main.tf
@@ -63,11 +63,11 @@ locals {
       name    = "[${upper(local.env)}] [${local.app}] ALB — Target Response Time High"
       type    = "metric alert"
       message = "ALB target has high average response time."
-      query   = "avg(last_1h):avg:aws.applicationelb.target_response_time.average{application:${local.app}, environment:${local.env}} > 0.35"
+      query   = "avg(last_1h):avg:aws.applicationelb.target_response_time.average{application:${local.app}, environment:${local.env}} > ${local.env == "test" ? 1 : 0.35}"
 
       thresholds = {
-        critical = 0.35
-        warning  = 0.25
+        critical = local.env == "test" ? 1 : 0.35
+        warning  = local.env == "test" ? 0.75 : 0.25
       }
 
       notify_no_data           = local.env != "test"