diff --git a/ops/services/20-microservices/ecs.tf b/ops/services/20-microservices/ecs.tf index ffce8f911..9da96987b 100644 --- a/ops/services/20-microservices/ecs.tf +++ b/ops/services/20-microservices/ecs.tf @@ -40,7 +40,7 @@ resource "aws_ecs_task_definition" "ecs_task" { } }, { name = "datadog-agent" - image = "public.ecr.aws/datadog/agent:7.80.3" + image = "public.ecr.aws/datadog/agent:7.81.0-fips" essential = false readonlyRootFilesystem = true diff --git a/ops/services/30-monitors/config/defaults.yml b/ops/services/30-monitors/config/defaults.yml index 3ea0f9b82..6eacf383c 100644 --- a/ops/services/30-monitors/config/defaults.yml +++ b/ops/services/30-monitors/config/defaults.yml @@ -10,7 +10,7 @@ enabled: sqs: false sns: false lambda: false - s3: true + s3: false rds: true ecs: notify_no_data: true @@ -29,5 +29,6 @@ s3: no_data_timeframe_minutes: 10 timeframe: "last_5m" rds: + notify_no_data: true no_data_timeframe_minutes: 10 timeframe: "last_10m" diff --git a/ops/services/30-monitors/config/test.yml b/ops/services/30-monitors/config/test.yml index b44bbc656..09e90ed20 100644 --- a/ops/services/30-monitors/config/test.yml +++ b/ops/services/30-monitors/config/test.yml @@ -5,3 +5,5 @@ notifications: victorops: false ecs: notify_no_data: false # test shuts down overnight +rds: + notify_no_data: false # test shuts down overnight diff --git a/ops/services/30-monitors/main.tf b/ops/services/30-monitors/main.tf index 7ba2c518a..f6284a858 100644 --- a/ops/services/30-monitors/main.tf +++ b/ops/services/30-monitors/main.tf @@ -5,7 +5,7 @@ module "platform" { app = local.app env = local.env service = local.service - root_module = "https://github.com/CMSgov/bluebutton-web-server/tree/main/ops/services/${basename(abspath(path.module))}" + root_module = "https://github.com/CMSgov/bluebutton-web-server/tree/main/ops/services/${basename(abspath(path.module))}" ssm_hierarchy_roots = ["bb"] } @@ -22,8 +22,8 @@ data "aws_secretsmanager_secret_version" "datadog_cicd_application_key" { } locals { - env = terraform.workspace - service = "monitors" + env = terraform.workspace + service = "monitors" default_tags = module.platform.default_tags @@ -49,9 +49,105 @@ locals { } module "common_datadog_monitors" { - source = "github.com/CMSgov/cdap/terraform/modules/datadog_monitors?ref=6ded520857376f46bb317dca898e5df6a9ecc93b" + source = "github.com/CMSgov/cdap/terraform/modules/datadog_monitors?ref=0aba1af484320d0d121d804c05f36cf1a4d978c9" - app = local.app - env = local.env - monitor_config = local.monitor_config + app = local.app + env = local.env + monitor_config = local.monitor_config + custom_monitors = local.custom_monitors +} + +locals { + custom_monitors = [ + { + name = "[${upper(local.env)}] [${local.app}] ALB — Target Response Time High" + type = "metric alert" + message = "ALB target has high average response time." + query = "avg(last_1h):avg:aws.applicationelb.target_response_time.average{application:${local.app}, environment:${local.env}} > ${local.env == "test" ? 1 : 0.35}" + + thresholds = { + critical = local.env == "test" ? 1 : 0.35 + warning = local.env == "test" ? 0.75 : 0.25 + } + + notify_no_data = local.env != "test" + no_data_timeframe_minute = 60 + + # TODO are we sure about this? + require_full_window = false + }, + { + # TODO need PR from CDAP for this to work + create = local.env != "test" + + name = "[${upper(local.env)}] [${local.app}] ALB — Request Rate Low" + type = "metric alert" + message = "Request rate / throughput is low." + query = "avg(last_1h):sum:aws.applicationelb.request_count{application:${local.app}, environment:${local.env}}.as_rate() < ${local.env == "prod" ? 0.1 : 0.001}" + + thresholds = { + critical = local.env == "prod" ? 0.1 : 0.001 + warning = local.env == "prod" ? 0.5 : 0.005 + } + + notify_no_data = true + no_data_timeframe_minute = 60 + + require_full_window = false + }, + { + # TODO need PR from CDAP for this to work + create = local.env != "test" + + name = "[${upper(local.env)}] [${local.app}] ALB — Maximum Unhealthy Hosts" + type = "metric alert" + message = "Maximum number of unhealthy hosts passed threshold." + query = "max(last_1h):sum:aws.applicationelb.un_healthy_host_count.maximum{application:${local.app}, environment:${local.env}} >= 1" + + thresholds = { + # TODO should this be 2 or something? + critical = 1 + warning = 0.5 + } + + notify_no_data = true + no_data_timeframe_minute = 60 + + require_full_window = false + }, + { + name = "[${upper(local.env)}] [${local.app}] ALB — Target 5xx Count High" + type = "metric alert" + message = "High number of 5xx response codes generated by the target." + query = "sum(last_1h):sum:aws.applicationelb.httpcode_target_5xx{application:${local.app}, environment:${local.env}}.as_count() > 1000" + + thresholds = { + critical = 1000 + warning = 100 + } + + # TODO when editing this, why does the on_missing_data appear as "Show No data" even though that's not the default for counts? + notify_no_data = local.env != "test" # TODO should probably be false + no_data_timeframe_minute = 60 + + require_full_window = false + }, + { + name = "[${upper(local.env)}] [${local.app}] APM — Error Rate High" + type = "metric alert" + message = "Service ${local.app} has high error rate." + # TODO what evaluation window? + query = "sum(last_1h):sum:trace.django.request.errors{env:${local.env},service:${local.app},span.kind:server}.as_count() / sum:trace.django.request.hits{env:${local.env},service:${local.app},span.kind:server}.as_count() > 0.02" + + thresholds = { + critical = 0.02 + warning = 0.01 + } + + notify_no_data = local.env != "test" + no_data_timeframe_minute = 60 + + require_full_window = false + }, + ] } diff --git a/ops/services/40-dashboards/main.tf b/ops/services/40-dashboards/main.tf index 4b3eed228..98e6b7bdc 100644 --- a/ops/services/40-dashboards/main.tf +++ b/ops/services/40-dashboards/main.tf @@ -5,7 +5,7 @@ module "platform" { app = local.app env = local.env service = local.service - root_module = "https://github.com/CMSgov/bluebutton-web-server/tree/main/ops/services/${basename(abspath(path.module))}" + root_module = "https://github.com/CMSgov/bluebutton-web-server/tree/main/ops/services/${basename(abspath(path.module))}" ssm_hierarchy_roots = ["bb"] } @@ -22,8 +22,8 @@ data "aws_secretsmanager_secret_version" "datadog_cicd_application_key" { } locals { - env = terraform.workspace - service = "dashboards" + env = terraform.workspace + service = "dashboards" default_tags = module.platform.default_tags @@ -31,7 +31,7 @@ locals { } module "datadog_dashboard" { - source = "github.com/CMSgov/cdap/terraform/modules/datadog_dashboard?ref=6ded520857376f46bb317dca898e5df6a9ecc93b" + source = "github.com/CMSgov/cdap/terraform/modules/datadog_dashboard?ref=0aba1af484320d0d121d804c05f36cf1a4d978c9" app = local.app runbook_url = "https://github.com/CMSgov/bluebutton-web-server/blob/master/ops/services/RUNBOOK.md" @@ -43,7 +43,7 @@ module "datadog_dashboard" { sns = false sqs = false aurora = true - s3 = true + s3 = false apm = true } @@ -58,5 +58,7 @@ module "datadog_dashboard" { apm = "1h" } + apm_primary_operation = "django.request" + count = local.create_dashboards ? 1 : 0 }