Streams userdata log to s3 via kinesis to improve delivery reliability

.github/actions/test/action.yml

@@ -67,6 +67,45 @@ runs:
         fi
         echo "ARTIFACT_LOCATION=${ARTIFACT_LOCATION}"
 
+        # Flush Firehose streams so all data is written to S3
+        FIREHOSE_STREAMS=$(terraform output -json firehose_delivery_stream_names 2>/dev/null | jq -r '.[] // empty' || true)
+
+        if [ -n "$FIREHOSE_STREAMS" ]; then
+          echo "Flushing Firehose streams..."
+
+          for stream_name in $FIREHOSE_STREAMS; do
+            echo "Reducing buffering interval for stream: $stream_name"
+
+            # Get current delivery stream config and version
+            STREAM_DESC=$(aws firehose describe-delivery-stream --delivery-stream-name "$stream_name" 2>/dev/null || true)
+
+            if [ -n "$STREAM_DESC" ]; then
+              VERSION_ID=$(echo "$STREAM_DESC" | jq -r '.DeliveryStreamDescription.VersionId')
+              DEST_ID=$(echo "$STREAM_DESC" | jq -r '.DeliveryStreamDescription.Destinations[0].DestinationId')
+
+              # Update with reduced time interval (60s) to flush quickly while keeping max size (128 MiB)
+              # Since destination type is not changing, only BufferingHints need to be specified; other config is merged
+              aws firehose update-destination \
+                --delivery-stream-name "$stream_name" \
+                --current-delivery-stream-version-id "$VERSION_ID" \
+                --destination-id "$DEST_ID" \
+                --extended-s3-destination-update '{
+                  "BufferingHints": {
+                    "SizeInMBs": 128,
+                    "IntervalInSeconds": 60
+                  }
+                }' 2>/dev/null || echo "Warning: Could not update stream $stream_name"
+            fi
+          done
+
+          echo "Waiting 60 seconds for Firehose to flush buffered data..."
+          for i in 60 50 40 30 20 10; do
+            echo "  $i seconds remaining..."
+            sleep 10
+          done
+          echo "  Flush complete."
+        fi
+
         if [ "${TF_DESTROY_AFTER_TEST}" = "true" ]; then
           # destroy resources
           terraform destroy -no-color -input=false -auto-approve

README.md

@@ -72,6 +72,9 @@ aws_instance.source_build["centos8stream"]: Still creating... [1m10s elapsed]
 | Name | Type |
 |------|------|
 | [aws_ami.amis](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
+| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
+| [aws_iam_instance_profile.builds](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_instance_profile) | data source |
+| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source |
 | [aws_subnet.tfi](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source |
 | [aws_vpc.tfi](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc) | data source |
 | [http_http.ip](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source |
@@ -120,6 +123,7 @@ aws_instance.source_build["centos8stream"]: Still creating... [1m10s elapsed]
 | <a name="output_build_id"></a> [build\_id](#output\_build\_id) | n/a |
 | <a name="output_build_slug"></a> [build\_slug](#output\_build\_slug) | n/a |
 | <a name="output_builders"></a> [builders](#output\_builders) | n/a |
+| <a name="output_firehose_delivery_stream_names"></a> [firehose\_delivery\_stream\_names](#output\_firehose\_delivery\_stream\_names) | Kinesis Firehose delivery stream names for flushing before destroy |
 | <a name="output_private_key"></a> [private\_key](#output\_private\_key) | n/a |
 | <a name="output_public_key"></a> [public\_key](#output\_public\_key) | n/a |
 | <a name="output_source_builds"></a> [source\_builds](#output\_source\_builds) | n/a |

main.tf

@@ -259,6 +259,12 @@ locals {
   source_builds        = toset(var.source_builds)
   builders             = local.standalone_source == "builder" ? toset([for s in local.standalone_builds : local.build_info[s].platform.builder]) : toset([])
   unique_builds_needed = setunion(local.standalone_builds, local.source_builds, local.builders)
+
+  firehose_stream_configs = merge(
+    { for os in local.builders : "builder-${os}" => { build_type = local.build_type_builder, os = os, short_type = "bld" } },
+    { for os in local.standalone_builds : "standalone_build-${os}" => { build_type = local.build_type_standalone, os = os, short_type = "sta" } },
+    { for os in local.source_builds : "source_build-${os}" => { build_type = local.build_type_source, os = os, short_type = "src" } }
+  )
 }
 
 data "aws_ami" "amis" {
@@ -288,6 +294,15 @@ data "aws_vpc" "tfi" {
   id = data.aws_subnet.tfi.vpc_id
 }
 
+data "aws_partition" "current" {}
+
+data "aws_caller_identity" "current" {}
+
+data "aws_iam_instance_profile" "builds" {
+  count = var.instance_profile == "" ? 0 : 1
+  name  = var.instance_profile
+}
+
 data "http" "ip" {
   url = local.url_local_ip
 }
@@ -349,6 +364,107 @@ resource "aws_security_group" "builds" {
   }
 }
 
+resource "aws_iam_role" "firehose" {
+  name = "${local.resource_name}-firehose"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Sid    = "AllowFirehoseServiceAssumeRole"
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Principal = {
+          Service = "firehose.amazonaws.com"
+        }
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy" "firehose_s3" {
+  name = "${local.resource_name}-firehose-s3"
+  role = aws_iam_role.firehose.id
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Sid = "AllowFirehoseS3Access"
+        Action = [
+          "s3:AbortMultipartUpload",
+          "s3:GetBucketLocation",
+          "s3:GetObject",
+          "s3:ListBucket",
+          "s3:ListBucketMultipartUploads",
+          "s3:PutObject"
+        ]
+        Effect = "Allow"
+        Resource = [
+          "arn:${data.aws_partition.current.partition}:s3:::${var.s3_bucket}",
+          "arn:${data.aws_partition.current.partition}:s3:::${var.s3_bucket}/*"
+        ]
+      },
+      {
+        Sid = "AllowFirehoseCloudWatchLogsPutEvents"
+        Action = [
+          "logs:PutLogEvents"
+        ]
+        Effect   = "Allow"
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+resource "aws_kinesis_firehose_delivery_stream" "userdata_logs" {
+  for_each = local.firehose_stream_configs
+
+  name        = "${local.resource_name}-${each.value.short_type}-${each.value.os}-logs"
+  destination = "extended_s3"
+
+  extended_s3_configuration {
+    role_arn            = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:role/${aws_iam_role_policy.firehose_s3.role}"
+    bucket_arn          = "arn:${data.aws_partition.current.partition}:s3:::${var.s3_bucket}"
+    prefix              = "!{timestamp:yyyyMMdd}/${local.date_hm}-${local.build_id}/${each.key}/kinesis/"
+    error_output_prefix = "!{timestamp:yyyyMMdd}/${local.date_hm}-${local.build_id}/${each.key}/kinesis_errors/!{firehose:error-output-type}/"
+    buffering_size      = 128
+    buffering_interval  = 900
+    compression_format  = "UNCOMPRESSED"
+    file_extension      = ".log"
+  }
+}
+
+resource "aws_iam_role_policy" "instance_profile_firehose_put" {
+  count = var.instance_profile == "" ? 0 : 1
+
+  name = "${local.resource_name}-firehose-put"
+  role = data.aws_iam_instance_profile.builds[0].role_name
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Sid = "AllowInstanceFirehosePutRecords"
+        Action = [
+          "firehose:PutRecord",
+          "firehose:PutRecordBatch"
+        ]
+        Effect   = "Allow"
+        Resource = [for stream in aws_kinesis_firehose_delivery_stream.userdata_logs : stream.arn]
+      },
+      {
+        Sid = "AllowInstancePublishAgentMetrics"
+        Action = [
+          "cloudwatch:PutMetricData"
+        ]
+        Effect   = "Allow"
+        Resource = "*"
+      }
+    ]
+  })
+}
+
 resource "aws_instance" "builder" {
   for_each = local.builders
 
@@ -371,10 +487,11 @@ resource "aws_instance" "builder" {
             local.template_vars.base,
             local.template_vars[local.build_info[each.key].platform.key],
             {
-              build_os    = each.key
-              build_type  = local.build_type_builder
-              build_label = format(local.format_str_build_label, local.build_type_builder, each.key)
-              password    = local.build_info[each.key].platform.connection_password
+              build_os                      = each.key
+              build_type                    = local.build_type_builder
+              build_label                   = format(local.format_str_build_label, local.build_type_builder, each.key)
+              firehose_delivery_stream_name = aws_kinesis_firehose_delivery_stream.userdata_logs["builder-${each.key}"].name
+              password                      = local.build_info[each.key].platform.connection_password
             }
           )
         ))
@@ -388,10 +505,11 @@ resource "aws_instance" "builder" {
         local.template_vars.base,
         local.template_vars[local.build_info[each.key].platform.key],
         {
-          build_os    = each.key
-          build_type  = local.build_type_builder
-          build_label = format(local.format_str_build_label, local.build_type_builder, each.key)
-          password    = local.build_info[each.key].platform.connection_password
+          build_os                      = each.key
+          build_type                    = local.build_type_builder
+          build_label                   = format(local.format_str_build_label, local.build_type_builder, each.key)
+          firehose_delivery_stream_name = aws_kinesis_firehose_delivery_stream.userdata_logs["builder-${each.key}"].name
+          password                      = local.build_info[each.key].platform.connection_password
         }
       )
     )
@@ -497,10 +615,11 @@ resource "aws_instance" "standalone_build" {
             local.template_vars.base,
             local.template_vars[local.build_info[each.key].platform.key],
             {
-              build_os    = each.key
-              build_type  = local.build_type_standalone
-              build_label = format(local.format_str_build_label, local.build_type_standalone, each.key)
-              password    = local.build_info[each.key].platform.connection_password
+              build_os                      = each.key
+              build_type                    = local.build_type_standalone
+              build_label                   = format(local.format_str_build_label, local.build_type_standalone, each.key)
+              firehose_delivery_stream_name = aws_kinesis_firehose_delivery_stream.userdata_logs["standalone_build-${each.key}"].name
+              password                      = local.build_info[each.key].platform.connection_password
             }
           )
         ))
@@ -514,10 +633,11 @@ resource "aws_instance" "standalone_build" {
         local.template_vars.base,
         local.template_vars[local.build_info[each.key].platform.key],
         {
-          build_os    = each.key
-          build_type  = local.build_type_standalone
-          build_label = format(local.format_str_build_label, local.build_type_standalone, each.key)
-          password    = local.build_info[each.key].platform.connection_password
+          build_os                      = each.key
+          build_type                    = local.build_type_standalone
+          build_label                   = format(local.format_str_build_label, local.build_type_standalone, each.key)
+          firehose_delivery_stream_name = aws_kinesis_firehose_delivery_stream.userdata_logs["standalone_build-${each.key}"].name
+          password                      = local.build_info[each.key].platform.connection_password
         }
       )
     )
@@ -629,10 +749,11 @@ resource "aws_instance" "source_build" {
             local.template_vars.base,
             local.template_vars[local.build_info[each.key].platform.key],
             {
-              build_os    = each.key
-              build_type  = local.build_type_source
-              build_label = format(local.format_str_build_label, local.build_type_source, each.key)
-              password    = local.build_info[each.key].platform.connection_password
+              build_os                      = each.key
+              build_type                    = local.build_type_source
+              build_label                   = format(local.format_str_build_label, local.build_type_source, each.key)
+              firehose_delivery_stream_name = aws_kinesis_firehose_delivery_stream.userdata_logs["source_build-${each.key}"].name
+              password                      = local.build_info[each.key].platform.connection_password
             }
           )
         ))
@@ -646,10 +767,11 @@ resource "aws_instance" "source_build" {
         local.template_vars.base,
         local.template_vars[local.build_info[each.key].platform.key],
         {
-          build_os    = each.key
-          build_type  = local.build_type_source
-          build_label = format(local.format_str_build_label, local.build_type_source, each.key)
-          password    = local.build_info[each.key].platform.connection_password
+          build_os                      = each.key
+          build_type                    = local.build_type_source
+          build_label                   = format(local.format_str_build_label, local.build_type_source, each.key)
+          firehose_delivery_stream_name = aws_kinesis_firehose_delivery_stream.userdata_logs["source_build-${each.key}"].name
+          password                      = local.build_info[each.key].platform.connection_password
         }
       )
     )

outputs.tf

@@ -43,3 +43,10 @@ output "public_key" {
 output "build_slug" {
   value = local.build_slug
 }
+
+output "firehose_delivery_stream_names" {
+  description = "Kinesis Firehose delivery stream names for flushing before destroy"
+  value = {
+    for key, stream in aws_kinesis_firehose_delivery_stream.userdata_logs : key => stream.name
+  }
+}