From af0d13c7a601679dd580c26f46350884b7ced915 Mon Sep 17 00:00:00 2001
From: Loren Gordon <8457307+lorengordon@users.noreply.github.com>
Date: Wed, 13 May 2026 11:55:28 -0700
Subject: [PATCH 1/2] Streams userdata log to s3 via kinesis to improve
 delivery reliability

---
 main.tf                    | 170 +++++++++++++++++++++++++++++++------
 templates/lx_userdata.sh   | 127 +++++++++++++++++++++++++--
 templates/win_userdata.ps1 | 105 ++++++++++++++++++++---
 3 files changed, 359 insertions(+), 43 deletions(-)

diff --git a/main.tf b/main.tf
index 7b83f8f..ede1693 100644
--- a/main.tf
+++ b/main.tf
@@ -259,6 +259,12 @@ locals {
   source_builds        = toset(var.source_builds)
   builders             = local.standalone_source == "builder" ? toset([for s in local.standalone_builds : local.build_info[s].platform.builder]) : toset([])
   unique_builds_needed = setunion(local.standalone_builds, local.source_builds, local.builders)
+
+  firehose_stream_configs = merge(
+    { for os in local.builders : "builder-${os}" => { build_type = local.build_type_builder, os = os, short_type = "bld" } },
+    { for os in local.standalone_builds : "standalone_build-${os}" => { build_type = local.build_type_standalone, os = os, short_type = "sta" } },
+    { for os in local.source_builds : "source_build-${os}" => { build_type = local.build_type_source, os = os, short_type = "src" } }
+  )
 }
 
 data "aws_ami" "amis" {
@@ -288,6 +294,15 @@ data "aws_vpc" "tfi" {
   id = data.aws_subnet.tfi.vpc_id
 }
 
+data "aws_partition" "current" {}
+
+data "aws_caller_identity" "current" {}
+
+data "aws_iam_instance_profile" "builds" {
+  count = var.instance_profile == "" ? 0 : 1
+  name  = var.instance_profile
+}
+
 data "http" "ip" {
   url = local.url_local_ip
 }
@@ -349,6 +364,107 @@ resource "aws_security_group" "builds" {
   }
 }
 
+resource "aws_iam_role" "firehose" {
+  name = "${local.resource_name}-firehose"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Sid    = "AllowFirehoseServiceAssumeRole"
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Principal = {
+          Service = "firehose.amazonaws.com"
+        }
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role_policy" "firehose_s3" {
+  name = "${local.resource_name}-firehose-s3"
+  role = aws_iam_role.firehose.id
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Sid = "AllowFirehoseS3Access"
+        Action = [
+          "s3:AbortMultipartUpload",
+          "s3:GetBucketLocation",
+          "s3:GetObject",
+          "s3:ListBucket",
+          "s3:ListBucketMultipartUploads",
+          "s3:PutObject"
+        ]
+        Effect = "Allow"
+        Resource = [
+          "arn:${data.aws_partition.current.partition}:s3:::${var.s3_bucket}",
+          "arn:${data.aws_partition.current.partition}:s3:::${var.s3_bucket}/*"
+        ]
+      },
+      {
+        Sid = "AllowFirehoseCloudWatchLogsPutEvents"
+        Action = [
+          "logs:PutLogEvents"
+        ]
+        Effect   = "Allow"
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+resource "aws_kinesis_firehose_delivery_stream" "userdata_logs" {
+  for_each = local.firehose_stream_configs
+
+  name        = "${local.resource_name}-${each.value.short_type}-${each.value.os}-logs"
+  destination = "extended_s3"
+
+  extended_s3_configuration {
+    role_arn            = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:role/${aws_iam_role_policy.firehose_s3.role}"
+    bucket_arn          = "arn:${data.aws_partition.current.partition}:s3:::${var.s3_bucket}"
+    prefix              = "!{timestamp:yyyyMMdd}/${local.date_hm}-${local.build_id}/${each.key}/kinesis/"
+    error_output_prefix = "!{timestamp:yyyyMMdd}/${local.date_hm}-${local.build_id}/${each.key}/kinesis_errors/!{firehose:error-output-type}/"
+    buffering_size      = 128
+    buffering_interval  = 900
+    compression_format  = "UNCOMPRESSED"
+    file_extension      = ".log"
+  }
+}
+
+resource "aws_iam_role_policy" "instance_profile_firehose_put" {
+  count = var.instance_profile == "" ? 0 : 1
+
+  name = "${local.resource_name}-firehose-put"
+  role = data.aws_iam_instance_profile.builds[0].role_name
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Sid = "AllowInstanceFirehosePutRecords"
+        Action = [
+          "firehose:PutRecord",
+          "firehose:PutRecordBatch"
+        ]
+        Effect   = "Allow"
+        Resource = [for stream in aws_kinesis_firehose_delivery_stream.userdata_logs : stream.arn]
+      },
+      {
+        Sid = "AllowInstancePublishAgentMetrics"
+        Action = [
+          "cloudwatch:PutMetricData"
+        ]
+        Effect   = "Allow"
+        Resource = "*"
+      }
+    ]
+  })
+}
+
 resource "aws_instance" "builder" {
   for_each = local.builders
 
@@ -371,10 +487,11 @@ resource "aws_instance" "builder" {
             local.template_vars.base,
             local.template_vars[local.build_info[each.key].platform.key],
             {
-              build_os    = each.key
-              build_type  = local.build_type_builder
-              build_label = format(local.format_str_build_label, local.build_type_builder, each.key)
-              password    = local.build_info[each.key].platform.connection_password
+              build_os                      = each.key
+              build_type                    = local.build_type_builder
+              build_label                   = format(local.format_str_build_label, local.build_type_builder, each.key)
+              firehose_delivery_stream_name = aws_kinesis_firehose_delivery_stream.userdata_logs["builder-${each.key}"].name
+              password                      = local.build_info[each.key].platform.connection_password
             }
           )
         ))
@@ -388,10 +505,11 @@ resource "aws_instance" "builder" {
         local.template_vars.base,
         local.template_vars[local.build_info[each.key].platform.key],
         {
-          build_os    = each.key
-          build_type  = local.build_type_builder
-          build_label = format(local.format_str_build_label, local.build_type_builder, each.key)
-          password    = local.build_info[each.key].platform.connection_password
+          build_os                      = each.key
+          build_type                    = local.build_type_builder
+          build_label                   = format(local.format_str_build_label, local.build_type_builder, each.key)
+          firehose_delivery_stream_name = aws_kinesis_firehose_delivery_stream.userdata_logs["builder-${each.key}"].name
+          password                      = local.build_info[each.key].platform.connection_password
         }
       )
     )
@@ -497,10 +615,11 @@ resource "aws_instance" "standalone_build" {
             local.template_vars.base,
             local.template_vars[local.build_info[each.key].platform.key],
             {
-              build_os    = each.key
-              build_type  = local.build_type_standalone
-              build_label = format(local.format_str_build_label, local.build_type_standalone, each.key)
-              password    = local.build_info[each.key].platform.connection_password
+              build_os                      = each.key
+              build_type                    = local.build_type_standalone
+              build_label                   = format(local.format_str_build_label, local.build_type_standalone, each.key)
+              firehose_delivery_stream_name = aws_kinesis_firehose_delivery_stream.userdata_logs["standalone_build-${each.key}"].name
+              password                      = local.build_info[each.key].platform.connection_password
             }
           )
         ))
@@ -514,10 +633,11 @@ resource "aws_instance" "standalone_build" {
         local.template_vars.base,
         local.template_vars[local.build_info[each.key].platform.key],
         {
-          build_os    = each.key
-          build_type  = local.build_type_standalone
-          build_label = format(local.format_str_build_label, local.build_type_standalone, each.key)
-          password    = local.build_info[each.key].platform.connection_password
+          build_os                      = each.key
+          build_type                    = local.build_type_standalone
+          build_label                   = format(local.format_str_build_label, local.build_type_standalone, each.key)
+          firehose_delivery_stream_name = aws_kinesis_firehose_delivery_stream.userdata_logs["standalone_build-${each.key}"].name
+          password                      = local.build_info[each.key].platform.connection_password
         }
       )
     )
@@ -629,10 +749,11 @@ resource "aws_instance" "source_build" {
             local.template_vars.base,
             local.template_vars[local.build_info[each.key].platform.key],
             {
-              build_os    = each.key
-              build_type  = local.build_type_source
-              build_label = format(local.format_str_build_label, local.build_type_source, each.key)
-              password    = local.build_info[each.key].platform.connection_password
+              build_os                      = each.key
+              build_type                    = local.build_type_source
+              build_label                   = format(local.format_str_build_label, local.build_type_source, each.key)
+              firehose_delivery_stream_name = aws_kinesis_firehose_delivery_stream.userdata_logs["source_build-${each.key}"].name
+              password                      = local.build_info[each.key].platform.connection_password
             }
           )
         ))
@@ -646,10 +767,11 @@ resource "aws_instance" "source_build" {
         local.template_vars.base,
         local.template_vars[local.build_info[each.key].platform.key],
         {
-          build_os    = each.key
-          build_type  = local.build_type_source
-          build_label = format(local.format_str_build_label, local.build_type_source, each.key)
-          password    = local.build_info[each.key].platform.connection_password
+          build_os                      = each.key
+          build_type                    = local.build_type_source
+          build_label                   = format(local.format_str_build_label, local.build_type_source, each.key)
+          firehose_delivery_stream_name = aws_kinesis_firehose_delivery_stream.userdata_logs["source_build-${each.key}"].name
+          password                      = local.build_info[each.key].platform.connection_password
         }
       )
     )
diff --git a/templates/lx_userdata.sh b/templates/lx_userdata.sh
index 09926a2..086b37a 100644
--- a/templates/lx_userdata.sh
+++ b/templates/lx_userdata.sh
@@ -17,6 +17,7 @@ github_artifact_repo_name="${github_artifact_repo_name}"
 github_artifact_repo_owner="${github_artifact_repo_owner}"
 github_artifact_run_id="${github_artifact_run_id}"
 github_artifact_token_ssm_parameter="${github_artifact_token_ssm_parameter}"
+firehose_delivery_stream_name="${firehose_delivery_stream_name}"
 port="${port}"
 release_prefix="${release_prefix}"
 scan_slug="${scan_slug}"
@@ -43,15 +44,12 @@ export AWS_DEFAULT_REGION="$aws_region"
 echo "------------------------------- $build_label ---------------------"
 
 debug-2s3() {
-  ## With as few dependencies as possible, immediately upload the debug and log
-  ## files to S3. Calling this multiple times will simply overwrite the
-  ## previously uploaded logs.
+  ## Keep an incremental local debug log. Final artifact upload handles S3
+  ## persistence, which avoids network calls on every log write.
   local msg="$1"
 
   local debug_file="$temp_dir/debug.log"
   echo "$msg" >> "$debug_file"
-  aws s3 cp "$debug_file" "s3://$build_slug/$build_label/" > /dev/null 2>&1 || true
-  aws s3 cp "$userdata_log" "s3://$build_slug/$build_label/" > /dev/null 2>&1 || true
 }
 
 check-metadata-availability() {
@@ -59,6 +57,123 @@ check-metadata-availability() {
   try_cmd 50 curl -sSL $metadata_loopback_az
 }
 
+configure-kinesis-agent() {
+  local config_path="/etc/aws-kinesis/agent.json"
+  local src_dir="/tmp/aws-kinesis-agent-src"
+  local original_pwd=""
+
+  if [[ -z "$firehose_delivery_stream_name" ]]; then
+    write-tfi "Kinesis Agent setup skipped: firehose delivery stream name was not provided"
+    return 0
+  fi
+
+  if command -v dnf > /dev/null 2>&1; then
+    try_cmd 3 dnf -y install initscripts || {
+      write-tfi "Initscripts install failed via dnf"
+      return 0
+    }
+    try_cmd 3 dnf -y install aws-kinesis-agent || {
+      write-tfi "Kinesis Agent install failed via dnf"
+      return 0
+    }
+  elif command -v apt-get > /dev/null 2>&1; then
+    try_cmd 3 apt-get -y update || true
+    try_cmd 3 apt-get -y install curl tar openjdk-11-jdk-headless || {
+      write-tfi "Kinesis Agent install failed: unable to install prerequisites via apt-get"
+      return 0
+    }
+
+    try_cmd 3 curl -fsSL https://github.com/awslabs/amazon-kinesis-agent/archive/refs/heads/master.tar.gz -o /tmp/aws-kinesis-agent-src.tar.gz || {
+      write-tfi "Kinesis Agent install failed: unable to download source archive"
+      return 0
+    }
+
+    try_cmd 1 rm -rf "$src_dir" || {
+      write-tfi "Kinesis Agent install failed: unable to clean source directory"
+      return 0
+    }
+
+    try_cmd 1 mkdir -p "$src_dir" || {
+      write-tfi "Kinesis Agent install failed: unable to prepare source directory"
+      return 0
+    }
+
+    try_cmd 1 tar -xzf /tmp/aws-kinesis-agent-src.tar.gz --strip-components=1 -C "$src_dir" || {
+      write-tfi "Kinesis Agent install failed: unable to extract source archive"
+      return 0
+    }
+
+    original_pwd=$(pwd)
+    cd "$src_dir" || {
+      write-tfi "Kinesis Agent install failed: unable to enter source directory"
+      return 0
+    }
+
+    try_cmd 1 bash ./setup --install || {
+      cd "$original_pwd" > /dev/null 2>&1 || true
+      write-tfi "Kinesis Agent install failed via setup script"
+      return 0
+    }
+
+    cd "$original_pwd" > /dev/null 2>&1 || true
+  else
+    write-tfi "Kinesis Agent setup skipped: package manager not available"
+    return 0
+  fi
+
+  chmod 644 "$userdata_log" > /dev/null 2>&1 || true
+
+  cat > "$config_path" << EOF
+{
+  "cloudwatch.emitMetrics": true,
+  "firehose.endpoint": "firehose.$aws_region.amazonaws.com",
+  "maxConnections": 1,
+  "maxSendingThreads": 1,
+  "flows": [
+    {
+      "filePattern": "$userdata_log",
+      "deliveryStream": "$firehose_delivery_stream_name",
+      "initialPosition": "START_OF_FILE",
+      "maxBufferAgeMillis": 1000,
+      "maxBufferSizeRecords": 1
+    }
+  ]
+}
+EOF
+
+  if command -v systemctl > /dev/null 2>&1; then
+    try_cmd 1 systemctl stop aws-kinesis-agent > /dev/null 2>&1 || true
+    try_cmd 1 systemctl start aws-kinesis-agent > /dev/null 2>&1 || {
+      if pgrep -f "com.amazon.kinesis.streaming.agent.Agent" > /dev/null 2>&1; then
+        write-tfi "Kinesis Agent service reported start failure, but agent process is running"
+      else
+        write-tfi "Kinesis Agent start failed"
+      fi
+      try_cmd 1 systemctl status aws-kinesis-agent --no-pager -l || true
+      [[ -f /var/log/aws-kinesis-agent/aws-kinesis-agent.log ]] && tail -n 80 /var/log/aws-kinesis-agent/aws-kinesis-agent.log || true
+      return 0
+    }
+  elif command -v service > /dev/null 2>&1; then
+    try_cmd 1 service aws-kinesis-agent stop > /dev/null 2>&1 || true
+    try_cmd 1 service aws-kinesis-agent start > /dev/null 2>&1 || {
+      if pgrep -f "com.amazon.kinesis.streaming.agent.Agent" > /dev/null 2>&1; then
+        write-tfi "Kinesis Agent service reported start failure, but agent process is running"
+      else
+        write-tfi "Kinesis Agent start failed"
+      fi
+      try_cmd 1 service aws-kinesis-agent status || true
+      [[ -f /var/log/aws-kinesis-agent/aws-kinesis-agent.log ]] && tail -n 80 /var/log/aws-kinesis-agent/aws-kinesis-agent.log || true
+      return 0
+    }
+  else
+    write-tfi "Kinesis Agent installed and configured; restart command not found"
+    return 0
+  fi
+
+  write-tfi "Kinesis Agent configured for stream $firehose_delivery_stream_name"
+  return 0
+}
+
 write-tfi() {
   local msg=""
   local result=""
@@ -563,6 +678,8 @@ start=$(date +%s)
 userdata_status_code=0
 userdata_status_message="Success"
 
+configure-kinesis-agent
+
 # shellcheck disable=SC1083,SC2288
 %{ if build_type == build_type_builder }
 
diff --git a/templates/win_userdata.ps1 b/templates/win_userdata.ps1
index f0bc05f..09e13cb 100644
--- a/templates/win_userdata.ps1
+++ b/templates/win_userdata.ps1
@@ -15,6 +15,7 @@ $GitHubArtifactRepoOwner = "${github_artifact_repo_owner}"
 $GitHubArtifactRepoName = "${github_artifact_repo_name}"
 $GitHubArtifactRunId = "${github_artifact_run_id}"
 $GitHubArtifactTokenSsmParameter = "${github_artifact_token_ssm_parameter}"
+$FirehoseDeliveryStream = "${firehose_delivery_stream_name}"
 $WinUser = "${user}"
 $PypiUrl = "${url_pypi}"
 $DebugMode = "${debug}"
@@ -41,8 +42,6 @@ function Debug-2S3 {
   $DebugFileName = "debug.log"
   $DebugFile = "$TempDir\$DebugFileName"
   "$(Get-Date): $Msg" | Out-File $DebugFile -Append -Encoding utf8
-  Write-S3Object -BucketName "$BuildBucket" -Key "$${BuildKeyPrefix}/$${BuildLabel}/$${DebugFileName}" -File "$DebugFile"
-  Write-S3Object -BucketName "$BuildBucket" -Key "$${BuildKeyPrefix}/$${BuildLabel}/$${UserdataLogFileName}" -File "$UserdataLogFile"
 }
 
 function Check-Metadata {
@@ -114,9 +113,7 @@ function Test-Command {
             [string]$CommandDescription,
             [int]$AttemptNumber,
             [int]$MaxAttempts,
-            [int]$IntervalSeconds,
-            [string]$S3Bucket,
-            [string]$S3Key
+            [int]$IntervalSeconds
           )
 
           # Use a FileStream append with shared read/write access to reduce cross-process contention.
@@ -158,16 +155,8 @@ function Test-Command {
             $SecondsRunning = $HeartbeatCount * $IntervalSeconds
 
             Add-HeartbeatLogLine -Path "$LogPath" -Message "$(Get-Date): Command still running at $${SecondsRunning}s (attempt $AttemptNumber/$MaxAttempts) [$CommandDescription]"
-
-            # Best effort: upload userdata log periodically so diagnostics survive instance termination.
-            try {
-              Write-S3Object -BucketName "$S3Bucket" -Key "$S3Key" -File "$LogPath" -ErrorAction Stop
-            }
-            catch {
-              # Intentionally continue heartbeat even when S3 sync is unavailable.
-            }
           }
-        } -ArgumentList "$UserdataLogFile", "$Description", $Attempt, $Tries, $HeartbeatSeconds, "$BuildBucket", "$BuildKeyPrefix/$BuildLabel/$UserdataLogFileName"
+        } -ArgumentList "$UserdataLogFile", "$Description", $Attempt, $Tries, $HeartbeatSeconds
       }
       catch {
         Write-Tfi ("Unable to start heartbeat logger for command [{0}]: {1}" -f $Description, [String]$_.Exception.Message)
@@ -646,6 +635,93 @@ function Install-WatchmakerFromGitHubArtifact {
   }
 }
 
+function Configure-KinesisAgent {
+  if ([string]::IsNullOrEmpty($FirehoseDeliveryStream)) {
+    Write-Tfi "Kinesis Agent setup skipped: firehose delivery stream name was not provided"
+    return
+  }
+
+  $KinesisTapService = Get-Service -Name "AWSKinesisTap" -ErrorAction SilentlyContinue
+  if ($null -eq $KinesisTapService) {
+    try {
+      $InstallerScript = Join-Path $TempDir "InstallKinesisAgent.ps1"
+      Test-Command -Description "Download InstallKinesisAgent.ps1" -Command {
+        Invoke-WebRequest -Uri "https://s3-us-west-2.amazonaws.com/kinesis-agent-windows/downloads/InstallKinesisAgent.ps1" -OutFile $InstallerScript -UseBasicParsing
+      }
+      Test-Command -Description "Run InstallKinesisAgent.ps1" -Command {
+        & $InstallerScript
+      }
+      $KinesisTapService = Get-Service -Name "AWSKinesisTap" -ErrorAction SilentlyContinue
+    }
+    catch {
+      Write-Tfi "Kinesis Agent install failed: $([String]$_.Exception.Message)"
+      return
+    }
+  }
+
+  if ($null -eq $KinesisTapService) {
+    Write-Tfi "Kinesis Agent setup skipped: AWSKinesisTap service is not available"
+    return
+  }
+
+  try {
+    $ConfigDir = Join-Path $Env:ProgramFiles "Amazon\AWSKinesisTap"
+    New-Item -Path $ConfigDir -ItemType Directory -Force | Out-Null
+    $ConfigPath = Join-Path $ConfigDir "appsettings.json"
+
+    $UserdataDir = Split-Path -Path $UserdataLogFile -Parent
+    $UserdataLeaf = Split-Path -Path $UserdataLogFile -Leaf
+
+    $KinesisConfig = [ordered]@{
+      Sources = @(
+        [ordered]@{
+          Id             = "UserdataLogSource"
+          SourceType     = "DirectorySource"
+          Directory      = $UserdataDir
+          FileNameFilter = $UserdataLeaf
+          RecordParser   = "SingleLine"
+          InitialPosition = "0"
+        }
+      )
+      Sinks = @(
+        [ordered]@{
+          Id                  = "UserdataFirehoseSink"
+          SinkType            = "KinesisFirehose"
+          StreamName          = $FirehoseDeliveryStream
+          Region              = "${aws_region}"
+          QueueType           = "file"
+          ParallelUploadCount = 1
+        }
+      )
+      Pipes = @(
+        [ordered]@{
+          Id        = "UserdataToFirehose"
+          SourceRef = "UserdataLogSource"
+          SinkRef   = "UserdataFirehoseSink"
+        }
+      )
+    }
+
+    $KinesisConfig | ConvertTo-Json -Depth 8 | Out-File $ConfigPath -Encoding utf8
+
+    if ($KinesisTapService.Status -eq "Running") {
+      Test-Command -Description "Restart AWSKinesisTap service" -Command {
+        Restart-Service -Name "AWSKinesisTap" -ErrorAction Stop
+      }
+    }
+    else {
+      Test-Command -Description "Start AWSKinesisTap service" -Command {
+        Start-Service -Name "AWSKinesisTap" -ErrorAction Stop
+      }
+    }
+
+    Write-Tfi "Kinesis Agent configured for stream $FirehoseDeliveryStream" $true
+  }
+  catch {
+    Write-Tfi "Kinesis Agent configuration failed: $([String]$_.Exception.Message)"
+  }
+}
+
 try {
   $ErrorActionPreference = "Stop"
   $StartDate = Get-Date
@@ -657,6 +733,7 @@ try {
   $UserdataStatus = New-UserdataStatus -Code 1 -Message "Error: Build not completed (should never see this error)"
   [Net.ServicePointManager]::SecurityProtocol = "Tls12, Tls13"
   Check-Metadata
+  Configure-KinesisAgent
   Write-Tfi "Start Build ============"
 
 %{~ if build_type == build_type_builder }

From b5882cee2e8f28b4e425ed2697807685473f9811 Mon Sep 17 00:00:00 2001
From: Loren Gordon <8457307+lorengordon@users.noreply.github.com>
Date: Wed, 13 May 2026 11:56:37 -0700
Subject: [PATCH 2/2] Flushes the kinesis stream to s3 before deleting
 resources

---
 .github/actions/test/action.yml | 39 +++++++++++++++++++++++++++++++++
 README.md                       |  4 ++++
 outputs.tf                      |  7 ++++++
 3 files changed, 50 insertions(+)

diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml
index 167a35d..be37fa0 100644
--- a/.github/actions/test/action.yml
+++ b/.github/actions/test/action.yml
@@ -67,6 +67,45 @@ runs:
         fi
         echo "ARTIFACT_LOCATION=${ARTIFACT_LOCATION}"
 
+        # Flush Firehose streams so all data is written to S3
+        FIREHOSE_STREAMS=$(terraform output -json firehose_delivery_stream_names 2>/dev/null | jq -r '.[] // empty' || true)
+
+        if [ -n "$FIREHOSE_STREAMS" ]; then
+          echo "Flushing Firehose streams..."
+
+          for stream_name in $FIREHOSE_STREAMS; do
+            echo "Reducing buffering interval for stream: $stream_name"
+
+            # Get current delivery stream config and version
+            STREAM_DESC=$(aws firehose describe-delivery-stream --delivery-stream-name "$stream_name" 2>/dev/null || true)
+
+            if [ -n "$STREAM_DESC" ]; then
+              VERSION_ID=$(echo "$STREAM_DESC" | jq -r '.DeliveryStreamDescription.VersionId')
+              DEST_ID=$(echo "$STREAM_DESC" | jq -r '.DeliveryStreamDescription.Destinations[0].DestinationId')
+
+              # Update with reduced time interval (60s) to flush quickly while keeping max size (128 MiB)
+              # Since destination type is not changing, only BufferingHints need to be specified; other config is merged
+              aws firehose update-destination \
+                --delivery-stream-name "$stream_name" \
+                --current-delivery-stream-version-id "$VERSION_ID" \
+                --destination-id "$DEST_ID" \
+                --extended-s3-destination-update '{
+                  "BufferingHints": {
+                    "SizeInMBs": 128,
+                    "IntervalInSeconds": 60
+                  }
+                }' 2>/dev/null || echo "Warning: Could not update stream $stream_name"
+            fi
+          done
+
+          echo "Waiting 60 seconds for Firehose to flush buffered data..."
+          for i in 60 50 40 30 20 10; do
+            echo "  $i seconds remaining..."
+            sleep 10
+          done
+          echo "  Flush complete."
+        fi
+
         if [ "${TF_DESTROY_AFTER_TEST}" = "true" ]; then
           # destroy resources
           terraform destroy -no-color -input=false -auto-approve
diff --git a/README.md b/README.md
index cb2bb7a..9d6f9e2 100644
--- a/README.md
+++ b/README.md
@@ -72,6 +72,9 @@ aws_instance.source_build["centos8stream"]: Still creating... [1m10s elapsed]
 | Name | Type |
 |------|------|
 | [aws_ami.amis](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
+| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
+| [aws_iam_instance_profile.builds](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_instance_profile) | data source |
+| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source |
 | [aws_subnet.tfi](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source |
 | [aws_vpc.tfi](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc) | data source |
 | [http_http.ip](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source |
@@ -120,6 +123,7 @@ aws_instance.source_build["centos8stream"]: Still creating... [1m10s elapsed]
 | <a name="output_build_id"></a> [build\_id](#output\_build\_id) | n/a |
 | <a name="output_build_slug"></a> [build\_slug](#output\_build\_slug) | n/a |
 | <a name="output_builders"></a> [builders](#output\_builders) | n/a |
+| <a name="output_firehose_delivery_stream_names"></a> [firehose\_delivery\_stream\_names](#output\_firehose\_delivery\_stream\_names) | Kinesis Firehose delivery stream names for flushing before destroy |
 | <a name="output_private_key"></a> [private\_key](#output\_private\_key) | n/a |
 | <a name="output_public_key"></a> [public\_key](#output\_public\_key) | n/a |
 | <a name="output_source_builds"></a> [source\_builds](#output\_source\_builds) | n/a |
diff --git a/outputs.tf b/outputs.tf
index e94c35c..dcff77e 100644
--- a/outputs.tf
+++ b/outputs.tf
@@ -43,3 +43,10 @@ output "public_key" {
 output "build_slug" {
   value = local.build_slug
 }
+
+output "firehose_delivery_stream_names" {
+  description = "Kinesis Firehose delivery stream names for flushing before destroy"
+  value = {
+    for key, stream in aws_kinesis_firehose_delivery_stream.userdata_logs : key => stream.name
+  }
+}