diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml
index 167a35d..be37fa0 100644
--- a/.github/actions/test/action.yml
+++ b/.github/actions/test/action.yml
@@ -67,6 +67,45 @@ runs:
fi
echo "ARTIFACT_LOCATION=${ARTIFACT_LOCATION}"
+ # Flush Firehose streams so all data is written to S3
+ FIREHOSE_STREAMS=$(terraform output -json firehose_delivery_stream_names 2>/dev/null | jq -r '.[] // empty' || true)
+
+ if [ -n "$FIREHOSE_STREAMS" ]; then
+ echo "Flushing Firehose streams..."
+
+ for stream_name in $FIREHOSE_STREAMS; do
+ echo "Reducing buffering interval for stream: $stream_name"
+
+ # Get current delivery stream config and version
+ STREAM_DESC=$(aws firehose describe-delivery-stream --delivery-stream-name "$stream_name" 2>/dev/null || true)
+
+ if [ -n "$STREAM_DESC" ]; then
+ VERSION_ID=$(echo "$STREAM_DESC" | jq -r '.DeliveryStreamDescription.VersionId')
+ DEST_ID=$(echo "$STREAM_DESC" | jq -r '.DeliveryStreamDescription.Destinations[0].DestinationId')
+
+ # Update with reduced time interval (60s) to flush quickly while keeping max size (128 MiB)
+ # Since destination type is not changing, only BufferingHints need to be specified; other config is merged
+ aws firehose update-destination \
+ --delivery-stream-name "$stream_name" \
+ --current-delivery-stream-version-id "$VERSION_ID" \
+ --destination-id "$DEST_ID" \
+ --extended-s3-destination-update '{
+ "BufferingHints": {
+ "SizeInMBs": 128,
+ "IntervalInSeconds": 60
+ }
+ }' 2>/dev/null || echo "Warning: Could not update stream $stream_name"
+ fi
+ done
+
+ echo "Waiting 60 seconds for Firehose to flush buffered data..."
+ for i in 60 50 40 30 20 10; do
+ echo " $i seconds remaining..."
+ sleep 10
+ done
+ echo " Flush complete."
+ fi
+
if [ "${TF_DESTROY_AFTER_TEST}" = "true" ]; then
# destroy resources
terraform destroy -no-color -input=false -auto-approve
diff --git a/README.md b/README.md
index cb2bb7a..9d6f9e2 100644
--- a/README.md
+++ b/README.md
@@ -72,6 +72,9 @@ aws_instance.source_build["centos8stream"]: Still creating... [1m10s elapsed]
| Name | Type |
|------|------|
| [aws_ami.amis](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
+| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
+| [aws_iam_instance_profile.builds](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_instance_profile) | data source |
+| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source |
| [aws_subnet.tfi](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source |
| [aws_vpc.tfi](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc) | data source |
| [http_http.ip](https://registry.terraform.io/providers/hashicorp/http/latest/docs/data-sources/http) | data source |
@@ -120,6 +123,7 @@ aws_instance.source_build["centos8stream"]: Still creating... [1m10s elapsed]
| [build\_id](#output\_build\_id) | n/a |
| [build\_slug](#output\_build\_slug) | n/a |
| [builders](#output\_builders) | n/a |
+| [firehose\_delivery\_stream\_names](#output\_firehose\_delivery\_stream\_names) | Kinesis Firehose delivery stream names for flushing before destroy |
| [private\_key](#output\_private\_key) | n/a |
| [public\_key](#output\_public\_key) | n/a |
| [source\_builds](#output\_source\_builds) | n/a |
diff --git a/main.tf b/main.tf
index 7b83f8f..ede1693 100644
--- a/main.tf
+++ b/main.tf
@@ -259,6 +259,12 @@ locals {
source_builds = toset(var.source_builds)
builders = local.standalone_source == "builder" ? toset([for s in local.standalone_builds : local.build_info[s].platform.builder]) : toset([])
unique_builds_needed = setunion(local.standalone_builds, local.source_builds, local.builders)
+
+ firehose_stream_configs = merge(
+ { for os in local.builders : "builder-${os}" => { build_type = local.build_type_builder, os = os, short_type = "bld" } },
+ { for os in local.standalone_builds : "standalone_build-${os}" => { build_type = local.build_type_standalone, os = os, short_type = "sta" } },
+ { for os in local.source_builds : "source_build-${os}" => { build_type = local.build_type_source, os = os, short_type = "src" } }
+ )
}
data "aws_ami" "amis" {
@@ -288,6 +294,15 @@ data "aws_vpc" "tfi" {
id = data.aws_subnet.tfi.vpc_id
}
+data "aws_partition" "current" {}
+
+data "aws_caller_identity" "current" {}
+
+data "aws_iam_instance_profile" "builds" {
+ count = var.instance_profile == "" ? 0 : 1
+ name = var.instance_profile
+}
+
data "http" "ip" {
url = local.url_local_ip
}
@@ -349,6 +364,107 @@ resource "aws_security_group" "builds" {
}
}
+resource "aws_iam_role" "firehose" {
+ name = "${local.resource_name}-firehose"
+
+ assume_role_policy = jsonencode({
+ Version = "2012-10-17"
+ Statement = [
+ {
+ Sid = "AllowFirehoseServiceAssumeRole"
+ Action = "sts:AssumeRole"
+ Effect = "Allow"
+ Principal = {
+ Service = "firehose.amazonaws.com"
+ }
+ }
+ ]
+ })
+}
+
+resource "aws_iam_role_policy" "firehose_s3" {
+ name = "${local.resource_name}-firehose-s3"
+ role = aws_iam_role.firehose.id
+
+ policy = jsonencode({
+ Version = "2012-10-17"
+ Statement = [
+ {
+ Sid = "AllowFirehoseS3Access"
+ Action = [
+ "s3:AbortMultipartUpload",
+ "s3:GetBucketLocation",
+ "s3:GetObject",
+ "s3:ListBucket",
+ "s3:ListBucketMultipartUploads",
+ "s3:PutObject"
+ ]
+ Effect = "Allow"
+ Resource = [
+ "arn:${data.aws_partition.current.partition}:s3:::${var.s3_bucket}",
+ "arn:${data.aws_partition.current.partition}:s3:::${var.s3_bucket}/*"
+ ]
+ },
+ {
+ Sid = "AllowFirehoseCloudWatchLogsPutEvents"
+ Action = [
+ "logs:PutLogEvents"
+ ]
+ Effect = "Allow"
+ Resource = "*"
+ }
+ ]
+ })
+}
+
+resource "aws_kinesis_firehose_delivery_stream" "userdata_logs" {
+ for_each = local.firehose_stream_configs
+
+ name = "${local.resource_name}-${each.value.short_type}-${each.value.os}-logs"
+ destination = "extended_s3"
+
+ extended_s3_configuration {
+ role_arn = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:role/${aws_iam_role_policy.firehose_s3.role}"
+ bucket_arn = "arn:${data.aws_partition.current.partition}:s3:::${var.s3_bucket}"
+ prefix = "!{timestamp:yyyyMMdd}/${local.date_hm}-${local.build_id}/${each.key}/kinesis/"
+ error_output_prefix = "!{timestamp:yyyyMMdd}/${local.date_hm}-${local.build_id}/${each.key}/kinesis_errors/!{firehose:error-output-type}/"
+ buffering_size = 128
+ buffering_interval = 900
+ compression_format = "UNCOMPRESSED"
+ file_extension = ".log"
+ }
+}
+
+resource "aws_iam_role_policy" "instance_profile_firehose_put" {
+ count = var.instance_profile == "" ? 0 : 1
+
+ name = "${local.resource_name}-firehose-put"
+ role = data.aws_iam_instance_profile.builds[0].role_name
+
+ policy = jsonencode({
+ Version = "2012-10-17"
+ Statement = [
+ {
+ Sid = "AllowInstanceFirehosePutRecords"
+ Action = [
+ "firehose:PutRecord",
+ "firehose:PutRecordBatch"
+ ]
+ Effect = "Allow"
+ Resource = [for stream in aws_kinesis_firehose_delivery_stream.userdata_logs : stream.arn]
+ },
+ {
+ Sid = "AllowInstancePublishAgentMetrics"
+ Action = [
+ "cloudwatch:PutMetricData"
+ ]
+ Effect = "Allow"
+ Resource = "*"
+ }
+ ]
+ })
+}
+
resource "aws_instance" "builder" {
for_each = local.builders
@@ -371,10 +487,11 @@ resource "aws_instance" "builder" {
local.template_vars.base,
local.template_vars[local.build_info[each.key].platform.key],
{
- build_os = each.key
- build_type = local.build_type_builder
- build_label = format(local.format_str_build_label, local.build_type_builder, each.key)
- password = local.build_info[each.key].platform.connection_password
+ build_os = each.key
+ build_type = local.build_type_builder
+ build_label = format(local.format_str_build_label, local.build_type_builder, each.key)
+ firehose_delivery_stream_name = aws_kinesis_firehose_delivery_stream.userdata_logs["builder-${each.key}"].name
+ password = local.build_info[each.key].platform.connection_password
}
)
))
@@ -388,10 +505,11 @@ resource "aws_instance" "builder" {
local.template_vars.base,
local.template_vars[local.build_info[each.key].platform.key],
{
- build_os = each.key
- build_type = local.build_type_builder
- build_label = format(local.format_str_build_label, local.build_type_builder, each.key)
- password = local.build_info[each.key].platform.connection_password
+ build_os = each.key
+ build_type = local.build_type_builder
+ build_label = format(local.format_str_build_label, local.build_type_builder, each.key)
+ firehose_delivery_stream_name = aws_kinesis_firehose_delivery_stream.userdata_logs["builder-${each.key}"].name
+ password = local.build_info[each.key].platform.connection_password
}
)
)
@@ -497,10 +615,11 @@ resource "aws_instance" "standalone_build" {
local.template_vars.base,
local.template_vars[local.build_info[each.key].platform.key],
{
- build_os = each.key
- build_type = local.build_type_standalone
- build_label = format(local.format_str_build_label, local.build_type_standalone, each.key)
- password = local.build_info[each.key].platform.connection_password
+ build_os = each.key
+ build_type = local.build_type_standalone
+ build_label = format(local.format_str_build_label, local.build_type_standalone, each.key)
+ firehose_delivery_stream_name = aws_kinesis_firehose_delivery_stream.userdata_logs["standalone_build-${each.key}"].name
+ password = local.build_info[each.key].platform.connection_password
}
)
))
@@ -514,10 +633,11 @@ resource "aws_instance" "standalone_build" {
local.template_vars.base,
local.template_vars[local.build_info[each.key].platform.key],
{
- build_os = each.key
- build_type = local.build_type_standalone
- build_label = format(local.format_str_build_label, local.build_type_standalone, each.key)
- password = local.build_info[each.key].platform.connection_password
+ build_os = each.key
+ build_type = local.build_type_standalone
+ build_label = format(local.format_str_build_label, local.build_type_standalone, each.key)
+ firehose_delivery_stream_name = aws_kinesis_firehose_delivery_stream.userdata_logs["standalone_build-${each.key}"].name
+ password = local.build_info[each.key].platform.connection_password
}
)
)
@@ -629,10 +749,11 @@ resource "aws_instance" "source_build" {
local.template_vars.base,
local.template_vars[local.build_info[each.key].platform.key],
{
- build_os = each.key
- build_type = local.build_type_source
- build_label = format(local.format_str_build_label, local.build_type_source, each.key)
- password = local.build_info[each.key].platform.connection_password
+ build_os = each.key
+ build_type = local.build_type_source
+ build_label = format(local.format_str_build_label, local.build_type_source, each.key)
+ firehose_delivery_stream_name = aws_kinesis_firehose_delivery_stream.userdata_logs["source_build-${each.key}"].name
+ password = local.build_info[each.key].platform.connection_password
}
)
))
@@ -646,10 +767,11 @@ resource "aws_instance" "source_build" {
local.template_vars.base,
local.template_vars[local.build_info[each.key].platform.key],
{
- build_os = each.key
- build_type = local.build_type_source
- build_label = format(local.format_str_build_label, local.build_type_source, each.key)
- password = local.build_info[each.key].platform.connection_password
+ build_os = each.key
+ build_type = local.build_type_source
+ build_label = format(local.format_str_build_label, local.build_type_source, each.key)
+ firehose_delivery_stream_name = aws_kinesis_firehose_delivery_stream.userdata_logs["source_build-${each.key}"].name
+ password = local.build_info[each.key].platform.connection_password
}
)
)
diff --git a/outputs.tf b/outputs.tf
index e94c35c..dcff77e 100644
--- a/outputs.tf
+++ b/outputs.tf
@@ -43,3 +43,10 @@ output "public_key" {
output "build_slug" {
value = local.build_slug
}
+
+output "firehose_delivery_stream_names" {
+ description = "Kinesis Firehose delivery stream names for flushing before destroy"
+ value = {
+ for key, stream in aws_kinesis_firehose_delivery_stream.userdata_logs : key => stream.name
+ }
+}
diff --git a/templates/lx_userdata.sh b/templates/lx_userdata.sh
index 09926a2..086b37a 100644
--- a/templates/lx_userdata.sh
+++ b/templates/lx_userdata.sh
@@ -17,6 +17,7 @@ github_artifact_repo_name="${github_artifact_repo_name}"
github_artifact_repo_owner="${github_artifact_repo_owner}"
github_artifact_run_id="${github_artifact_run_id}"
github_artifact_token_ssm_parameter="${github_artifact_token_ssm_parameter}"
+firehose_delivery_stream_name="${firehose_delivery_stream_name}"
port="${port}"
release_prefix="${release_prefix}"
scan_slug="${scan_slug}"
@@ -43,15 +44,12 @@ export AWS_DEFAULT_REGION="$aws_region"
echo "------------------------------- $build_label ---------------------"
debug-2s3() {
- ## With as few dependencies as possible, immediately upload the debug and log
- ## files to S3. Calling this multiple times will simply overwrite the
- ## previously uploaded logs.
+ ## Keep an incremental local debug log. Final artifact upload handles S3
+ ## persistence, which avoids network calls on every log write.
local msg="$1"
local debug_file="$temp_dir/debug.log"
echo "$msg" >> "$debug_file"
- aws s3 cp "$debug_file" "s3://$build_slug/$build_label/" > /dev/null 2>&1 || true
- aws s3 cp "$userdata_log" "s3://$build_slug/$build_label/" > /dev/null 2>&1 || true
}
check-metadata-availability() {
@@ -59,6 +57,123 @@ check-metadata-availability() {
try_cmd 50 curl -sSL $metadata_loopback_az
}
+configure-kinesis-agent() {
+ local config_path="/etc/aws-kinesis/agent.json"
+ local src_dir="/tmp/aws-kinesis-agent-src"
+ local original_pwd=""
+
+ if [[ -z "$firehose_delivery_stream_name" ]]; then
+ write-tfi "Kinesis Agent setup skipped: firehose delivery stream name was not provided"
+ return 0
+ fi
+
+ if command -v dnf > /dev/null 2>&1; then
+ try_cmd 3 dnf -y install initscripts || {
+ write-tfi "Initscripts install failed via dnf"
+ return 0
+ }
+ try_cmd 3 dnf -y install aws-kinesis-agent || {
+ write-tfi "Kinesis Agent install failed via dnf"
+ return 0
+ }
+ elif command -v apt-get > /dev/null 2>&1; then
+ try_cmd 3 apt-get -y update || true
+ try_cmd 3 apt-get -y install curl tar openjdk-11-jdk-headless || {
+ write-tfi "Kinesis Agent install failed: unable to install prerequisites via apt-get"
+ return 0
+ }
+
+ try_cmd 3 curl -fsSL https://github.com/awslabs/amazon-kinesis-agent/archive/refs/heads/master.tar.gz -o /tmp/aws-kinesis-agent-src.tar.gz || {
+ write-tfi "Kinesis Agent install failed: unable to download source archive"
+ return 0
+ }
+
+ try_cmd 1 rm -rf "$src_dir" || {
+ write-tfi "Kinesis Agent install failed: unable to clean source directory"
+ return 0
+ }
+
+ try_cmd 1 mkdir -p "$src_dir" || {
+ write-tfi "Kinesis Agent install failed: unable to prepare source directory"
+ return 0
+ }
+
+ try_cmd 1 tar -xzf /tmp/aws-kinesis-agent-src.tar.gz --strip-components=1 -C "$src_dir" || {
+ write-tfi "Kinesis Agent install failed: unable to extract source archive"
+ return 0
+ }
+
+ original_pwd=$(pwd)
+ cd "$src_dir" || {
+ write-tfi "Kinesis Agent install failed: unable to enter source directory"
+ return 0
+ }
+
+ try_cmd 1 bash ./setup --install || {
+ cd "$original_pwd" > /dev/null 2>&1 || true
+ write-tfi "Kinesis Agent install failed via setup script"
+ return 0
+ }
+
+ cd "$original_pwd" > /dev/null 2>&1 || true
+ else
+ write-tfi "Kinesis Agent setup skipped: package manager not available"
+ return 0
+ fi
+
+ chmod 644 "$userdata_log" > /dev/null 2>&1 || true
+
+ cat > "$config_path" << EOF
+{
+ "cloudwatch.emitMetrics": true,
+ "firehose.endpoint": "firehose.$aws_region.amazonaws.com",
+ "maxConnections": 1,
+ "maxSendingThreads": 1,
+ "flows": [
+ {
+ "filePattern": "$userdata_log",
+ "deliveryStream": "$firehose_delivery_stream_name",
+ "initialPosition": "START_OF_FILE",
+ "maxBufferAgeMillis": 1000,
+ "maxBufferSizeRecords": 1
+ }
+ ]
+}
+EOF
+
+ if command -v systemctl > /dev/null 2>&1; then
+ try_cmd 1 systemctl stop aws-kinesis-agent > /dev/null 2>&1 || true
+ try_cmd 1 systemctl start aws-kinesis-agent > /dev/null 2>&1 || {
+ if pgrep -f "com.amazon.kinesis.streaming.agent.Agent" > /dev/null 2>&1; then
+ write-tfi "Kinesis Agent service reported start failure, but agent process is running"
+ else
+ write-tfi "Kinesis Agent start failed"
+ fi
+ try_cmd 1 systemctl status aws-kinesis-agent --no-pager -l || true
+ [[ -f /var/log/aws-kinesis-agent/aws-kinesis-agent.log ]] && tail -n 80 /var/log/aws-kinesis-agent/aws-kinesis-agent.log || true
+ return 0
+ }
+ elif command -v service > /dev/null 2>&1; then
+ try_cmd 1 service aws-kinesis-agent stop > /dev/null 2>&1 || true
+ try_cmd 1 service aws-kinesis-agent start > /dev/null 2>&1 || {
+ if pgrep -f "com.amazon.kinesis.streaming.agent.Agent" > /dev/null 2>&1; then
+ write-tfi "Kinesis Agent service reported start failure, but agent process is running"
+ else
+ write-tfi "Kinesis Agent start failed"
+ fi
+ try_cmd 1 service aws-kinesis-agent status || true
+ [[ -f /var/log/aws-kinesis-agent/aws-kinesis-agent.log ]] && tail -n 80 /var/log/aws-kinesis-agent/aws-kinesis-agent.log || true
+ return 0
+ }
+ else
+ write-tfi "Kinesis Agent installed and configured; restart command not found"
+ return 0
+ fi
+
+ write-tfi "Kinesis Agent configured for stream $firehose_delivery_stream_name"
+ return 0
+}
+
write-tfi() {
local msg=""
local result=""
@@ -563,6 +678,8 @@ start=$(date +%s)
userdata_status_code=0
userdata_status_message="Success"
+configure-kinesis-agent
+
# shellcheck disable=SC1083,SC2288
%{ if build_type == build_type_builder }
diff --git a/templates/win_userdata.ps1 b/templates/win_userdata.ps1
index f0bc05f..09e13cb 100644
--- a/templates/win_userdata.ps1
+++ b/templates/win_userdata.ps1
@@ -15,6 +15,7 @@ $GitHubArtifactRepoOwner = "${github_artifact_repo_owner}"
$GitHubArtifactRepoName = "${github_artifact_repo_name}"
$GitHubArtifactRunId = "${github_artifact_run_id}"
$GitHubArtifactTokenSsmParameter = "${github_artifact_token_ssm_parameter}"
+$FirehoseDeliveryStream = "${firehose_delivery_stream_name}"
$WinUser = "${user}"
$PypiUrl = "${url_pypi}"
$DebugMode = "${debug}"
@@ -41,8 +42,6 @@ function Debug-2S3 {
$DebugFileName = "debug.log"
$DebugFile = "$TempDir\$DebugFileName"
"$(Get-Date): $Msg" | Out-File $DebugFile -Append -Encoding utf8
- Write-S3Object -BucketName "$BuildBucket" -Key "$${BuildKeyPrefix}/$${BuildLabel}/$${DebugFileName}" -File "$DebugFile"
- Write-S3Object -BucketName "$BuildBucket" -Key "$${BuildKeyPrefix}/$${BuildLabel}/$${UserdataLogFileName}" -File "$UserdataLogFile"
}
function Check-Metadata {
@@ -114,9 +113,7 @@ function Test-Command {
[string]$CommandDescription,
[int]$AttemptNumber,
[int]$MaxAttempts,
- [int]$IntervalSeconds,
- [string]$S3Bucket,
- [string]$S3Key
+ [int]$IntervalSeconds
)
# Use a FileStream append with shared read/write access to reduce cross-process contention.
@@ -158,16 +155,8 @@ function Test-Command {
$SecondsRunning = $HeartbeatCount * $IntervalSeconds
Add-HeartbeatLogLine -Path "$LogPath" -Message "$(Get-Date): Command still running at $${SecondsRunning}s (attempt $AttemptNumber/$MaxAttempts) [$CommandDescription]"
-
- # Best effort: upload userdata log periodically so diagnostics survive instance termination.
- try {
- Write-S3Object -BucketName "$S3Bucket" -Key "$S3Key" -File "$LogPath" -ErrorAction Stop
- }
- catch {
- # Intentionally continue heartbeat even when S3 sync is unavailable.
- }
}
- } -ArgumentList "$UserdataLogFile", "$Description", $Attempt, $Tries, $HeartbeatSeconds, "$BuildBucket", "$BuildKeyPrefix/$BuildLabel/$UserdataLogFileName"
+ } -ArgumentList "$UserdataLogFile", "$Description", $Attempt, $Tries, $HeartbeatSeconds
}
catch {
Write-Tfi ("Unable to start heartbeat logger for command [{0}]: {1}" -f $Description, [String]$_.Exception.Message)
@@ -646,6 +635,93 @@ function Install-WatchmakerFromGitHubArtifact {
}
}
+function Configure-KinesisAgent {
+ if ([string]::IsNullOrEmpty($FirehoseDeliveryStream)) {
+ Write-Tfi "Kinesis Agent setup skipped: firehose delivery stream name was not provided"
+ return
+ }
+
+ $KinesisTapService = Get-Service -Name "AWSKinesisTap" -ErrorAction SilentlyContinue
+ if ($null -eq $KinesisTapService) {
+ try {
+ $InstallerScript = Join-Path $TempDir "InstallKinesisAgent.ps1"
+ Test-Command -Description "Download InstallKinesisAgent.ps1" -Command {
+ Invoke-WebRequest -Uri "https://s3-us-west-2.amazonaws.com/kinesis-agent-windows/downloads/InstallKinesisAgent.ps1" -OutFile $InstallerScript -UseBasicParsing
+ }
+ Test-Command -Description "Run InstallKinesisAgent.ps1" -Command {
+ & $InstallerScript
+ }
+ $KinesisTapService = Get-Service -Name "AWSKinesisTap" -ErrorAction SilentlyContinue
+ }
+ catch {
+ Write-Tfi "Kinesis Agent install failed: $([String]$_.Exception.Message)"
+ return
+ }
+ }
+
+ if ($null -eq $KinesisTapService) {
+ Write-Tfi "Kinesis Agent setup skipped: AWSKinesisTap service is not available"
+ return
+ }
+
+ try {
+ $ConfigDir = Join-Path $Env:ProgramFiles "Amazon\AWSKinesisTap"
+ New-Item -Path $ConfigDir -ItemType Directory -Force | Out-Null
+ $ConfigPath = Join-Path $ConfigDir "appsettings.json"
+
+ $UserdataDir = Split-Path -Path $UserdataLogFile -Parent
+ $UserdataLeaf = Split-Path -Path $UserdataLogFile -Leaf
+
+ $KinesisConfig = [ordered]@{
+ Sources = @(
+ [ordered]@{
+ Id = "UserdataLogSource"
+ SourceType = "DirectorySource"
+ Directory = $UserdataDir
+ FileNameFilter = $UserdataLeaf
+ RecordParser = "SingleLine"
+ InitialPosition = "0"
+ }
+ )
+ Sinks = @(
+ [ordered]@{
+ Id = "UserdataFirehoseSink"
+ SinkType = "KinesisFirehose"
+ StreamName = $FirehoseDeliveryStream
+ Region = "${aws_region}"
+ QueueType = "file"
+ ParallelUploadCount = 1
+ }
+ )
+ Pipes = @(
+ [ordered]@{
+ Id = "UserdataToFirehose"
+ SourceRef = "UserdataLogSource"
+ SinkRef = "UserdataFirehoseSink"
+ }
+ )
+ }
+
+ $KinesisConfig | ConvertTo-Json -Depth 8 | Out-File $ConfigPath -Encoding utf8
+
+ if ($KinesisTapService.Status -eq "Running") {
+ Test-Command -Description "Restart AWSKinesisTap service" -Command {
+ Restart-Service -Name "AWSKinesisTap" -ErrorAction Stop
+ }
+ }
+ else {
+ Test-Command -Description "Start AWSKinesisTap service" -Command {
+ Start-Service -Name "AWSKinesisTap" -ErrorAction Stop
+ }
+ }
+
+ Write-Tfi "Kinesis Agent configured for stream $FirehoseDeliveryStream" $true
+ }
+ catch {
+ Write-Tfi "Kinesis Agent configuration failed: $([String]$_.Exception.Message)"
+ }
+}
+
try {
$ErrorActionPreference = "Stop"
$StartDate = Get-Date
@@ -657,6 +733,7 @@ try {
$UserdataStatus = New-UserdataStatus -Code 1 -Message "Error: Build not completed (should never see this error)"
[Net.ServicePointManager]::SecurityProtocol = "Tls12, Tls13"
Check-Metadata
+ Configure-KinesisAgent
Write-Tfi "Start Build ============"
%{~ if build_type == build_type_builder }