From 473694bce26c7d840bfb9555e54ca5ff4e988a1f Mon Sep 17 00:00:00 2001 From: Benji Date: Thu, 23 Jul 2026 17:54:01 +0000 Subject: [PATCH 01/18] feat: add SNS/SQS messaging infrastructure for hyperfleet notifications Replace DynamoDB Streams with SNS/SQS for bidirectional change notifications between the operator (RC) and kube-applier (MC). Add two new Terraform modules: kube-applier-mc-messaging (specs SQS + status SNS on MC side) and kube-applier-rc-messaging (specs SNS + status SQS queues on RC side, with cross-account subscriptions and IAM policies). Wire both into existing management-cluster and kube-applier-dynamodb-provisioning configs, and update buildspec scripts to exchange ARNs between the two stacks. Co-Authored-By: Claude Sonnet 4.6 --- .../provision-kube-applier-dynamodb.sh | 45 +++ .../main.tf | 30 ++ .../outputs.tf | 17 + .../variables.tf | 22 ++ terraform/config/management-cluster/main.tf | 25 ++ .../config/management-cluster/outputs.tf | 16 + .../config/management-cluster/variables.tf | 10 + .../modules/kube-applier-mc-messaging/main.tf | 284 ++++++++++++++++ .../kube-applier-mc-messaging/outputs.tf | 28 ++ .../kube-applier-mc-messaging/variables.tf | 44 +++ .../kube-applier-mc-messaging/versions.tf | 10 + .../modules/kube-applier-rc-messaging/main.tf | 306 ++++++++++++++++++ .../kube-applier-rc-messaging/outputs.tf | 28 ++ .../kube-applier-rc-messaging/variables.tf | 60 ++++ .../kube-applier-rc-messaging/versions.tf | 10 + 15 files changed, 935 insertions(+) create mode 100644 terraform/modules/kube-applier-mc-messaging/main.tf create mode 100644 terraform/modules/kube-applier-mc-messaging/outputs.tf create mode 100644 terraform/modules/kube-applier-mc-messaging/variables.tf create mode 100644 terraform/modules/kube-applier-mc-messaging/versions.tf create mode 100644 terraform/modules/kube-applier-rc-messaging/main.tf create mode 100644 terraform/modules/kube-applier-rc-messaging/outputs.tf create mode 100644 terraform/modules/kube-applier-rc-messaging/variables.tf create mode 100644 terraform/modules/kube-applier-rc-messaging/versions.tf diff --git a/scripts/buildspec/provision-kube-applier-dynamodb.sh b/scripts/buildspec/provision-kube-applier-dynamodb.sh index 68549156d..6d581ce82 100755 --- a/scripts/buildspec/provision-kube-applier-dynamodb.sh +++ b/scripts/buildspec/provision-kube-applier-dynamodb.sh @@ -31,6 +31,49 @@ _RC_REGIONAL_ID=$(jq -r '.regional_id // "regional"' "$_RC_CONFIG_FILE") # ── Switch to RC account ──────────────────────────────────────────────────── use_rc_account +# ── Read MC messaging outputs from management-cluster state ──────────────── +# The mc-messaging module (specs SQS queue + status SNS topic) is provisioned +# by the MC management-cluster pipeline step, which may run in parallel with +# this step. Retry until the outputs appear or we time out (45 min), matching +# the pattern used for OIDC outputs in provision-infra-mc.sh. +# +# On destroy, skip the read and pass empty strings so the rc-messaging module +# is cleanly removed without blocking on stale MC outputs. +if [ "${DELETE_FLAG}" != "true" ]; then + _MC_STATE_BUCKET="terraform-state-${TARGET_ACCOUNT_ID}-${TARGET_REGION}" + _MC_STATE_KEY="management-cluster/${CLUSTER_ID}.tfstate" + _MC_TF_DIR="terraform/config/management-cluster" + + # Init the MC state backend (read-only; we only run `terraform output`) + (cd "$_MC_TF_DIR" && terraform init -reconfigure \ + -backend-config="bucket=${_MC_STATE_BUCKET}" \ + -backend-config="key=${_MC_STATE_KEY}" \ + -backend-config="region=${TARGET_REGION}" \ + -backend-config="use_lockfile=true" >/dev/null 2>&1) + + _MSG_MAX_RETRIES=90 + _MSG_RETRY_DELAY=30 + _MSG_RETRY_COUNT=0 + TF_VAR_mc_specs_queue_arn="" + TF_VAR_mc_status_sns_topic_arn="" + while [ $_MSG_RETRY_COUNT -lt $_MSG_MAX_RETRIES ]; do + _MSG_RETRY_COUNT=$((_MSG_RETRY_COUNT + 1)) + TF_VAR_mc_specs_queue_arn=$(cd "$_MC_TF_DIR" && terraform output -raw kube_applier_specs_queue_arn 2>/dev/null || true) + TF_VAR_mc_status_sns_topic_arn=$(cd "$_MC_TF_DIR" && terraform output -raw kube_applier_status_topic_arn 2>/dev/null || true) + if [ -n "${TF_VAR_mc_specs_queue_arn}" ] && [ -n "${TF_VAR_mc_status_sns_topic_arn}" ]; then + break + fi + echo "MC messaging outputs not ready (attempt ${_MSG_RETRY_COUNT}/${_MSG_MAX_RETRIES}), retrying in ${_MSG_RETRY_DELAY}s..." + sleep "$_MSG_RETRY_DELAY" + done + if [ -z "${TF_VAR_mc_specs_queue_arn}" ] || [ -z "${TF_VAR_mc_status_sns_topic_arn}" ]; then + echo "INFO: MC messaging outputs missing after $((_MSG_MAX_RETRIES * _MSG_RETRY_DELAY / 60))+ minutes — rc-messaging module will be skipped." + TF_VAR_mc_specs_queue_arn="" + TF_VAR_mc_status_sns_topic_arn="" + fi + export TF_VAR_mc_specs_queue_arn TF_VAR_mc_status_sns_topic_arn +fi + # ── Terraform apply ──────────────────────────────────────────────────────── _RC_STATE_BUCKET="terraform-state-${RESOLVED_REGIONAL_ACCOUNT_ID}-${TARGET_REGION}" export TF_STATE_BUCKET="${_RC_STATE_BUCKET}" @@ -43,6 +86,8 @@ export TF_VAR_mc_aws_account_id="${TARGET_ACCOUNT_ID}" export TF_VAR_rc_id="${_RC_REGIONAL_ID}" TF_VAR_enable_pitr=$(parseBool '.kube_applier_dynamodb_enable_pitr' false "$DEPLOY_CONFIG_FILE") export TF_VAR_enable_pitr +TF_VAR_operator_replica_count=$(jq -r '.operator_replica_count // 3' "$DEPLOY_CONFIG_FILE") +export TF_VAR_operator_replica_count export TF_VAR_app_code="${APP_CODE}" export TF_VAR_service_phase="${SERVICE_PHASE}" export TF_VAR_cost_center="${COST_CENTER}" diff --git a/terraform/config/kube-applier-dynamodb-provisioning/main.tf b/terraform/config/kube-applier-dynamodb-provisioning/main.tf index cd58e1c98..bc510d3d5 100644 --- a/terraform/config/kube-applier-dynamodb-provisioning/main.tf +++ b/terraform/config/kube-applier-dynamodb-provisioning/main.tf @@ -90,3 +90,33 @@ resource "aws_iam_role_policy" "hyperfleet_operator_dynamodb" { ] }) } + +# ============================================================================= +# kube-applier RC-side Messaging (SNS/SQS cross-account notifications) +# +# Creates the specs SNS topic in the RC account (the operator publishes here +# after writing a desire document) and the per-replica status SQS queues +# (the operator polls its own queue for status notifications from kube-applier). +# +# Also creates the cross-account SNS→SQS subscriptions for both directions: +# Specs: RC specs SNS topic → MC specs SQS queue (mc_specs_queue_arn) +# Status: MC status SNS topic → each RC operator SQS queue +# +# mc_status_sns_topic_arn and mc_specs_queue_arn are read from the MC +# management-cluster terraform state by the buildspec script. When empty +# (e.g. during the initial bootstrap run before MC messaging is provisioned) +# the module is skipped. +# ============================================================================= + +module "kube_applier_rc_messaging" { + count = var.mc_status_sns_topic_arn != "" && var.mc_specs_queue_arn != "" ? 1 : 0 + source = "../../modules/kube-applier-rc-messaging" + + mc_name = var.mc_name + mc_aws_account_id = var.mc_aws_account_id + rc_id = var.rc_id + aws_region = var.region + operator_replica_count = var.operator_replica_count + mc_status_sns_topic_arn = var.mc_status_sns_topic_arn + mc_specs_queue_arn = var.mc_specs_queue_arn +} diff --git a/terraform/config/kube-applier-dynamodb-provisioning/outputs.tf b/terraform/config/kube-applier-dynamodb-provisioning/outputs.tf index dcbc20536..4ff97f6a4 100644 --- a/terraform/config/kube-applier-dynamodb-provisioning/outputs.tf +++ b/terraform/config/kube-applier-dynamodb-provisioning/outputs.tf @@ -12,3 +12,20 @@ output "status_readdesires_stream_arn" { description = "Stream ARN for the status-readdesires table" value = module.kube_applier_dynamodb.status_readdesires_stream_arn } + +# ============================================================================= +# Messaging Outputs +# Read by the MC management-cluster buildspec to pass rc_specs_sns_topic_arn +# into the MC terraform run. +# ============================================================================= + +output "specs_sns_topic_arn" { + description = "ARN of the RC-account specs SNS topic for this MC (operator publishes here after writing a desire document). Empty when messaging is not yet provisioned." + value = length(module.kube_applier_rc_messaging) > 0 ? module.kube_applier_rc_messaging[0].specs_topic_arn : "" +} + +output "status_sqs_queue_urls" { + description = "URLs of the RC-account operator status SQS queues (one per replica). Empty list when messaging is not yet provisioned." + value = length(module.kube_applier_rc_messaging) > 0 ? module.kube_applier_rc_messaging[0].status_queue_urls : [] +} + diff --git a/terraform/config/kube-applier-dynamodb-provisioning/variables.tf b/terraform/config/kube-applier-dynamodb-provisioning/variables.tf index 0a5969c03..e1464861b 100644 --- a/terraform/config/kube-applier-dynamodb-provisioning/variables.tf +++ b/terraform/config/kube-applier-dynamodb-provisioning/variables.tf @@ -53,3 +53,25 @@ variable "environment" { description = "Environment name (staging, production, etc.)" type = string } + +# ============================================================================= +# kube-applier Messaging Variables +# ============================================================================= + +variable "mc_status_sns_topic_arn" { + description = "ARN of the MC-account SNS topic for status change notifications (read from MC management-cluster terraform state). The RC-side operator SQS queues subscribe to this topic." + type = string + default = "" +} + +variable "mc_specs_queue_arn" { + description = "ARN of the MC-account SQS queue for specs change notifications (read from MC management-cluster terraform state). The RC specs SNS topic delivers messages to this queue." + type = string + default = "" +} + +variable "operator_replica_count" { + description = "Number of hyperfleet-operator replicas. One status SQS queue is created per replica in the RC account." + type = number + default = 3 +} diff --git a/terraform/config/management-cluster/main.tf b/terraform/config/management-cluster/main.tf index 63ff13b14..e6c4da725 100755 --- a/terraform/config/management-cluster/main.tf +++ b/terraform/config/management-cluster/main.tf @@ -187,3 +187,28 @@ module "kube_applier" { rc_aws_account_id = var.regional_aws_account_id aws_region = var.region } + +# ============================================================================= +# kube-applier MC-side Messaging (SNS/SQS cross-account notifications) +# +# Creates the specs SQS queue (receives notifications from the RC specs SNS +# topic when the operator writes a new desire document) and the status SNS +# topic (kube-applier publishes here after writing a status document so the +# RC-side operator queues are notified immediately). +# +# rc_specs_sns_topic_arn is read from the RC kube-applier-dynamodb terraform +# state by the buildspec script and passed in as TF_VAR_rc_specs_sns_topic_arn. +# When empty (e.g. during initial bootstrap before the RC run completes) the +# module is skipped and messaging falls back to 5-minute safety polling. +# ============================================================================= + +module "kube_applier_mc_messaging" { + count = var.rc_specs_sns_topic_arn != "" ? 1 : 0 + source = "../../modules/kube-applier-mc-messaging" + + mc_name = var.management_id + rc_aws_account_id = var.regional_aws_account_id + rc_specs_sns_topic_arn = var.rc_specs_sns_topic_arn + eks_cluster_name = module.management_cluster.cluster_name + aws_region = var.region +} diff --git a/terraform/config/management-cluster/outputs.tf b/terraform/config/management-cluster/outputs.tf index ec584de3c..a0bea2e40 100755 --- a/terraform/config/management-cluster/outputs.tf +++ b/terraform/config/management-cluster/outputs.tf @@ -166,3 +166,19 @@ output "kube_applier_role_arn" { description = "IAM role ARN for the kube-applier-aws controller" value = module.kube_applier.kube_applier_role_arn } + +# ============================================================================= +# kube-applier Messaging Outputs +# Read by the RC kube-applier-dynamodb buildspec to wire cross-account +# SNS subscriptions. +# ============================================================================= + +output "kube_applier_specs_queue_arn" { + description = "ARN of the MC-side specs SQS queue (receives RC specs SNS notifications). Empty when messaging is not yet provisioned." + value = length(module.kube_applier_mc_messaging) > 0 ? module.kube_applier_mc_messaging[0].specs_queue_arn : "" +} + +output "kube_applier_status_topic_arn" { + description = "ARN of the MC-side status SNS topic (kube-applier publishes here after writing status). Empty when messaging is not yet provisioned." + value = length(module.kube_applier_mc_messaging) > 0 ? module.kube_applier_mc_messaging[0].status_topic_arn : "" +} diff --git a/terraform/config/management-cluster/variables.tf b/terraform/config/management-cluster/variables.tf index e9b0dbc54..74a010b66 100755 --- a/terraform/config/management-cluster/variables.tf +++ b/terraform/config/management-cluster/variables.tf @@ -139,3 +139,13 @@ variable "oidc_cloudfront_domain" { default = "" } +# ============================================================================= +# kube-applier Messaging Variables +# ============================================================================= + +variable "rc_specs_sns_topic_arn" { + description = "ARN of the RC-account SNS topic for specs change notifications (read from RC kube-applier-dynamodb terraform state). kube-applier subscribes the MC-side SQS queue to this topic." + type = string + default = "" +} + diff --git a/terraform/modules/kube-applier-mc-messaging/main.tf b/terraform/modules/kube-applier-mc-messaging/main.tf new file mode 100644 index 000000000..071872c91 --- /dev/null +++ b/terraform/modules/kube-applier-mc-messaging/main.tf @@ -0,0 +1,284 @@ +# ============================================================================= +# kube-applier-mc-messaging Module +# +# Provisions the MC-side messaging resources for the two-way SNS/SQS +# notification system between the hyperfleet-operator (RC account) and +# kube-applier-aws (MC account). +# +# Specs path (RC → MC): The RC account publishes to an SNS topic when it +# writes a new desire document. This module creates the SQS queue in the MC +# account that receives those notifications. kube-applier polls this queue +# instead of DynamoDB Streams. +# +# Status path (MC → RC): kube-applier publishes to an SNS topic in the MC +# account after writing a status document. This module creates that topic. +# The RC account provisions the corresponding SQS queues and subscriptions. +# +# Resource naming: +# Specs SQS queue: ${mc_name}-specs-notifications (MC account) +# Status SNS topic: ${mc_name}-status-notifications (MC account) +# KMS key alias: alias/${mc_name}-kube-applier-messaging +# ============================================================================= + +data "aws_caller_identity" "current" {} +data "aws_partition" "current" {} + +locals { + common_tags = merge( + var.tags, + { + ManagedBy = "terraform" + Module = "kube-applier-mc-messaging" + ManagementCluster = var.mc_name + } + ) + + # IAM role ARN for the kube-applier pod in this MC account + kube_applier_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${var.mc_name}-kube-applier" +} + +# ============================================================================= +# KMS Key — shared encryption key for MC-side messaging resources +# ============================================================================= + +resource "aws_kms_key" "messaging" { + description = "KMS key for ${var.mc_name} kube-applier messaging (SQS + SNS)" + deletion_window_in_days = 7 + enable_key_rotation = true + rotation_period_in_days = 90 + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "EnableRootAccess" + Effect = "Allow" + Principal = { + AWS = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:root" + } + Action = "kms:*" + Resource = "*" + }, + { + # SNS must be able to encrypt/decrypt when delivering messages to SQS + Sid = "AllowSNSDelivery" + Effect = "Allow" + Principal = { + Service = "sns.amazonaws.com" + } + Action = [ + "kms:Decrypt", + "kms:GenerateDataKey", + ] + Resource = "*" + Condition = { + StringEquals = { + "aws:SourceAccount" = data.aws_caller_identity.current.account_id + } + } + }, + { + # SQS must be able to use the key when the queue is encrypted + Sid = "AllowSQS" + Effect = "Allow" + Principal = { + Service = "sqs.amazonaws.com" + } + Action = [ + "kms:Decrypt", + "kms:GenerateDataKey", + ] + Resource = "*" + Condition = { + StringEquals = { + "aws:SourceAccount" = data.aws_caller_identity.current.account_id + } + } + }, + ] + }) + + tags = merge(local.common_tags, { + Name = "${var.mc_name}-kube-applier-messaging" + }) +} + +resource "aws_kms_alias" "messaging" { + name = "alias/${var.mc_name}-kube-applier-messaging" + target_key_id = aws_kms_key.messaging.key_id +} + +# ============================================================================= +# Specs SQS Queue (specs path receiver — RC SNS → MC SQS) +# +# kube-applier polls this queue for notifications that the operator has written +# a new desire document. On receipt, it immediately re-queues the affected +# documentID for reconciliation instead of waiting for the 5-minute safety poll. +# ============================================================================= + +resource "aws_sqs_queue" "specs" { + name = "${var.mc_name}-specs-notifications" + kms_master_key_id = aws_kms_key.messaging.id + message_retention_seconds = 300 # 5 minutes — notifications are ephemeral wake-up signals + visibility_timeout_seconds = 30 + receive_wait_time_seconds = 20 # long-polling + + tags = merge(local.common_tags, { + Name = "${var.mc_name}-specs-notifications" + Direction = "specs-rc-to-mc" + }) +} + +# Allow the RC-account specs SNS topic to deliver messages to this queue. +# AWS requires both an identity-based policy on the SNS topic AND a +# resource-based policy on the SQS queue for cross-account delivery. +resource "aws_sqs_queue_policy" "specs" { + queue_url = aws_sqs_queue.specs.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Sid = "AllowRCSpecsSNSDelivery" + Effect = "Allow" + Principal = { + Service = "sns.amazonaws.com" + } + Action = "sqs:SendMessage" + Resource = aws_sqs_queue.specs.arn + Condition = { + ArnEquals = { + "aws:SourceArn" = var.rc_specs_sns_topic_arn + } + } + }] + }) +} + +# ============================================================================= +# Status SNS Topic (status path sender — MC SNS → RC SQS) +# +# kube-applier publishes a lightweight notification here after successfully +# writing a status document. The RC account subscribes its per-replica operator +# SQS queues to this topic for cross-account delivery. +# ============================================================================= + +resource "aws_sns_topic" "status" { + name = "${var.mc_name}-status-notifications" + kms_master_key_id = aws_kms_key.messaging.id + + tags = merge(local.common_tags, { + Name = "${var.mc_name}-status-notifications" + Direction = "status-mc-to-rc" + }) +} + +# Allow the kube-applier pod role to publish status notifications. +# The RC account (as subscriber) is also given sns:Subscribe so that the +# subscription created in the RC account's kube-applier-rc-messaging module +# can be confirmed without requiring manual approval. +resource "aws_sns_topic_policy" "status" { + arn = aws_sns_topic.status.arn + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "AllowKubeApplierPublish" + Effect = "Allow" + Principal = { + AWS = local.kube_applier_role_arn + } + Action = "sns:Publish" + Resource = aws_sns_topic.status.arn + }, + { + # Allow the RC account to create cross-account SQS subscriptions. + # Without this, aws_sns_topic_subscription from the RC module would + # fail with an AuthorizationError even if the SQS queue policy permits + # delivery. + Sid = "AllowRCAccountSubscribe" + Effect = "Allow" + Principal = { + AWS = "arn:${data.aws_partition.current.partition}:iam::${var.rc_aws_account_id}:root" + } + Action = [ + "sns:Subscribe", + "sns:Unsubscribe", + ] + Resource = aws_sns_topic.status.arn + }, + ] + }) +} + +# ============================================================================= +# IAM: extend kube-applier role with messaging permissions +# +# The kube-applier role is created by the kube-applier module. We add a +# supplementary inline policy here so that all messaging IAM is co-located +# with the messaging infrastructure rather than scattered across modules. +# ============================================================================= + +resource "aws_iam_role_policy" "kube_applier_messaging" { + name = "${var.mc_name}-kube-applier-messaging" + role = "${var.mc_name}-kube-applier" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "SpecsQueueReceive" + Effect = "Allow" + Action = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + ] + Resource = aws_sqs_queue.specs.arn + }, + { + Sid = "StatusTopicPublish" + Effect = "Allow" + Action = [ + "sns:Publish", + ] + Resource = aws_sns_topic.status.arn + }, + { + Sid = "MessagingKMSAccess" + Effect = "Allow" + Action = [ + "kms:Decrypt", + "kms:GenerateDataKey", + ] + Resource = aws_kms_key.messaging.arn + }, + ] + }) +} + +# ============================================================================= +# SSM Parameters — surface queue URL and topic ARN for app config +# ============================================================================= + +resource "aws_ssm_parameter" "specs_queue_url" { + name = "/${var.mc_name}/messaging/specs-queue-url" + description = "SQS queue URL for specs change notifications (RC → MC)" + type = "String" + value = aws_sqs_queue.specs.url + + tags = merge(local.common_tags, { + Name = "${var.mc_name}-specs-queue-url" + }) +} + +resource "aws_ssm_parameter" "status_topic_arn" { + name = "/${var.mc_name}/messaging/status-topic-arn" + description = "SNS topic ARN for status change notifications (MC → RC)" + type = "String" + value = aws_sns_topic.status.arn + + tags = merge(local.common_tags, { + Name = "${var.mc_name}-status-topic-arn" + }) +} diff --git a/terraform/modules/kube-applier-mc-messaging/outputs.tf b/terraform/modules/kube-applier-mc-messaging/outputs.tf new file mode 100644 index 000000000..a31c72790 --- /dev/null +++ b/terraform/modules/kube-applier-mc-messaging/outputs.tf @@ -0,0 +1,28 @@ +# ============================================================================= +# kube-applier-mc-messaging Module Outputs +# ============================================================================= + +output "specs_queue_arn" { + description = "ARN of the specs SQS queue that receives notifications from the RC specs SNS topic" + value = aws_sqs_queue.specs.arn +} + +output "specs_queue_url" { + description = "URL of the specs SQS queue for use by kube-applier" + value = aws_sqs_queue.specs.url +} + +output "status_topic_arn" { + description = "ARN of the status SNS topic that kube-applier publishes to after writing status documents" + value = aws_sns_topic.status.arn +} + +output "status_topic_name" { + description = "Name of the status SNS topic" + value = aws_sns_topic.status.name +} + +output "kms_key_arn" { + description = "ARN of the KMS key used to encrypt MC-side messaging resources" + value = aws_kms_key.messaging.arn +} diff --git a/terraform/modules/kube-applier-mc-messaging/variables.tf b/terraform/modules/kube-applier-mc-messaging/variables.tf new file mode 100644 index 000000000..a164f4b62 --- /dev/null +++ b/terraform/modules/kube-applier-mc-messaging/variables.tf @@ -0,0 +1,44 @@ +# ============================================================================= +# kube-applier-mc-messaging Module - Input Variables +# ============================================================================= + +variable "mc_name" { + description = "Management cluster identifier (e.g., 'mc01'). Used as a prefix for resource names." + type = string + + validation { + condition = can(regex("^[a-z0-9-]+$", var.mc_name)) + error_message = "mc_name must contain only lowercase letters, numbers, and hyphens" + } +} + +variable "rc_aws_account_id" { + description = "AWS account ID of the regional cluster. Used to scope IAM and queue policies." + type = string + + validation { + condition = can(regex("^[0-9]{12}$", var.rc_aws_account_id)) + error_message = "rc_aws_account_id must be a 12-digit AWS account ID" + } +} + +variable "rc_specs_sns_topic_arn" { + description = "ARN of the specs SNS topic in the RC account that will deliver notifications to the specs SQS queue in this MC." + type = string +} + +variable "eks_cluster_name" { + description = "Name of the EKS management cluster. Used for the Pod Identity association." + type = string +} + +variable "aws_region" { + description = "AWS region where resources are created." + type = string +} + +variable "tags" { + description = "Additional tags to apply to resources." + type = map(string) + default = {} +} diff --git a/terraform/modules/kube-applier-mc-messaging/versions.tf b/terraform/modules/kube-applier-mc-messaging/versions.tf new file mode 100644 index 000000000..ddfcb0e05 --- /dev/null +++ b/terraform/modules/kube-applier-mc-messaging/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } +} diff --git a/terraform/modules/kube-applier-rc-messaging/main.tf b/terraform/modules/kube-applier-rc-messaging/main.tf new file mode 100644 index 000000000..4d15522e6 --- /dev/null +++ b/terraform/modules/kube-applier-rc-messaging/main.tf @@ -0,0 +1,306 @@ +# ============================================================================= +# kube-applier-rc-messaging Module +# +# Provisions the RC-side messaging resources for the two-way SNS/SQS +# notification system between the hyperfleet-operator (RC account) and +# kube-applier-aws (MC account). +# +# Specs path (RC → MC): The hyperfleet-operator publishes to an SNS topic in +# the RC account after writing a desire document. This module creates that +# topic and subscribes the MC-side specs SQS queue (mc_specs_queue_arn) to +# it, forming the cross-account delivery link. +# +# Status path (MC → RC): kube-applier publishes status notifications to an SNS +# topic in the MC account. This module creates one SQS queue per operator +# replica in the RC account and subscribes each to the MC status SNS topic +# (mc_status_sns_topic_arn). Each operator pod drains its own queue. +# +# Resource naming: +# Specs SNS topic: ${mc_name}-specs-notifications (RC account) +# Status SQS queues: ${rc_id}-hyperfleet-operator-{0..N-1} (RC account) +# KMS key alias: alias/${mc_name}-kube-applier-messaging +# +# Incremental IAM pattern: +# Like the existing ${mc_name}-dynamodb-access policy, this module attaches +# a per-MC inline policy (${mc_name}-messaging-access) to the shared +# ${rc_id}-hyperfleet-operator role. Each MC pipeline run adds its own +# policy, so parallel per-MC state files never collide. +# ============================================================================= + +data "aws_caller_identity" "current" {} +data "aws_partition" "current" {} + +locals { + common_tags = merge( + var.tags, + { + ManagedBy = "terraform" + Module = "kube-applier-rc-messaging" + ManagementCluster = var.mc_name + } + ) + + # Ordinal indices for the operator replica queues (0-based) + replica_indices = range(var.operator_replica_count) + + # IAM role for the hyperfleet-operator (RC account, shared across all MCs) + hyperfleet_operator_role_name = "${var.rc_id}-hyperfleet-operator" +} + +# ============================================================================= +# KMS Key — shared encryption key for RC-side messaging resources (per MC) +# ============================================================================= + +resource "aws_kms_key" "messaging" { + description = "KMS key for ${var.mc_name} kube-applier messaging (SNS + SQS) in RC account" + deletion_window_in_days = 7 + enable_key_rotation = true + rotation_period_in_days = 90 + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "EnableRootAccess" + Effect = "Allow" + Principal = { + AWS = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:root" + } + Action = "kms:*" + Resource = "*" + }, + { + # SNS must be able to encrypt/decrypt when delivering messages to SQS + Sid = "AllowSNSDelivery" + Effect = "Allow" + Principal = { + Service = "sns.amazonaws.com" + } + Action = [ + "kms:Decrypt", + "kms:GenerateDataKey", + ] + Resource = "*" + Condition = { + StringEquals = { + "aws:SourceAccount" = data.aws_caller_identity.current.account_id + } + } + }, + { + # SQS must be able to use the key + Sid = "AllowSQS" + Effect = "Allow" + Principal = { + Service = "sqs.amazonaws.com" + } + Action = [ + "kms:Decrypt", + "kms:GenerateDataKey", + ] + Resource = "*" + Condition = { + StringEquals = { + "aws:SourceAccount" = data.aws_caller_identity.current.account_id + } + } + }, + ] + }) + + tags = merge(local.common_tags, { + Name = "${var.mc_name}-kube-applier-messaging" + }) +} + +resource "aws_kms_alias" "messaging" { + name = "alias/${var.mc_name}-kube-applier-messaging" + target_key_id = aws_kms_key.messaging.key_id +} + +# ============================================================================= +# Specs SNS Topic (specs path sender — RC SNS → MC SQS) +# +# The hyperfleet-operator publishes a lightweight notification here after +# writing an ApplyDesire or ReadDesire document. The cross-account subscription +# below delivers that notification to the MC-side SQS queue. +# ============================================================================= + +resource "aws_sns_topic" "specs" { + name = "${var.mc_name}-specs-notifications" + kms_master_key_id = aws_kms_key.messaging.id + + tags = merge(local.common_tags, { + Name = "${var.mc_name}-specs-notifications" + Direction = "specs-rc-to-mc" + }) +} + +# Allow the hyperfleet-operator pod role to publish specs notifications. +# Also allow the MC account to confirm the cross-account SQS subscription. +resource "aws_sns_topic_policy" "specs" { + arn = aws_sns_topic.specs.arn + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "AllowHyperfleetOperatorPublish" + Effect = "Allow" + Principal = { + AWS = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:role/${local.hyperfleet_operator_role_name}" + } + Action = "sns:Publish" + Resource = aws_sns_topic.specs.arn + }, + { + # The SNS subscription is created from the RC account (this module), + # targeting the MC SQS queue. AWS auto-confirms SQS subscriptions, so + # no MC-account confirmation action is needed here. + Sid = "AllowSNSToDeliverToSQS" + Effect = "Allow" + Principal = { + Service = "sns.amazonaws.com" + } + Action = "sns:Publish" + Resource = aws_sns_topic.specs.arn + Condition = { + StringEquals = { + "aws:SourceAccount" = data.aws_caller_identity.current.account_id + } + } + }, + ] + }) +} + +# Cross-account SNS→SQS subscription: RC specs topic → MC specs queue. +# AWS auto-confirms SQS subscriptions, so no ConfirmSubscription step is needed. +resource "aws_sns_topic_subscription" "specs_to_mc_queue" { + topic_arn = aws_sns_topic.specs.arn + protocol = "sqs" + endpoint = var.mc_specs_queue_arn + + # Raw delivery passes the JSON notification body directly without the SNS + # envelope wrapper, simplifying parsing in the kube-applier consumer. + raw_message_delivery = true +} + +# ============================================================================= +# Status SQS Queues (status path receiver — MC SNS → RC SQS) +# +# One queue per hyperfleet-operator pod replica. Each pod polls only its own +# queue (named after its hostname, e.g. hyperfleet-operator-2), eliminating +# competing-consumer problems and making queue drain deterministic on scale-down. +# ============================================================================= + +resource "aws_sqs_queue" "status" { + count = var.operator_replica_count + + name = "${var.rc_id}-hyperfleet-operator-${count.index}" + kms_master_key_id = aws_kms_key.messaging.id + message_retention_seconds = 300 # 5 minutes — notifications are ephemeral wake-up signals + visibility_timeout_seconds = 30 + receive_wait_time_seconds = 20 # long-polling + + tags = merge(local.common_tags, { + Name = "${var.rc_id}-hyperfleet-operator-${count.index}" + Direction = "status-mc-to-rc" + Replica = tostring(count.index) + }) +} + +# Allow the MC-account status SNS topic to deliver messages to each queue. +resource "aws_sqs_queue_policy" "status" { + count = var.operator_replica_count + queue_url = aws_sqs_queue.status[count.index].id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Sid = "AllowMCStatusSNSDelivery" + Effect = "Allow" + Principal = { + Service = "sns.amazonaws.com" + } + Action = "sqs:SendMessage" + Resource = aws_sqs_queue.status[count.index].arn + Condition = { + ArnEquals = { + "aws:SourceArn" = var.mc_status_sns_topic_arn + } + } + }] + }) +} + +# Cross-account SNS→SQS subscriptions: MC status topic → each RC operator queue. +resource "aws_sns_topic_subscription" "status_to_rc_queues" { + count = var.operator_replica_count + + topic_arn = var.mc_status_sns_topic_arn + protocol = "sqs" + endpoint = aws_sqs_queue.status[count.index].arn + raw_message_delivery = true +} + +# ============================================================================= +# IAM: extend hyperfleet-operator role with messaging permissions (per-MC) +# +# Follows the same incremental pattern as ${mc_name}-dynamodb-access: each MC +# pipeline run attaches its own named policy to the shared operator role. +# Parallel per-MC state files never collide because policy names are unique. +# ============================================================================= + +resource "aws_iam_role_policy" "hyperfleet_operator_messaging" { + name = "${var.mc_name}-messaging-access" + role = local.hyperfleet_operator_role_name + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "SpecsTopicPublish" + Effect = "Allow" + Action = [ + "sns:Publish", + ] + Resource = aws_sns_topic.specs.arn + }, + { + Sid = "StatusQueuesReceive" + Effect = "Allow" + Action = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + ] + Resource = aws_sqs_queue.status[*].arn + }, + { + Sid = "MessagingKMSAccess" + Effect = "Allow" + Action = [ + "kms:Decrypt", + "kms:GenerateDataKey", + ] + Resource = aws_kms_key.messaging.arn + }, + ] + }) +} + +# ============================================================================= +# SSM Parameter — specs topic ARN for operator configuration +# ============================================================================= + +resource "aws_ssm_parameter" "specs_topic_arn" { + name = "/${var.rc_id}/${var.mc_name}/messaging/specs-topic-arn" + description = "SNS topic ARN for ${var.mc_name} specs change notifications (RC → MC)" + type = "String" + value = aws_sns_topic.specs.arn + + tags = merge(local.common_tags, { + Name = "${var.rc_id}-${var.mc_name}-specs-topic-arn" + }) +} diff --git a/terraform/modules/kube-applier-rc-messaging/outputs.tf b/terraform/modules/kube-applier-rc-messaging/outputs.tf new file mode 100644 index 000000000..7d2b50b92 --- /dev/null +++ b/terraform/modules/kube-applier-rc-messaging/outputs.tf @@ -0,0 +1,28 @@ +# ============================================================================= +# kube-applier-rc-messaging Module Outputs +# ============================================================================= + +output "specs_topic_arn" { + description = "ARN of the specs SNS topic in the RC account that the hyperfleet-operator publishes to" + value = aws_sns_topic.specs.arn +} + +output "specs_topic_name" { + description = "Name of the specs SNS topic" + value = aws_sns_topic.specs.name +} + +output "status_queue_arns" { + description = "ARNs of the RC-side status SQS queues (one per operator replica, indexed 0..N-1)" + value = aws_sqs_queue.status[*].arn +} + +output "status_queue_urls" { + description = "URLs of the RC-side status SQS queues (one per operator replica, indexed 0..N-1)" + value = aws_sqs_queue.status[*].url +} + +output "kms_key_arn" { + description = "ARN of the KMS key used to encrypt RC-side messaging resources for this MC" + value = aws_kms_key.messaging.arn +} diff --git a/terraform/modules/kube-applier-rc-messaging/variables.tf b/terraform/modules/kube-applier-rc-messaging/variables.tf new file mode 100644 index 000000000..9fac47661 --- /dev/null +++ b/terraform/modules/kube-applier-rc-messaging/variables.tf @@ -0,0 +1,60 @@ +# ============================================================================= +# kube-applier-rc-messaging Module - Input Variables +# ============================================================================= + +variable "mc_name" { + description = "Management cluster identifier (e.g., 'mc01'). Used as a prefix for per-MC resource names." + type = string + + validation { + condition = can(regex("^[a-z0-9-]+$", var.mc_name)) + error_message = "mc_name must contain only lowercase letters, numbers, and hyphens" + } +} + +variable "mc_aws_account_id" { + description = "AWS account ID of the management cluster. Used to scope IAM and queue policies." + type = string + + validation { + condition = can(regex("^[0-9]{12}$", var.mc_aws_account_id)) + error_message = "mc_aws_account_id must be a 12-digit AWS account ID" + } +} + +variable "rc_id" { + description = "Regional cluster identifier for resource naming (e.g., 'regional'). Used to name the operator SQS queues." + type = string +} + +variable "aws_region" { + description = "AWS region where resources are created." + type = string +} + +variable "operator_replica_count" { + description = "Number of hyperfleet-operator pod replicas. One SQS queue is created per replica so that each pod drains its own queue without competing consumers." + type = number + default = 3 + + validation { + condition = var.operator_replica_count >= 1 && var.operator_replica_count <= 10 + error_message = "operator_replica_count must be between 1 and 10" + } +} + +variable "mc_status_sns_topic_arn" { + description = "ARN of the status SNS topic in the MC account. The RC-side SQS queues subscribe to this topic for cross-account delivery." + type = string +} + +variable "mc_specs_queue_arn" { + description = "ARN of the specs SQS queue in the MC account. The RC specs SNS topic subscribes this queue so the operator's publish triggers delivery to kube-applier." + type = string +} + +variable "tags" { + description = "Additional tags to apply to resources." + type = map(string) + default = {} +} diff --git a/terraform/modules/kube-applier-rc-messaging/versions.tf b/terraform/modules/kube-applier-rc-messaging/versions.tf new file mode 100644 index 000000000..ddfcb0e05 --- /dev/null +++ b/terraform/modules/kube-applier-rc-messaging/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + } +} From ec7c43d0bee99385f4d9717d047fc22370e151e2 Mon Sep 17 00:00:00 2001 From: Benji Date: Mon, 27 Jul 2026 13:10:14 +0000 Subject: [PATCH 02/18] fix: use kms:GenerateDataKey* wildcard in messaging KMS policies Co-Authored-By: Claude Sonnet 4.6 --- .../modules/kube-applier-mc-messaging/main.tf | 16 +++------------- .../modules/kube-applier-rc-messaging/main.tf | 16 +++------------- 2 files changed, 6 insertions(+), 26 deletions(-) diff --git a/terraform/modules/kube-applier-mc-messaging/main.tf b/terraform/modules/kube-applier-mc-messaging/main.tf index 071872c91..de5fa1902 100644 --- a/terraform/modules/kube-applier-mc-messaging/main.tf +++ b/terraform/modules/kube-applier-mc-messaging/main.tf @@ -68,17 +68,7 @@ resource "aws_kms_key" "messaging" { } Action = [ "kms:Decrypt", - "kms:GenerateDataKey", - ] - Resource = "*" - Condition = { - StringEquals = { - "aws:SourceAccount" = data.aws_caller_identity.current.account_id - } - } - }, - { - # SQS must be able to use the key when the queue is encrypted + "kms:GenerateDataKey*", Sid = "AllowSQS" Effect = "Allow" Principal = { @@ -86,7 +76,7 @@ resource "aws_kms_key" "messaging" { } Action = [ "kms:Decrypt", - "kms:GenerateDataKey", + "kms:GenerateDataKey*", ] Resource = "*" Condition = { @@ -249,7 +239,7 @@ resource "aws_iam_role_policy" "kube_applier_messaging" { Effect = "Allow" Action = [ "kms:Decrypt", - "kms:GenerateDataKey", + "kms:GenerateDataKey*", ] Resource = aws_kms_key.messaging.arn }, diff --git a/terraform/modules/kube-applier-rc-messaging/main.tf b/terraform/modules/kube-applier-rc-messaging/main.tf index 4d15522e6..244f5edff 100644 --- a/terraform/modules/kube-applier-rc-messaging/main.tf +++ b/terraform/modules/kube-applier-rc-messaging/main.tf @@ -78,17 +78,7 @@ resource "aws_kms_key" "messaging" { } Action = [ "kms:Decrypt", - "kms:GenerateDataKey", - ] - Resource = "*" - Condition = { - StringEquals = { - "aws:SourceAccount" = data.aws_caller_identity.current.account_id - } - } - }, - { - # SQS must be able to use the key + "kms:GenerateDataKey*", Sid = "AllowSQS" Effect = "Allow" Principal = { @@ -96,7 +86,7 @@ resource "aws_kms_key" "messaging" { } Action = [ "kms:Decrypt", - "kms:GenerateDataKey", + "kms:GenerateDataKey*", ] Resource = "*" Condition = { @@ -282,7 +272,7 @@ resource "aws_iam_role_policy" "hyperfleet_operator_messaging" { Effect = "Allow" Action = [ "kms:Decrypt", - "kms:GenerateDataKey", + "kms:GenerateDataKey*", ] Resource = aws_kms_key.messaging.arn }, From b203d7dcc144d8c16c72e956965b9cbd2e03e98d Mon Sep 17 00:00:00 2001 From: Benji Date: Mon, 27 Jul 2026 15:38:44 +0000 Subject: [PATCH 03/18] fix: close missing ] in AllowSNSDelivery Action array in KMS key policies Both the RC and MC messaging modules had a missing closing bracket after kms:GenerateDataKey* in the AllowSNSDelivery statement, causing the SNS and SQS statements to be merged into a single malformed block. Co-Authored-By: Claude Sonnet 4.6 --- terraform/modules/kube-applier-mc-messaging/main.tf | 9 +++++++++ terraform/modules/kube-applier-rc-messaging/main.tf | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/terraform/modules/kube-applier-mc-messaging/main.tf b/terraform/modules/kube-applier-mc-messaging/main.tf index de5fa1902..edb742bb5 100644 --- a/terraform/modules/kube-applier-mc-messaging/main.tf +++ b/terraform/modules/kube-applier-mc-messaging/main.tf @@ -69,6 +69,15 @@ resource "aws_kms_key" "messaging" { Action = [ "kms:Decrypt", "kms:GenerateDataKey*", + ] + Resource = "*" + Condition = { + StringEquals = { + "aws:SourceAccount" = data.aws_caller_identity.current.account_id + } + } + }, + { Sid = "AllowSQS" Effect = "Allow" Principal = { diff --git a/terraform/modules/kube-applier-rc-messaging/main.tf b/terraform/modules/kube-applier-rc-messaging/main.tf index 244f5edff..6e77705c8 100644 --- a/terraform/modules/kube-applier-rc-messaging/main.tf +++ b/terraform/modules/kube-applier-rc-messaging/main.tf @@ -79,6 +79,15 @@ resource "aws_kms_key" "messaging" { Action = [ "kms:Decrypt", "kms:GenerateDataKey*", + ] + Resource = "*" + Condition = { + StringEquals = { + "aws:SourceAccount" = data.aws_caller_identity.current.account_id + } + } + }, + { Sid = "AllowSQS" Effect = "Allow" Principal = { From b7b4a3c88ba1d90a5b8bbcaed5bda051f7f04202 Mon Sep 17 00:00:00 2001 From: Benji Date: Mon, 27 Jul 2026 15:45:16 +0000 Subject: [PATCH 04/18] chore: update kube-applier-aws and hyperfleet-operator image tags kube-applier-aws: quay.io/psav/kube-applier-aws:252920c hyperfleet-operator: quay.io/psav/hyperfleet-operator:6d51902 Co-Authored-By: Claude Sonnet 4.6 --- .../management-cluster/kube-applier/values.yaml | 2 +- .../config/regional-cluster/hyperfleet/values.yaml | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/argocd/config/management-cluster/kube-applier/values.yaml b/argocd/config/management-cluster/kube-applier/values.yaml index 75c1cbe9b..108d9fb3e 100644 --- a/argocd/config/management-cluster/kube-applier/values.yaml +++ b/argocd/config/management-cluster/kube-applier/values.yaml @@ -7,7 +7,7 @@ kubeApplier: image: registry: "quay.io" repository: "psav/kube-applier-aws" - tag: "remove-delete-desire-18d87ca" + tag: "252920c" pullPolicy: IfNotPresent # Deployment configuration diff --git a/argocd/config/regional-cluster/hyperfleet/values.yaml b/argocd/config/regional-cluster/hyperfleet/values.yaml index 197c2ea23..446979b45 100644 --- a/argocd/config/regional-cluster/hyperfleet/values.yaml +++ b/argocd/config/regional-cluster/hyperfleet/values.yaml @@ -8,19 +8,21 @@ hyperfleet: project: default source: - repoURL: https://github.com/typeid/hyperfleet-operator.git - targetRevision: main - path: charts/hyperfleet-operator + repoURL: https://github.com/rrp-bot/rosa-hyperfleet-api.git + targetRevision: feature/sns-sqs + path: hyperfleet-operator/charts syncPolicy: automated: prune: true selfHeal: true + sqsQueueUrlPrefix: "" + helmValues: image: - repository: quay.io/redhat-user-workloads/rosa-tenant/hyperfleet-operator - tag: "42e0e77996c25cd4daaf4655f7d882fff0724227" + repository: quay.io/psav/hyperfleet-operator + tag: "9ae1c99" pullPolicy: Always replicaCount: 2 From ceac24def36c0ce090028cedd91030384db8bd7d Mon Sep 17 00:00:00 2001 From: Benji Date: Mon, 27 Jul 2026 16:41:06 +0000 Subject: [PATCH 05/18] fix: remove invalid sns:Unsubscribe action from MC status topic policy sns:Unsubscribe is not a valid SNS resource-based policy action and causes AWS to reject the policy with 'action out of service scope'. Subscribers can always remove their own subscriptions without an explicit grant. Co-Authored-By: Claude Sonnet 4.6 --- terraform/modules/kube-applier-mc-messaging/main.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/terraform/modules/kube-applier-mc-messaging/main.tf b/terraform/modules/kube-applier-mc-messaging/main.tf index edb742bb5..321e9fb2c 100644 --- a/terraform/modules/kube-applier-mc-messaging/main.tf +++ b/terraform/modules/kube-applier-mc-messaging/main.tf @@ -202,7 +202,6 @@ resource "aws_sns_topic_policy" "status" { } Action = [ "sns:Subscribe", - "sns:Unsubscribe", ] Resource = aws_sns_topic.status.arn }, From e62760f41c5207c8ef3c019e64c89e3fd0f10a60 Mon Sep 17 00:00:00 2001 From: Benji Date: Mon, 27 Jul 2026 19:21:26 +0000 Subject: [PATCH 06/18] feat: wire kube-applier SQS queue URL and SNS topic ARN through annotation pattern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass specs_queue_url and status_topic_arn from MC Terraform outputs through the established annotation pipeline (Terraform output → bootstrap-argocd.sh → ECS task env → local-cluster-identity annotation → ApplicationSet valuesObject → kube-applier Helm chart → deployment args). - bootstrap-argocd.sh: read kube_applier_specs_queue_url and kube_applier_status_topic_arn from terraform output -json for MC; pass both as KUBE_APPLIER_SQS_QUEUE_URL / KUBE_APPLIER_SNS_STATUS_TOPIC_ARN to ECS task - ecs-bootstrap/variables.tf: add kube_applier_specs_queue_url and kube_applier_status_topic_arn input variables - ecs-bootstrap/main.tf: inject both as container env vars and as annotations on the local-cluster-identity cluster secret - config/management-cluster/main.tf: pass new vars to ecs-bootstrap module - config/management-cluster/outputs.tf: add kube_applier_specs_queue_url output - applicationset.yaml.j2: add MC-only kubeApplier.config valuesObject block reading the two new annotations; fix pre-existing argo-cd targetGroup indent - deploy/*/argocd-bootstrap-management-cluster/applicationset.yaml: re-rendered - kube-applier/values.yaml: add sqsQueueUrl and snsStatusTopicArn fields - kube-applier/deployment.yaml: add --sqs-queue-url and --sns-status-topic-arn args (conditional on non-empty value) - kube-applier-mc-messaging/main.tf: remove SSM parameters (never consumed) Co-Authored-By: Claude Sonnet 4.6 --- .../kube-applier/templates/deployment.yaml | 6 +++++ .../kube-applier/values.yaml | 5 ++++ .../argocd-bootstrap/applicationset.yaml.j2 | 7 +++++ .../applicationset.yaml | 5 ++++ .../applicationset.yaml | 5 ++++ scripts/bootstrap-argocd.sh | 13 +++++++++- terraform/config/management-cluster/main.tf | 3 +++ .../config/management-cluster/outputs.tf | 10 ++++++- terraform/modules/ecs-bootstrap/main.tf | 10 +++++++ terraform/modules/ecs-bootstrap/variables.tf | 12 +++++++++ .../modules/kube-applier-mc-messaging/main.tf | 26 ------------------- 11 files changed, 74 insertions(+), 28 deletions(-) diff --git a/argocd/config/management-cluster/kube-applier/templates/deployment.yaml b/argocd/config/management-cluster/kube-applier/templates/deployment.yaml index ebe9261a1..65117c39b 100644 --- a/argocd/config/management-cluster/kube-applier/templates/deployment.yaml +++ b/argocd/config/management-cluster/kube-applier/templates/deployment.yaml @@ -45,6 +45,12 @@ spec: - --leader-election-id={{ .Values.kubeApplier.config.leaderElectionId }} - --log-verbosity={{ .Values.kubeApplier.config.logVerbosity }} - --exit-on-panic={{ .Values.kubeApplier.config.exitOnPanic }} + {{- if .Values.kubeApplier.config.sqsQueueUrl }} + - --sqs-queue-url={{ .Values.kubeApplier.config.sqsQueueUrl }} + {{- end }} + {{- if .Values.kubeApplier.config.snsStatusTopicArn }} + - --sns-status-topic-arn={{ .Values.kubeApplier.config.snsStatusTopicArn }} + {{- end }} {{- if .Values.kubeApplier.config.awsEndpointUrl }} - --aws-endpoint-url={{ .Values.kubeApplier.config.awsEndpointUrl }} {{- end }} diff --git a/argocd/config/management-cluster/kube-applier/values.yaml b/argocd/config/management-cluster/kube-applier/values.yaml index 108d9fb3e..133a7e191 100644 --- a/argocd/config/management-cluster/kube-applier/values.yaml +++ b/argocd/config/management-cluster/kube-applier/values.yaml @@ -32,6 +32,11 @@ kubeApplier: leaderElectionId: "kube-applier" logVerbosity: 4 exitOnPanic: true + # Injected at runtime from the local-cluster-identity secret annotations via the + # ArgoCD ApplicationSet valuesObject. These must not be set here — they are always + # provided by the annotation pattern (MC Terraform output → bootstrap → annotation → here). + sqsQueueUrl: "" + snsStatusTopicArn: "" serviceAccount: name: kube-applier diff --git a/config/templates/argocd-bootstrap/applicationset.yaml.j2 b/config/templates/argocd-bootstrap/applicationset.yaml.j2 index 3644f07e4..4eea5f267 100644 --- a/config/templates/argocd-bootstrap/applicationset.yaml.j2 +++ b/config/templates/argocd-bootstrap/applicationset.yaml.j2 @@ -130,6 +130,13 @@ spec: sre: targetGroup: arn: '{{ '{{ .metadata.annotations.sre_prometheus_target_group_arn }}' }}' +{%- if cluster_type == 'management-cluster' %} + + kubeApplier: + config: + sqsQueueUrl: '{{ '{{ .metadata.annotations.kube_applier_specs_queue_url }}' }}' + snsStatusTopicArn: '{{ '{{ .metadata.annotations.kube_applier_status_topic_arn }}' }}' +{%- endif %} path: '{{ '{{ .path.path }}' }}' repoURL: '{{ '{{ .metadata.annotations.git_repo }}' }}' targetRevision: '{% if pinned %}{{ pinned }}{% else %}{{ "{{ .metadata.annotations.git_revision }}" }}{% endif %}' diff --git a/deploy/ephemeral/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml b/deploy/ephemeral/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml index 404b249c5..596232bc1 100644 --- a/deploy/ephemeral/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml +++ b/deploy/ephemeral/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml @@ -129,6 +129,11 @@ spec: sre: targetGroup: arn: '{{ .metadata.annotations.sre_prometheus_target_group_arn }}' + + kubeApplier: + config: + sqsQueueUrl: '{{ .metadata.annotations.kube_applier_specs_queue_url }}' + snsStatusTopicArn: '{{ .metadata.annotations.kube_applier_status_topic_arn }}' path: '{{ .path.path }}' repoURL: '{{ .metadata.annotations.git_repo }}' targetRevision: '{{ .metadata.annotations.git_revision }}' diff --git a/deploy/integration/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml b/deploy/integration/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml index 13d2a9eb3..d05b9a5e9 100644 --- a/deploy/integration/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml +++ b/deploy/integration/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml @@ -129,6 +129,11 @@ spec: sre: targetGroup: arn: '{{ .metadata.annotations.sre_prometheus_target_group_arn }}' + + kubeApplier: + config: + sqsQueueUrl: '{{ .metadata.annotations.kube_applier_specs_queue_url }}' + snsStatusTopicArn: '{{ .metadata.annotations.kube_applier_status_topic_arn }}' path: '{{ .path.path }}' repoURL: '{{ .metadata.annotations.git_repo }}' targetRevision: '{{ .metadata.annotations.git_revision }}' diff --git a/scripts/bootstrap-argocd.sh b/scripts/bootstrap-argocd.sh index 259260b27..49707ebfb 100755 --- a/scripts/bootstrap-argocd.sh +++ b/scripts/bootstrap-argocd.sh @@ -108,6 +108,15 @@ else REDIS_ENDPOINT="" fi +KUBE_APPLIER_SQS_QUEUE_URL="${KUBE_APPLIER_SQS_QUEUE_URL:-}" +KUBE_APPLIER_SNS_STATUS_TOPIC_ARN="${KUBE_APPLIER_SNS_STATUS_TOPIC_ARN:-}" + +# For management clusters, read the messaging outputs from terraform state. +if [[ "$CLUSTER_TYPE" == "management-cluster" ]]; then + KUBE_APPLIER_SQS_QUEUE_URL=$(echo "$OUTPUTS" | jq -r '.kube_applier_specs_queue_url.value // ""') + KUBE_APPLIER_SNS_STATUS_TOPIC_ARN=$(echo "$OUTPUTS" | jq -r '.kube_applier_status_topic_arn.value // ""') +fi + RHOBS_API_URL="${RHOBS_API_URL:-}" DNS_ZONE_OPERATOR_ROLE_ARN="${DNS_ZONE_OPERATOR_ROLE_ARN:-}" @@ -149,7 +158,9 @@ RUN_TASK_OUTPUT=$(aws ecs run-task \ {\"name\": \"SRE_THANOS_TARGET_GROUP_ARN\", \"value\": \"$SRE_THANOS_TARGET_GROUP_ARN\"}, {\"name\": \"SRE_ALB_DNS_NAME\", \"value\": \"$SRE_ALB_DNS_NAME\"}, {\"name\": \"SRE_DOMAIN\", \"value\": \"$SRE_DOMAIN\"}, - {\"name\": \"REDIS_ENDPOINT\", \"value\": \"$REDIS_ENDPOINT\"} + {\"name\": \"REDIS_ENDPOINT\", \"value\": \"$REDIS_ENDPOINT\"}, + {\"name\": \"KUBE_APPLIER_SQS_QUEUE_URL\", \"value\": \"$KUBE_APPLIER_SQS_QUEUE_URL\"}, + {\"name\": \"KUBE_APPLIER_SNS_STATUS_TOPIC_ARN\", \"value\": \"$KUBE_APPLIER_SNS_STATUS_TOPIC_ARN\"} ] }] }" 2>&1) diff --git a/terraform/config/management-cluster/main.tf b/terraform/config/management-cluster/main.tf index e6c4da725..9baf0ba0e 100755 --- a/terraform/config/management-cluster/main.tf +++ b/terraform/config/management-cluster/main.tf @@ -68,6 +68,9 @@ module "ecs_bootstrap" { repository_url = var.repository_url repository_branch = var.repository_branch + + kube_applier_specs_queue_url = length(module.kube_applier_mc_messaging) > 0 ? module.kube_applier_mc_messaging[0].specs_queue_url : "" + kube_applier_status_topic_arn = length(module.kube_applier_mc_messaging) > 0 ? module.kube_applier_mc_messaging[0].status_topic_arn : "" } # ============================================================================= diff --git a/terraform/config/management-cluster/outputs.tf b/terraform/config/management-cluster/outputs.tf index a0bea2e40..dde10cf65 100755 --- a/terraform/config/management-cluster/outputs.tf +++ b/terraform/config/management-cluster/outputs.tf @@ -169,7 +169,10 @@ output "kube_applier_role_arn" { # ============================================================================= # kube-applier Messaging Outputs -# Read by the RC kube-applier-dynamodb buildspec to wire cross-account +# Read by bootstrap-argocd.sh to wire the queue URL and topic ARN into the +# ArgoCD cluster secret annotations, which the ApplicationSet then passes +# to the kube-applier Helm chart as --sqs-queue-url and --sns-status-topic-arn. +# Also read by the RC kube-applier-dynamodb buildspec to wire cross-account # SNS subscriptions. # ============================================================================= @@ -178,6 +181,11 @@ output "kube_applier_specs_queue_arn" { value = length(module.kube_applier_mc_messaging) > 0 ? module.kube_applier_mc_messaging[0].specs_queue_arn : "" } +output "kube_applier_specs_queue_url" { + description = "URL of the MC-side specs SQS queue (polled by kube-applier for spec change notifications). Empty when messaging is not yet provisioned." + value = length(module.kube_applier_mc_messaging) > 0 ? module.kube_applier_mc_messaging[0].specs_queue_url : "" +} + output "kube_applier_status_topic_arn" { description = "ARN of the MC-side status SNS topic (kube-applier publishes here after writing status). Empty when messaging is not yet provisioned." value = length(module.kube_applier_mc_messaging) > 0 ? module.kube_applier_mc_messaging[0].status_topic_arn : "" diff --git a/terraform/modules/ecs-bootstrap/main.tf b/terraform/modules/ecs-bootstrap/main.tf index 622d2db08..a422db5b9 100644 --- a/terraform/modules/ecs-bootstrap/main.tf +++ b/terraform/modules/ecs-bootstrap/main.tf @@ -226,6 +226,8 @@ resource "aws_ecs_task_definition" "bootstrap" { sre_alb_dns_name: "$SRE_ALB_DNS_NAME" sre_domain: "$SRE_DOMAIN" redis_endpoint: "$REDIS_ENDPOINT" + kube_applier_specs_queue_url: "$KUBE_APPLIER_SQS_QUEUE_URL" + kube_applier_status_topic_arn: "$KUBE_APPLIER_SNS_STATUS_TOPIC_ARN" type: Opaque stringData: name: in-cluster @@ -298,6 +300,14 @@ resource "aws_ecs_task_definition" "bootstrap" { { name = "REDIS_ENDPOINT" value = var.redis_endpoint + }, + { + name = "KUBE_APPLIER_SQS_QUEUE_URL" + value = var.kube_applier_specs_queue_url + }, + { + name = "KUBE_APPLIER_SNS_STATUS_TOPIC_ARN" + value = var.kube_applier_status_topic_arn } ] diff --git a/terraform/modules/ecs-bootstrap/variables.tf b/terraform/modules/ecs-bootstrap/variables.tf index 07e002171..ea4f24f39 100644 --- a/terraform/modules/ecs-bootstrap/variables.tf +++ b/terraform/modules/ecs-bootstrap/variables.tf @@ -82,3 +82,15 @@ variable "redis_endpoint" { default = "" } +variable "kube_applier_specs_queue_url" { + description = "URL of the MC-side SQS queue that kube-applier polls for spec change notifications. Written as an annotation on the local-cluster-identity secret so the ApplicationSet can pass it to the kube-applier Helm chart." + type = string + default = "" +} + +variable "kube_applier_status_topic_arn" { + description = "ARN of the MC-side SNS topic that kube-applier publishes status updates to. Written as an annotation on the local-cluster-identity secret so the ApplicationSet can pass it to the kube-applier Helm chart." + type = string + default = "" +} + diff --git a/terraform/modules/kube-applier-mc-messaging/main.tf b/terraform/modules/kube-applier-mc-messaging/main.tf index 321e9fb2c..102556e8d 100644 --- a/terraform/modules/kube-applier-mc-messaging/main.tf +++ b/terraform/modules/kube-applier-mc-messaging/main.tf @@ -254,29 +254,3 @@ resource "aws_iam_role_policy" "kube_applier_messaging" { ] }) } - -# ============================================================================= -# SSM Parameters — surface queue URL and topic ARN for app config -# ============================================================================= - -resource "aws_ssm_parameter" "specs_queue_url" { - name = "/${var.mc_name}/messaging/specs-queue-url" - description = "SQS queue URL for specs change notifications (RC → MC)" - type = "String" - value = aws_sqs_queue.specs.url - - tags = merge(local.common_tags, { - Name = "${var.mc_name}-specs-queue-url" - }) -} - -resource "aws_ssm_parameter" "status_topic_arn" { - name = "/${var.mc_name}/messaging/status-topic-arn" - description = "SNS topic ARN for status change notifications (MC → RC)" - type = "String" - value = aws_sns_topic.status.arn - - tags = merge(local.common_tags, { - Name = "${var.mc_name}-status-topic-arn" - }) -} From b2e970d93cf01174bf474ce053a30168cd5229e4 Mon Sep 17 00:00:00 2001 From: Benji Date: Mon, 27 Jul 2026 21:16:16 +0000 Subject: [PATCH 07/18] feat: symmetric SNS/SQS wiring via CLI subscriptions in register.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove circular-dependency count gates from both messaging modules and their configs. Each module now constructs the other account's ARNs locally from predictable patterns — no cross-module variable threading. Both SNS→SQS subscriptions are created at the end of register.sh (Stage 4), the first point where all four resources are guaranteed to exist. The script already runs under RC account credentials; the MC status topic policy grants sns:Subscribe to the RC account so both subscriptions can be wired from one place without account switching. AWS auto-removes subscriptions when their SNS topic is deleted, so no explicit teardown step is needed. Co-Authored-By: Claude Sonnet 4.6 --- .../provision-kube-applier-dynamodb.sh | 43 ----------------- scripts/buildspec/register.sh | 44 +++++++++++++++++ .../main.tf | 26 +++++----- .../outputs.tf | 8 ++-- .../variables.tf | 16 ------- terraform/config/management-cluster/main.tf | 14 +++--- .../config/management-cluster/outputs.tf | 12 ++--- .../config/management-cluster/variables.tf | 11 ----- .../modules/kube-applier-mc-messaging/main.tf | 7 ++- .../kube-applier-mc-messaging/variables.tf | 5 -- .../modules/kube-applier-rc-messaging/main.tf | 48 +++---------------- .../kube-applier-rc-messaging/variables.tf | 10 ---- 12 files changed, 84 insertions(+), 160 deletions(-) diff --git a/scripts/buildspec/provision-kube-applier-dynamodb.sh b/scripts/buildspec/provision-kube-applier-dynamodb.sh index 6d581ce82..d8dad743b 100755 --- a/scripts/buildspec/provision-kube-applier-dynamodb.sh +++ b/scripts/buildspec/provision-kube-applier-dynamodb.sh @@ -31,49 +31,6 @@ _RC_REGIONAL_ID=$(jq -r '.regional_id // "regional"' "$_RC_CONFIG_FILE") # ── Switch to RC account ──────────────────────────────────────────────────── use_rc_account -# ── Read MC messaging outputs from management-cluster state ──────────────── -# The mc-messaging module (specs SQS queue + status SNS topic) is provisioned -# by the MC management-cluster pipeline step, which may run in parallel with -# this step. Retry until the outputs appear or we time out (45 min), matching -# the pattern used for OIDC outputs in provision-infra-mc.sh. -# -# On destroy, skip the read and pass empty strings so the rc-messaging module -# is cleanly removed without blocking on stale MC outputs. -if [ "${DELETE_FLAG}" != "true" ]; then - _MC_STATE_BUCKET="terraform-state-${TARGET_ACCOUNT_ID}-${TARGET_REGION}" - _MC_STATE_KEY="management-cluster/${CLUSTER_ID}.tfstate" - _MC_TF_DIR="terraform/config/management-cluster" - - # Init the MC state backend (read-only; we only run `terraform output`) - (cd "$_MC_TF_DIR" && terraform init -reconfigure \ - -backend-config="bucket=${_MC_STATE_BUCKET}" \ - -backend-config="key=${_MC_STATE_KEY}" \ - -backend-config="region=${TARGET_REGION}" \ - -backend-config="use_lockfile=true" >/dev/null 2>&1) - - _MSG_MAX_RETRIES=90 - _MSG_RETRY_DELAY=30 - _MSG_RETRY_COUNT=0 - TF_VAR_mc_specs_queue_arn="" - TF_VAR_mc_status_sns_topic_arn="" - while [ $_MSG_RETRY_COUNT -lt $_MSG_MAX_RETRIES ]; do - _MSG_RETRY_COUNT=$((_MSG_RETRY_COUNT + 1)) - TF_VAR_mc_specs_queue_arn=$(cd "$_MC_TF_DIR" && terraform output -raw kube_applier_specs_queue_arn 2>/dev/null || true) - TF_VAR_mc_status_sns_topic_arn=$(cd "$_MC_TF_DIR" && terraform output -raw kube_applier_status_topic_arn 2>/dev/null || true) - if [ -n "${TF_VAR_mc_specs_queue_arn}" ] && [ -n "${TF_VAR_mc_status_sns_topic_arn}" ]; then - break - fi - echo "MC messaging outputs not ready (attempt ${_MSG_RETRY_COUNT}/${_MSG_MAX_RETRIES}), retrying in ${_MSG_RETRY_DELAY}s..." - sleep "$_MSG_RETRY_DELAY" - done - if [ -z "${TF_VAR_mc_specs_queue_arn}" ] || [ -z "${TF_VAR_mc_status_sns_topic_arn}" ]; then - echo "INFO: MC messaging outputs missing after $((_MSG_MAX_RETRIES * _MSG_RETRY_DELAY / 60))+ minutes — rc-messaging module will be skipped." - TF_VAR_mc_specs_queue_arn="" - TF_VAR_mc_status_sns_topic_arn="" - fi - export TF_VAR_mc_specs_queue_arn TF_VAR_mc_status_sns_topic_arn -fi - # ── Terraform apply ──────────────────────────────────────────────────────── _RC_STATE_BUCKET="terraform-state-${RESOLVED_REGIONAL_ACCOUNT_ID}-${TARGET_REGION}" export TF_STATE_BUCKET="${_RC_STATE_BUCKET}" diff --git a/scripts/buildspec/register.sh b/scripts/buildspec/register.sh index 117f76466..6598e792b 100755 --- a/scripts/buildspec/register.sh +++ b/scripts/buildspec/register.sh @@ -155,3 +155,47 @@ if [ "$REG_OK" != "true" ]; then cat /tmp/register-response.json >&2 exit 1 fi + +# Wire SNS→SQS subscriptions. +# +# Both subscriptions are created here — after API registration succeeds — because +# this is the first point in the pipeline where all four resources are guaranteed +# to exist: +# Stage 1 (Deploy MC): MC SQS + MC SNS created +# Stage 2 (Provision-KubeApplier-DynamoDB): RC SNS + RC SQS created +# Stage 4 (Register, this script): safe to subscribe +# +# This script already runs under RC account credentials (use_rc_account above). +# The MC status topic policy grants sns:Subscribe to the RC account +# (AllowRCAccountSubscribe), so both subscriptions can be created from here. +# +# AWS automatically removes subscriptions when their SNS topic is deleted, so +# no explicit teardown is needed — Terraform destroying a topic cleans up its +# subscriptions for free. + +SPECS_TOPIC_ARN="arn:aws:sns:${TARGET_REGION}:${RESOLVED_REGIONAL_ACCOUNT_ID}:${CLUSTER_ID}-specs-notifications" +SPECS_QUEUE_ARN="arn:aws:sqs:${TARGET_REGION}:${TARGET_ACCOUNT_ID}:${CLUSTER_ID}-specs-notifications" +STATUS_TOPIC_ARN="arn:aws:sns:${TARGET_REGION}:${TARGET_ACCOUNT_ID}:${CLUSTER_ID}-status-notifications" +OPERATOR_REPLICA_COUNT=$(jq -r '.operator_replica_count // 3' "$DEPLOY_CONFIG_FILE") + +echo "Subscribing specs queue to specs topic" +aws sns subscribe \ + --topic-arn "$SPECS_TOPIC_ARN" \ + --protocol sqs \ + --notification-endpoint "$SPECS_QUEUE_ARN" \ + --attributes '{"RawMessageDelivery":"true"}' \ + --region "$TARGET_REGION" + +echo "Subscribing ${OPERATOR_REPLICA_COUNT} operator replica queue(s) to status topic" +for i in $(seq 0 $((OPERATOR_REPLICA_COUNT - 1))); do + STATUS_QUEUE_ARN="arn:aws:sqs:${TARGET_REGION}:${RESOLVED_REGIONAL_ACCOUNT_ID}:${RC_REGIONAL_ID}-hyperfleet-operator-${i}" + echo " Subscribing replica ${i}: ${STATUS_QUEUE_ARN}" + aws sns subscribe \ + --topic-arn "$STATUS_TOPIC_ARN" \ + --protocol sqs \ + --notification-endpoint "$STATUS_QUEUE_ARN" \ + --attributes '{"RawMessageDelivery":"true"}' \ + --region "$TARGET_REGION" +done + +echo "SNS→SQS subscriptions wired successfully" diff --git a/terraform/config/kube-applier-dynamodb-provisioning/main.tf b/terraform/config/kube-applier-dynamodb-provisioning/main.tf index bc510d3d5..ec84c8dc1 100644 --- a/terraform/config/kube-applier-dynamodb-provisioning/main.tf +++ b/terraform/config/kube-applier-dynamodb-provisioning/main.tf @@ -98,25 +98,21 @@ resource "aws_iam_role_policy" "hyperfleet_operator_dynamodb" { # after writing a desire document) and the per-replica status SQS queues # (the operator polls its own queue for status notifications from kube-applier). # -# Also creates the cross-account SNS→SQS subscriptions for both directions: -# Specs: RC specs SNS topic → MC specs SQS queue (mc_specs_queue_arn) -# Status: MC status SNS topic → each RC operator SQS queue +# Both modules use predictable ARNs to reference cross-account resources, so +# there is no dependency on MC Terraform outputs and no count gate. Both sides +# can be provisioned independently in a single pipeline run. # -# mc_status_sns_topic_arn and mc_specs_queue_arn are read from the MC -# management-cluster terraform state by the buildspec script. When empty -# (e.g. during the initial bootstrap run before MC messaging is provisioned) -# the module is skipped. +# Subscriptions (RC SNS → MC SQS and MC SNS → RC SQS × N) are created by +# the Register buildspec step, after both modules have run and all four +# resources are guaranteed to exist. # ============================================================================= module "kube_applier_rc_messaging" { - count = var.mc_status_sns_topic_arn != "" && var.mc_specs_queue_arn != "" ? 1 : 0 source = "../../modules/kube-applier-rc-messaging" - mc_name = var.mc_name - mc_aws_account_id = var.mc_aws_account_id - rc_id = var.rc_id - aws_region = var.region - operator_replica_count = var.operator_replica_count - mc_status_sns_topic_arn = var.mc_status_sns_topic_arn - mc_specs_queue_arn = var.mc_specs_queue_arn + mc_name = var.mc_name + mc_aws_account_id = var.mc_aws_account_id + rc_id = var.rc_id + aws_region = var.region + operator_replica_count = var.operator_replica_count } diff --git a/terraform/config/kube-applier-dynamodb-provisioning/outputs.tf b/terraform/config/kube-applier-dynamodb-provisioning/outputs.tf index 4ff97f6a4..095a2679d 100644 --- a/terraform/config/kube-applier-dynamodb-provisioning/outputs.tf +++ b/terraform/config/kube-applier-dynamodb-provisioning/outputs.tf @@ -20,12 +20,12 @@ output "status_readdesires_stream_arn" { # ============================================================================= output "specs_sns_topic_arn" { - description = "ARN of the RC-account specs SNS topic for this MC (operator publishes here after writing a desire document). Empty when messaging is not yet provisioned." - value = length(module.kube_applier_rc_messaging) > 0 ? module.kube_applier_rc_messaging[0].specs_topic_arn : "" + description = "ARN of the RC-account specs SNS topic for this MC (operator publishes here after writing a desire document)." + value = module.kube_applier_rc_messaging.specs_topic_arn } output "status_sqs_queue_urls" { - description = "URLs of the RC-account operator status SQS queues (one per replica). Empty list when messaging is not yet provisioned." - value = length(module.kube_applier_rc_messaging) > 0 ? module.kube_applier_rc_messaging[0].status_queue_urls : [] + description = "URLs of the RC-account operator status SQS queues (one per replica)." + value = module.kube_applier_rc_messaging.status_queue_urls } diff --git a/terraform/config/kube-applier-dynamodb-provisioning/variables.tf b/terraform/config/kube-applier-dynamodb-provisioning/variables.tf index e1464861b..038daa7ea 100644 --- a/terraform/config/kube-applier-dynamodb-provisioning/variables.tf +++ b/terraform/config/kube-applier-dynamodb-provisioning/variables.tf @@ -54,22 +54,6 @@ variable "environment" { type = string } -# ============================================================================= -# kube-applier Messaging Variables -# ============================================================================= - -variable "mc_status_sns_topic_arn" { - description = "ARN of the MC-account SNS topic for status change notifications (read from MC management-cluster terraform state). The RC-side operator SQS queues subscribe to this topic." - type = string - default = "" -} - -variable "mc_specs_queue_arn" { - description = "ARN of the MC-account SQS queue for specs change notifications (read from MC management-cluster terraform state). The RC specs SNS topic delivers messages to this queue." - type = string - default = "" -} - variable "operator_replica_count" { description = "Number of hyperfleet-operator replicas. One status SQS queue is created per replica in the RC account." type = number diff --git a/terraform/config/management-cluster/main.tf b/terraform/config/management-cluster/main.tf index 9baf0ba0e..42aa9cd92 100755 --- a/terraform/config/management-cluster/main.tf +++ b/terraform/config/management-cluster/main.tf @@ -69,8 +69,8 @@ module "ecs_bootstrap" { repository_url = var.repository_url repository_branch = var.repository_branch - kube_applier_specs_queue_url = length(module.kube_applier_mc_messaging) > 0 ? module.kube_applier_mc_messaging[0].specs_queue_url : "" - kube_applier_status_topic_arn = length(module.kube_applier_mc_messaging) > 0 ? module.kube_applier_mc_messaging[0].status_topic_arn : "" + kube_applier_specs_queue_url = module.kube_applier_mc_messaging.specs_queue_url + kube_applier_status_topic_arn = module.kube_applier_mc_messaging.status_topic_arn } # ============================================================================= @@ -206,12 +206,10 @@ module "kube_applier" { # ============================================================================= module "kube_applier_mc_messaging" { - count = var.rc_specs_sns_topic_arn != "" ? 1 : 0 source = "../../modules/kube-applier-mc-messaging" - mc_name = var.management_id - rc_aws_account_id = var.regional_aws_account_id - rc_specs_sns_topic_arn = var.rc_specs_sns_topic_arn - eks_cluster_name = module.management_cluster.cluster_name - aws_region = var.region + mc_name = var.management_id + rc_aws_account_id = var.regional_aws_account_id + eks_cluster_name = module.management_cluster.cluster_name + aws_region = var.region } diff --git a/terraform/config/management-cluster/outputs.tf b/terraform/config/management-cluster/outputs.tf index dde10cf65..180b6dddf 100755 --- a/terraform/config/management-cluster/outputs.tf +++ b/terraform/config/management-cluster/outputs.tf @@ -177,16 +177,16 @@ output "kube_applier_role_arn" { # ============================================================================= output "kube_applier_specs_queue_arn" { - description = "ARN of the MC-side specs SQS queue (receives RC specs SNS notifications). Empty when messaging is not yet provisioned." - value = length(module.kube_applier_mc_messaging) > 0 ? module.kube_applier_mc_messaging[0].specs_queue_arn : "" + description = "ARN of the MC-side specs SQS queue (receives RC specs SNS notifications)." + value = module.kube_applier_mc_messaging.specs_queue_arn } output "kube_applier_specs_queue_url" { - description = "URL of the MC-side specs SQS queue (polled by kube-applier for spec change notifications). Empty when messaging is not yet provisioned." - value = length(module.kube_applier_mc_messaging) > 0 ? module.kube_applier_mc_messaging[0].specs_queue_url : "" + description = "URL of the MC-side specs SQS queue (polled by kube-applier for spec change notifications)." + value = module.kube_applier_mc_messaging.specs_queue_url } output "kube_applier_status_topic_arn" { - description = "ARN of the MC-side status SNS topic (kube-applier publishes here after writing status). Empty when messaging is not yet provisioned." - value = length(module.kube_applier_mc_messaging) > 0 ? module.kube_applier_mc_messaging[0].status_topic_arn : "" + description = "ARN of the MC-side status SNS topic (kube-applier publishes here after writing status)." + value = module.kube_applier_mc_messaging.status_topic_arn } diff --git a/terraform/config/management-cluster/variables.tf b/terraform/config/management-cluster/variables.tf index 74a010b66..694f25827 100755 --- a/terraform/config/management-cluster/variables.tf +++ b/terraform/config/management-cluster/variables.tf @@ -138,14 +138,3 @@ variable "oidc_cloudfront_domain" { type = string default = "" } - -# ============================================================================= -# kube-applier Messaging Variables -# ============================================================================= - -variable "rc_specs_sns_topic_arn" { - description = "ARN of the RC-account SNS topic for specs change notifications (read from RC kube-applier-dynamodb terraform state). kube-applier subscribes the MC-side SQS queue to this topic." - type = string - default = "" -} - diff --git a/terraform/modules/kube-applier-mc-messaging/main.tf b/terraform/modules/kube-applier-mc-messaging/main.tf index 102556e8d..fa6a68d9f 100644 --- a/terraform/modules/kube-applier-mc-messaging/main.tf +++ b/terraform/modules/kube-applier-mc-messaging/main.tf @@ -35,6 +35,11 @@ locals { # IAM role ARN for the kube-applier pod in this MC account kube_applier_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${var.mc_name}-kube-applier" + + # RC specs SNS topic ARN — predictable, constructed from known values. + # Used in the SQS queue policy so delivery is authorised from day one, + # before the RC messaging module has run. + rc_specs_sns_topic_arn = "arn:aws:sns:${var.aws_region}:${var.rc_aws_account_id}:${var.mc_name}-specs-notifications" } # ============================================================================= @@ -146,7 +151,7 @@ resource "aws_sqs_queue_policy" "specs" { Resource = aws_sqs_queue.specs.arn Condition = { ArnEquals = { - "aws:SourceArn" = var.rc_specs_sns_topic_arn + "aws:SourceArn" = local.rc_specs_sns_topic_arn } } }] diff --git a/terraform/modules/kube-applier-mc-messaging/variables.tf b/terraform/modules/kube-applier-mc-messaging/variables.tf index a164f4b62..a32267814 100644 --- a/terraform/modules/kube-applier-mc-messaging/variables.tf +++ b/terraform/modules/kube-applier-mc-messaging/variables.tf @@ -22,11 +22,6 @@ variable "rc_aws_account_id" { } } -variable "rc_specs_sns_topic_arn" { - description = "ARN of the specs SNS topic in the RC account that will deliver notifications to the specs SQS queue in this MC." - type = string -} - variable "eks_cluster_name" { description = "Name of the EKS management cluster. Used for the Pod Identity association." type = string diff --git a/terraform/modules/kube-applier-rc-messaging/main.tf b/terraform/modules/kube-applier-rc-messaging/main.tf index 6e77705c8..c94b67349 100644 --- a/terraform/modules/kube-applier-rc-messaging/main.tf +++ b/terraform/modules/kube-applier-rc-messaging/main.tf @@ -45,6 +45,12 @@ locals { # IAM role for the hyperfleet-operator (RC account, shared across all MCs) hyperfleet_operator_role_name = "${var.rc_id}-hyperfleet-operator" + + # MC ARNs — predictable, constructed from known values. + # Used in policies and subscriptions so the RC module is self-contained + # and does not depend on outputs from the MC Terraform apply. + mc_specs_queue_arn = "arn:aws:sqs:${var.aws_region}:${var.mc_aws_account_id}:${var.mc_name}-specs-notifications" + mc_status_sns_topic_arn = "arn:aws:sns:${var.aws_region}:${var.mc_aws_account_id}:${var.mc_name}-status-notifications" } # ============================================================================= @@ -136,7 +142,6 @@ resource "aws_sns_topic" "specs" { } # Allow the hyperfleet-operator pod role to publish specs notifications. -# Also allow the MC account to confirm the cross-account SQS subscription. resource "aws_sns_topic_policy" "specs" { arn = aws_sns_topic.specs.arn @@ -152,39 +157,10 @@ resource "aws_sns_topic_policy" "specs" { Action = "sns:Publish" Resource = aws_sns_topic.specs.arn }, - { - # The SNS subscription is created from the RC account (this module), - # targeting the MC SQS queue. AWS auto-confirms SQS subscriptions, so - # no MC-account confirmation action is needed here. - Sid = "AllowSNSToDeliverToSQS" - Effect = "Allow" - Principal = { - Service = "sns.amazonaws.com" - } - Action = "sns:Publish" - Resource = aws_sns_topic.specs.arn - Condition = { - StringEquals = { - "aws:SourceAccount" = data.aws_caller_identity.current.account_id - } - } - }, ] }) } -# Cross-account SNS→SQS subscription: RC specs topic → MC specs queue. -# AWS auto-confirms SQS subscriptions, so no ConfirmSubscription step is needed. -resource "aws_sns_topic_subscription" "specs_to_mc_queue" { - topic_arn = aws_sns_topic.specs.arn - protocol = "sqs" - endpoint = var.mc_specs_queue_arn - - # Raw delivery passes the JSON notification body directly without the SNS - # envelope wrapper, simplifying parsing in the kube-applier consumer. - raw_message_delivery = true -} - # ============================================================================= # Status SQS Queues (status path receiver — MC SNS → RC SQS) # @@ -226,23 +202,13 @@ resource "aws_sqs_queue_policy" "status" { Resource = aws_sqs_queue.status[count.index].arn Condition = { ArnEquals = { - "aws:SourceArn" = var.mc_status_sns_topic_arn + "aws:SourceArn" = local.mc_status_sns_topic_arn } } }] }) } -# Cross-account SNS→SQS subscriptions: MC status topic → each RC operator queue. -resource "aws_sns_topic_subscription" "status_to_rc_queues" { - count = var.operator_replica_count - - topic_arn = var.mc_status_sns_topic_arn - protocol = "sqs" - endpoint = aws_sqs_queue.status[count.index].arn - raw_message_delivery = true -} - # ============================================================================= # IAM: extend hyperfleet-operator role with messaging permissions (per-MC) # diff --git a/terraform/modules/kube-applier-rc-messaging/variables.tf b/terraform/modules/kube-applier-rc-messaging/variables.tf index 9fac47661..bced11cda 100644 --- a/terraform/modules/kube-applier-rc-messaging/variables.tf +++ b/terraform/modules/kube-applier-rc-messaging/variables.tf @@ -43,16 +43,6 @@ variable "operator_replica_count" { } } -variable "mc_status_sns_topic_arn" { - description = "ARN of the status SNS topic in the MC account. The RC-side SQS queues subscribe to this topic for cross-account delivery." - type = string -} - -variable "mc_specs_queue_arn" { - description = "ARN of the specs SQS queue in the MC account. The RC specs SNS topic subscribes this queue so the operator's publish triggers delivery to kube-applier." - type = string -} - variable "tags" { description = "Additional tags to apply to resources." type = map(string) From d7d61ab01bf42ad97d395c28c407841848969e18 Mon Sep 17 00:00:00 2001 From: Benji Date: Tue, 28 Jul 2026 10:36:16 +0000 Subject: [PATCH 08/18] feat: wire --sqs-queue-url-prefix to hyperfleet-operator via ApplicationSet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hyperfleet-operator StatefulSet pods call os.Hostname() to get their ordinal and append it to a --sqs-queue-url-prefix flag to form their full SQS queue URL. This flag was never passed through the ArgoCD chart wiring, causing pod crashes when they could not find their queue. The prefix follows the predictable pattern: https://sqs..amazonaws.com//-hyperfleet-operator- All three values (aws_region, aws_account_id, cluster_name) are already present on the RC cluster secret as labels/annotations, so the prefix is constructed inline in the ApplicationSet valuesObject — no new Terraform outputs, bootstrap wiring, or ECS variables needed. Changes: - applicationset.yaml.j2: add hyperfleet.sqsQueueUrlPrefix to RC valuesObject - hyperfleet/values.yaml: add sqsQueueUrlPrefix default - hyperfleet/templates/application.yaml: pipe prefix into external chart values - Re-render deploy/ applicationsets Co-Authored-By: Claude Sonnet 4.6 --- .../regional-cluster/hyperfleet/templates/application.yaml | 1 + argocd/config/regional-cluster/hyperfleet/values.yaml | 2 ++ config/templates/argocd-bootstrap/applicationset.yaml.j2 | 5 +++++ .../argocd-bootstrap-regional-cluster/applicationset.yaml | 3 +++ .../argocd-bootstrap-regional-cluster/applicationset.yaml | 3 +++ 5 files changed, 14 insertions(+) diff --git a/argocd/config/regional-cluster/hyperfleet/templates/application.yaml b/argocd/config/regional-cluster/hyperfleet/templates/application.yaml index e0afd45ed..b6806595a 100644 --- a/argocd/config/regional-cluster/hyperfleet/templates/application.yaml +++ b/argocd/config/regional-cluster/hyperfleet/templates/application.yaml @@ -19,6 +19,7 @@ spec: {{- end }} awsRegion: {{ .Values.global.aws_region }} baseDomain: {{ .Values.baseDomain }} + sqsQueueUrlPrefix: {{ index .Values "hyperfleet" "sqsQueueUrlPrefix" | default "" | quote }} postgres: secretName: {{ .Values.global.postgres_secret_name | default "hyperfleet-db-dsn" }} secretKey: {{ .Values.global.postgres_secret_key | default "dsn" }} diff --git a/argocd/config/regional-cluster/hyperfleet/values.yaml b/argocd/config/regional-cluster/hyperfleet/values.yaml index 446979b45..e8ec0e837 100644 --- a/argocd/config/regional-cluster/hyperfleet/values.yaml +++ b/argocd/config/regional-cluster/hyperfleet/values.yaml @@ -7,6 +7,8 @@ hyperfleet: project: default + sqsQueueUrlPrefix: "" + source: repoURL: https://github.com/rrp-bot/rosa-hyperfleet-api.git targetRevision: feature/sns-sqs diff --git a/config/templates/argocd-bootstrap/applicationset.yaml.j2 b/config/templates/argocd-bootstrap/applicationset.yaml.j2 index 4eea5f267..bbb5904de 100644 --- a/config/templates/argocd-bootstrap/applicationset.yaml.j2 +++ b/config/templates/argocd-bootstrap/applicationset.yaml.j2 @@ -136,6 +136,11 @@ spec: config: sqsQueueUrl: '{{ '{{ .metadata.annotations.kube_applier_specs_queue_url }}' }}' snsStatusTopicArn: '{{ '{{ .metadata.annotations.kube_applier_status_topic_arn }}' }}' +{%- endif %} +{%- if cluster_type == 'regional-cluster' %} + + hyperfleet: + sqsQueueUrlPrefix: 'https://sqs.{{ '{{ .metadata.labels.aws_region }}' }}.amazonaws.com/{{ '{{ .metadata.annotations.aws_account_id }}' }}/{{ '{{ .metadata.labels.cluster_name }}' }}-hyperfleet-operator-' {%- endif %} path: '{{ '{{ .path.path }}' }}' repoURL: '{{ '{{ .metadata.annotations.git_repo }}' }}' diff --git a/deploy/ephemeral/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml b/deploy/ephemeral/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml index fdff0e336..cecd48f28 100644 --- a/deploy/ephemeral/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml +++ b/deploy/ephemeral/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml @@ -127,6 +127,9 @@ spec: sre: targetGroup: arn: '{{ .metadata.annotations.sre_prometheus_target_group_arn }}' + + hyperfleet: + sqsQueueUrlPrefix: 'https://sqs.{{ .metadata.labels.aws_region }}.amazonaws.com/{{ .metadata.annotations.aws_account_id }}/{{ .metadata.labels.cluster_name }}-hyperfleet-operator-' path: '{{ .path.path }}' repoURL: '{{ .metadata.annotations.git_repo }}' targetRevision: '{{ .metadata.annotations.git_revision }}' diff --git a/deploy/integration/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml b/deploy/integration/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml index e39904232..96b2de2e9 100644 --- a/deploy/integration/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml +++ b/deploy/integration/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml @@ -127,6 +127,9 @@ spec: sre: targetGroup: arn: '{{ .metadata.annotations.sre_prometheus_target_group_arn }}' + + hyperfleet: + sqsQueueUrlPrefix: 'https://sqs.{{ .metadata.labels.aws_region }}.amazonaws.com/{{ .metadata.annotations.aws_account_id }}/{{ .metadata.labels.cluster_name }}-hyperfleet-operator-' path: '{{ .path.path }}' repoURL: '{{ .metadata.annotations.git_repo }}' targetRevision: '{{ .metadata.annotations.git_revision }}' From 9b2f032613abdd2928305f5322d66d56dc690d2e Mon Sep 17 00:00:00 2001 From: Benji Date: Tue, 28 Jul 2026 16:07:34 +0000 Subject: [PATCH 09/18] fix: use correct image repo quay.io/openshift-online/hyperfleet-operator quay.io/psav/hyperfleet-operator is from the dead typeid repo and does not contain the /compactor binary. The rosa-hyperfleet-api Makefile publishes to quay.io/openshift-online/hyperfleet-operator. Co-Authored-By: Claude Sonnet 4.6 --- argocd/config/regional-cluster/hyperfleet/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argocd/config/regional-cluster/hyperfleet/values.yaml b/argocd/config/regional-cluster/hyperfleet/values.yaml index e8ec0e837..18ce34fa0 100644 --- a/argocd/config/regional-cluster/hyperfleet/values.yaml +++ b/argocd/config/regional-cluster/hyperfleet/values.yaml @@ -23,7 +23,7 @@ hyperfleet: helmValues: image: - repository: quay.io/psav/hyperfleet-operator + repository: quay.io/openshift-online/hyperfleet-operator tag: "9ae1c99" pullPolicy: Always From 82bfa59daacc47e1fd72afbeff408a8d5be2cf01 Mon Sep 17 00:00:00 2001 From: Benji Date: Tue, 28 Jul 2026 16:42:41 +0000 Subject: [PATCH 10/18] fix: use rrp-bot quay org for hyperfleet-operator image Co-Authored-By: Claude Sonnet 4.6 --- argocd/config/regional-cluster/hyperfleet/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argocd/config/regional-cluster/hyperfleet/values.yaml b/argocd/config/regional-cluster/hyperfleet/values.yaml index 18ce34fa0..a859d99c4 100644 --- a/argocd/config/regional-cluster/hyperfleet/values.yaml +++ b/argocd/config/regional-cluster/hyperfleet/values.yaml @@ -23,7 +23,7 @@ hyperfleet: helmValues: image: - repository: quay.io/openshift-online/hyperfleet-operator + repository: quay.io/rrp-bot/hyperfleet-operator tag: "9ae1c99" pullPolicy: Always From 629c9a2727fadf08d183088a6989dc4dc8c3031a Mon Sep 17 00:00:00 2001 From: Benji Date: Tue, 28 Jul 2026 17:04:23 +0000 Subject: [PATCH 11/18] fix: revert to quay.io/psav/hyperfleet-operator image repo Co-Authored-By: Claude Sonnet 4.6 --- argocd/config/regional-cluster/hyperfleet/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argocd/config/regional-cluster/hyperfleet/values.yaml b/argocd/config/regional-cluster/hyperfleet/values.yaml index a859d99c4..e8ec0e837 100644 --- a/argocd/config/regional-cluster/hyperfleet/values.yaml +++ b/argocd/config/regional-cluster/hyperfleet/values.yaml @@ -23,7 +23,7 @@ hyperfleet: helmValues: image: - repository: quay.io/rrp-bot/hyperfleet-operator + repository: quay.io/psav/hyperfleet-operator tag: "9ae1c99" pullPolicy: Always From af5a39c7ec5cb52f6eba87dc7fa38b455aaeeaa9 Mon Sep 17 00:00:00 2001 From: Benji Date: Thu, 30 Jul 2026 13:44:50 +0000 Subject: [PATCH 12/18] fix: add AllowRCAccountSubscribe to RC specs SNS topic policy register.sh runs as OrganizationAccountAccessRole in the RC account and calls aws sns subscribe on the RC-owned specs topic. Without an explicit sns:Subscribe grant in the topic policy, AWS returns 'pending confirmation' even when the MC SQS queue policy is correct. The MC status topic already has this pattern (AllowRCAccountSubscribe) which is why the status subscriptions confirm immediately. Mirror it on the specs topic. Co-Authored-By: Claude Sonnet 4.6 --- .../modules/kube-applier-rc-messaging/main.tf | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/terraform/modules/kube-applier-rc-messaging/main.tf b/terraform/modules/kube-applier-rc-messaging/main.tf index c94b67349..8b0928132 100644 --- a/terraform/modules/kube-applier-rc-messaging/main.tf +++ b/terraform/modules/kube-applier-rc-messaging/main.tf @@ -142,6 +142,9 @@ resource "aws_sns_topic" "specs" { } # Allow the hyperfleet-operator pod role to publish specs notifications. +# The RC account root is also granted sns:Subscribe so that register.sh +# (running as OrganizationAccountAccessRole) can wire the cross-account +# subscription to the MC-side SQS queue without getting pending confirmation. resource "aws_sns_topic_policy" "specs" { arn = aws_sns_topic.specs.arn @@ -157,6 +160,20 @@ resource "aws_sns_topic_policy" "specs" { Action = "sns:Publish" Resource = aws_sns_topic.specs.arn }, + { + # Allow the RC account to create the cross-account SQS subscription. + # register.sh runs as OrganizationAccountAccessRole (an RC account + # principal) and calls aws sns subscribe on this topic. Without an + # explicit Allow for sns:Subscribe the call returns pending confirmation + # even though the queue policy is correct. + Sid = "AllowRCAccountSubscribe" + Effect = "Allow" + Principal = { + AWS = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:root" + } + Action = "sns:Subscribe" + Resource = aws_sns_topic.specs.arn + }, ] }) } From 5406f001de95ae725b7d48a70a570c089b85eceb Mon Sep 17 00:00:00 2001 From: Benji Date: Thu, 30 Jul 2026 14:17:18 +0000 Subject: [PATCH 13/18] chore: bump image tags to latest feature/sns-sqs tips hyperfleet-operator: 9ae1c99 -> bfe629c (rosa-hyperfleet-api) kube-applier-aws: 252920c -> 9106cf5 Also remove duplicate sqsQueueUrlPrefix key in hyperfleet values.yaml. Co-Authored-By: Claude Sonnet 4.6 --- argocd/config/management-cluster/kube-applier/values.yaml | 2 +- argocd/config/regional-cluster/hyperfleet/values.yaml | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/argocd/config/management-cluster/kube-applier/values.yaml b/argocd/config/management-cluster/kube-applier/values.yaml index 133a7e191..39060c643 100644 --- a/argocd/config/management-cluster/kube-applier/values.yaml +++ b/argocd/config/management-cluster/kube-applier/values.yaml @@ -7,7 +7,7 @@ kubeApplier: image: registry: "quay.io" repository: "psav/kube-applier-aws" - tag: "252920c" + tag: "9106cf5" pullPolicy: IfNotPresent # Deployment configuration diff --git a/argocd/config/regional-cluster/hyperfleet/values.yaml b/argocd/config/regional-cluster/hyperfleet/values.yaml index e8ec0e837..23b0d4acb 100644 --- a/argocd/config/regional-cluster/hyperfleet/values.yaml +++ b/argocd/config/regional-cluster/hyperfleet/values.yaml @@ -19,12 +19,10 @@ hyperfleet: prune: true selfHeal: true - sqsQueueUrlPrefix: "" - helmValues: image: repository: quay.io/psav/hyperfleet-operator - tag: "9ae1c99" + tag: "bfe629c" pullPolicy: Always replicaCount: 2 From 8f3419b926c36bcd5dea37cfaeff703d0bf44109 Mon Sep 17 00:00:00 2001 From: Benji Date: Thu, 30 Jul 2026 14:39:51 +0000 Subject: [PATCH 14/18] fix: allow RC account SNS service to use MC KMS key for cross-account delivery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For the specs subscription (RC SNS → MC SQS), SNS acts on behalf of the RC account when encrypting messages into the MC-side KMS-encrypted SQS queue. The aws:SourceAccount condition on AllowSNSDelivery was MC-only, so the KMS key denied the RC-account SNS service principal, preventing subscription auto-confirmation (pending confirmation). Add rc_aws_account_id to the StringEquals condition so both MC-local and cross-account SNS delivery can use the key. Co-Authored-By: Claude Sonnet 4.6 --- terraform/modules/kube-applier-mc-messaging/main.tf | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/terraform/modules/kube-applier-mc-messaging/main.tf b/terraform/modules/kube-applier-mc-messaging/main.tf index fa6a68d9f..1beeb7b9a 100644 --- a/terraform/modules/kube-applier-mc-messaging/main.tf +++ b/terraform/modules/kube-applier-mc-messaging/main.tf @@ -65,7 +65,9 @@ resource "aws_kms_key" "messaging" { Resource = "*" }, { - # SNS must be able to encrypt/decrypt when delivering messages to SQS + # SNS must be able to encrypt/decrypt when delivering messages to SQS. + # For cross-account delivery (RC SNS → MC SQS), SNS acts on behalf of + # the RC account, so aws:SourceAccount will be the RC account ID. Sid = "AllowSNSDelivery" Effect = "Allow" Principal = { @@ -78,7 +80,10 @@ resource "aws_kms_key" "messaging" { Resource = "*" Condition = { StringEquals = { - "aws:SourceAccount" = data.aws_caller_identity.current.account_id + "aws:SourceAccount" = [ + data.aws_caller_identity.current.account_id, + var.rc_aws_account_id, + ] } } }, From ff2d46f3c47983e62398525302f2344da2396e53 Mon Sep 17 00:00:00 2001 From: Benji Date: Thu, 30 Jul 2026 18:01:35 +0000 Subject: [PATCH 15/18] =?UTF-8?q?fix:=20subscribe=20specs=20SNS=E2=86=92SQ?= =?UTF-8?q?S=20from=20MC=20account=20for=20auto-confirmation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AWS only auto-confirms an SNS→SQS subscription when the subscribe caller is from the same account as the queue. The specs SQS queue is in the MC account, so register.sh must call sns:Subscribe as the MC account (use_mc_account), not the RC account. Correspondingly, the RC specs topic policy now grants sns:Subscribe to the MC account root (AllowMCAccountSubscribe) instead of the RC account root — the RC account already owns the topic and never needed an explicit Allow to subscribe to its own resources. Co-Authored-By: Claude Sonnet 4.6 --- scripts/buildspec/register.sh | 26 ++++++++++++++----- .../modules/kube-applier-rc-messaging/main.tf | 24 ++++++++++------- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/scripts/buildspec/register.sh b/scripts/buildspec/register.sh index 6598e792b..fb37e5189 100755 --- a/scripts/buildspec/register.sh +++ b/scripts/buildspec/register.sh @@ -161,13 +161,21 @@ fi # Both subscriptions are created here — after API registration succeeds — because # this is the first point in the pipeline where all four resources are guaranteed # to exist: -# Stage 1 (Deploy MC): MC SQS + MC SNS created +# Stage 1 (Deploy MC): MC SQS + MC SNS created # Stage 2 (Provision-KubeApplier-DynamoDB): RC SNS + RC SQS created -# Stage 4 (Register, this script): safe to subscribe +# Stage 4 (Register, this script): safe to subscribe # -# This script already runs under RC account credentials (use_rc_account above). -# The MC status topic policy grants sns:Subscribe to the RC account -# (AllowRCAccountSubscribe), so both subscriptions can be created from here. +# AWS only auto-confirms an SNS→SQS subscription when the caller is from the +# same account as the queue. The two subscriptions therefore need different +# caller identities: +# +# Specs (RC SNS → MC SQS): call subscribe from the MC account, which owns +# the specs SQS queue. The RC specs topic policy grants sns:Subscribe to +# the MC account root (AllowMCAccountSubscribe). +# +# Status (MC SNS → RC SQS): call subscribe from the RC account, which owns +# the status SQS queues. The MC status topic policy grants sns:Subscribe +# to the RC account root (AllowRCAccountSubscribe). # # AWS automatically removes subscriptions when their SNS topic is deleted, so # no explicit teardown is needed — Terraform destroying a topic cleans up its @@ -178,7 +186,9 @@ SPECS_QUEUE_ARN="arn:aws:sqs:${TARGET_REGION}:${TARGET_ACCOUNT_ID}:${CLUSTER_ID} STATUS_TOPIC_ARN="arn:aws:sns:${TARGET_REGION}:${TARGET_ACCOUNT_ID}:${CLUSTER_ID}-status-notifications" OPERATOR_REPLICA_COUNT=$(jq -r '.operator_replica_count // 3' "$DEPLOY_CONFIG_FILE") -echo "Subscribing specs queue to specs topic" +# Specs subscription: must be called from the MC account (queue owner). +echo "Subscribing specs queue to specs topic (as MC account)" +use_mc_account aws sns subscribe \ --topic-arn "$SPECS_TOPIC_ARN" \ --protocol sqs \ @@ -186,7 +196,9 @@ aws sns subscribe \ --attributes '{"RawMessageDelivery":"true"}' \ --region "$TARGET_REGION" -echo "Subscribing ${OPERATOR_REPLICA_COUNT} operator replica queue(s) to status topic" +# Status subscriptions: must be called from the RC account (queue owner). +echo "Subscribing ${OPERATOR_REPLICA_COUNT} operator replica queue(s) to status topic (as RC account)" +use_rc_account for i in $(seq 0 $((OPERATOR_REPLICA_COUNT - 1))); do STATUS_QUEUE_ARN="arn:aws:sqs:${TARGET_REGION}:${RESOLVED_REGIONAL_ACCOUNT_ID}:${RC_REGIONAL_ID}-hyperfleet-operator-${i}" echo " Subscribing replica ${i}: ${STATUS_QUEUE_ARN}" diff --git a/terraform/modules/kube-applier-rc-messaging/main.tf b/terraform/modules/kube-applier-rc-messaging/main.tf index 8b0928132..b6da896dc 100644 --- a/terraform/modules/kube-applier-rc-messaging/main.tf +++ b/terraform/modules/kube-applier-rc-messaging/main.tf @@ -142,9 +142,12 @@ resource "aws_sns_topic" "specs" { } # Allow the hyperfleet-operator pod role to publish specs notifications. -# The RC account root is also granted sns:Subscribe so that register.sh -# (running as OrganizationAccountAccessRole) can wire the cross-account -# subscription to the MC-side SQS queue without getting pending confirmation. +# The MC account root is also granted sns:Subscribe so that register.sh +# (running as OrganizationAccountAccessRole in the MC account) can create the +# cross-account subscription and have it auto-confirmed. AWS only auto-confirms +# SNS→SQS subscriptions when the caller is from the same account as the queue; +# since the specs SQS queue is in the MC account, the subscribe call must come +# from the MC account. resource "aws_sns_topic_policy" "specs" { arn = aws_sns_topic.specs.arn @@ -161,15 +164,16 @@ resource "aws_sns_topic_policy" "specs" { Resource = aws_sns_topic.specs.arn }, { - # Allow the RC account to create the cross-account SQS subscription. - # register.sh runs as OrganizationAccountAccessRole (an RC account - # principal) and calls aws sns subscribe on this topic. Without an - # explicit Allow for sns:Subscribe the call returns pending confirmation - # even though the queue policy is correct. - Sid = "AllowRCAccountSubscribe" + # Allow the MC account to create the cross-account SQS subscription. + # register.sh assumes OrganizationAccountAccessRole in the MC account + # before calling aws sns subscribe on this topic. AWS only auto-confirms + # an SNS→SQS subscription when the subscriber caller is from the same + # account as the queue — so the subscribe call must come from the MC + # account (which owns the specs SQS queue), not the RC account. + Sid = "AllowMCAccountSubscribe" Effect = "Allow" Principal = { - AWS = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:root" + AWS = "arn:${data.aws_partition.current.partition}:iam::${var.mc_aws_account_id}:root" } Action = "sns:Subscribe" Resource = aws_sns_topic.specs.arn From f7d38328d70ff4f9bde535d9453f13d6c33f6cc6 Mon Sep 17 00:00:00 2001 From: Benji Date: Thu, 30 Jul 2026 18:44:22 +0000 Subject: [PATCH 16/18] chore: bump hyperfleet-operator image tag to c756faf Co-Authored-By: Claude Sonnet 4.6 --- argocd/config/regional-cluster/hyperfleet/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argocd/config/regional-cluster/hyperfleet/values.yaml b/argocd/config/regional-cluster/hyperfleet/values.yaml index 23b0d4acb..928751eea 100644 --- a/argocd/config/regional-cluster/hyperfleet/values.yaml +++ b/argocd/config/regional-cluster/hyperfleet/values.yaml @@ -22,7 +22,7 @@ hyperfleet: helmValues: image: repository: quay.io/psav/hyperfleet-operator - tag: "bfe629c" + tag: "c756faf" pullPolicy: Always replicaCount: 2 From d1b2919d05de199ee91fe0e81272b5ed66d06fd6 Mon Sep 17 00:00:00 2001 From: Benji Date: Fri, 31 Jul 2026 15:47:19 +0000 Subject: [PATCH 17/18] chore: bump kube-applier image tag to 1f8498a (add SNS publish logging) Co-Authored-By: Claude Sonnet 4.6 --- argocd/config/management-cluster/kube-applier/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argocd/config/management-cluster/kube-applier/values.yaml b/argocd/config/management-cluster/kube-applier/values.yaml index 39060c643..43f525828 100644 --- a/argocd/config/management-cluster/kube-applier/values.yaml +++ b/argocd/config/management-cluster/kube-applier/values.yaml @@ -7,7 +7,7 @@ kubeApplier: image: registry: "quay.io" repository: "psav/kube-applier-aws" - tag: "9106cf5" + tag: "1f8498a" pullPolicy: IfNotPresent # Deployment configuration From 7ea3944188eab455901d0d653856c505d372429c Mon Sep 17 00:00:00 2001 From: Benji Date: Fri, 31 Jul 2026 17:04:54 +0000 Subject: [PATCH 18/18] fix(rc-messaging): allow MC account in RC KMS AllowSNSDelivery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For the status path (MC SNS → RC SQS), SNS delivers on behalf of the MC account. The RC KMS key must permit aws:SourceAccount for both the RC and MC accounts so SNS can encrypt messages before writing to the RC SQS queues. Without this, SNS silently drops all status notifications. Co-Authored-By: Claude Sonnet 4.6 --- terraform/modules/kube-applier-rc-messaging/main.tf | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/terraform/modules/kube-applier-rc-messaging/main.tf b/terraform/modules/kube-applier-rc-messaging/main.tf index b6da896dc..9407ed6a2 100644 --- a/terraform/modules/kube-applier-rc-messaging/main.tf +++ b/terraform/modules/kube-applier-rc-messaging/main.tf @@ -76,7 +76,10 @@ resource "aws_kms_key" "messaging" { Resource = "*" }, { - # SNS must be able to encrypt/decrypt when delivering messages to SQS + # SNS must be able to encrypt/decrypt when delivering messages to SQS. + # For cross-account delivery (MC SNS → RC SQS), SNS acts on behalf of + # the MC account, so aws:SourceAccount will be the MC account ID. + # Both RC and MC account IDs are required. Sid = "AllowSNSDelivery" Effect = "Allow" Principal = { @@ -89,7 +92,10 @@ resource "aws_kms_key" "messaging" { Resource = "*" Condition = { StringEquals = { - "aws:SourceAccount" = data.aws_caller_identity.current.account_id + "aws:SourceAccount" = [ + data.aws_caller_identity.current.account_id, + var.mc_aws_account_id, + ] } } },