feat(keycloak): add Flatcar bootstrap spike

aws/keycloak-flatcar-spike/.gitignore

@@ -0,0 +1,10 @@
+.terraform/
+*.tfstate
+*.tfstate.*
+tfplan
+crash.log
+crash.*.log
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json

aws/keycloak-flatcar-spike/Justfile

@@ -0,0 +1,29 @@
+set shell := ["bash", "-euo", "pipefail", "-c"]
+
+default:
+    @just --list
+
+# Offline validation: no backend or AWS credentials required.
+check:
+    ./scripts/check.sh
+
+# Format Tofu files in place.
+fmt:
+    tofu fmt -recursive
+
+# Initialize the working directory against the S3 backend.
+init:
+    test -n "${GLAB_AWS_STATE_BUCKET:-}" || { echo "Set GLAB_AWS_STATE_BUCKET to the pre-created S3 backend bucket." >&2; exit 1; }
+    tofu init -reconfigure -backend-config="bucket=${GLAB_AWS_STATE_BUCKET}"
+
+# Render and save a plan to `tfplan`.
+plan:
+    tofu plan -out=tfplan
+
+# Apply the saved plan produced by `just plan`.
+apply:
+    tofu apply tfplan
+
+# Show outputs from the last applied state.
+output:
+    tofu output

aws/keycloak-flatcar-spike/README.md

@@ -0,0 +1,32 @@
+# AWS Keycloak Flatcar Spike
+
+Temporary OpenTofu root for proving the Flatcar bootstrap path before moving the live Keycloak host off AL2023.
+
+This stack intentionally creates only a sidecar EC2 instance:
+
+- Flatcar stable arm64 in the existing lab public subnet
+- outbound-only security group
+- temporary IAM role and instance profile
+- encrypted 16 GiB gp3 root volume
+- no DNS records
+- no persistent application data volume
+
+On boot, Ignition enables `glab-keycloak-bootstrap.service`. The unit runs the pinned `labctl` container with host networking and writes the decrypted `stack_env` field from `services/keycloak/bootstrap.sops.yaml` to `/run/glab/keycloak/stack.env`.
+
+## Validation
+
+```sh
+just check
+AWS_PROFILE=lab-admin AWS_REGION=us-west-2 just init
+AWS_PROFILE=lab-admin AWS_REGION=us-west-2 just plan
+```
+
+After apply, inspect only non-secret metadata through SSM:
+
+```sh
+systemctl is-active glab-keycloak-bootstrap.service
+stat -c '%a %s %n' /run/glab/keycloak/stack.env
+cut -d= -f1 /run/glab/keycloak/stack.env | sort
+```
+
+Do not print the full `/run/glab/keycloak/stack.env` contents.

aws/keycloak-flatcar-spike/backend.tf

@@ -0,0 +1,8 @@
+terraform {
+  backend "s3" {
+    key          = "aws/keycloak-flatcar-spike.tfstate"
+    region       = "us-west-2"
+    encrypt      = true
+    use_lockfile = true
+  }
+}

aws/keycloak-flatcar-spike/compute.tf

@@ -0,0 +1,27 @@
+resource "aws_instance" "flatcar" {
+  ami                         = var.flatcar_ami_id
+  associate_public_ip_address = true
+  iam_instance_profile        = aws_iam_instance_profile.flatcar.name
+  instance_type               = var.instance_type
+  subnet_id                   = data.aws_subnet.public.id
+  user_data                   = local.ignition_config
+  user_data_replace_on_change = true
+  vpc_security_group_ids      = [aws_security_group.flatcar.id]
+
+  metadata_options {
+    http_endpoint               = "enabled"
+    http_put_response_hop_limit = 2
+    http_tokens                 = "required"
+  }
+
+  root_block_device {
+    delete_on_termination = true
+    encrypted             = true
+    volume_size           = var.root_volume_size
+    volume_type           = "gp3"
+  }
+
+  tags = merge(local.common_tags, {
+    Name = var.instance_name
+  })
+}

aws/keycloak-flatcar-spike/data.tf

@@ -0,0 +1,22 @@
+data "aws_lambda_function" "github_token_broker" {
+  function_name = var.github_token_broker_function_name
+}
+
+data "aws_subnet" "public" {
+  filter {
+    name   = "tag:Name"
+    values = [var.public_subnet_name]
+  }
+
+  filter {
+    name   = "vpc-id"
+    values = [data.aws_vpc.lab.id]
+  }
+}
+
+data "aws_vpc" "lab" {
+  filter {
+    name   = "tag:Name"
+    values = [var.vpc_name]
+  }
+}

aws/keycloak-flatcar-spike/iam.tf

@@ -0,0 +1,30 @@
+resource "aws_iam_role" "flatcar" {
+  assume_role_policy = local.flatcar_assume_role_policy
+  name               = var.iam_role_name
+
+  tags = merge(local.common_tags, {
+    Name = var.iam_role_name
+  })
+}
+
+resource "aws_iam_role_policy" "github_token_broker_invoke" {
+  name   = "${var.iam_role_name}-github-token-broker-invoke"
+  policy = local.github_token_broker_invoke_policy
+  role   = aws_iam_role.flatcar.id
+}
+
+resource "aws_iam_role_policy" "sops_keycloak_decrypt" {
+  name   = "${var.iam_role_name}-sops-keycloak-decrypt"
+  policy = local.sops_keycloak_decrypt_policy
+  role   = aws_iam_role.flatcar.id
+}
+
+resource "aws_iam_role_policy_attachment" "ssm_managed_instance_core" {
+  policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+  role       = aws_iam_role.flatcar.name
+}
+
+resource "aws_iam_instance_profile" "flatcar" {
+  name = var.iam_role_name
+  role = aws_iam_role.flatcar.name
+}

aws/keycloak-flatcar-spike/locals.tf

@@ -0,0 +1,98 @@
+locals {
+  common_tags = merge(var.tags, {
+    "glab:project" = "glab"
+    "glab:domain"  = "aws"
+    "glab:purpose" = "keycloak-flatcar-spike"
+  })
+
+  flatcar_assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Principal = {
+          Service = "ec2.amazonaws.com"
+        }
+      },
+    ]
+  })
+
+  github_token_broker_invoke_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = [
+          "lambda:InvokeFunction",
+        ]
+        Effect = "Allow"
+        Resource = [
+          data.aws_lambda_function.github_token_broker.arn,
+        ]
+        Sid = "AllowInvokeGitHubTokenBroker"
+      },
+    ]
+  })
+
+  sops_keycloak_decrypt_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = [
+          "kms:Decrypt",
+        ]
+        Condition = {
+          StringEquals = {
+            "kms:EncryptionContext:Repo"  = var.sops_kms_context_repo
+            "kms:EncryptionContext:Scope" = var.sops_kms_context_scope
+          }
+        }
+        Effect = "Allow"
+        Resource = [
+          var.sops_kms_key_arn,
+        ]
+        Sid = "AllowDecryptKeycloakSopsSecrets"
+      },
+    ]
+  })
+
+  bootstrap_unit_name = "glab-keycloak-bootstrap.service"
+
+  bootstrap_unit = <<-UNIT
+    [Unit]
+    Description=Fetch Keycloak bootstrap secrets with labctl
+    Wants=network-online.target
+    After=network-online.target docker.service
+    Requires=docker.service
+
+    [Service]
+    Type=oneshot
+    RemainAfterExit=yes
+    ExecStartPre=/usr/bin/mkdir -p ${var.bootstrap_runtime_dir}
+    ExecStartPre=/usr/bin/chmod 0700 ${var.bootstrap_runtime_dir}
+    ExecStartPre=/usr/bin/rm -f ${var.bootstrap_output_path}
+    ExecStart=/usr/bin/docker run --rm --network host --user 0:0 -v ${var.bootstrap_runtime_dir}:${var.bootstrap_runtime_dir} ${var.labctl_image} secrets get ${var.bootstrap_secret_path} --source github --field ${var.bootstrap_field} --output ${var.bootstrap_output_path} --aws-region ${var.aws_region} --broker-function ${var.github_token_broker_function_name}
+
+    [Install]
+    WantedBy=multi-user.target
+  UNIT
+
+  ignition_config = jsonencode({
+    ignition = {
+      version = "3.3.0"
+    }
+    systemd = {
+      units = [
+        {
+          enabled = true
+          name    = "amazon-ssm-agent.service"
+        },
+        {
+          contents = local.bootstrap_unit
+          enabled  = true
+          name     = local.bootstrap_unit_name
+        },
+      ]
+    }
+  })
+}

aws/keycloak-flatcar-spike/moon.yml

@@ -0,0 +1,24 @@
+layer: 'application'
+tags:
+  - 'aws'
+  - 'tofu'
+
+project:
+  title: 'AWS Keycloak Flatcar Spike'
+  description: 'Temporary OpenTofu stack for proving Flatcar Keycloak bootstrap with labctl.'
+  owner: 'GilmanLab'
+  maintainers:
+    - 'josh'
+
+tasks:
+  check:
+    command: './scripts/check.sh'
+    toolchains: 'system'
+    inputs:
+      - '*.tf'
+      - '.terraform.lock.hcl'
+      - 'scripts/check.sh'
+      - 'tests/**/*.tftest.hcl'
+    options:
+      cache: false
+      runInCI: true

aws/keycloak-flatcar-spike/network.tf

@@ -0,0 +1,21 @@
+resource "aws_security_group" "flatcar" {
+  description = "Outbound-only security group for the temporary Flatcar Keycloak spike."
+  name        = var.security_group_name
+  vpc_id      = data.aws_vpc.lab.id
+
+  tags = merge(local.common_tags, {
+    Name = var.security_group_name
+  })
+}
+
+resource "aws_vpc_security_group_egress_rule" "flatcar_ipv4" {
+  cidr_ipv4         = "0.0.0.0/0"
+  ip_protocol       = "-1"
+  security_group_id = aws_security_group.flatcar.id
+}
+
+resource "aws_vpc_security_group_egress_rule" "flatcar_ipv6" {
+  cidr_ipv6         = "::/0"
+  ip_protocol       = "-1"
+  security_group_id = aws_security_group.flatcar.id
+}

aws/keycloak-flatcar-spike/outputs.tf

@@ -0,0 +1,34 @@
+output "bootstrap_unit_name" {
+  description = "Systemd unit that fetches the Keycloak bootstrap secret."
+  value       = local.bootstrap_unit_name
+}
+
+output "github_token_broker_function_arn" {
+  description = "ARN of the GitHub token broker Lambda invoked by the spike instance."
+  value       = data.aws_lambda_function.github_token_broker.arn
+}
+
+output "iam_role_arn" {
+  description = "IAM role ARN used by the Flatcar spike instance."
+  value       = aws_iam_role.flatcar.arn
+}
+
+output "instance_id" {
+  description = "EC2 instance ID of the Flatcar spike host."
+  value       = aws_instance.flatcar.id
+}
+
+output "private_ip" {
+  description = "Private IPv4 address of the Flatcar spike host."
+  value       = aws_instance.flatcar.private_ip
+}
+
+output "public_ip" {
+  description = "Public IPv4 address associated with the Flatcar spike host for outbound bootstrap traffic."
+  value       = aws_instance.flatcar.public_ip
+}
+
+output "security_group_id" {
+  description = "Security group attached to the Flatcar spike host."
+  value       = aws_security_group.flatcar.id
+}

aws/keycloak-flatcar-spike/providers.tf

@@ -0,0 +1,10 @@
+provider "aws" {
+  region = var.aws_region
+
+  default_tags {
+    tags = {
+      "glab:managed-by" = "tofu"
+      "glab:stack"      = "aws/keycloak-flatcar-spike"
+    }
+  }
+}

aws/keycloak-flatcar-spike/scripts/check.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+TF_DATA_DIR="$(mktemp -d)"
+trap 'rm -rf "$TF_DATA_DIR"' EXIT
+export TF_DATA_DIR
+
+tofu fmt -check -recursive
+tofu init -backend=false -input=false
+tofu validate
+tofu test