GilmanLab · jmgilman · Apr 30, 2026 · Apr 30, 2026 · Apr 30, 2026
diff --git a/aws/keycloak/README.md b/aws/keycloak/README.md
@@ -13,9 +13,9 @@ This stack creates:
 - a GitHub token broker Lambda, sourced from `meigma/github-token-broker`, for short-lived `GilmanLab/secrets` access
 - Keycloak instance-role permission to invoke the token broker
 - Keycloak instance-role permission to decrypt only SOPS secrets with `Repo=GilmanLab/secrets` and `Scope=keycloak`
-- Ignition-managed systemd units for Postgres, Keycloak, Traefik, and bootstrap secret fetches
+- Ignition-managed systemd units for Postgres, Keycloak, Traefik, bootstrap secret fetches, and first-boot realm configuration
 
-This stack intentionally stops at starting Keycloak. It does **not** configure realms, GitHub OIDC, clients, backups, Synology sync, Kubernetes OIDC, Argo CD, Grafana, IAM Identity Center federation, or `keycloak-config-cli`.
+This stack configures only the initial `lab` realm and one local admin account with password plus WebAuthn/YubiKey enrollment. It does **not** configure GitHub OIDC, service clients, backups, Synology sync, Kubernetes OIDC, Argo CD, Grafana, or IAM Identity Center federation.
 
 Traefik obtains the `id.glab.lol` certificate from Let's Encrypt through DNS-01. Cloudflare delegates `_acme-challenge.id.glab.lol` to the public Route 53 `acme.glab.lol` zone, and the Keycloak instance role may mutate only the delegated TXT record for this hostname.
 
@@ -36,6 +36,24 @@ secrets get services/keycloak/bootstrap.sops.yaml \
 
 Plaintext bootstrap material is written only under `/run/glab/keycloak`. The persistent data volume stores Postgres state in `/var/lib/keycloak/postgres` and ACME state in `/var/lib/keycloak/acme`; the Postgres directory is owned by the container's Postgres UID.
 
+`glab-keycloak-config.service` runs once after Keycloak is healthy. It fetches
+`services/keycloak/admin.sops.yaml` through the same broker-backed `labctl`
+path, writes `/run/glab/keycloak/admin.env`, then runs pinned
+`keycloak-config-cli` to create the `lab` realm, local admin user, and
+touch-only WebAuthn policy. The service writes
+`/var/lib/keycloak/config/lab-realm-imported` after a successful import so it
+does not run again on reboot.
+
+After enrolling and validating the admin YubiKey in the browser, disable the
+temporary master bootstrap admin manually:
+
+```sh
+aws ssm send-command \
+  --document-name AWS-RunShellScript \
+  --targets Key=instanceids,Values="$(tofu output -raw instance_id)" \
+  --parameters commands='["systemctl start glab-keycloak-disable-bootstrap-admin.service"]'
+```
+
 ## Prerequisites
 
 - OpenTofu `>= 1.10`
@@ -50,6 +68,7 @@ Plaintext bootstrap material is written only under `/run/glab/keycloak`. The per
 - `gh` and `sha256sum` on the apply host so the broker module can download and verify the pinned release asset
 - Cloudflare delegates `acme.glab.lol` to the `aws/lab-foundation` `acme_zone_name_servers` output and sets `_acme-challenge.id.glab.lol` as a CNAME to `_acme-challenge.id.acme.glab.lol`
 - `secrets/services/keycloak/bootstrap.sops.yaml` exists in `GilmanLab/secrets` with SOPS KMS context `Repo=GilmanLab/secrets` and `Scope=keycloak`
+- `secrets/services/keycloak/admin.sops.yaml` exists in `GilmanLab/secrets` with SOPS KMS context `Repo=GilmanLab/secrets` and `Scope=keycloak`
 
 The expected local operator flow is to export both AWS values via `direnv`.
 
@@ -96,9 +115,12 @@ aws ssm send-command \
   --targets Key=instanceids,Values="$(tofu output -raw instance_id)" \
   --parameters commands='[
     "systemctl is-active glab-keycloak-bootstrap.service glab-keycloak-postgres.service glab-keycloak.service glab-keycloak-traefik.service",
+    "systemctl is-active glab-keycloak-config.service || systemctl status --no-pager glab-keycloak-config.service",
     "findmnt /var/lib/keycloak",
     "stat -c %a\\ %s\\ %n /run/glab/keycloak/stack.env",
+    "stat -c %a\\ %s\\ %n /run/glab/keycloak/admin.env",
     "cut -d= -f1 /run/glab/keycloak/stack.env | sort",
+    "cut -d= -f1 /run/glab/keycloak/admin.env | sort",
     "curl -fsS http://127.0.0.1:9000/health/ready"
   ]'
 ```
diff --git a/aws/keycloak/locals.tf b/aws/keycloak/locals.tf
@@ -13,19 +13,28 @@ locals {
   postgres_state_dir            = "${var.data_dir}/postgres"
   keycloak_env_path             = "${var.bootstrap_runtime_dir}/keycloak.env"
   helper_script_dir             = "${var.runtime_dir}/bin"
+  keycloak_config_dir           = "${var.data_dir}/config"
+  keycloak_config_marker_path   = "${local.keycloak_config_dir}/lab-realm-imported"
+  keycloak_realm_config_path    = "${var.bootstrap_runtime_dir}/lab-realm.json"
   data_volume_device_path       = "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_${replace(aws_ebs_volume.keycloak_data.id, "-", "")}"
 
   traefik_dynamic_config = templatefile("${path.module}/templates/traefik_dynamic.yml.tftpl", {
     private_hostname             = var.private_hostname
     traefik_certificate_resolver = local.traefik_certificate_resolver
   })
 
-  bootstrap_unit_name = "glab-keycloak-bootstrap.service"
-  data_unit_name      = "glab-keycloak-data.service"
-  network_unit_name   = "glab-keycloak-network.service"
-  postgres_unit_name  = "glab-keycloak-postgres.service"
-  keycloak_unit_name  = "glab-keycloak.service"
-  traefik_unit_name   = "glab-keycloak-traefik.service"
+  lab_realm_config = templatefile("${path.module}/templates/keycloak/lab-realm.json.tftpl", {
+    private_hostname = var.private_hostname
+  })
+
+  bootstrap_unit_name               = "glab-keycloak-bootstrap.service"
+  config_unit_name                  = "glab-keycloak-config.service"
+  data_unit_name                    = "glab-keycloak-data.service"
+  disable_bootstrap_admin_unit_name = "glab-keycloak-disable-bootstrap-admin.service"
+  network_unit_name                 = "glab-keycloak-network.service"
+  postgres_unit_name                = "glab-keycloak-postgres.service"
+  keycloak_unit_name                = "glab-keycloak.service"
+  traefik_unit_name                 = "glab-keycloak-traefik.service"
 
   runtime_template_vars = {
     acme_ca_server                    = var.acme_ca_server
@@ -36,14 +45,24 @@ locals {
     bootstrap_runtime_dir             = var.bootstrap_runtime_dir
     bootstrap_secret_path             = var.bootstrap_secret_path
     bootstrap_unit_name               = local.bootstrap_unit_name
+    config_field                      = var.config_field
+    config_output_path                = var.config_output_path
+    config_secret_path                = var.config_secret_path
+    config_unit_name                  = local.config_unit_name
     data_dir                          = var.data_dir
     data_unit_name                    = local.data_unit_name
     data_volume_device_path           = local.data_volume_device_path
     data_volume_label                 = var.data_volume_label
+    disable_bootstrap_admin_unit_name = local.disable_bootstrap_admin_unit_name
     github_token_broker_function_name = var.github_token_broker_function_name
     helper_script_dir                 = local.helper_script_dir
+    keycloak_config_cli_image         = var.keycloak_config_cli_image
+    keycloak_config_dir               = local.keycloak_config_dir
+    keycloak_config_marker_path       = local.keycloak_config_marker_path
     keycloak_env_path                 = local.keycloak_env_path
     keycloak_image                    = var.keycloak_image
+    keycloak_realm_config_path        = local.keycloak_realm_config_path
+    lab_realm_config_gzip_base64      = base64gzip(local.lab_realm_config)
     keycloak_unit_name                = local.keycloak_unit_name
     labctl_image                      = var.labctl_image
     network_unit_name                 = local.network_unit_name
@@ -66,13 +85,20 @@ locals {
   wait_postgres_script  = templatefile("${path.module}/templates/scripts/wait-postgres.sh.tftpl", local.runtime_template_vars)
   run_keycloak_script   = templatefile("${path.module}/templates/scripts/run-keycloak.sh.tftpl", local.runtime_template_vars)
   run_traefik_script    = templatefile("${path.module}/templates/scripts/run-traefik.sh.tftpl", local.runtime_template_vars)
+  run_config_script     = templatefile("${path.module}/templates/scripts/run-keycloak-config.sh.tftpl", local.runtime_template_vars)
+  disable_admin_script  = templatefile("${path.module}/templates/scripts/disable-bootstrap-admin.sh.tftpl", local.runtime_template_vars)
 
   data_unit      = templatefile("${path.module}/templates/systemd/${local.data_unit_name}.tftpl", local.runtime_template_vars)
   network_unit   = templatefile("${path.module}/templates/systemd/${local.network_unit_name}.tftpl", local.runtime_template_vars)
   bootstrap_unit = templatefile("${path.module}/templates/systemd/${local.bootstrap_unit_name}.tftpl", local.runtime_template_vars)
   postgres_unit  = templatefile("${path.module}/templates/systemd/${local.postgres_unit_name}.tftpl", local.runtime_template_vars)
   keycloak_unit  = templatefile("${path.module}/templates/systemd/${local.keycloak_unit_name}.tftpl", local.runtime_template_vars)
   traefik_unit   = templatefile("${path.module}/templates/systemd/${local.traefik_unit_name}.tftpl", local.runtime_template_vars)
+  config_unit    = templatefile("${path.module}/templates/systemd/${local.config_unit_name}.tftpl", local.runtime_template_vars)
+  disable_admin_unit = templatefile(
+    "${path.module}/templates/systemd/${local.disable_bootstrap_admin_unit_name}.tftpl",
+    local.runtime_template_vars,
+  )
 
   ignition_files = [
     {
@@ -124,9 +150,23 @@ locals {
         source = "data:text/plain;charset=utf-8;base64,${base64encode(local.run_traefik_script)}"
       }
     },
+    {
+      path = "${local.helper_script_dir}/run-keycloak-config.sh"
+      mode = 493
+      contents = {
+        source = "data:text/plain;charset=utf-8;base64,${base64encode(local.run_config_script)}"
+      }
+    },
+    {
+      path = "${local.helper_script_dir}/disable-bootstrap-admin.sh"
+      mode = 493
+      contents = {
+        source = "data:text/plain;charset=utf-8;base64,${base64encode(local.disable_admin_script)}"
+      }
+    },
   ]
 
-  ignition_config = jsonencode({
+  ignition_payload_config = jsonencode({
     ignition = {
       version = "3.3.0"
     }
@@ -178,12 +218,33 @@ locals {
           enabled  = true
           name     = local.keycloak_unit_name
         },
+        {
+          contents = local.config_unit
+          enabled  = true
+          name     = local.config_unit_name
+        },
         {
           contents = local.traefik_unit
           enabled  = true
           name     = local.traefik_unit_name
         },
+        {
+          contents = local.disable_admin_unit
+          name     = local.disable_bootstrap_admin_unit_name
+        },
       ]
     }
   })
+
+  ignition_config = jsonencode({
+    ignition = {
+      version = "3.3.0"
+      config = {
+        replace = {
+          source      = "data:application/vnd.coreos.ignition+json;base64,${base64gzip(local.ignition_payload_config)}"
+          compression = "gzip"
+        }
+      }
+    }
+  })
 }
diff --git a/aws/keycloak/templates/keycloak/lab-realm.json.tftpl b/aws/keycloak/templates/keycloak/lab-realm.json.tftpl
@@ -0,0 +1,137 @@
+{
+  "realm": "lab",
+  "enabled": true,
+  "displayName": "glab",
+  "registrationAllowed": false,
+  "resetPasswordAllowed": false,
+  "rememberMe": false,
+  "loginWithEmailAllowed": false,
+  "duplicateEmailsAllowed": false,
+  "verifyEmail": false,
+  "browserFlow": "glab browser",
+  "webAuthnPolicyRpEntityName": "glab",
+  "webAuthnPolicyRpId": "${private_hostname}",
+  "webAuthnPolicySignatureAlgorithms": [
+    "ES256",
+    "RS256"
+  ],
+  "webAuthnPolicyAttestationConveyancePreference": "none",
+  "webAuthnPolicyAuthenticatorAttachment": "not specified",
+  "webAuthnPolicyRequireResidentKey": "not specified",
+  "webAuthnPolicyUserVerificationRequirement": "discouraged",
+  "webAuthnPolicyCreateTimeout": 0,
+  "webAuthnPolicyAvoidSameAuthenticatorRegister": true,
+  "authenticationFlows": [
+    {
+      "alias": "glab browser",
+      "description": "Browser flow for local password plus WebAuthn when enrolled.",
+      "providerId": "basic-flow",
+      "topLevel": true,
+      "builtIn": false,
+      "authenticationExecutions": [
+        {
+          "authenticator": "auth-cookie",
+          "requirement": "ALTERNATIVE",
+          "priority": 10,
+          "userSetupAllowed": false,
+          "authenticatorFlow": false
+        },
+        {
+          "authenticator": "identity-provider-redirector",
+          "requirement": "ALTERNATIVE",
+          "priority": 20,
+          "userSetupAllowed": false,
+          "authenticatorFlow": false
+        },
+        {
+          "flowAlias": "glab browser forms",
+          "requirement": "ALTERNATIVE",
+          "priority": 30,
+          "userSetupAllowed": false,
+          "authenticatorFlow": true
+        }
+      ]
+    },
+    {
+      "alias": "glab browser forms",
+      "description": "Local password form with conditional WebAuthn second factor.",
+      "providerId": "basic-flow",
+      "topLevel": false,
+      "builtIn": false,
+      "authenticationExecutions": [
+        {
+          "authenticator": "auth-username-password-form",
+          "requirement": "REQUIRED",
+          "priority": 10,
+          "userSetupAllowed": false,
+          "authenticatorFlow": false
+        },
+        {
+          "flowAlias": "glab webauthn conditional",
+          "requirement": "CONDITIONAL",
+          "priority": 20,
+          "userSetupAllowed": false,
+          "authenticatorFlow": true
+        }
+      ]
+    },
+    {
+      "alias": "glab webauthn conditional",
+      "description": "Require WebAuthn when the user has a WebAuthn credential.",
+      "providerId": "basic-flow",
+      "topLevel": false,
+      "builtIn": false,
+      "authenticationExecutions": [
+        {
+          "authenticator": "conditional-user-configured",
+          "requirement": "REQUIRED",
+          "priority": 10,
+          "userSetupAllowed": false,
+          "authenticatorFlow": false
+        },
+        {
+          "authenticator": "webauthn-authenticator",
+          "requirement": "REQUIRED",
+          "priority": 20,
+          "userSetupAllowed": false,
+          "authenticatorFlow": false
+        }
+      ]
+    }
+  ],
+  "requiredActions": [
+    {
+      "alias": "webauthn-register",
+      "name": "WebAuthn Register",
+      "providerId": "webauthn-register",
+      "enabled": true,
+      "defaultAction": false,
+      "priority": 20,
+      "config": {}
+    }
+  ],
+  "users": [
+    {
+      "username": "$(env:KEYCLOAK_LOCAL_ADMIN_USERNAME)",
+      "email": "$(env:KEYCLOAK_LOCAL_ADMIN_EMAIL)",
+      "emailVerified": true,
+      "enabled": true,
+      "requiredActions": [
+        "webauthn-register"
+      ],
+      "credentials": [
+        {
+          "type": "password",
+          "userLabel": "initial",
+          "value": "$(env:KEYCLOAK_LOCAL_ADMIN_PASSWORD)",
+          "temporary": false
+        }
+      ],
+      "clientRoles": {
+        "realm-management": [
+          "realm-admin"
+        ]
+      }
+    }
+  ]
+}
diff --git a/aws/keycloak/templates/scripts/disable-bootstrap-admin.sh.tftpl b/aws/keycloak/templates/scripts/disable-bootstrap-admin.sh.tftpl
@@ -0,0 +1,50 @@
+#!/bin/sh
+set -eu
+
+. '${bootstrap_output_path}'
+
+deadline=$(( $(date +%s) + 300 ))
+until /usr/bin/curl -fsS http://127.0.0.1:9000/health/ready >/dev/null; do
+  if [ "$(date +%s)" -ge "$deadline" ]; then
+    echo 'Timed out waiting for Keycloak readiness.' >&2
+    exit 1
+  fi
+  sleep 5
+done
+
+/usr/bin/docker run --rm \
+  --network keycloak \
+  -e KC_BOOTSTRAP_ADMIN_USERNAME="$KC_BOOTSTRAP_ADMIN_USERNAME" \
+  -e KC_BOOTSTRAP_ADMIN_PASSWORD="$KC_BOOTSTRAP_ADMIN_PASSWORD" \
+  --entrypoint /bin/sh \
+  '${keycloak_image}' \
+  -c 'set -eu
+    /opt/keycloak/bin/kcadm.sh config credentials \
+      --server http://keycloak:8080 \
+      --realm master \
+      --user "$KC_BOOTSTRAP_ADMIN_USERNAME" \
+      --password "$KC_BOOTSTRAP_ADMIN_PASSWORD" >/dev/null
+
+    user_id="$(
+      /opt/keycloak/bin/kcadm.sh get users \
+        -r master \
+        -q username="$KC_BOOTSTRAP_ADMIN_USERNAME" \
+        --fields id \
+        --format csv \
+        --noquotes |
+      {
+        line=
+        IFS= read -r line || true
+        if [ "$line" = "id" ]; then
+          IFS= read -r line || true
+        fi
+        printf "%s" "$line"
+      }
+    )"
+    if [ -z "$user_id" ]; then
+      echo "Could not find bootstrap admin user." >&2
+      exit 1
+    fi
+
+    /opt/keycloak/bin/kcadm.sh update "users/$user_id" -r master -s enabled=false
+  '