api: real actuator health probes + raise k8s startup budget

platform/cluster/flux/apps/stateless/api/deployment.yaml

@@ -210,30 +210,52 @@ spec:
           ports:
             - containerPort: 8080
               name: http
-          # All probes are tcpSocket rather than httpGet because
-          # SecurityConfig.kt registers Spring's HttpsRedirectFilter in
-          # production (`app.security.require-https=true`, enforced by
-          # ProductionSecurityHardeningGuard which fails startup if you
-          # set it to false outside dev/test). The filter throws HTTP
-          # 500 on any in-cluster HTTP request whose port has no
-          # corresponding HTTPS port mapping — the kubelet probe is
-          # exactly that. tcpSocket avoids the filter entirely; the
-          # probe just checks Tomcat is bound to 8080. Edge HTTP
-          # behaviour is unaffected (Traefik terminates TLS and
-          # forwards X-Forwarded-Proto).
+            # Spring Boot actuator runs on a separate management port
+            # (management.server.port=8081 in application.yaml). Probes
+            # target this port so they bypass the main filter chain's
+            # HTTPS redirect entirely; SecurityConfig.actuatorChain
+            # additionally permits unauthenticated access to /actuator/**.
+            - containerPort: 8081
+              name: management
+          # Real HTTP health probes against Spring Boot's
+          # /actuator/health/{liveness,readiness} group endpoints.
+          # Tomcat binding alone (the old tcpSocket probe) said nothing
+          # about whether Flyway, Hibernate, Vault and the OAuth2
+          # Authorization Server had finished initializing — the api
+          # was being marked Ready while in-flight requests still hit
+          # half-initialized beans. The actuator probes are wired to
+          # Spring's AvailabilityState, which only flips to
+          # ACCEPTING_TRAFFIC after the ApplicationStartedEvent fires
+          # and all HealthIndicators (db, diskSpace, ping, …) report UP.
+          #
+          # startupProbe is generous on purpose: cold boot with Flyway
+          # migrations, Hibernate metadata scan, and Vault Agent
+          # sidecar handshake can take several minutes on first deploy
+          # of a new image, and we'd rather Kubernetes wait than
+          # crashloop a healthy-but-slow pod. failureThreshold * period
+          # = 5s * 180 = 15 minutes before the kubelet gives up.
+          # Once startup optimisation work lands, this can be tightened.
           startupProbe:
-            tcpSocket:
-              port: http
+            httpGet:
+              path: /actuator/health/readiness
+              port: management
             periodSeconds: 5
-            failureThreshold: 60
+            failureThreshold: 180
+            timeoutSeconds: 3
           readinessProbe:
-            tcpSocket:
-              port: http
+            httpGet:
+              path: /actuator/health/readiness
+              port: management
+            periodSeconds: 10
             timeoutSeconds: 5
+            failureThreshold: 3
           livenessProbe:
-            tcpSocket:
-              port: http
+            httpGet:
+              path: /actuator/health/liveness
+              port: management
+            periodSeconds: 20
             timeoutSeconds: 5
+            failureThreshold: 6
           resources:
             requests:
               cpu: 500m

services/api/src/main/kotlin/net/blueshell/api/platform/config/SecurityConfig.kt

@@ -7,6 +7,7 @@ import net.blueshell.api.infrastructure.security.permission.CompositePermissionE
 import net.blueshell.api.shared.enums.Role
 import org.springframework.beans.factory.ObjectProvider
 import org.springframework.beans.factory.annotation.Value
+import org.springframework.boot.security.autoconfigure.actuate.web.servlet.EndpointRequest
 import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
@@ -88,6 +89,21 @@ class SecurityConfig(
         return tokenRepository
     }
 
+    // Dedicated chain for Spring Boot actuator endpoints. Lives at @Order(0)
+    // so it runs before the @Order(3) authChain that calls redirectToHttps —
+    // kubelet probes speak plain HTTP, and a 302 to https from a permitAll
+    // path would still fail the probe. CSRF disabled (probes have no token)
+    // and anyRequest().permitAll() so in-cluster scrapers (kubelet,
+    // Prometheus, Gatus) can reach health/prometheus without a JWT.
+    @Bean
+    @Order(0)
+    fun actuatorChain(http: HttpSecurity): SecurityFilterChain {
+        http.securityMatcher(EndpointRequest.toAnyEndpoint())
+            .csrf { it.disable() }
+            .authorizeHttpRequests { it.anyRequest().permitAll() }
+        return http.build()
+    }
+
     @Bean
     @Order(3)
     fun authChain(

services/api/src/main/kotlin/net/blueshell/api/platform/web/MainController.kt

@@ -8,13 +8,14 @@ import org.springframework.web.bind.annotation.RestController
 @RestController
 @Tag(name = "Health")
 class MainController {
-    // `/health` is the single source of truth. The public Gatus probe at
+    // `/health` is the external-facing endpoint: the public Gatus probe at
     // `https://v2.esa-blueshell.nl/api/health` arrives here as `/health`
-    // because Traefik's apex `PathPrefix(/api)` rule now strips the
-    // `/api` prefix uniformly (see apps/edge/ingressroutes/api.yaml).
-    // The pod-level k8s probes hit port 8080 directly via tcpSocket
-    // (no path), so changing this controller's path no longer needs
-    // a lockstep manifest update.
+    // because Traefik's apex `PathPrefix(/api)` rule strips the `/api`
+    // prefix uniformly (see apps/edge/ingressroutes/api.yaml).
+    // Pod-level k8s probes use Spring Boot's
+    // /actuator/health/{liveness,readiness} on the management port (8081)
+    // instead, so they reflect real AvailabilityState rather than a
+    // hardcoded `true`.
     @GetMapping("/health")
     @PermitAll
     fun healthCheck(): Boolean {