From cedcd48554d5f8642851172d5a554b995a4fc07d Mon Sep 17 00:00:00 2001 From: "j.w.jonkers" Date: Mon, 11 May 2026 09:20:22 +0200 Subject: [PATCH] api: replace tcpSocket probes with real actuator health endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kubelet was marking the api Ready as soon as Tomcat bound port 8080, well before Flyway migrations, Hibernate metadata scan, Vault Agent handshake and the OAuth2 Authorization Server had finished initializing — in-flight requests could land on half-initialized beans. Switch liveness/readiness to httpGet probes against /actuator/health/{liveness,readiness} on the management port (8081), which only report UP once AvailabilityState and all HealthIndicators agree. A dedicated @Order(0) SecurityFilterChain matching EndpointRequest.toAnyEndpoint() permits unauthenticated access and bypasses the HTTPS redirect that breaks plain-HTTP kubelet probes. Startup budget bumped to 15 min (5s * 180) so slow cold boots aren't crashlooped while startup optimisation lands. --- .../flux/apps/stateless/api/deployment.yaml | 58 +++++++++++++------ .../api/platform/config/SecurityConfig.kt | 16 +++++ .../api/platform/web/MainController.kt | 13 +++-- 3 files changed, 63 insertions(+), 24 deletions(-) diff --git a/platform/cluster/flux/apps/stateless/api/deployment.yaml b/platform/cluster/flux/apps/stateless/api/deployment.yaml index 1b4c60a5a..67b409f4f 100644 --- a/platform/cluster/flux/apps/stateless/api/deployment.yaml +++ b/platform/cluster/flux/apps/stateless/api/deployment.yaml @@ -210,30 +210,52 @@ spec: ports: - containerPort: 8080 name: http - # All probes are tcpSocket rather than httpGet because - # SecurityConfig.kt registers Spring's HttpsRedirectFilter in - # production (`app.security.require-https=true`, enforced by - # ProductionSecurityHardeningGuard which fails startup if you - # set it to false outside dev/test). The filter throws HTTP - # 500 on any in-cluster HTTP request whose port has no - # corresponding HTTPS port mapping — the kubelet probe is - # exactly that. tcpSocket avoids the filter entirely; the - # probe just checks Tomcat is bound to 8080. Edge HTTP - # behaviour is unaffected (Traefik terminates TLS and - # forwards X-Forwarded-Proto). + # Spring Boot actuator runs on a separate management port + # (management.server.port=8081 in application.yaml). Probes + # target this port so they bypass the main filter chain's + # HTTPS redirect entirely; SecurityConfig.actuatorChain + # additionally permits unauthenticated access to /actuator/**. + - containerPort: 8081 + name: management + # Real HTTP health probes against Spring Boot's + # /actuator/health/{liveness,readiness} group endpoints. + # Tomcat binding alone (the old tcpSocket probe) said nothing + # about whether Flyway, Hibernate, Vault and the OAuth2 + # Authorization Server had finished initializing — the api + # was being marked Ready while in-flight requests still hit + # half-initialized beans. The actuator probes are wired to + # Spring's AvailabilityState, which only flips to + # ACCEPTING_TRAFFIC after the ApplicationStartedEvent fires + # and all HealthIndicators (db, diskSpace, ping, …) report UP. + # + # startupProbe is generous on purpose: cold boot with Flyway + # migrations, Hibernate metadata scan, and Vault Agent + # sidecar handshake can take several minutes on first deploy + # of a new image, and we'd rather Kubernetes wait than + # crashloop a healthy-but-slow pod. failureThreshold * period + # = 5s * 180 = 15 minutes before the kubelet gives up. + # Once startup optimisation work lands, this can be tightened. startupProbe: - tcpSocket: - port: http + httpGet: + path: /actuator/health/readiness + port: management periodSeconds: 5 - failureThreshold: 60 + failureThreshold: 180 + timeoutSeconds: 3 readinessProbe: - tcpSocket: - port: http + httpGet: + path: /actuator/health/readiness + port: management + periodSeconds: 10 timeoutSeconds: 5 + failureThreshold: 3 livenessProbe: - tcpSocket: - port: http + httpGet: + path: /actuator/health/liveness + port: management + periodSeconds: 20 timeoutSeconds: 5 + failureThreshold: 6 resources: requests: cpu: 500m diff --git a/services/api/src/main/kotlin/net/blueshell/api/platform/config/SecurityConfig.kt b/services/api/src/main/kotlin/net/blueshell/api/platform/config/SecurityConfig.kt index 5aab82ee2..0c4d5073e 100644 --- a/services/api/src/main/kotlin/net/blueshell/api/platform/config/SecurityConfig.kt +++ b/services/api/src/main/kotlin/net/blueshell/api/platform/config/SecurityConfig.kt @@ -7,6 +7,7 @@ import net.blueshell.api.infrastructure.security.permission.CompositePermissionE import net.blueshell.api.shared.enums.Role import org.springframework.beans.factory.ObjectProvider import org.springframework.beans.factory.annotation.Value +import org.springframework.boot.security.autoconfigure.actuate.web.servlet.EndpointRequest import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @@ -88,6 +89,21 @@ class SecurityConfig( return tokenRepository } + // Dedicated chain for Spring Boot actuator endpoints. Lives at @Order(0) + // so it runs before the @Order(3) authChain that calls redirectToHttps — + // kubelet probes speak plain HTTP, and a 302 to https from a permitAll + // path would still fail the probe. CSRF disabled (probes have no token) + // and anyRequest().permitAll() so in-cluster scrapers (kubelet, + // Prometheus, Gatus) can reach health/prometheus without a JWT. + @Bean + @Order(0) + fun actuatorChain(http: HttpSecurity): SecurityFilterChain { + http.securityMatcher(EndpointRequest.toAnyEndpoint()) + .csrf { it.disable() } + .authorizeHttpRequests { it.anyRequest().permitAll() } + return http.build() + } + @Bean @Order(3) fun authChain( diff --git a/services/api/src/main/kotlin/net/blueshell/api/platform/web/MainController.kt b/services/api/src/main/kotlin/net/blueshell/api/platform/web/MainController.kt index c0555ef34..90454616c 100644 --- a/services/api/src/main/kotlin/net/blueshell/api/platform/web/MainController.kt +++ b/services/api/src/main/kotlin/net/blueshell/api/platform/web/MainController.kt @@ -8,13 +8,14 @@ import org.springframework.web.bind.annotation.RestController @RestController @Tag(name = "Health") class MainController { - // `/health` is the single source of truth. The public Gatus probe at + // `/health` is the external-facing endpoint: the public Gatus probe at // `https://v2.esa-blueshell.nl/api/health` arrives here as `/health` - // because Traefik's apex `PathPrefix(/api)` rule now strips the - // `/api` prefix uniformly (see apps/edge/ingressroutes/api.yaml). - // The pod-level k8s probes hit port 8080 directly via tcpSocket - // (no path), so changing this controller's path no longer needs - // a lockstep manifest update. + // because Traefik's apex `PathPrefix(/api)` rule strips the `/api` + // prefix uniformly (see apps/edge/ingressroutes/api.yaml). + // Pod-level k8s probes use Spring Boot's + // /actuator/health/{liveness,readiness} on the management port (8081) + // instead, so they reflect real AvailabilityState rather than a + // hardcoded `true`. @GetMapping("/health") @PermitAll fun healthCheck(): Boolean {