Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
{{- if and .Values.spiffe .Values.spiffe.enabled .Values.spiffe.operatorAuth .Values.spiffe.operatorAuth.enabled }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: operator-spiffe-helper-config
namespace: {{ include "kagenti-operator.namespace" . }}
labels:
{{- include "kagenti-operator.labels" . | nindent 4 }}
data:
config.hcl: |
agent_address = "/spiffe-workload-api/spire-agent.sock"
cmd = "/bin/chmod"
cmd_args = "644,/opt/jwt_svid.token"
renew_signal = ""
cert_dir = ""
svid_file_name = ""
svid_bundle_file_name = ""
# jwt_audience MUST match Keycloak's realm issuer URL exactly.
# The JWT-SVID's aud claim must match the "iss" claim in tokens issued by Keycloak.
# Use the public/external URL, not internal K8s service names, as Keycloak's issuer
# typically uses the external hostname.
jwt_svids = [{
jwt_audience = "{{ .Values.spiffe.operatorAuth.jwtAudience | default (printf "%s/realms/%s" .Values.keycloak.publicUrl .Values.keycloak.realm) }}"
jwt_svid_file_name = "/opt/jwt_svid.token"
}]
{{- end }}
51 changes: 50 additions & 1 deletion charts/kagenti-operator/templates/manager/manager.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,11 @@ spec:
- "--credential-wait-timeout={{ .Values.authbridgeConfig.credentialWaitTimeout }}"
{{- end }}
{{- end }}
{{- if and .Values.spiffe .Values.spiffe.enabled .Values.spiffe.operatorAuth .Values.spiffe.operatorAuth.enabled }}
- "--use-spiffe-auth=true"
- "--jwt-svid-path={{ .Values.spiffe.operatorAuth.jwtSVIDPath | default "/opt/jwt_svid.token" }}"
- "--operator-client-id=spiffe://{{ .Values.signatureVerification.spireTrustDomain | default "localtest.me" }}/ns/{{ .Release.Namespace }}/sa/{{ .Values.controllerManager.serviceAccountName }}"
{{- end }}
command:
- {{ .Values.controllerManager.container.cmd }}
image: {{ .Values.controllerManager.container.image.repository }}:{{ .Values.controllerManager.container.image.tag }}
Expand Down Expand Up @@ -150,6 +155,42 @@ spec:
mountPath: /spiffe-workload-api
readOnly: true
{{- end }}
{{- if and .Values.spiffe .Values.spiffe.enabled .Values.spiffe.operatorAuth .Values.spiffe.operatorAuth.enabled }}
- name: jwt-svid
mountPath: /opt
readOnly: true
{{- end }}
{{- if and .Values.spiffe .Values.spiffe.enabled .Values.spiffe.operatorAuth .Values.spiffe.operatorAuth.enabled }}
- name: spiffe-helper
image: ghcr.io/kagenti/kagenti-extensions/spiffe-helper:latest
imagePullPolicy: IfNotPresent
args:
- "-config"
- "/etc/spiffe-helper/config.hcl"
volumeMounts:
- name: spiffe-workload-api
mountPath: /spiffe-workload-api
readOnly: true
- name: spiffe-helper-config
mountPath: /etc/spiffe-helper
readOnly: true
- name: jwt-svid
mountPath: /opt
securityContext:
allowPrivilegeEscalation: false
runAsNonRoot: true
runAsUser: 1000
capabilities:
drop:
- ALL
resources:
requests:
cpu: 10m
memory: 32Mi
limits:
cpu: 100m
memory: 64Mi
{{- end }}
securityContext:
{{- toYaml .Values.controllerManager.securityContext | nindent 8 }}
serviceAccountName: {{ .Values.controllerManager.serviceAccountName }}
Expand All @@ -173,9 +214,17 @@ spec:
secret:
secretName: kagenti-operator-metrics-server-cert
{{- end }}
{{- if .Values.verifiedFetch.enabled }}
{{- if or .Values.verifiedFetch.enabled (and .Values.spiffe .Values.spiffe.enabled .Values.spiffe.operatorAuth .Values.spiffe.operatorAuth.enabled) }}
- name: spiffe-workload-api
csi:
driver: "csi.spiffe.io"
readOnly: true
{{- end }}
{{- if and .Values.spiffe .Values.spiffe.enabled .Values.spiffe.operatorAuth .Values.spiffe.operatorAuth.enabled }}
- name: spiffe-helper-config
configMap:
name: operator-spiffe-helper-config
- name: jwt-svid
emptyDir:
medium: Memory
{{- end }}
14 changes: 14 additions & 0 deletions charts/kagenti-operator/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,20 @@ signatureVerification:
# How far before SVID expiry to trigger proactive workload restart
svidExpiryGracePeriod: "30m"

# [SPIFFE OPERATOR AUTH]: Operator authentication to Keycloak using JWT-SVID
# When enabled, the operator uses its SPIFFE identity (JWT-SVID) to authenticate
# to Keycloak instead of admin credentials. Requires Keycloak SPIFFE IdP configured.
spiffe:
enabled: false
operatorAuth:
enabled: false
# JWT audience must match Keycloak's realm issuer URL exactly.
# If not set, defaults to: {{ .Values.keycloak.publicUrl }}/realms/{{ .Values.keycloak.realm }}
# Check Keycloak's .well-known/openid-configuration for the correct issuer value.
jwtAudience: ""
# Path to JWT-SVID file written by spiffe-helper sidecar
jwtSVIDPath: "/opt/jwt_svid.token"

# Feature gates — highest-priority layer in the injection precedence chain.
# Set globalEnabled to false to disable ALL sidecar injection (kill switch).
# Set individual gates to false to disable specific sidecars cluster-wide.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ package controller
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"time"

Expand All @@ -20,6 +22,7 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/retry"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
Expand Down Expand Up @@ -72,7 +75,24 @@ type ClientRegistrationReconciler struct {
SpireTrustDomain string
// KeycloakAdminTokenCache caches admin password-grant tokens by Keycloak URL and credentials to
// avoid a token request on every reconcile. If nil, PasswordGrantToken is used without caching.
// Only used when UseSpiffeAuth is false.
KeycloakAdminTokenCache *keycloak.CachedAdminTokenProvider

// UseSpiffeAuth enables JWT-SVID authentication instead of admin credentials.
// When true, the operator authenticates to Keycloak with its JWT-SVID and uses
// the Admin API with manage-clients role. When false, uses admin credentials.
UseSpiffeAuth bool

// JWTSVIDPath is the file path to read the operator's JWT-SVID from.
// Only used when UseSpiffeAuth is true. Default: /opt/jwt_svid.token
JWTSVIDPath string

// OperatorClientID is the operator's SPIFFE ID (e.g., spiffe://localtest.me/ns/kagenti-operator-system/sa/...).
// Only used when UseSpiffeAuth is true.
OperatorClientID string

// Recorder emits Kubernetes Events to surface configuration issues visible in kubectl describe.
Recorder record.EventRecorder
}

func (r *ClientRegistrationReconciler) uncachedReader() client.Reader {
Expand Down Expand Up @@ -228,19 +248,6 @@ func (r *ClientRegistrationReconciler) reconcileOne(
return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
}

adminUser, adminPass, err := r.resolveKeycloakAdminCredentials(ctx)
if err != nil {
if apierrors.IsNotFound(err) {
logger.Info("waiting for Keycloak admin secret")
return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
}
return ctrl.Result{}, err
}
if adminUser == "" || adminPass == "" {
logger.Info("Keycloak admin secret missing credentials")
return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
}

spireEnabled := strings.EqualFold(strings.TrimSpace(ab.SpireEnabled), "true")
clientName := ns + "/" + workloadName
clientID, err := resolveKeycloakClientID(ns, workloadName, template.Spec.ServiceAccountName, spireEnabled, r.SpireTrustDomain)
Expand All @@ -258,14 +265,84 @@ func (r *ClientRegistrationReconciler) reconcileOne(

kc := keycloak.Admin{BaseURL: ab.KeycloakURL, HTTPClient: keycloak.DefaultHTTPClient()}
var token string
if r.KeycloakAdminTokenCache != nil {
token, err = r.KeycloakAdminTokenCache.Token(ctx, &kc, adminUser, adminPass)

// Authenticate to Keycloak: use JWT-SVID if enabled, otherwise admin credentials
if r.UseSpiffeAuth {
// SPIFFE authentication path
// Validate OperatorClientID before file I/O to fail fast on misconfiguration
if r.OperatorClientID == "" {
err := fmt.Errorf("OperatorClientID not configured")
logger.Error(err, "SPIFFE auth requires OperatorClientID")
if r.Recorder != nil {
r.Recorder.Event(owner, corev1.EventTypeWarning, "OperatorClientIDMissing",
"UseSpiffeAuth=true but OperatorClientID is empty. Check operator configuration.")
}
return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
}

jwtSVIDPath := r.JWTSVIDPath
if jwtSVIDPath == "" {
jwtSVIDPath = "/opt/jwt_svid.token"
}

// Path traversal protection: only allow reading from designated directories
cleanPath := filepath.Clean(jwtSVIDPath)
if !strings.HasPrefix(cleanPath, "/opt/") && !strings.HasPrefix(cleanPath, "/var/run/secrets/") {
err := fmt.Errorf("JWT-SVID path %q outside allowed directories (/opt/, /var/run/secrets/)", jwtSVIDPath)
logger.Error(err, "invalid JWT-SVID path")
if r.Recorder != nil {
r.Recorder.Eventf(owner, corev1.EventTypeWarning, "InvalidJWTSVIDPath",
"JWT-SVID path %q rejected: must be under /opt/ or /var/run/secrets/", jwtSVIDPath)
}
return ctrl.Result{}, err // fail permanently on config error
}

jwtSVID, err := os.ReadFile(cleanPath)
if err != nil {
logger.Error(err, "read JWT-SVID failed", "path", cleanPath)
if r.Recorder != nil {
r.Recorder.Eventf(owner, corev1.EventTypeWarning, "JWTSVIDReadFailed",
"Failed to read JWT-SVID from %s: %v. Check spiffe-helper sidecar configuration.", cleanPath, err)
}
return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
}

// WARNING: JWT-SVID is a bearer token - must never appear in logs or error messages
// to prevent token exposure. All code paths must handle jwtSVID as sensitive data.
token, err = kc.JWTSVIDGrantToken(ctx, ab.KeycloakRealm, r.OperatorClientID, string(jwtSVID))

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[MUST-FIX] JWT-SVID token exposure risk

The JWT-SVID is a bearer token and must never appear in logs. If JWTSVIDGrantToken() fails or debug logging is enabled, the token could leak.

Fix: Add a code comment and ensure error messages don't include the token:

// WARNING: jwtSVID is a bearer token - never log its contents
token, err = kc.JWTSVIDGrantToken(ctx, ab.KeycloakRealm, r.OperatorClientID, string(jwtSVID))
if err != nil {
    // Do NOT include jwtSVID in error context
    logger.Error(err, "Keycloak JWT-SVID authentication failed", 
        "realm", ab.KeycloakRealm, "clientId", r.OperatorClientID)
    return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
}

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[SUGGESTION] Add basic JWT format validation

Validate the JWT-SVID has the expected structure before sending to Keycloak. This provides faster feedback for malformed tokens.

Fix:

import "bytes"

// Basic JWT format check (header.payload.signature)
if bytes.Count(jwtSVID, []byte{'.'}) != 2 {
    logger.Error(nil, "invalid JWT-SVID format", "path", jwtSVIDPath)
    return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
}

if err != nil {
logger.Error(err, "Keycloak JWT-SVID authentication failed")
if r.Recorder != nil {
r.Recorder.Event(owner, corev1.EventTypeWarning, "KeycloakAuthFailed",
"Failed to authenticate to Keycloak with JWT-SVID. Check SPIFFE IdP configuration in Keycloak.")
}
return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
}
logger.V(1).Info("authenticated with JWT-SVID", "clientId", r.OperatorClientID)
} else {
token, err = kc.PasswordGrantToken(ctx, adminUser, adminPass)
}
if err != nil {
logger.Error(err, "Keycloak admin token failed")
return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
// Admin credentials path (legacy)
adminUser, adminPass, err := r.resolveKeycloakAdminCredentials(ctx)
if err != nil {
if apierrors.IsNotFound(err) {
logger.Info("waiting for Keycloak admin secret")
return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
}
return ctrl.Result{}, err
}
if adminUser == "" || adminPass == "" {
logger.Info("Keycloak admin secret missing credentials")
return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
}
if r.KeycloakAdminTokenCache != nil {
token, err = r.KeycloakAdminTokenCache.Token(ctx, &kc, adminUser, adminPass)
} else {
token, err = kc.PasswordGrantToken(ctx, adminUser, adminPass)
}
if err != nil {
logger.Error(err, "Keycloak admin token failed")
return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
}
logger.V(1).Info("authenticated with admin credentials")
}
agentClientUUID, clientSecret, err := kc.RegisterOrFetchClientWithToken(ctx, token, keycloak.ClientRegistrationParams{
Realm: ab.KeycloakRealm,
Expand Down
42 changes: 42 additions & 0 deletions kagenti-operator/internal/keycloak/admin.go
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,48 @@ func (a *Admin) PasswordGrantToken(ctx context.Context, adminUser, adminPass str
return token, err
}

// JWTSVIDGrantToken authenticates using JWT-SVID and returns an access token.
// The clientID must be the operator's SPIFFE ID (e.g., spiffe://localtest.me/ns/kagenti-operator-system/sa/...).
// The operator client must be configured in Keycloak with:
// - clientAuthenticatorType: "federated-jwt"
// - attributes.jwt.credential.issuer: SPIFFE IdP alias
// - attributes.jwt.credential.sub: operator's SPIFFE ID
// - Service account with manage-clients role
func (a *Admin) JWTSVIDGrantToken(ctx context.Context, realm, clientID, jwtSVID string) (string, error) {
base := trimBaseURL(a.BaseURL)
form := url.Values{}
form.Set("grant_type", "client_credentials")
form.Set("client_id", clientID)
form.Set("client_assertion_type", "urn:ietf:params:oauth:client-assertion-type:jwt-spiffe")
form.Set("client_assertion", jwtSVID)

req, err := http.NewRequestWithContext(ctx, http.MethodPost,
base+"/realms/"+url.PathEscape(realm)+"/protocol/openid-connect/token",
strings.NewReader(form.Encode()))
if err != nil {
return "", err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

resp, err := a.httpc().Do(req)
if err != nil {
return "", err
}
defer func() { _ = resp.Body.Close() }()
body, _ := io.ReadAll(resp.Body)
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("keycloak JWT-SVID token: status %d: %s", resp.StatusCode, truncate(body, 512))
}
var tr adminTokenResponse
if err := json.Unmarshal(body, &tr); err != nil {
return "", fmt.Errorf("keycloak JWT-SVID token decode: %w", err)
}
if tr.AccessToken == "" {
return "", fmt.Errorf("keycloak JWT-SVID token: empty access_token")
}
return tr.AccessToken, nil
}

// RegisterOrFetchClient ensures an OAuth client exists and returns its internal UUID and client secret value.
func (a *Admin) RegisterOrFetchClient(ctx context.Context, adminUser, adminPass string, p ClientRegistrationParams) (internalID, secret string, err error) {
token, _, err := a.adminToken(ctx, adminUser, adminPass)
Expand Down
Loading