Refactor Keycloak direct-access-grants into shared helper; harden eval port-forwards; add cluster/dry flags to ibac comparison

exgentic_a2a_runner/analyze-run.sh

@@ -27,10 +27,23 @@ WINDOW="${WINDOW:-3h}"
 MLFLOW_NAMESPACE="${MLFLOW_NAMESPACE:-}"
 MLFLOW_SERVICE="${MLFLOW_SERVICE:-}"
 MLFLOW_REMOTE_PORT="${MLFLOW_REMOTE_PORT:-}"
-MLFLOW_LOCAL_PORT="${MLFLOW_LOCAL_PORT:-8080}"
+# Local port for the MLflow port-forward. Must NOT be 8080: the kind ingress
+# serves keycloak.localtest.me (and other *.localtest.me hosts) on 8080, and
+# keycloak.localtest.me resolves to 127.0.0.1 — so binding the port-forward to
+# localhost:8080 would shadow Keycloak, and the secret-mode token request (a
+# password grant against Keycloak) would hit MLflow instead.
+MLFLOW_LOCAL_PORT="${MLFLOW_LOCAL_PORT:-8085}"
 MLFLOW_TLS="${MLFLOW_TLS:-}"
 MLFLOW_WORKSPACE="${MLFLOW_WORKSPACE:-}"
 AUTH_MODE="${AUTH_MODE:-}"
+# secret-mode (kind) auth: a password grant against the mlflow Keycloak client as
+# an MLflow user. MLflow's mlflow-oidc-auth authorizes reads from its own user DB
+# (where "admin" is seeded as a global admin), NOT from the token's Keycloak group
+# claim — so the user must be one MLflow knows. Defaults to admin; override with
+# MLFLOW_USER. The password defaults to the rossoctl-test-user secret in the
+# keycloak namespace (which holds admin's password); override with KEYCLOAK_PASSWORD.
+MLFLOW_USER="${MLFLOW_USER:-admin}"
+KEYCLOAK_PASSWORD="${KEYCLOAK_PASSWORD:-}"
 KUBECTL_BIN="${KUBECTL_BIN:-kubectl}"
 # `whoami -t` is an OpenShift (oc) extension, not a kubectl subcommand, so the
 # token command is separate from KUBECTL_BIN. Override with OC_BIN if needed.
@@ -43,6 +56,8 @@ EXPERIMENT_FILTER=""
 COMPARE_EXPERIMENTS=""
 CLUSTER_MODE=""
 INGRESS_DOMAIN=""
+# Directory to save the raw downloaded traces JSON into. Empty = don't save.
+SAVE_TRACES_DIR="${SAVE_TRACES_DIR:-}"
 
 usage() {
     cat << EOF
@@ -62,6 +77,7 @@ Options:
     --mlflow-tls               MLflow serves HTTPS on the forwarded port
     --mlflow-workspace NAME    Send x-mlflow-workspace header
     --auth-mode MODE           Token source: secret (rossoctl oauth secret) or oc-token (oc whoami -t)
+    --save-traces DIR          Save the raw downloaded traces JSON into DIR (created if needed)
     -h, --help                 Show this help message
 
 The MLflow location, TLS, workspace, auth mode, and experiment id all DEFAULT
@@ -89,10 +105,15 @@ Examples:
     $0 --openshift apps.mycluster.example.com
     $0 --openshift apps.mycluster.example.com --experiment-id 3 --compare baseline,test1
     $0 -u http://mlflow.localtest.me:8080 --window 2d
+    $0 --window 6h --save-traces ./traces
 EOF
     exit 1
 }
 
+# Capture the original invocation so the auth-failure hint can print the exact
+# command to re-run (the arg loop below consumes "$@" via shift).
+ORIGINAL_INVOCATION=("$0" "$@")
+
 while [[ $# -gt 0 ]]; do
     case $1 in
         -u|--url)           MLFLOW_URL="$2"; shift 2 ;;
@@ -106,6 +127,7 @@ while [[ $# -gt 0 ]]; do
         --mlflow-tls)       MLFLOW_TLS="true"; shift ;;
         --mlflow-workspace) MLFLOW_WORKSPACE="$2"; shift 2 ;;
         --auth-mode)        AUTH_MODE="$2"; shift 2 ;;
+        --save-traces|-save-traces) SAVE_TRACES_DIR="$2"; shift 2 ;;
         --kind)             CLUSTER_MODE="kind"; shift ;;
         --openshift)
             CLUSTER_MODE="openshift"
@@ -207,6 +229,15 @@ if ! WINDOW_MS=$(parse_window_ms "$WINDOW"); then
     exit 1
 fi
 
+# If --save-traces was given, make sure the target directory exists (create it
+# if needed) so the downloader's output can be written there.
+if [ -n "$SAVE_TRACES_DIR" ]; then
+    if ! mkdir -p "$SAVE_TRACES_DIR" 2>/dev/null; then
+        echo "Error: could not create traces directory '$SAVE_TRACES_DIR'"
+        exit 1
+    fi
+fi
+
 echo "=== MLflow Trace Analysis ==="
 echo "Cluster mode: $CLUSTER_MODE"
 if [ "$USE_PORT_FORWARD" = "true" ]; then
@@ -226,6 +257,9 @@ fi
 if [ -n "$COMPARE_EXPERIMENTS" ]; then
     echo "Comparing Experiments: $COMPARE_EXPERIMENTS"
 fi
+if [ -n "$SAVE_TRACES_DIR" ]; then
+    echo "Saving traces to: $SAVE_TRACES_DIR"
+fi
 echo ""
 
 # --- Verify kubectl points at the cluster matching CLUSTER_MODE ---
@@ -238,6 +272,14 @@ source "$SCRIPT_DIR/libsh/check-kubectl-context.sh"
 check_kubectl_context
 echo ""
 
+# urls.sh provides keycloak_api_url (CLUSTER_MODE must be exported, done above);
+# keycloak-direct-access.sh provides enable_direct_access_grants. Both are used
+# by the secret-mode token flow (password grant against the mlflow client).
+# shellcheck source=libsh/urls.sh
+source "$SCRIPT_DIR/libsh/urls.sh"
+# shellcheck source=libsh/keycloak-direct-access.sh
+source "$SCRIPT_DIR/libsh/keycloak-direct-access.sh"
+
 # --- Helper functions ---
 
 OAUTH_TOKEN=""
@@ -277,10 +319,21 @@ cleanup_port_forward() {
     fi
 }
 
-# secret mode: rossoctl's client-credentials flow. Reads mlflow-oauth-secret and
-# execs into the MLflow pod to exchange it for an access token.
+# secret mode: password (direct-access) grant against the mlflow Keycloak client.
+#
+# Why not client_credentials (the previous approach): that mints a token for the
+# mlflow *service account*, which mlflow-oidc-auth does not grant experiment reads
+# to, so the traces API returns 403. mlflow-oidc-auth authorizes from its own user
+# DB, where "admin" is seeded as a global admin — so we obtain a token for a real
+# MLflow user (default: admin) instead. The mlflow client already carries a groups
+# protocol mapper and is the confidential client MLflow trusts.
+#
+# The client id/secret come from mlflow-oauth-secret; the user password defaults to
+# the rossoctl-test-user secret (admin's password). The token endpoint is built from
+# keycloak_api_url (the OIDC_TOKEN_URL in the secret points at the in-cluster
+# Keycloak service, which is not reachable from the laptop).
 get_token_from_secret() {
-    echo "Obtaining OAuth token via mlflow-oauth-secret..."
+    echo "Obtaining OAuth token via password grant against the mlflow client..."
 
     # Note: under `set -e`, a failing command substitution aborts the script
     # before the following `if` can run. Capture status explicitly so the
@@ -293,36 +346,43 @@ get_token_from_secret() {
         return 1
     fi
 
-    local client_id client_secret token_url
+    local client_id client_secret
     client_id=$(echo "$secret_json" | jq -r '.data["OIDC_CLIENT_ID"]' | base64 -d) || true
     client_secret=$(echo "$secret_json" | jq -r '.data["OIDC_CLIENT_SECRET"]' | base64 -d) || true
-    token_url=$(echo "$secret_json" | jq -r '.data["OIDC_TOKEN_URL"]' | base64 -d) || true
-
-    if [ -z "$client_id" ] || [ -z "$client_secret" ] || [ -z "$token_url" ]; then
-        echo "Error: Could not extract OAuth credentials from secret"
+    if [ -z "$client_id" ] || [ -z "$client_secret" ]; then
+        echo "Error: Could not extract OIDC client id/secret from mlflow-oauth-secret"
         return 1
     fi
 
-    local mlflow_pod
-    mlflow_pod=$("$KUBECTL_BIN" get pod -n "$MLFLOW_NAMESPACE" -l app=mlflow -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) || true
-    if [ -z "$mlflow_pod" ]; then
-        echo "Error: Could not find MLflow pod"
+    # Resolve the MLflow user's password: explicit KEYCLOAK_PASSWORD wins, else the
+    # rossoctl-test-user secret (holds admin's password) in the keycloak namespace.
+    local user_password="$KEYCLOAK_PASSWORD"
+    if [ -z "$user_password" ]; then
+        user_password=$("$KUBECTL_BIN" get secret rossoctl-test-user -n keycloak \
+            -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || true)
+    fi
+    if [ -z "$user_password" ]; then
+        echo "Error: Could not resolve a password for MLflow user '$MLFLOW_USER'"
+        echo "Hint: set KEYCLOAK_PASSWORD, or confirm the rossoctl-test-user secret exists in the keycloak namespace"
         return 1
     fi
 
+    # The mlflow client needs Direct Access Grants enabled for the password grant.
+    local KEYCLOAK_API
+    KEYCLOAK_API="$(keycloak_api_url)"
+    export KEYCLOAK_API
+    enable_direct_access_grants "$client_id"
+
+    local token_url="$KEYCLOAK_API/realms/rossoctl/protocol/openid-connect/token"
+    echo "Requesting token for user '$MLFLOW_USER' (client '$client_id')..."
     local token_response
-    token_response=$("$KUBECTL_BIN" exec -n "$MLFLOW_NAMESPACE" "$mlflow_pod" -- \
-        python3 -c "
-import urllib.request, urllib.parse, json
-data = urllib.parse.urlencode({
-    'grant_type': 'client_credentials',
-    'client_id': '${client_id}',
-    'client_secret': '${client_secret}'
-}).encode()
-req = urllib.request.Request('${token_url}', data=data, headers={'Content-Type': 'application/x-www-form-urlencoded'})
-resp = urllib.request.urlopen(req)
-print(resp.read().decode())
-" 2>/dev/null) || true
+    token_response=$(curl -s -X POST "$token_url" \
+        -H "Content-Type: application/x-www-form-urlencoded" \
+        -d "grant_type=password" \
+        -d "client_id=${client_id}" \
+        -d "client_secret=${client_secret}" \
+        -d "username=${MLFLOW_USER}" \
+        -d "password=${user_password}" 2>/dev/null) || true
 
     OAUTH_TOKEN=$(echo "$token_response" | jq -r '.access_token' 2>/dev/null) || true
     if [ -z "$OAUTH_TOKEN" ] || [ "$OAUTH_TOKEN" = "null" ]; then
@@ -425,4 +485,41 @@ if [ -n "$COMPARE_EXPERIMENTS" ]; then
     PYTHON_ARGS="--compare"
 fi
 
-python3 "$SCRIPT_DIR/download_mlflow_traces.py" | python3 "$SCRIPT_DIR/analyze_traces.py" $PYTHON_ARGS
+# download_mlflow_traces.py exits 75 when MLflow rejects the token (a valid
+# token still gets 403 until the user has logged into the MLflow UI once, which
+# is what populates mlflow-oidc-auth's permission DB). Capture the downloader's
+# status via PIPESTATUS so we can print an actionable hint instead of a raw
+# HTTP 403 traceback.
+set +e
+if [ -n "$SAVE_TRACES_DIR" ]; then
+    # tee the downloader's stdout (the raw traces JSON) into a timestamped file
+    # in SAVE_TRACES_DIR before it is piped to the analyzer, so both the saved
+    # copy and the analysis come from the same download.
+    SAVE_TRACES_FILE="$SAVE_TRACES_DIR/traces-$(date +%Y%m%d-%H%M%S).json"
+    python3 "$SCRIPT_DIR/download_mlflow_traces.py" \
+        | tee "$SAVE_TRACES_FILE" \
+        | python3 "$SCRIPT_DIR/analyze_traces.py" $PYTHON_ARGS
+    DOWNLOAD_STATUS=${PIPESTATUS[0]}
+else
+    python3 "$SCRIPT_DIR/download_mlflow_traces.py" | python3 "$SCRIPT_DIR/analyze_traces.py" $PYTHON_ARGS
+    DOWNLOAD_STATUS=${PIPESTATUS[0]}
+fi
+set -e
+
+if [ -n "$SAVE_TRACES_DIR" ] && [ "$DOWNLOAD_STATUS" -eq 0 ]; then
+    echo ""
+    echo "✓ Saved traces to $SAVE_TRACES_FILE"
+fi
+
+if [ "$DOWNLOAD_STATUS" -eq 75 ]; then
+    echo ""
+    echo "MLflow authentication succeeded but access was denied."
+    echo "Log into the MLflow UI once (this registers your user with MLflow's"
+    echo "permission system), then re-run the analysis:"
+    echo ""
+    printf '    '; printf '%q ' "${ORIGINAL_INVOCATION[@]}"; echo
+    echo ""
+    exit 75
+fi
+
+exit "$DOWNLOAD_STATUS"

exgentic_a2a_runner/delete-all-deployments.sh

@@ -100,6 +100,8 @@ export CLUSTER_MODE INGRESS_DOMAIN
 source "$SCRIPT_DIR/libsh/urls.sh"
 # shellcheck source=libsh/check-kubectl-context.sh
 source "$SCRIPT_DIR/libsh/check-kubectl-context.sh"
+# shellcheck source=libsh/keycloak-direct-access.sh
+source "$SCRIPT_DIR/libsh/keycloak-direct-access.sh"
 check_kubectl_context
 
 ROSSOCTL_API="$(rossoctl_api_url)"
@@ -117,6 +119,10 @@ echo ""
 # Step 1: Get Keycloak authentication token
 echo "Step 1: Getting Keycloak authentication token..."
 
+# The token requests below use grant_type=password against the rossoctl client,
+# which requires Direct Access Grants to be enabled on that client.
+enable_direct_access_grants
+
 if [ "$KEYCLOAK_PASSWORD" = "unknown" ]; then
     echo "Step 1.5: Attempting to fetch Keycloak password from cluster..."
     ROSSOCTL_PASSWORD=$("$KUBECTL_BIN" get secret rossoctl-test-user -n keycloak -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || echo "")

exgentic_a2a_runner/deploy-agent.sh

@@ -240,6 +240,8 @@ source "$SCRIPT_DIR/libsh/urls.sh"
 KUBECTL_BIN="${KUBECTL_BIN:-kubectl}"
 # shellcheck source=libsh/check-kubectl-context.sh
 source "$SCRIPT_DIR/libsh/check-kubectl-context.sh"
+# shellcheck source=libsh/keycloak-direct-access.sh
+source "$SCRIPT_DIR/libsh/keycloak-direct-access.sh"
 check_kubectl_context
 
 ROSSOCTL_API="$(rossoctl_api_url)"
@@ -331,35 +333,7 @@ fi
 
 # Step 2: Enable Direct Access Grants for rossoctl client if needed
 echo "Step 2: Enabling Direct Access Grants for rossoctl client..."
-
-# Get admin token first (use "admin" password for master realm)
-ADMIN_TOKEN_RESPONSE=$(curl -s -X POST "$KEYCLOAK_API/realms/master/protocol/openid-connect/token" \
-    -H "Content-Type: application/x-www-form-urlencoded" \
-    -d "username=admin" \
-    -d "password=admin" \
-    -d "grant_type=password" \
-    -d "client_id=admin-cli" 2>/dev/null || echo "TOKEN_ERROR")
-
-if [ "$ADMIN_TOKEN_RESPONSE" != "TOKEN_ERROR" ]; then
-    ADMIN_TOKEN=$(echo "$ADMIN_TOKEN_RESPONSE" | grep -o '"access_token":"[^"]*"' | sed 's/"access_token":"\([^"]*\)"/\1/')
-
-    if [ -n "$ADMIN_TOKEN" ]; then
-        # Get rossoctl client configuration
-        CLIENT_CONFIG=$(curl -s "$KEYCLOAK_API/admin/realms/rossoctl/clients?clientId=rossoctl" \
-            -H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null)
-
-        CLIENT_ID=$(echo "$CLIENT_CONFIG" | grep -o '"id":"[^"]*"' | head -1 | sed 's/"id":"\([^"]*\)"/\1/')
-
-        if [ -n "$CLIENT_ID" ]; then
-            # Enable direct access grants
-            curl -s -X PUT "$KEYCLOAK_API/admin/realms/rossoctl/clients/$CLIENT_ID" \
-                -H "Authorization: Bearer $ADMIN_TOKEN" \
-                -H "Content-Type: application/json" \
-                -d '{"directAccessGrantsEnabled": true}' >/dev/null 2>&1
-            echo "✓ Direct access grants enabled for rossoctl client"
-        fi
-    fi
-fi
+enable_direct_access_grants
 
 echo ""
 

exgentic_a2a_runner/deploy-benchmark.sh

@@ -122,6 +122,8 @@ source "$SCRIPT_DIR_BENCH/libsh/urls.sh"
 KUBECTL_BIN="${KUBECTL_BIN:-kubectl}"
 # shellcheck source=libsh/check-kubectl-context.sh
 source "$SCRIPT_DIR_BENCH/libsh/check-kubectl-context.sh"
+# shellcheck source=libsh/keycloak-direct-access.sh
+source "$SCRIPT_DIR_BENCH/libsh/keycloak-direct-access.sh"
 check_kubectl_context
 
 # Default to Exgentic registry, can be overridden with environment variable
@@ -195,55 +197,7 @@ fi
 
 # Step 4: Enable Direct Access Grants for rossoctl client if needed
 echo "Step 4: Enabling Direct Access Grants for rossoctl client..."
-
-# Resolve master-realm admin credentials: prefer env vars, fall back to the
-# keycloak-initial-admin secret (RHBK operator), then defaults.
-KEYCLOAK_ADMIN_USERNAME="${KEYCLOAK_ADMIN_USERNAME:-}"
-KEYCLOAK_ADMIN_PASSWORD="${KEYCLOAK_ADMIN_PASSWORD:-}"
-if [ -z "$KEYCLOAK_ADMIN_USERNAME" ] || [ -z "$KEYCLOAK_ADMIN_PASSWORD" ]; then
-    KC_ADMIN_USERNAME=$(kubectl get secret keycloak-initial-admin -n keycloak \
-        -o jsonpath='{.data.username}' 2>/dev/null | base64 -d 2>/dev/null || true)
-    KC_ADMIN_PASSWORD=$(kubectl get secret keycloak-initial-admin -n keycloak \
-        -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null || true)
-    KEYCLOAK_ADMIN_USERNAME="${KEYCLOAK_ADMIN_USERNAME:-${KC_ADMIN_USERNAME:-admin}}"
-    KEYCLOAK_ADMIN_PASSWORD="${KEYCLOAK_ADMIN_PASSWORD:-${KC_ADMIN_PASSWORD:-admin}}"
-fi
-
-ADMIN_TOKEN_RESPONSE=$(curl -s -X POST "$KEYCLOAK_API/realms/master/protocol/openid-connect/token" \
-    -H "Content-Type: application/x-www-form-urlencoded" \
-    -d "username=${KEYCLOAK_ADMIN_USERNAME}" \
-    -d "password=${KEYCLOAK_ADMIN_PASSWORD}" \
-    -d "grant_type=password" \
-    -d "client_id=admin-cli" 2>/dev/null) || true
-
-ADMIN_TOKEN=$(echo "$ADMIN_TOKEN_RESPONSE" | grep -o '"access_token":"[^"]*"' | sed 's/"access_token":"\([^"]*\)"/\1/')
-if [ -z "$ADMIN_TOKEN" ]; then
-    echo "Error: Could not obtain master-realm admin token from Keycloak"
-    echo "  Response: $ADMIN_TOKEN_RESPONSE"
-    echo "  Set KEYCLOAK_ADMIN_PASSWORD in your .env if the master realm admin password is not 'admin'."
-    exit 1
-fi
-
-CLIENT_CONFIG=$(curl -s "$KEYCLOAK_API/admin/realms/rossoctl/clients?clientId=rossoctl" \
-    -H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null)
-CLIENT_ID=$(echo "$CLIENT_CONFIG" | grep -o '"id":"[^"]*"' | head -1 | sed 's/"id":"\([^"]*\)"/\1/')
-if [ -z "$CLIENT_ID" ]; then
-    echo "Error: Could not find rossoctl client ID in Keycloak"
-    echo "  Response: $CLIENT_CONFIG"
-    exit 1
-fi
-
-PUT_CODE=$(curl -s -o /tmp/kc_put_response.txt -w "%{http_code}" \
-    -X PUT "$KEYCLOAK_API/admin/realms/rossoctl/clients/$CLIENT_ID" \
-    -H "Authorization: Bearer $ADMIN_TOKEN" \
-    -H "Content-Type: application/json" \
-    -d '{"directAccessGrantsEnabled": true}' 2>/dev/null) || PUT_CODE="000"
-if [ "$PUT_CODE" != "204" ] && [ "$PUT_CODE" != "200" ]; then
-    echo "Error: Failed to enable direct access grants for rossoctl client (HTTP $PUT_CODE)"
-    echo "  Response: $(cat /tmp/kc_put_response.txt 2>/dev/null)"
-    exit 1
-fi
-echo "✓ Direct access grants enabled for rossoctl client"
+enable_direct_access_grants
 
 echo ""