[patch] Retry for rbac permissions when timeout

Author: unnati-solanki-git

src/mas/devops/tekton.py

@@ -20,7 +20,7 @@
 
 from kubeconfig import kubectl
 from openshift.dynamic import DynamicClient
-from openshift.dynamic.exceptions import NotFoundError, UnprocessibleEntityError
+from openshift.dynamic.exceptions import NotFoundError, UnprocessibleEntityError, ApiException
 
 from jinja2 import Environment, FileSystemLoader
 
@@ -1152,7 +1152,51 @@ def prepareInstallRBAC(dynClient: DynamicClient, namespace: str, instanceId: str
 
             logger.debug(f"Applying RBAC resource {kind}/{name} in namespace {namespace} for instance {instanceId}")
             resourceAPI = dynClient.resources.get(api_version=apiVersion, kind=kind)
-            if namespace:
-                resourceAPI.apply(body=resourceBody, namespace=namespace)
-            else:
-                resourceAPI.apply(body=resourceBody)
+
+            # Retry logic for transient API server errors (5 minute timeout)
+            max_retries = 30  # 30 retries with exponential backoff = ~5 minutes
+            retry_delay = 2  # Initial delay in seconds
+            max_delay = 30  # Maximum delay between retries
+
+            for attempt in range(max_retries):
+                try:
+                    if namespace:
+                        resourceAPI.apply(body=resourceBody, namespace=namespace)
+                    else:
+                        resourceAPI.apply(body=resourceBody)
+                    break  # Success, exit retry loop
+
+                except ApiException as e:
+                    # Check if it's a retryable error (429, 503, 504, or API server shutdown)
+                    is_retryable = (
+                        e.status in [429, 503, 504] or
+                        "apiserver is shutting down" in str(e).lower() or
+                        "connection refused" in str(e).lower() or
+                        "too many requests" in str(e).lower()
+                    )
+
+                    if is_retryable and attempt < max_retries - 1:
+                        # Exponential backoff with max delay cap
+                        wait_time = min(retry_delay * (2 ** attempt), max_delay)
+                        logger.warning(
+                            f"API server temporarily unavailable for {kind}/{name} "
+                            f"(attempt {attempt + 1}/{max_retries}). "
+                            f"Retrying in {wait_time}s... Error: {e.status} - {str(e)[:100]}"
+                        )
+                        sleep(wait_time)
+                    elif is_retryable:
+                        # Exhausted all retries
+                        logger.error(
+                            f"Failed to apply RBAC resource {kind}/{name} after {max_retries} attempts "
+                            f"(~5 minutes). API server may be unavailable."
+                        )
+                        raise
+                    else:
+                        # Non-retryable error (permissions, invalid resource, etc.)
+                        logger.error(f"Failed to apply RBAC resource {kind}/{name}: {e}")
+                        raise
+
+                except Exception as e:
+                    # Catch any other unexpected errors
+                    logger.error(f"Unexpected error applying RBAC resource {kind}/{name}: {e}")
+                    raise