From d5bf46cd16d8f416150783744af0b3680c8e3cb4 Mon Sep 17 00:00:00 2001 From: Fabrizio Siciliano Date: Sat, 4 Jul 2026 12:20:21 +0200 Subject: [PATCH 1/7] Added password_expiry to user model, setup scheduled job run, started work on rotation manager --- litellm/constants.py | 14 ++ litellm/models/user.py | 1 + litellm/proxy/_types.py | 1 - .../user_credentials_rotation_manager.py | 169 ++++++++++++++++++ litellm/proxy/proxy_server.py | 45 +++++ 5 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 litellm/proxy/common_utils/user_credentials_rotation_manager.py diff --git a/litellm/constants.py b/litellm/constants.py index aeb74a65839..960697cae46 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1374,6 +1374,19 @@ # ``ProxyLogging._handle_logging_proxy_only_error``. LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL = "litellm_no_upstream_llm_call" +# User credentials Rotation Constants +LITELLM_USER_CREDENTIALS_ROTATION_ENABLED = os.getenv("LITELLM_USER_CREDENTIALS_ROTATION_ENABLED", false) +LITELLM_USER_CREDENTIALS_ROTATION_CHECK_INTERVAL_SECONDS = int( + os.getenv("LITELLM_USER_CREDENTIALS_ROTATION_CHECK_INTERVAL_SECONDS", 86400) +) # 24 hours default +LITELLM_USER_CREDENTIALS_ROTATION_GRACE_PERIOD: str = os.getenv( + "LITELLM_USER_CREDENTIALS_ROTATION_GRACE_PERIOD", "" +) # Duration to keep old key valid after rotation (e.g. "24h", "2d"); empty = immediate revoke (default) +LITELLM_USER_CREDENTIALS_ROTATION_LOCK_TTL_SECONDS = int( + os.getenv("LITELLM_USER_CREDENTIALS_ROTATION_LOCK_TTL_SECONDS", 600) +) # 10 minutes default — caps the deadlock window if a pod crashes mid-rotation + + # Key Rotation Constants LITELLM_KEY_ROTATION_ENABLED = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false") LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS = int( @@ -1423,6 +1436,7 @@ MAVVRIK_FOCUS_EXPORT_JOB_NAME = "mavvrik_focus_export_usage_data" CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int(os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000)) SPEND_LOG_CLEANUP_JOB_NAME = "spend_log_cleanup" +USER_CREDENTIALS_ROTATION_JOB_NAME = "litellm_user_credentials_rotation_job" KEY_ROTATION_JOB_NAME = "litellm_key_rotation_job" EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME = "litellm_expired_ui_session_key_cleanup_job" SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500)) diff --git a/litellm/models/user.py b/litellm/models/user.py index cd7e9db4aec..bef27026536 100644 --- a/litellm/models/user.py +++ b/litellm/models/user.py @@ -25,6 +25,7 @@ class LiteLLM_UserTable(LiteLLMPydanticObjectBase): organization_id: Optional[str] = None object_permission_id: Optional[str] = None password: Optional[str] = Field(default=None, exclude=True) + password_expiry: Optional[str] = Field(default=None) teams: List[str] = [] user_role: Optional[str] = None max_budget: Optional[float] = None diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d84588a4c24..1aca1d115b0 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2738,7 +2738,6 @@ class LiteLLM_UserTableFiltered(BaseModel): # done to avoid exposing sensitive user_id: str user_email: Optional[str] = None - class LiteLLM_UserTableWithKeyCount(LiteLLM_UserTable): key_count: int = 0 diff --git a/litellm/proxy/common_utils/user_credentials_rotation_manager.py b/litellm/proxy/common_utils/user_credentials_rotation_manager.py new file mode 100644 index 00000000000..76e85b59d81 --- /dev/null +++ b/litellm/proxy/common_utils/user_credentials_rotation_manager.py @@ -0,0 +1,169 @@ +""" +User Credentials Rotation Manager - Automated user credentials rotation based on rotation schedules + +Handles finding user credentials that need rotation based on their individual schedules. +""" + +from datetime import datetime, timezone +from typing import List + +from litellm._logging import verbose_proxy_logger +from litellm.constants import ( + LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, + LITELLM_USER_CREDENTIALS_ROTATION_GRACE_PERIOD, + LITELLM_USER_CREDENTIALS_ROTATION_LOCK_TTL_SECONDS, +) +from litellm.proxy._types import ( + GenerateKeyResponse, + LiteLLM_UserTablePasswordExpiryFiltered, + RegenerateKeyRequest, +) +from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks +from litellm.proxy.management_endpoints.key_management_endpoints import ( + _calculate_key_rotation_time, + regenerate_key_fn, +) +from litellm.proxy.utils import PrismaClient +from litellm.repositories.table_repositories import ( + DeprecatedVerificationTokenRepository, +) +from litellm.repositories.user_repository import ( + UserRepository, +) + + +class KeyRotationManager: + """ + Manages automated user credentials rotation based + on individual user credentials rotation schedules. + """ + + def __init__(self, prisma_client: PrismaClient, pod_lock_manager=None): + self.prisma_client = prisma_client + self.pod_lock_manager = pod_lock_manager + + async def process_rotations(self): + """ + Main entry point - find and rotate user credentials that are due for rotation. + Uses PodLockManager to ensure only one pod runs rotation in multi-pod deployments. + """ + from litellm.constants import USER_CREDENTIALS_ROTATION_JOB_NAME + + lock_acquired = False + try: + # If we have a pod lock manager with Redis, try to acquire the lock + if self.pod_lock_manager and self.pod_lock_manager.redis_cache: + # Use a dedicated lock TTL (default 600s) instead of the check interval + # (which defaults to 86400s / 24h). Using the check interval would create + # a 24-hour deadlock window if a pod crashes before releasing the lock. + lock_ttl = max( + LITELLM_USER_CREDENTIALS_ROTATION_LOCK_TTL_SECONDS, 300 + ) # At least 5 minutes, configurable via LITELLM_USER_CREDENTIALS_ROTATION_LOCK_TTL_SECONDS + lock_acquired = ( + await self.pod_lock_manager.acquire_lock( + cronjob_id=USER_CREDENTIALS_ROTATION_JOB_NAME, + ttl=lock_ttl, + ) + or False + ) + if not lock_acquired: + verbose_proxy_logger.warning( + "User Credentials rotation: another pod is already running rotation " + "or Redis lock acquisition failed — skipping this cycle. " + "User Credentials will be rotated on the next cycle." + ) + return + + verbose_proxy_logger.info("Starting scheduled user credentials rotation check...") + + # Find credentials that are due for rotation + user_credentials_to_rotate = await self._find_credentials_needing_rotation() + + if not user_credentials_to_rotate: + verbose_proxy_logger.debug("No user credentials are due for rotation at this time") + return + + verbose_proxy_logger.info(f"Found {len(user_credentials_to_rotate)} keys due for rotation") + + # Notify each user + for key in user_credentials_to_rotate: + try: + await self._rotate_key(key) + key_identifier = key.key_name or (key.token[:8] + "..." if key.token else "unknown") + verbose_proxy_logger.info(f"Successfully rotated key: {key_identifier}") + except Exception as e: + key_identifier = key.key_name or (key.token[:8] + "..." if key.token else "unknown") + verbose_proxy_logger.error(f"Failed to rotate key {key_identifier}: {e}") + + except Exception as e: + verbose_proxy_logger.error(f"Key rotation process failed: {e}") + finally: + # Only release the lock if it was actually acquired + if lock_acquired and self.pod_lock_manager and self.pod_lock_manager.redis_cache: + await self.pod_lock_manager.release_lock( + cronjob_id=USER_CREDENTIALS_ROTATION_JOB_NAME, + ) + + async def _find_credentials_needing_rotation(self) -> List[LiteLLM_UserTablePasswordExpiryFiltered]: + """ + Find user_credentials that are due for rotation based on their password_expiry timestamp. + + Logic: + - password_expiry is not null AND password_expiry <= now + """ + now = datetime.now(timezone.utc) + + credentials_with_rotation = await UserRepository(self.prisma_client).table.find_many( + where={ + "password_expiry": { "lte": now } # Users whose passwords expiry timestamp has passed + } + ) + + return credentials_with_rotation + + async def _rotate_key(self, key: LiteLLM_VerificationToken): + """ + Notify users whose password is expired + """ + # Create regenerate request with grace period for seamless cutover + regenerate_request = RegenerateKeyRequest( + key=key.token or "", + key_alias=key.key_alias, # Pass key alias to ensure correct secret is updated in AWS Secrets Manager + grace_period=LITELLM_KEY_ROTATION_GRACE_PERIOD or None, + ) + + # Create a system user for key rotation + from litellm.proxy._types import UserAPIKeyAuth + + system_user = UserAPIKeyAuth.get_litellm_internal_jobs_user_api_key_auth() + + # Use existing regenerate key function + response = await regenerate_key_fn( + data=regenerate_request, + user_api_key_dict=system_user, + litellm_changed_by=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, + ) + + # Update the NEW key with rotation info (regenerate_key_fn creates a new token) + if isinstance(response, GenerateKeyResponse) and response.token_id and key.rotation_interval: + # Calculate next rotation time using helper function + now = datetime.now(timezone.utc) + next_rotation_time = _calculate_key_rotation_time(key.rotation_interval) + await VerificationTokenRepository(self.prisma_client).table.update( + where={"token": response.token_id}, + data={ + "rotation_count": (key.rotation_count or 0) + 1, + "last_rotation_at": now, + "key_rotation_at": next_rotation_time, + }, + ) + + # Call the existing rotation hook for notifications, audit logs, etc. + if isinstance(response, GenerateKeyResponse): + await KeyManagementEventHooks.async_key_rotated_hook( + data=regenerate_request, + existing_key_row=key, + response=response, + user_api_key_dict=system_user, + litellm_changed_by=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index adbba821bf3..c4bc2a6c119 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7729,6 +7729,51 @@ async def _initialize_spend_tracking_background_jobs(cls, scheduler: AsyncIOSche from litellm.integrations.prometheus import PrometheusLogger PrometheusLogger.initialize_budget_metrics_cron_job(scheduler=scheduler) + + + ######################################################## + # User credentials Rotation Background Job + ######################################################## + + from litellm.constants import ( + LITELLM_USER_CREDENTIALS_ROTATION_ENABLED, + LITELLM_USER_CREDENTIALS_ROTATION_CHECK_INTERVAL_SECONDS, + ) + + user_credentials_enabled: Optional[bool] = str_to_bool(LITELLM_USER_CREDENTIALS_ROTATION_ENABLED) + verbose_proxy_logger.debug(f"user_credentials_enabled: {user_credentials_enabled}") + + # TODO: needs to be scheduled also ONLY when it's not behind SSO + if user_credentials_enabled is True: + try: + from litellm.proxy.common_utils.user_credentials_rotation_manager import ( + UserCredentialsRotationManager, + ) + + if prisma_client is not None: + # Reuse the PodLockManager from db_spend_update_writer + pod_lock_manager = proxy_logging_obj.db_spend_update_writer.pod_lock_manager + user_credentials_rotation_manager = UserCredentialsRotationManager( + prisma_client, + pod_lock_manager=pod_lock_manager, + ) + verbose_proxy_logger.debug( + f"User Credentials rotation background job scheduled every {LITELLM_USER_CREDENTIALS_ROTATION_CHECK_INTERVAL_SECONDS} seconds (LITELLM_USER_CREDENTIALS_ROTATION_ENABLED=true)"" + ) + scheduler.add_job( + user_credentials_rotation_manager.process_rotations, + "interval", + seconds=LITELLM_USER_CREDENTIALS_ROTATION_CHECK_INTERVAL_SECONDS, + id="user_credentials_rotation_job" + ) + else: + verbose_proxy_logger.warning("User Credentials rotation enabled but prisma_client not available") + except Exception as e: + verbose_proxy_logger.warning(f"Failed to setup user credentials rotation job: {e}") + else: + verbose_proxy_logger.debug("User Credentials rotation disabled (set LITELLM_USER_CREDENTIALS_ROTATION_ENABLED=true to enable)") + + ######################################################## # Key Rotation Background Job ######################################################## From a2d152fdce0eecba64ac6cd9bd003463923aad8d Mon Sep 17 00:00:00 2001 From: Fabrizio Siciliano Date: Sat, 4 Jul 2026 12:31:57 +0200 Subject: [PATCH 2/7] implemented user_credentials_rotation_manager.py, needs notification email setup --- .../user_credentials_rotation_manager.py | 72 +++++-------------- 1 file changed, 16 insertions(+), 56 deletions(-) diff --git a/litellm/proxy/common_utils/user_credentials_rotation_manager.py b/litellm/proxy/common_utils/user_credentials_rotation_manager.py index 76e85b59d81..467bd910394 100644 --- a/litellm/proxy/common_utils/user_credentials_rotation_manager.py +++ b/litellm/proxy/common_utils/user_credentials_rotation_manager.py @@ -15,7 +15,7 @@ ) from litellm.proxy._types import ( GenerateKeyResponse, - LiteLLM_UserTablePasswordExpiryFiltered, + LiteLLM_UserTableFiltered, RegenerateKeyRequest, ) from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks @@ -38,9 +38,10 @@ class KeyRotationManager: on individual user credentials rotation schedules. """ - def __init__(self, prisma_client: PrismaClient, pod_lock_manager=None): + def __init__(self, prisma_client: PrismaClient, pod_lock_manager=None, notify_users_on_expiration=False): self.prisma_client = prisma_client self.pod_lock_manager = pod_lock_manager + self.notify_users_on_expiration async def process_rotations(self): """ @@ -85,18 +86,17 @@ async def process_rotations(self): verbose_proxy_logger.info(f"Found {len(user_credentials_to_rotate)} keys due for rotation") - # Notify each user - for key in user_credentials_to_rotate: - try: - await self._rotate_key(key) - key_identifier = key.key_name or (key.token[:8] + "..." if key.token else "unknown") - verbose_proxy_logger.info(f"Successfully rotated key: {key_identifier}") - except Exception as e: - key_identifier = key.key_name or (key.token[:8] + "..." if key.token else "unknown") - verbose_proxy_logger.error(f"Failed to rotate key {key_identifier}: {e}") + if self.notify_users_on_expiration: + # Notify each user + for user in user_credentials_to_rotate: + try: + await self._notify_user(user) + verbose_proxy_logger.info(f"Successfully notified user: {user.user_id}") + except Exception as e: + verbose_proxy_logger.error(f"Failed to notify user {user.user_id}: {e}") except Exception as e: - verbose_proxy_logger.error(f"Key rotation process failed: {e}") + verbose_proxy_logger.error(f"User password expired process failed: {e}") finally: # Only release the lock if it was actually acquired if lock_acquired and self.pod_lock_manager and self.pod_lock_manager.redis_cache: @@ -104,7 +104,7 @@ async def process_rotations(self): cronjob_id=USER_CREDENTIALS_ROTATION_JOB_NAME, ) - async def _find_credentials_needing_rotation(self) -> List[LiteLLM_UserTablePasswordExpiryFiltered]: + async def _find_credentials_needing_rotation(self) -> List[LiteLLM_UserTableFiltered]: """ Find user_credentials that are due for rotation based on their password_expiry timestamp. @@ -121,49 +121,9 @@ async def _find_credentials_needing_rotation(self) -> List[LiteLLM_UserTablePass return credentials_with_rotation - async def _rotate_key(self, key: LiteLLM_VerificationToken): + async def _notify_user(self, user: LiteLLM_UserTableFiltered): """ Notify users whose password is expired """ - # Create regenerate request with grace period for seamless cutover - regenerate_request = RegenerateKeyRequest( - key=key.token or "", - key_alias=key.key_alias, # Pass key alias to ensure correct secret is updated in AWS Secrets Manager - grace_period=LITELLM_KEY_ROTATION_GRACE_PERIOD or None, - ) - - # Create a system user for key rotation - from litellm.proxy._types import UserAPIKeyAuth - - system_user = UserAPIKeyAuth.get_litellm_internal_jobs_user_api_key_auth() - - # Use existing regenerate key function - response = await regenerate_key_fn( - data=regenerate_request, - user_api_key_dict=system_user, - litellm_changed_by=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, - ) - - # Update the NEW key with rotation info (regenerate_key_fn creates a new token) - if isinstance(response, GenerateKeyResponse) and response.token_id and key.rotation_interval: - # Calculate next rotation time using helper function - now = datetime.now(timezone.utc) - next_rotation_time = _calculate_key_rotation_time(key.rotation_interval) - await VerificationTokenRepository(self.prisma_client).table.update( - where={"token": response.token_id}, - data={ - "rotation_count": (key.rotation_count or 0) + 1, - "last_rotation_at": now, - "key_rotation_at": next_rotation_time, - }, - ) - - # Call the existing rotation hook for notifications, audit logs, etc. - if isinstance(response, GenerateKeyResponse): - await KeyManagementEventHooks.async_key_rotated_hook( - data=regenerate_request, - existing_key_row=key, - response=response, - user_api_key_dict=system_user, - litellm_changed_by=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, - ) + # TODO: implement email notification + return From 4bbb4eaf56ae576c91e298756877bb649b53fcc9 Mon Sep 17 00:00:00 2001 From: Fabrizio Siciliano Date: Wed, 8 Jul 2026 10:54:50 +0200 Subject: [PATCH 3/7] feat(proxy): add user credentials rotation --- litellm/constants.py | 13 +- litellm/models/user.py | 6 +- litellm/proxy/_types.py | 10 ++ litellm/proxy/auth/login_utils.py | 22 +++ .../user_credentials_rotation_manager.py | 165 ++++++++---------- .../example_config_yaml/bad_schema.prisma | 1 + .../internal_user_endpoints.py | 33 +++- litellm/proxy/proxy_server.py | 38 ++-- schema.prisma | 1 + 9 files changed, 179 insertions(+), 110 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 5a05f125983..303cfe2f419 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1383,13 +1383,18 @@ LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL = "litellm_no_upstream_llm_call" # User credentials Rotation Constants -LITELLM_USER_CREDENTIALS_ROTATION_ENABLED = os.getenv("LITELLM_USER_CREDENTIALS_ROTATION_ENABLED", false) +LITELLM_USER_CREDENTIALS_ROTATION_ENABLED = os.getenv( + "LITELLM_USER_CREDENTIALS_ROTATION_ENABLED", "false" +) LITELLM_USER_CREDENTIALS_ROTATION_CHECK_INTERVAL_SECONDS = int( os.getenv("LITELLM_USER_CREDENTIALS_ROTATION_CHECK_INTERVAL_SECONDS", 86400) ) # 24 hours default -LITELLM_USER_CREDENTIALS_ROTATION_GRACE_PERIOD: str = os.getenv( - "LITELLM_USER_CREDENTIALS_ROTATION_GRACE_PERIOD", "" -) # Duration to keep old key valid after rotation (e.g. "24h", "2d"); empty = immediate revoke (default) +LITELLM_USER_CREDENTIALS_ROTATION_INTERVAL = os.getenv( + "LITELLM_USER_CREDENTIALS_ROTATION_INTERVAL", "" +) +LITELLM_USER_CREDENTIALS_ROTATION_REMINDER_DAYS = int( + os.getenv("LITELLM_USER_CREDENTIALS_ROTATION_REMINDER_DAYS", 10) +) LITELLM_USER_CREDENTIALS_ROTATION_LOCK_TTL_SECONDS = int( os.getenv("LITELLM_USER_CREDENTIALS_ROTATION_LOCK_TTL_SECONDS", 600) ) # 10 minutes default — caps the deadlock window if a pod crashes mid-rotation diff --git a/litellm/models/user.py b/litellm/models/user.py index bef27026536..9e8c54ea713 100644 --- a/litellm/models/user.py +++ b/litellm/models/user.py @@ -6,7 +6,7 @@ """ from datetime import datetime -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional from pydantic import ConfigDict, Field, model_validator @@ -25,7 +25,7 @@ class LiteLLM_UserTable(LiteLLMPydanticObjectBase): organization_id: Optional[str] = None object_permission_id: Optional[str] = None password: Optional[str] = Field(default=None, exclude=True) - password_expiry: Optional[str] = Field(default=None) + password_expiry: Optional[datetime] = Field(default=None) teams: List[str] = [] user_role: Optional[str] = None max_budget: Optional[float] = None @@ -51,7 +51,7 @@ class LiteLLM_UserTable(LiteLLMPydanticObjectBase): @model_validator(mode="before") @classmethod - def set_model_info(cls, values): + def set_model_info(cls, values: Dict[str, Any]) -> Dict[str, Any]: if values.get("spend") is None: values.update({"spend": 0.0}) if values.get("models") is None: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 3906975a88b..bf2c45461e7 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1516,6 +1516,7 @@ class NewUserRequestTeam(LiteLLMPydanticObjectBase): class NewUserRequest(GenerateRequestBase): max_budget: Optional[float] = None + password_expiry: Optional[datetime] = None user_email: Optional[str] = None user_alias: Optional[str] = None user_role: Optional[ @@ -1553,6 +1554,7 @@ class NewUserResponse(GenerateKeyResponse): class UpdateUserRequestNoUserIDorEmail(GenerateRequestBase): # shared with BulkUpdateUserRequest password: Optional[str] = None + password_expiry: Optional[datetime] = None spend: Optional[float] = None metadata: Optional[dict] = None user_alias: Optional[str] = None @@ -2233,6 +2235,14 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): description="sends alerts if requests hang for 5min+", ) ui_access_mode: Optional[Literal["admin_only", "all"]] = Field("all", description="Control access to the Proxy UI") + user_credentials_rotation_interval: Optional[str] = Field( + None, + description="How often non-SSO username/password users must rotate credentials, e.g. '30d'", + ) + user_credentials_rotation_reminder_days: Optional[int] = Field( + None, + description="How many days before password_expiry to send daily reminder emails for non-SSO users", + ) allowed_routes: Optional[List] = Field(None, description="Proxy API Endpoints you want users to be able to access") reject_clientside_metadata_tags: Optional[bool] = Field( None, diff --git a/litellm/proxy/auth/login_utils.py b/litellm/proxy/auth/login_utils.py index 11f12e597b9..ca7f0058833 100644 --- a/litellm/proxy/auth/login_utils.py +++ b/litellm/proxy/auth/login_utils.py @@ -7,6 +7,7 @@ import os import secrets +from datetime import datetime, timezone from typing import Literal, Optional, cast from fastapi import HTTPException @@ -103,6 +104,26 @@ def __init__( self.login_method = login_method +def _get_password_expiry(user_row: LiteLLM_UserTable) -> Optional[datetime]: + password_expiry = getattr(user_row, "password_expiry", None) + if isinstance(password_expiry, datetime): + return password_expiry if password_expiry.tzinfo is not None else password_expiry.replace(tzinfo=timezone.utc) + return None + + +def _raise_if_password_expired(user_row: LiteLLM_UserTable) -> None: + password_expiry = _get_password_expiry(user_row) + if password_expiry is None: + return + if datetime.now(timezone.utc) >= password_expiry: + raise ProxyException( + message="User credentials have expired. Please rotate your password before logging in again.", + type=ProxyErrorTypes.auth_error, + param="password_expiry", + code=401, + ) + + async def authenticate_user( username: str, password: str, @@ -264,6 +285,7 @@ async def authenticate_user( ) if verify_password(password, _password): + _raise_if_password_expired(_user_row) await _rehash_password_if_needed(_user_row.user_id, password, _password) if os.getenv("DATABASE_URL") is not None: response = await generate_key_helper_fn( diff --git a/litellm/proxy/common_utils/user_credentials_rotation_manager.py b/litellm/proxy/common_utils/user_credentials_rotation_manager.py index 467bd910394..6c35736aa8a 100644 --- a/litellm/proxy/common_utils/user_credentials_rotation_manager.py +++ b/litellm/proxy/common_utils/user_credentials_rotation_manager.py @@ -1,65 +1,34 @@ -""" -User Credentials Rotation Manager - Automated user credentials rotation based on rotation schedules - -Handles finding user credentials that need rotation based on their individual schedules. -""" - -from datetime import datetime, timezone -from typing import List +from datetime import datetime, timedelta, timezone +from typing import Any from litellm._logging import verbose_proxy_logger -from litellm.constants import ( - LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, - LITELLM_USER_CREDENTIALS_ROTATION_GRACE_PERIOD, - LITELLM_USER_CREDENTIALS_ROTATION_LOCK_TTL_SECONDS, -) -from litellm.proxy._types import ( - GenerateKeyResponse, - LiteLLM_UserTableFiltered, - RegenerateKeyRequest, -) -from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks -from litellm.proxy.management_endpoints.key_management_endpoints import ( - _calculate_key_rotation_time, - regenerate_key_fn, -) -from litellm.proxy.utils import PrismaClient -from litellm.repositories.table_repositories import ( - DeprecatedVerificationTokenRepository, -) -from litellm.repositories.user_repository import ( - UserRepository, -) +from litellm.constants import LITELLM_USER_CREDENTIALS_ROTATION_LOCK_TTL_SECONDS +from litellm.models.user import LiteLLM_UserTable +from litellm.proxy.utils import PrismaClient, send_email +from litellm.repositories.user_repository import UserRepository -class KeyRotationManager: - """ - Manages automated user credentials rotation based - on individual user credentials rotation schedules. - """ +UserCredentialsReminderRow = LiteLLM_UserTable - def __init__(self, prisma_client: PrismaClient, pod_lock_manager=None, notify_users_on_expiration=False): + +class UserCredentialsRotationManager: + def __init__( + self, + prisma_client: PrismaClient, + pod_lock_manager: Any = None, + reminder_days: int = 10, + ): self.prisma_client = prisma_client self.pod_lock_manager = pod_lock_manager - self.notify_users_on_expiration + self.reminder_days = max(reminder_days, 0) - async def process_rotations(self): - """ - Main entry point - find and rotate user credentials that are due for rotation. - Uses PodLockManager to ensure only one pod runs rotation in multi-pod deployments. - """ + async def process_rotations(self) -> None: from litellm.constants import USER_CREDENTIALS_ROTATION_JOB_NAME lock_acquired = False try: - # If we have a pod lock manager with Redis, try to acquire the lock if self.pod_lock_manager and self.pod_lock_manager.redis_cache: - # Use a dedicated lock TTL (default 600s) instead of the check interval - # (which defaults to 86400s / 24h). Using the check interval would create - # a 24-hour deadlock window if a pod crashes before releasing the lock. - lock_ttl = max( - LITELLM_USER_CREDENTIALS_ROTATION_LOCK_TTL_SECONDS, 300 - ) # At least 5 minutes, configurable via LITELLM_USER_CREDENTIALS_ROTATION_LOCK_TTL_SECONDS + lock_ttl = max(LITELLM_USER_CREDENTIALS_ROTATION_LOCK_TTL_SECONDS, 300) lock_acquired = ( await self.pod_lock_manager.acquire_lock( cronjob_id=USER_CREDENTIALS_ROTATION_JOB_NAME, @@ -69,61 +38,77 @@ async def process_rotations(self): ) if not lock_acquired: verbose_proxy_logger.warning( - "User Credentials rotation: another pod is already running rotation " - "or Redis lock acquisition failed — skipping this cycle. " - "User Credentials will be rotated on the next cycle." + "User credentials rotation: another pod is already running reminders or Redis lock acquisition failed; skipping this cycle" ) return - verbose_proxy_logger.info("Starting scheduled user credentials rotation check...") - - # Find credentials that are due for rotation - user_credentials_to_rotate = await self._find_credentials_needing_rotation() - - if not user_credentials_to_rotate: - verbose_proxy_logger.debug("No user credentials are due for rotation at this time") + users_to_notify = await self._find_users_needing_reminders() + if not users_to_notify: + verbose_proxy_logger.debug( + "No user credential rotation reminders are due at this time" + ) return - verbose_proxy_logger.info(f"Found {len(user_credentials_to_rotate)} keys due for rotation") - - if self.notify_users_on_expiration: - # Notify each user - for user in user_credentials_to_rotate: - try: - await self._notify_user(user) - verbose_proxy_logger.info(f"Successfully notified user: {user.user_id}") - except Exception as e: - verbose_proxy_logger.error(f"Failed to notify user {user.user_id}: {e}") - + for user in users_to_notify: + try: + await self._notify_user(user=user) + verbose_proxy_logger.info( + "Sent user credential rotation reminder to user_id=%s", + user.user_id, + ) + except Exception as e: + verbose_proxy_logger.error( + "Failed to send user credential rotation reminder to user_id=%s: %s", + user.user_id, + e, + ) except Exception as e: - verbose_proxy_logger.error(f"User password expired process failed: {e}") + verbose_proxy_logger.error( + "User credential rotation reminder process failed: %s", + e, + ) finally: - # Only release the lock if it was actually acquired - if lock_acquired and self.pod_lock_manager and self.pod_lock_manager.redis_cache: + if ( + lock_acquired + and self.pod_lock_manager + and self.pod_lock_manager.redis_cache + ): await self.pod_lock_manager.release_lock( cronjob_id=USER_CREDENTIALS_ROTATION_JOB_NAME, ) - async def _find_credentials_needing_rotation(self) -> List[LiteLLM_UserTableFiltered]: - """ - Find user_credentials that are due for rotation based on their password_expiry timestamp. - - Logic: - - password_expiry is not null AND password_expiry <= now - """ + async def _find_users_needing_reminders(self) -> list[UserCredentialsReminderRow]: now = datetime.now(timezone.utc) - - credentials_with_rotation = await UserRepository(self.prisma_client).table.find_many( + reminder_threshold = now + timedelta(days=self.reminder_days) + return await UserRepository(self.prisma_client).table.find_many( where={ - "password_expiry": { "lte": now } # Users whose passwords expiry timestamp has passed + "password": {"not": None}, + "password_expiry": { + "gte": now, + "lte": reminder_threshold, + }, + "user_email": {"not": None}, + "sso_user_id": None, } ) - return credentials_with_rotation + async def _notify_user(self, user: UserCredentialsReminderRow) -> None: + user_email = getattr(user, "user_email", None) + password_expiry = getattr(user, "password_expiry", None) + if not isinstance(user_email, str) or not user_email.strip() or password_expiry is None: + return + + subject = "LiteLLM password rotation reminder" + html = ( + "

Your LiteLLM password will expire soon.

" + f"

Password expiry: {password_expiry}

" + "

Please rotate your password before the expiry date to keep access.

" + ) + await send_email( + receiver_email=user_email, + subject=subject, + html=html, + ) + - async def _notify_user(self, user: LiteLLM_UserTableFiltered): - """ - Notify users whose password is expired - """ - # TODO: implement email notification - return +__all__ = ["UserCredentialsRotationManager"] diff --git a/litellm/proxy/example_config_yaml/bad_schema.prisma b/litellm/proxy/example_config_yaml/bad_schema.prisma index 5c631406a4f..cffba9e0a46 100644 --- a/litellm/proxy/example_config_yaml/bad_schema.prisma +++ b/litellm/proxy/example_config_yaml/bad_schema.prisma @@ -104,6 +104,7 @@ model LiteLLM_UserTable { team_id String? organization_id String? password String? + password_expiry DateTime? teams String[] @default([]) user_role String? max_budget Float? diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index abdbb317891..a10d3a01c66 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -15,9 +15,11 @@ import asyncio import json import traceback -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Union, cast +from litellm.litellm_core_utils.duration_parser import duration_in_seconds + import fastapi from fastapi import APIRouter, Depends, Header, HTTPException, Request, status @@ -76,10 +78,35 @@ router = APIRouter() +def _get_user_credentials_rotation_interval() -> Optional[str]: + from litellm.constants import LITELLM_USER_CREDENTIALS_ROTATION_INTERVAL + from litellm.proxy.proxy_server import general_settings + + configured_interval = general_settings.get( + "user_credentials_rotation_interval", + LITELLM_USER_CREDENTIALS_ROTATION_INTERVAL, + ) + if not isinstance(configured_interval, str): + return None + stripped_interval = configured_interval.strip() + return stripped_interval or None + + +def _calculate_password_expiry(rotation_interval: str) -> datetime: + return datetime.now(timezone.utc) + timedelta( + seconds=duration_in_seconds(rotation_interval) + ) + + def _hash_password_in_dict(data: dict) -> None: """Hash password field in-place if present.""" - if "password" in data and data["password"] is not None: - data["password"] = hash_password(data["password"]) + password = data.get("password") + if password is None: + return + data["password"] = hash_password(password) + rotation_interval = _get_user_credentials_rotation_interval() + if rotation_interval is not None: + data["password_expiry"] = _calculate_password_expiry(rotation_interval) def _strip_password_from_response(response) -> None: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6c061ae6bcd..da239034b0c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -261,6 +261,7 @@ def generate_feedback_box(): log_db_metrics, ) from litellm.proxy.auth.auth_utils import ( + _has_user_setup_sso, check_response_size_is_safe, is_request_body_safe, warn_once_if_custom_auth_skips_common_checks, @@ -7834,42 +7835,47 @@ async def _initialize_spend_tracking_background_jobs(cls, scheduler: AsyncIOSche ######################################################## from litellm.constants import ( - LITELLM_USER_CREDENTIALS_ROTATION_ENABLED, LITELLM_USER_CREDENTIALS_ROTATION_CHECK_INTERVAL_SECONDS, + LITELLM_USER_CREDENTIALS_ROTATION_ENABLED, + LITELLM_USER_CREDENTIALS_ROTATION_REMINDER_DAYS, ) user_credentials_enabled: Optional[bool] = str_to_bool(LITELLM_USER_CREDENTIALS_ROTATION_ENABLED) verbose_proxy_logger.debug(f"user_credentials_enabled: {user_credentials_enabled}") - # TODO: needs to be scheduled also ONLY when it's not behind SSO - if user_credentials_enabled is True: + if user_credentials_enabled is True and not _has_user_setup_sso(): try: from litellm.proxy.common_utils.user_credentials_rotation_manager import ( UserCredentialsRotationManager, ) if prisma_client is not None: - # Reuse the PodLockManager from db_spend_update_writer pod_lock_manager = proxy_logging_obj.db_spend_update_writer.pod_lock_manager + configured_reminder_days = general_settings.get( + "user_credentials_rotation_reminder_days", + LITELLM_USER_CREDENTIALS_ROTATION_REMINDER_DAYS, + ) + reminder_days = configured_reminder_days if isinstance(configured_reminder_days, int) else LITELLM_USER_CREDENTIALS_ROTATION_REMINDER_DAYS user_credentials_rotation_manager = UserCredentialsRotationManager( - prisma_client, + prisma_client, pod_lock_manager=pod_lock_manager, + reminder_days=reminder_days, ) verbose_proxy_logger.debug( - f"User Credentials rotation background job scheduled every {LITELLM_USER_CREDENTIALS_ROTATION_CHECK_INTERVAL_SECONDS} seconds (LITELLM_USER_CREDENTIALS_ROTATION_ENABLED=true)"" + f"User credentials rotation background job scheduled every {LITELLM_USER_CREDENTIALS_ROTATION_CHECK_INTERVAL_SECONDS} seconds" ) scheduler.add_job( user_credentials_rotation_manager.process_rotations, "interval", seconds=LITELLM_USER_CREDENTIALS_ROTATION_CHECK_INTERVAL_SECONDS, - id="user_credentials_rotation_job" + id="user_credentials_rotation_job", ) else: - verbose_proxy_logger.warning("User Credentials rotation enabled but prisma_client not available") + verbose_proxy_logger.warning("User credentials rotation enabled but prisma_client not available") except Exception as e: verbose_proxy_logger.warning(f"Failed to setup user credentials rotation job: {e}") else: - verbose_proxy_logger.debug("User Credentials rotation disabled (set LITELLM_USER_CREDENTIALS_ROTATION_ENABLED=true to enable)") + verbose_proxy_logger.debug("User credentials rotation disabled or skipped because SSO is enabled") ######################################################## @@ -13618,8 +13624,20 @@ async def claim_onboarding_link(data: InvitationClaim, request: Request): ) ### UPDATE USER OBJECT ### + from litellm.constants import LITELLM_USER_CREDENTIALS_ROTATION_INTERVAL + + configured_rotation_interval = general_settings.get( + "user_credentials_rotation_interval", + LITELLM_USER_CREDENTIALS_ROTATION_INTERVAL, + ) + password_expiry = None + if isinstance(configured_rotation_interval, str) and configured_rotation_interval.strip(): + password_expiry = litellm.utils.get_utc_datetime() + timedelta( + seconds=duration_in_seconds(configured_rotation_interval.strip()) + ) + user_update_data = {"password": hashed_pw, "password_expiry": password_expiry} user_obj = await tx.litellm_usertable.update( - where={"user_id": invite_obj.user_id}, data={"password": hashed_pw} + where={"user_id": invite_obj.user_id}, data=user_update_data ) if user_obj is None: diff --git a/schema.prisma b/schema.prisma index f9ab5e6aefd..7e99da0ddda 100644 --- a/schema.prisma +++ b/schema.prisma @@ -239,6 +239,7 @@ model LiteLLM_UserTable { organization_id String? object_permission_id String? password String? + password_expiry DateTime? teams String[] @default([]) user_role String? max_budget Float? From 8272139e0368ebe5de8b79312b566e0a2b43ba61 Mon Sep 17 00:00:00 2001 From: Fabrizio Siciliano Date: Wed, 8 Jul 2026 10:55:10 +0200 Subject: [PATCH 4/7] test(proxy): add tests for user credentials rotation --- .../proxy/auth/test_login_utils.py | 43 +++++++++++++++ .../test_internal_user_endpoints.py | 53 +++++++++++++++++++ .../proxy_server/test_routes_onboarding.py | 11 +++- 3 files changed, 106 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/auth/test_login_utils.py b/tests/test_litellm/proxy/auth/test_login_utils.py index 288e2533b72..3096df0e65d 100644 --- a/tests/test_litellm/proxy/auth/test_login_utils.py +++ b/tests/test_litellm/proxy/auth/test_login_utils.py @@ -6,6 +6,7 @@ """ import os +from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -252,6 +253,48 @@ async def test_authenticate_user_wrong_password(): assert "Invalid credentials" in exc_info.value.message +@pytest.mark.asyncio +async def test_authenticate_user_with_expired_password_rejected(): + """Expired DB-backed credentials should be blocked before a UI session is minted.""" + master_key = "sk-1234" + user_email = "test@example.com" + correct_password = "correct-password" + hashed_password = hash_token(token=correct_password) + + mock_user = LiteLLM_UserTable( + user_id="test-user-123", + user_email=user_email, + password=hashed_password, + password_expiry=datetime.now(timezone.utc) - timedelta(days=1), + user_role=LitellmUserRoles.INTERNAL_USER, + ) + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = AsyncMock( + return_value=mock_user + ) + + with patch.dict( + os.environ, + { + "DATABASE_URL": "postgresql://test:test@localhost/test", + "UI_USERNAME": "admin", + "UI_PASSWORD": "admin-password", + }, + ): + with pytest.raises(ProxyException) as exc_info: + await authenticate_user( + username=user_email, + password=correct_password, + master_key=master_key, + prisma_client=mock_prisma_client, + ) + + assert exc_info.value.type == ProxyErrorTypes.auth_error + assert exc_info.value.code == "401" + assert "expired" in exc_info.value.message.lower() + + @pytest.mark.asyncio async def test_authenticate_user_email_case_insensitive_login(): """Test that email lookup is case-insensitive during login""" diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py index 81e7fdbbf50..dfca62d5dd1 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py @@ -3572,6 +3572,59 @@ async def test_ghsa_wvg4_proxy_admin_can_update_user_budget(mocker): assert result is not None +@pytest.mark.asyncio +async def test_user_update_password_sets_password_expiry_when_rotation_interval_configured( + mocker, +): + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + _update_single_user_helper, + ) + + existing_user = mocker.MagicMock() + existing_user.model_dump.return_value = { + "user_id": "target-user", + "user_email": "target@example.com", + "password": None, + } + existing_user.user_id = "target-user" + existing_user.metadata = {} + + mock_prisma_client = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_first = mocker.AsyncMock( + return_value=existing_user + ) + mock_prisma_client.update_data = mocker.AsyncMock( + return_value={"user_id": "target-user", "password": "hashed-password"} + ) + mock_prisma_client.jsonify_object = mocker.MagicMock(side_effect=lambda x: x) + + mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + mocker.patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin") + mocker.patch( + "litellm.proxy.proxy_server.general_settings", + {"user_credentials_rotation_interval": "30d"}, + ) + mocker.patch( + "litellm.proxy.management_endpoints.internal_user_endpoints.duration_in_seconds", + return_value=30 * 24 * 60 * 60, + ) + + user_request = UpdateUserRequest(user_id="target-user", password="hunter2") + admin_caller = UserAPIKeyAuth( + user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN + ) + + await _update_single_user_helper( + user_request=user_request, + user_api_key_dict=admin_caller, + ) + + update_call = mock_prisma_client.update_data.await_args + data = update_call.kwargs["data"] + assert data["password"] != "hunter2" + assert data["password_expiry"] is not None + + @pytest.mark.asyncio async def test_admin_user_update_spend_invalidates_counter(mocker): """A direct /user/update spend change must invalidate the cross-pod diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_onboarding.py b/tests/test_litellm/proxy/proxy_server/test_routes_onboarding.py index 35ae9a3568e..fa8e7c2cc0a 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_onboarding.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_onboarding.py @@ -216,8 +216,13 @@ def test_claim_onboarding_link_happy(client, monkeypatch, mock_prisma): monkeypatch.setattr(ps, "prisma_client", mock_prisma) monkeypatch.setattr(ps, "master_key", "sk-master-test") - monkeypatch.setattr(ps, "general_settings", {}) + monkeypatch.setattr( + ps, + "general_settings", + {"user_credentials_rotation_interval": "30d"}, + ) monkeypatch.setattr(ps, "premium_user", False) + monkeypatch.setattr(ps, "duration_in_seconds", lambda duration: 30 * 24 * 60 * 60) # Avoid hitting generate_key_helper_fn (touches DB / many globals); patch # the helper directly so we focus on the route's own behavior. @@ -244,6 +249,10 @@ async def _fake_session_token(user_obj): assert body["token"] == "session-jwt-token" assert body["user_email"] == "alice@example.com" assert body["login_url"].endswith("/ui/?login=success") + update_call = mock_prisma.db.litellm_usertable.update.await_args + assert update_call.kwargs["where"] == {"user_id": "user-abc"} + assert "password" in update_call.kwargs["data"] + assert update_call.kwargs["data"]["password_expiry"] is not None def test_claim_onboarding_link_invalid_invite_401(client, monkeypatch, mock_prisma): From 01749ea47879175262247e46ba13a190e6da91f6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 8 Jul 2026 08:55:28 +0000 Subject: [PATCH 5/7] chore: sync schema.prisma copies from root --- litellm-proxy-extras/litellm_proxy_extras/schema.prisma | 1 + litellm/proxy/schema.prisma | 1 + 2 files changed, 2 insertions(+) diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index f9ab5e6aefd..7e99da0ddda 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -239,6 +239,7 @@ model LiteLLM_UserTable { organization_id String? object_permission_id String? password String? + password_expiry DateTime? teams String[] @default([]) user_role String? max_budget Float? diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index f9ab5e6aefd..7e99da0ddda 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -239,6 +239,7 @@ model LiteLLM_UserTable { organization_id String? object_permission_id String? password String? + password_expiry DateTime? teams String[] @default([]) user_role String? max_budget Float? From 3029070a5b3f320f615e016dc5c103f44f0b40de Mon Sep 17 00:00:00 2001 From: Fabrizio Siciliano Date: Wed, 8 Jul 2026 12:12:57 +0200 Subject: [PATCH 6/7] test(proxy): fix tests for user credentials rotation --- .../test_user_credentials_rotation_manager.py | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 tests/test_litellm/proxy/common_utils/test_user_credentials_rotation_manager.py diff --git a/tests/test_litellm/proxy/common_utils/test_user_credentials_rotation_manager.py b/tests/test_litellm/proxy/common_utils/test_user_credentials_rotation_manager.py new file mode 100644 index 00000000000..02cb4d2eabc --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_user_credentials_rotation_manager.py @@ -0,0 +1,110 @@ +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from litellm.proxy.common_utils.user_credentials_rotation_manager import ( + UserCredentialsRotationManager, +) + + +@pytest.mark.asyncio +async def test_find_users_needing_reminders_filters_active_window(): + mock_prisma = MagicMock() + mock_prisma.db.litellm_usertable.find_many = AsyncMock(return_value=[]) + + manager = UserCredentialsRotationManager(mock_prisma, reminder_days=10) + await manager._find_users_needing_reminders() + + where = mock_prisma.db.litellm_usertable.find_many.await_args.kwargs["where"] + now = datetime.now(timezone.utc) + lower_bound = where["password_expiry"]["gte"] + upper_bound = where["password_expiry"]["lte"] + + assert where["password"] == {"not": None} + assert where["user_email"] == {"not": None} + assert where["sso_user_id"] is None + assert lower_bound >= now - timedelta(seconds=5) + assert timedelta(days=9, hours=23) <= upper_bound - lower_bound <= timedelta(days=10, seconds=5) + + +@pytest.mark.asyncio +async def test_notify_user_sends_email_for_expiring_password(): + mock_prisma = MagicMock() + manager = UserCredentialsRotationManager(mock_prisma, reminder_days=10) + user = MagicMock() + user.user_email = "user@example.com" + user.password_expiry = datetime(2030, 1, 1, tzinfo=timezone.utc) + + with patch( + "litellm.proxy.common_utils.user_credentials_rotation_manager.send_email", + new_callable=AsyncMock, + ) as mock_send_email: + await manager._notify_user(user) + + mock_send_email.assert_awaited_once() + call = mock_send_email.await_args.kwargs + assert call["receiver_email"] == "user@example.com" + assert call["subject"] == "LiteLLM password rotation reminder" + assert "2030-01-01" in call["html"] + + +@pytest.mark.asyncio +async def test_process_rotations_skips_when_lock_not_acquired(): + mock_prisma = MagicMock() + mock_pod_lock_manager = MagicMock() + mock_pod_lock_manager.redis_cache = MagicMock() + mock_pod_lock_manager.acquire_lock = AsyncMock(return_value=False) + mock_pod_lock_manager.release_lock = AsyncMock() + + manager = UserCredentialsRotationManager( + mock_prisma, + pod_lock_manager=mock_pod_lock_manager, + reminder_days=10, + ) + manager._find_users_needing_reminders = AsyncMock() + + await manager.process_rotations() + + manager._find_users_needing_reminders.assert_not_awaited() + mock_pod_lock_manager.release_lock.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_process_rotations_acquires_and_releases_lock(): + mock_prisma = MagicMock() + mock_pod_lock_manager = MagicMock() + mock_pod_lock_manager.redis_cache = MagicMock() + mock_pod_lock_manager.acquire_lock = AsyncMock(return_value=True) + mock_pod_lock_manager.release_lock = AsyncMock() + + manager = UserCredentialsRotationManager( + mock_prisma, + pod_lock_manager=mock_pod_lock_manager, + reminder_days=10, + ) + manager._find_users_needing_reminders = AsyncMock(return_value=[]) + + await manager.process_rotations() + + mock_pod_lock_manager.acquire_lock.assert_awaited_once() + acquire_call = mock_pod_lock_manager.acquire_lock.await_args + assert acquire_call.kwargs["cronjob_id"] == "litellm_user_credentials_rotation_job" + assert acquire_call.kwargs["ttl"] >= 300 + mock_pod_lock_manager.release_lock.assert_awaited_once_with( + cronjob_id="litellm_user_credentials_rotation_job" + ) + + +@pytest.mark.asyncio +async def test_process_rotations_continues_after_notify_error(): + mock_prisma = MagicMock() + manager = UserCredentialsRotationManager(mock_prisma, reminder_days=10) + user_one = MagicMock(user_id="user-1") + user_two = MagicMock(user_id="user-2") + manager._find_users_needing_reminders = AsyncMock(return_value=[user_one, user_two]) + manager._notify_user = AsyncMock(side_effect=[Exception("boom"), None]) + + await manager.process_rotations() + + assert manager._notify_user.await_count == 2 From a9d9e59620f7d88f0b1345f53508bb4b6caab351 Mon Sep 17 00:00:00 2001 From: Fabrizio Siciliano Date: Wed, 8 Jul 2026 12:14:36 +0200 Subject: [PATCH 7/7] feat(api): update schema.d.ts file --- ui/litellm-dashboard/src/lib/http/schema.d.ts | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 496a0462ebe..933ffbb6516 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -22554,6 +22554,16 @@ export interface components { * @description If True and LiteLLM_SpendLogs has been converted to a range-partitioned table (db_scripts/partition_spend_logs.sql), retention cleanup drops expired partitions instead of deleting rows, and pre-creates upcoming partitions. Default is False. */ use_spend_logs_partitioning?: boolean | null; + /** + * User Credentials Rotation Interval + * @description How often non-SSO username/password users must rotate credentials, e.g. '30d' + */ + user_credentials_rotation_interval?: string | null; + /** + * User Credentials Rotation Reminder Days + * @description How many days before password_expiry to send daily reminder emails for non-SSO users + */ + user_credentials_rotation_reminder_days?: number | null; /** User Header Mappings */ user_header_mappings?: components["schemas"]["UserHeaderMapping"][] | null; /** @@ -25954,6 +25964,8 @@ export interface components { organization_id?: string | null; /** Organization Memberships */ organization_memberships?: components["schemas"]["LiteLLM_OrganizationMembershipTable"][] | null; + /** Password Expiry */ + password_expiry?: string | null; /** * Policies * @default [] @@ -26047,6 +26059,8 @@ export interface components { organization_id?: string | null; /** Organization Memberships */ organization_memberships?: components["schemas"]["LiteLLM_OrganizationMembershipTable"][] | null; + /** Password Expiry */ + password_expiry?: string | null; /** * Policies * @default [] @@ -27989,6 +28003,8 @@ export interface components { object_permission?: components["schemas"]["LiteLLM_ObjectPermissionBase"] | null; /** Organizations */ organizations?: string[] | null; + /** Password Expiry */ + password_expiry?: string | null; /** * Permissions * @default {} @@ -32176,6 +32192,8 @@ export interface components { object_permission?: components["schemas"]["LiteLLM_ObjectPermissionBase"] | null; /** Password */ password?: string | null; + /** Password Expiry */ + password_expiry?: string | null; /** * Permissions * @default {} @@ -32278,6 +32296,8 @@ export interface components { object_permission?: components["schemas"]["LiteLLM_ObjectPermissionBase"] | null; /** Password */ password?: string | null; + /** Password Expiry */ + password_expiry?: string | null; /** * Permissions * @default {}