diff --git a/docs/docs/reference/environment-variables.md b/docs/docs/reference/environment-variables.md index 4c5d44bd56..3c28ba3339 100644 --- a/docs/docs/reference/environment-variables.md +++ b/docs/docs/reference/environment-variables.md @@ -123,6 +123,7 @@ For more details on the options below, refer to the [server deployment](../guide - `DSTACK_DB_POOL_SIZE`{ #DSTACK_DB_POOL_SIZE } - The client DB connections pool size. Defaults to `20`, - `DSTACK_DB_MAX_OVERFLOW`{ #DSTACK_DB_MAX_OVERFLOW } - The client DB connections pool allowed overflow. Defaults to `20`. - `DSTACK_SERVER_BACKGROUND_PROCESSING_FACTOR`{ #DSTACK_SERVER_BACKGROUND_PROCESSING_FACTOR } - The number of background jobs for processing server resources. Increase if you need to process more resources per server replica quickly. Defaults to `1`. +- `DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED`{ #DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED } - Disables background processing if set to any value. Useful to run only web frontend and API server. ??? info "Internal environment variables" The following environment variables are intended for development purposes: diff --git a/src/dstack/_internal/server/app.py b/src/dstack/_internal/server/app.py index 6dd09d766c..8aff963c1a 100644 --- a/src/dstack/_internal/server/app.py +++ b/src/dstack/_internal/server/app.py @@ -151,7 +151,10 @@ async def lifespan(app: FastAPI): ) if settings.SERVER_S3_BUCKET is not None or settings.SERVER_GCS_BUCKET is not None: init_default_storage() - scheduler = start_background_tasks() + if settings.SERVER_BACKGROUND_PROCESSING_ENABLED: + scheduler = start_background_tasks() + else: + logger.info("Background processing is disabled") dstack_version = DSTACK_VERSION if DSTACK_VERSION else "(no version)" logger.info(f"The admin token is {admin.token.get_plaintext_or_error()}", {"show_path": False}) logger.info( @@ -161,7 +164,8 @@ async def lifespan(app: FastAPI): for func in _ON_STARTUP_HOOKS: await func(app) yield - scheduler.shutdown() + if settings.SERVER_BACKGROUND_PROCESSING_ENABLED: + scheduler.shutdown() await gateway_connections_pool.remove_all() service_conn_pool = await get_injector_from_app(app).get_service_connection_pool() await service_conn_pool.remove_all() diff --git a/src/dstack/_internal/server/settings.py b/src/dstack/_internal/server/settings.py index a60fc5e658..f025d71644 100644 --- a/src/dstack/_internal/server/settings.py +++ b/src/dstack/_internal/server/settings.py @@ -42,6 +42,11 @@ os.getenv("DSTACK_SERVER_BACKGROUND_PROCESSING_FACTOR", 1) ) +SERVER_BACKGROUND_PROCESSING_DISABLED = ( + os.getenv("DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED") is not None +) +SERVER_BACKGROUND_PROCESSING_ENABLED = not SERVER_BACKGROUND_PROCESSING_DISABLED + SERVER_EXECUTOR_MAX_WORKERS = int(os.getenv("DSTACK_SERVER_EXECUTOR_MAX_WORKERS", 128)) MAX_OFFERS_TRIED = int(os.getenv("DSTACK_SERVER_MAX_OFFERS_TRIED", 25)) @@ -113,5 +118,5 @@ UPDATE_DEFAULT_PROJECT = os.getenv("DSTACK_UPDATE_DEFAULT_PROJECT") is not None DO_NOT_UPDATE_DEFAULT_PROJECT = os.getenv("DSTACK_DO_NOT_UPDATE_DEFAULT_PROJECT") is not None -SKIP_GATEWAY_UPDATE = os.getenv("DSTACK_SKIP_GATEWAY_UPDATE", None) is not None -ENABLE_PROMETHEUS_METRICS = os.getenv("DSTACK_ENABLE_PROMETHEUS_METRICS", None) is not None +SKIP_GATEWAY_UPDATE = os.getenv("DSTACK_SKIP_GATEWAY_UPDATE") is not None +ENABLE_PROMETHEUS_METRICS = os.getenv("DSTACK_ENABLE_PROMETHEUS_METRICS") is not None