From e44b4c81a9544f1414a5cdc169c46c4dc197105c Mon Sep 17 00:00:00 2001 From: james-boydell Date: Mon, 14 Jul 2025 11:43:26 -0400 Subject: [PATCH 1/4] add server setting --- src/dstack/_internal/server/settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/dstack/_internal/server/settings.py b/src/dstack/_internal/server/settings.py index a60fc5e658..e0bc566534 100644 --- a/src/dstack/_internal/server/settings.py +++ b/src/dstack/_internal/server/settings.py @@ -115,3 +115,6 @@ DO_NOT_UPDATE_DEFAULT_PROJECT = os.getenv("DSTACK_DO_NOT_UPDATE_DEFAULT_PROJECT") is not None SKIP_GATEWAY_UPDATE = os.getenv("DSTACK_SKIP_GATEWAY_UPDATE", None) is not None ENABLE_PROMETHEUS_METRICS = os.getenv("DSTACK_ENABLE_PROMETHEUS_METRICS", None) is not None +SERVER_DISABLE_BACKGROUND_PROCESSING = ( + os.getenv("DSTACK_SERVER_DISABLE_BACKGROUND_PROCESSING", None) is not None +) From 7f54f64a2914542d81f49aeceeea02b911097241 Mon Sep 17 00:00:00 2001 From: james-boydell Date: Mon, 14 Jul 2025 11:43:50 -0400 Subject: [PATCH 2/4] add logic to disable background processing --- src/dstack/_internal/server/app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dstack/_internal/server/app.py b/src/dstack/_internal/server/app.py index 6dd09d766c..fdedf50b8f 100644 --- a/src/dstack/_internal/server/app.py +++ b/src/dstack/_internal/server/app.py @@ -151,7 +151,8 @@ async def lifespan(app: FastAPI): ) if settings.SERVER_S3_BUCKET is not None or settings.SERVER_GCS_BUCKET is not None: init_default_storage() - scheduler = start_background_tasks() + if settings.SERVER_DISABLE_BACKGROUND_PROCESSING is not None: + scheduler = start_background_tasks() dstack_version = DSTACK_VERSION if DSTACK_VERSION else "(no version)" logger.info(f"The admin token is {admin.token.get_plaintext_or_error()}", {"show_path": False}) logger.info( From 4aaea2580536f6303d2a615d9cfc65a3a8b9f4b4 Mon Sep 17 00:00:00 2001 From: james-boydell Date: Mon, 14 Jul 2025 11:43:59 -0400 Subject: [PATCH 3/4] update docs for new setting --- docs/docs/reference/environment-variables.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/docs/reference/environment-variables.md b/docs/docs/reference/environment-variables.md index 4c5d44bd56..c442f0afd4 100644 --- a/docs/docs/reference/environment-variables.md +++ b/docs/docs/reference/environment-variables.md @@ -123,6 +123,7 @@ For more details on the options below, refer to the [server deployment](../guide - `DSTACK_DB_POOL_SIZE`{ #DSTACK_DB_POOL_SIZE } - The client DB connections pool size. Defaults to `20`, - `DSTACK_DB_MAX_OVERFLOW`{ #DSTACK_DB_MAX_OVERFLOW } - The client DB connections pool allowed overflow. Defaults to `20`. - `DSTACK_SERVER_BACKGROUND_PROCESSING_FACTOR`{ #DSTACK_SERVER_BACKGROUND_PROCESSING_FACTOR } - The number of background jobs for processing server resources. Increase if you need to process more resources per server replica quickly. Defaults to `1`. +- `STACK_SERVER_DISABLE_BACKGROUND_PROCESSING`{ #DSTACK_SERVER_DISABLE_BACKGROUND_PROCESSING } - Disables background processing if set to any value. Useful to run only web frontend and API server. ??? info "Internal environment variables" The following environment variables are intended for development purposes: From 1dd19375db491fbd165ac23b005f5d0cb666099d Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Tue, 15 Jul 2025 10:51:00 +0500 Subject: [PATCH 4/4] Rename to DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED --- docs/docs/reference/environment-variables.md | 2 +- src/dstack/_internal/server/app.py | 7 +++++-- src/dstack/_internal/server/settings.py | 12 +++++++----- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/docs/reference/environment-variables.md b/docs/docs/reference/environment-variables.md index c442f0afd4..3c28ba3339 100644 --- a/docs/docs/reference/environment-variables.md +++ b/docs/docs/reference/environment-variables.md @@ -123,7 +123,7 @@ For more details on the options below, refer to the [server deployment](../guide - `DSTACK_DB_POOL_SIZE`{ #DSTACK_DB_POOL_SIZE } - The client DB connections pool size. Defaults to `20`, - `DSTACK_DB_MAX_OVERFLOW`{ #DSTACK_DB_MAX_OVERFLOW } - The client DB connections pool allowed overflow. Defaults to `20`. - `DSTACK_SERVER_BACKGROUND_PROCESSING_FACTOR`{ #DSTACK_SERVER_BACKGROUND_PROCESSING_FACTOR } - The number of background jobs for processing server resources. Increase if you need to process more resources per server replica quickly. Defaults to `1`. -- `STACK_SERVER_DISABLE_BACKGROUND_PROCESSING`{ #DSTACK_SERVER_DISABLE_BACKGROUND_PROCESSING } - Disables background processing if set to any value. Useful to run only web frontend and API server. +- `DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED`{ #DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED } - Disables background processing if set to any value. Useful to run only web frontend and API server. ??? info "Internal environment variables" The following environment variables are intended for development purposes: diff --git a/src/dstack/_internal/server/app.py b/src/dstack/_internal/server/app.py index fdedf50b8f..8aff963c1a 100644 --- a/src/dstack/_internal/server/app.py +++ b/src/dstack/_internal/server/app.py @@ -151,8 +151,10 @@ async def lifespan(app: FastAPI): ) if settings.SERVER_S3_BUCKET is not None or settings.SERVER_GCS_BUCKET is not None: init_default_storage() - if settings.SERVER_DISABLE_BACKGROUND_PROCESSING is not None: + if settings.SERVER_BACKGROUND_PROCESSING_ENABLED: scheduler = start_background_tasks() + else: + logger.info("Background processing is disabled") dstack_version = DSTACK_VERSION if DSTACK_VERSION else "(no version)" logger.info(f"The admin token is {admin.token.get_plaintext_or_error()}", {"show_path": False}) logger.info( @@ -162,7 +164,8 @@ async def lifespan(app: FastAPI): for func in _ON_STARTUP_HOOKS: await func(app) yield - scheduler.shutdown() + if settings.SERVER_BACKGROUND_PROCESSING_ENABLED: + scheduler.shutdown() await gateway_connections_pool.remove_all() service_conn_pool = await get_injector_from_app(app).get_service_connection_pool() await service_conn_pool.remove_all() diff --git a/src/dstack/_internal/server/settings.py b/src/dstack/_internal/server/settings.py index e0bc566534..f025d71644 100644 --- a/src/dstack/_internal/server/settings.py +++ b/src/dstack/_internal/server/settings.py @@ -42,6 +42,11 @@ os.getenv("DSTACK_SERVER_BACKGROUND_PROCESSING_FACTOR", 1) ) +SERVER_BACKGROUND_PROCESSING_DISABLED = ( + os.getenv("DSTACK_SERVER_BACKGROUND_PROCESSING_DISABLED") is not None +) +SERVER_BACKGROUND_PROCESSING_ENABLED = not SERVER_BACKGROUND_PROCESSING_DISABLED + SERVER_EXECUTOR_MAX_WORKERS = int(os.getenv("DSTACK_SERVER_EXECUTOR_MAX_WORKERS", 128)) MAX_OFFERS_TRIED = int(os.getenv("DSTACK_SERVER_MAX_OFFERS_TRIED", 25)) @@ -113,8 +118,5 @@ UPDATE_DEFAULT_PROJECT = os.getenv("DSTACK_UPDATE_DEFAULT_PROJECT") is not None DO_NOT_UPDATE_DEFAULT_PROJECT = os.getenv("DSTACK_DO_NOT_UPDATE_DEFAULT_PROJECT") is not None -SKIP_GATEWAY_UPDATE = os.getenv("DSTACK_SKIP_GATEWAY_UPDATE", None) is not None -ENABLE_PROMETHEUS_METRICS = os.getenv("DSTACK_ENABLE_PROMETHEUS_METRICS", None) is not None -SERVER_DISABLE_BACKGROUND_PROCESSING = ( - os.getenv("DSTACK_SERVER_DISABLE_BACKGROUND_PROCESSING", None) is not None -) +SKIP_GATEWAY_UPDATE = os.getenv("DSTACK_SKIP_GATEWAY_UPDATE") is not None +ENABLE_PROMETHEUS_METRICS = os.getenv("DSTACK_ENABLE_PROMETHEUS_METRICS") is not None