[Feature]: Default CLI log level is DEBUG; WARNING and above go to STDOUT, DEBUG logs to a file #2939

Author: peterschmidt85

docs/docs/guides/troubleshooting.md

@@ -14,8 +14,14 @@ If you have a question or need help, feel free to ask it in our [Discord server]
 Make sure to provide clear, detailed steps to reproduce the issue. 
 Include server logs, CLI outputs, and configuration samples. Avoid using screenshots for logs or errors—use text instead. 
 
-To get more detailed logs, make sure to set the `DSTACK_CLI_LOG_LEVEL` and `DSTACK_SERVER_LOG_LEVEL` 
-environment variables to `debug` when running the CLI and the server, respectively.
+#### Server logs
+
+To get more detailed server logs, set the `DSTACK_SERVER_LOG_LEVEL` 
+environment variables to `debug`. By default, it is set to `INFO`.
+
+#### CLI logs
+
+CLI logs are located in `~/.dstack/logs/cli`, and the default log level is `DEBUG`.
 
 > See these examples for well-reported issues: [this :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/issues/1640){:target="_blank"}
 and [this :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/issues/1551){:target="_blank"}.

docs/docs/reference/environment-variables.md

@@ -145,7 +145,7 @@ For more details on the options below, refer to the [server deployment](../guide
 
 The following environment variables are supported by the CLI.
 
-- `DSTACK_CLI_LOG_LEVEL`{ #DSTACK_CLI_LOG_LEVEL } – Configures CLI logging level. Defaults to `INFO`.
+- `DSTACK_CLI_LOG_LEVEL`{ #DSTACK_CLI_LOG_LEVEL } – Sets the logging level for CLI output to stdout. Defaults to `INFO`.
 
 Example:
 
@@ -157,4 +157,17 @@ $ DSTACK_CLI_LOG_LEVEL=debug dstack apply -f .dstack.yml
 
 </div>
 
+- `DSTACK_CLI_FILE_LOG_LEVEL`{ #DSTACK_CLI_FILE_LOG_LEVEL } – Sets the logging level for CLI log files. Defaults to `DEBUG`.
+
+<div class="termy">
+
+```shell
+$ find ~/.dstack/logs/cli/
+
+ ~/.dstack/logs/cli/latest.log
+ ~/.dstack/logs/cli/2025-07-31.log
+```
+
+</div>
+
 - `DSTACK_PROJECT`{ #DSTACK_PROJECT } – Has the same effect as `--project`. Defaults to `None`.

src/dstack/_internal/cli/utils/common.py

@@ -1,5 +1,6 @@
 import logging
 import os
+from datetime import datetime, timezone
 from typing import Any, Dict, Union
 
 from rich.console import Console
@@ -9,6 +10,7 @@
 
 from dstack._internal.cli.utils.rich import DstackRichHandler
 from dstack._internal.core.errors import CLIError, DstackError
+from dstack._internal.utils.common import get_dstack_dir
 
 _colors = {
     "secondary": "grey58",
@@ -37,10 +39,51 @@ def cli_error(e: DstackError) -> CLIError:
 
 def configure_logging():
     dstack_logger = logging.getLogger("dstack")
-    dstack_logger.setLevel(os.getenv("DSTACK_CLI_LOG_LEVEL", "INFO").upper())
-    handler = DstackRichHandler(console=console)
-    handler.setFormatter(logging.Formatter(fmt="%(message)s", datefmt="[%X]"))
-    dstack_logger.addHandler(handler)
+    dstack_logger.handlers.clear()
+
+    log_dir = get_dstack_dir() / "logs" / "cli"
+    log_file = log_dir / "latest.log"
+
+    if log_file.exists():
+        file_mtime = datetime.fromtimestamp(log_file.stat().st_mtime, tz=timezone.utc)
+        current_date = datetime.now(timezone.utc).date()
+
+        if file_mtime.date() < current_date:
+            date_str = file_mtime.strftime("%Y-%m-%d")
+            rotated_file = log_dir / f"{date_str}.log"
+
+            counter = 1
+            while rotated_file.exists():
+                rotated_file = log_dir / f"{date_str}-{counter}.log"
+                counter += 1
+
+            log_file.rename(rotated_file)
+
+    level_names = logging.getLevelNamesMapping()
+    stdout_level_name = os.getenv("DSTACK_CLI_LOG_LEVEL", "INFO").upper()
+    stdout_level = level_names[stdout_level_name]
+    dstack_logger.setLevel(stdout_level)
+
+    stdout_handler = DstackRichHandler(console=console)
+    stdout_handler.setFormatter(logging.Formatter(fmt="%(message)s", datefmt="[%X]"))
+    stdout_handler.setLevel(stdout_level)
+    dstack_logger.addHandler(stdout_handler)
+
+    file_level_name = os.getenv("DSTACK_CLI_FILE_LOG_LEVEL", "DEBUG").upper()
+    file_level = level_names[file_level_name]
+
+    log_dir.mkdir(parents=True, exist_ok=True)
+
+    file_handler = logging.FileHandler(log_file)
+    file_handler.setFormatter(
+        logging.Formatter(
+            fmt="%(asctime)s - %(name)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
+        )
+    )
+    file_handler.setLevel(file_level)
+    dstack_logger.addHandler(file_handler)
+
+    dstack_logger.setLevel(min(stdout_level, file_level))
 
 
 def confirm_ask(prompt, **kwargs) -> bool:

src/dstack/api/server/__init__.py

@@ -173,7 +173,7 @@ def _request(
                 raise ClientError(
                     f"Unexpected error: status code {resp.status_code}"
                     f" when requesting {resp.request.url}."
-                    " Check server logs or run with DSTACK_CLI_LOG_LEVEL=DEBUG to see more details"
+                    " Check the server logs for backend issues, and the CLI logs at (~/.dstack/logs/cli/latest.log) local CLI output"
                 )
         return resp