[Bug]: Empty logs cause server to spew python errors #2899
Description
Steps to reproduce
- Create some sort of run that generates no logs
- Attempt to view logs using the front end.
Actual behaviour
Front end does the right thing and displays "No logs"
But the server throws the following error:
ERROR: Exception in ASGI application
+ Exception Group Traceback (most recent call last):
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_utils.py", line 76, in collapse_excgroups
| yield
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 177, in __call__
| async with anyio.create_task_group() as task_group:
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 772, in __aexit__
| raise BaseExceptionGroup(
| ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception)
+-+---------------- 1 ----------------
| Traceback (most recent call last):
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/uvicorn/protocols/http/httptools_impl.py", line 409, in run_asgi
| result = await app( # type: ignore[func-returns-value]
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__
| return await self.app(scope, receive, send)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/fastapi/applications.py", line 1054, in __call__
| await super().__call__(scope, receive, send)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/applications.py", line 112, in __call__
| await self.middleware_stack(scope, receive, send)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/errors.py", line 187, in __call__
| raise exc
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/errors.py", line 165, in __call__
| await self.app(scope, receive, _send)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 176, in __call__
| with recv_stream, send_stream, collapse_excgroups():
| File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
| self.gen.throw(typ, value, traceback)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_utils.py", line 82, in collapse_excgroups
| raise exc
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 178, in __call__
| response = await self.dispatch_func(request, call_next)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/app.py", line 308, in check_client_version
| return await call_next(request)
| ^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 156, in call_next
| raise app_exc
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 141, in coro
| await self.app(scope, receive_or_disconnect, send_no_error)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 176, in __call__
| with recv_stream, send_stream, collapse_excgroups():
| File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
| self.gen.throw(typ, value, traceback)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_utils.py", line 82, in collapse_excgroups
| raise exc
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 178, in __call__
| response = await self.dispatch_func(request, call_next)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/app.py", line 278, in log_http_metrics
| response: Response = await call_next(request)
| ^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 156, in call_next
| raise app_exc
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 141, in coro
| await self.app(scope, receive_or_disconnect, send_no_error)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 176, in __call__
| with recv_stream, send_stream, collapse_excgroups():
| File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
| self.gen.throw(typ, value, traceback)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_utils.py", line 82, in collapse_excgroups
| raise exc
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 178, in __call__
| response = await self.dispatch_func(request, call_next)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/app.py", line 235, in log_request
| response: Response = await call_next(request)
| ^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 156, in call_next
| raise app_exc
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 141, in coro
| await self.app(scope, receive_or_disconnect, send_no_error)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/exceptions.py", line 62, in __call__
| await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
| raise exc
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
| await app(scope, receive, sender)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/routing.py", line 714, in __call__
| await self.middleware_stack(scope, receive, send)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/routing.py", line 734, in app
| await route.handle(scope, receive, send)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/routing.py", line 288, in handle
| await self.app(scope, receive, send)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/routing.py", line 76, in app
| await wrap_app_handling_exceptions(app, request)(scope, receive, send)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
| raise exc
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
| await app(scope, receive, sender)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/routing.py", line 73, in app
| response = await f(request)
| ^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/fastapi/routing.py", line 301, in app
| raw_response = await run_endpoint_function(
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/fastapi/routing.py", line 212, in run_endpoint_function
| return await dependant.call(**values)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/routers/logs.py", line 30, in poll_logs
| return await logs.poll_logs_async(project=project, request=body)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/services/logs/__init__.py", line 78, in poll_logs_async
| return await run_async(get_log_storage().poll_logs, project=project, request=request)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/utils/common.py", line 21, in run_async
| return await asyncio.get_running_loop().run_in_executor(None, func_with_args)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/usr/local/lib/python3.11/concurrent/futures/thread.py", line 58, in run
| result = self.fn(*self.args, **self.kwargs)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/services/logs/filelog.py", line 87, in poll_logs
| raise LogStorageError(f"Failed to read log file {log_file_path}: {e}")
| dstack._internal.server.services.logs.base.LogStorageError: Failed to read log file /root/.dstack/server/projects/main/logs/tasty-cheetah-6/7b0433a3-9344-46f5-a9df-e6473acb1458/job.log: [Errno 2] No such file or directory: '/root/.dstack/server/projects/main/logs/tasty-cheetah-6/7b0433a3-9344-46f5-a9df-e6473acb1458/job.log'
+------------------------------------
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/uvicorn/protocols/http/httptools_impl.py", line 409, in run_asgi
result = await app( # type: ignore[func-returns-value]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__
return await self.app(scope, receive, send)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/fastapi/applications.py", line 1054, in __call__
await super().__call__(scope, receive, send)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/applications.py", line 112, in __call__
await self.middleware_stack(scope, receive, send)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/errors.py", line 187, in __call__
raise exc
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/errors.py", line 165, in __call__
await self.app(scope, receive, _send)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 176, in __call__
with recv_stream, send_stream, collapse_excgroups():
File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
self.gen.throw(typ, value, traceback)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_utils.py", line 82, in collapse_excgroups
raise exc
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 178, in __call__
response = await self.dispatch_func(request, call_next)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/app.py", line 308, in check_client_version
return await call_next(request)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 156, in call_next
raise app_exc
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 141, in coro
await self.app(scope, receive_or_disconnect, send_no_error)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 176, in __call__
with recv_stream, send_stream, collapse_excgroups():
File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
self.gen.throw(typ, value, traceback)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_utils.py", line 82, in collapse_excgroups
raise exc
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 178, in __call__
response = await self.dispatch_func(request, call_next)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/app.py", line 278, in log_http_metrics
response: Response = await call_next(request)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 156, in call_next
raise app_exc
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 141, in coro
await self.app(scope, receive_or_disconnect, send_no_error)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 176, in __call__
with recv_stream, send_stream, collapse_excgroups():
File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
self.gen.throw(typ, value, traceback)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_utils.py", line 82, in collapse_excgroups
raise exc
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 178, in __call__
response = await self.dispatch_func(request, call_next)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/app.py", line 235, in log_request
response: Response = await call_next(request)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 156, in call_next
raise app_exc
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 141, in coro
await self.app(scope, receive_or_disconnect, send_no_error)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/exceptions.py", line 62, in __call__
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/routing.py", line 714, in __call__
await self.middleware_stack(scope, receive, send)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/routing.py", line 734, in app
await route.handle(scope, receive, send)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/routing.py", line 288, in handle
await self.app(scope, receive, send)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/routing.py", line 76, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/routing.py", line 73, in app
response = await f(request)
^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/fastapi/routing.py", line 301, in app
raw_response = await run_endpoint_function(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/fastapi/routing.py", line 212, in run_endpoint_function
return await dependant.call(**values)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/routers/logs.py", line 30, in poll_logs
return await logs.poll_logs_async(project=project, request=body)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/services/logs/__init__.py", line 78, in poll_logs_async
return await run_async(get_log_storage().poll_logs, project=project, request=request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/utils/common.py", line 21, in run_async
return await asyncio.get_running_loop().run_in_executor(None, func_with_args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/concurrent/futures/thread.py", line 58, in run
result = self.fn(*self.args, **self.kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/services/logs/filelog.py", line 87, in poll_logs
raise LogStorageError(f"Failed to read log file {log_file_path}: {e}")
dstack._internal.server.services.logs.base.LogStorageError: Failed to read log file /root/.dstack/server/projects/main/logs/tasty-cheetah-6/7b0433a3-9344-46f5-a9df-e6473acb1458/job.log: [Errno 2] No such file or directory: '/root/.dstack/server/projects/main/logs/tasty-cheetah-6/7b0433a3-9344-46f5-a9df-e6473acb1458/job.log'
Expected behaviour
Server should gracefully handle the state
Or, a log file should be generated regardless of the contents.
dstack version
0.19.18
Server logs
(This is just a copy of the log in "Actual Behavior")
: Exception in ASGI application
+ Exception Group Traceback (most recent call last):
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_utils.py", line 76, in collapse_excgroups
| yield
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 177, in __call__
| async with anyio.create_task_group() as task_group:
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 772, in __aexit__
| raise BaseExceptionGroup(
| ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception)
+-+---------------- 1 ----------------
| Traceback (most recent call last):
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/uvicorn/protocols/http/httptools_impl.py", line 409, in run_asgi
| result = await app( # type: ignore[func-returns-value]
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__
| return await self.app(scope, receive, send)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/fastapi/applications.py", line 1054, in __call__
| await super().__call__(scope, receive, send)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/applications.py", line 112, in __call__
| await self.middleware_stack(scope, receive, send)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/errors.py", line 187, in __call__
| raise exc
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/errors.py", line 165, in __call__
| await self.app(scope, receive, _send)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 176, in __call__
| with recv_stream, send_stream, collapse_excgroups():
| File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
| self.gen.throw(typ, value, traceback)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_utils.py", line 82, in collapse_excgroups
| raise exc
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 178, in __call__
| response = await self.dispatch_func(request, call_next)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/app.py", line 308, in check_client_version
| return await call_next(request)
| ^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 156, in call_next
| raise app_exc
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 141, in coro
| await self.app(scope, receive_or_disconnect, send_no_error)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 176, in __call__
| with recv_stream, send_stream, collapse_excgroups():
| File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
| self.gen.throw(typ, value, traceback)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_utils.py", line 82, in collapse_excgroups
| raise exc
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 178, in __call__
| response = await self.dispatch_func(request, call_next)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/app.py", line 278, in log_http_metrics
| response: Response = await call_next(request)
| ^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 156, in call_next
| raise app_exc
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 141, in coro
| await self.app(scope, receive_or_disconnect, send_no_error)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 176, in __call__
| with recv_stream, send_stream, collapse_excgroups():
| File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
| self.gen.throw(typ, value, traceback)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_utils.py", line 82, in collapse_excgroups
| raise exc
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 178, in __call__
| response = await self.dispatch_func(request, call_next)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/app.py", line 235, in log_request
| response: Response = await call_next(request)
| ^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 156, in call_next
| raise app_exc
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 141, in coro
| await self.app(scope, receive_or_disconnect, send_no_error)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/exceptions.py", line 62, in __call__
| await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
| raise exc
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
| await app(scope, receive, sender)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/routing.py", line 714, in __call__
| await self.middleware_stack(scope, receive, send)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/routing.py", line 734, in app
| await route.handle(scope, receive, send)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/routing.py", line 288, in handle
| await self.app(scope, receive, send)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/routing.py", line 76, in app
| await wrap_app_handling_exceptions(app, request)(scope, receive, send)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
| raise exc
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
| await app(scope, receive, sender)
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/routing.py", line 73, in app
| response = await f(request)
| ^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/fastapi/routing.py", line 301, in app
| raw_response = await run_endpoint_function(
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/fastapi/routing.py", line 212, in run_endpoint_function
| return await dependant.call(**values)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/routers/logs.py", line 30, in poll_logs
| return await logs.poll_logs_async(project=project, request=body)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/services/logs/__init__.py", line 78, in poll_logs_async
| return await run_async(get_log_storage().poll_logs, project=project, request=request)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/utils/common.py", line 21, in run_async
| return await asyncio.get_running_loop().run_in_executor(None, func_with_args)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/usr/local/lib/python3.11/concurrent/futures/thread.py", line 58, in run
| result = self.fn(*self.args, **self.kwargs)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
| File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/services/logs/filelog.py", line 87, in poll_logs
| raise LogStorageError(f"Failed to read log file {log_file_path}: {e}")
| dstack._internal.server.services.logs.base.LogStorageError: Failed to read log file /root/.dstack/server/projects/main/logs/tasty-cheetah-6/7b0433a3-9344-46f5-a9df-e6473acb1458/job.log: [Errno 2] No such file or directory: '/root/.dstack/server/projects/main/logs/tasty-cheetah-6/7b0433a3-9344-46f5-a9df-e6473acb1458/job.log'
+------------------------------------
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/uvicorn/protocols/http/httptools_impl.py", line 409, in run_asgi
result = await app( # type: ignore[func-returns-value]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__
return await self.app(scope, receive, send)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/fastapi/applications.py", line 1054, in __call__
await super().__call__(scope, receive, send)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/applications.py", line 112, in __call__
await self.middleware_stack(scope, receive, send)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/errors.py", line 187, in __call__
raise exc
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/errors.py", line 165, in __call__
await self.app(scope, receive, _send)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 176, in __call__
with recv_stream, send_stream, collapse_excgroups():
File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
self.gen.throw(typ, value, traceback)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_utils.py", line 82, in collapse_excgroups
raise exc
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 178, in __call__
response = await self.dispatch_func(request, call_next)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/app.py", line 308, in check_client_version
return await call_next(request)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 156, in call_next
raise app_exc
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 141, in coro
await self.app(scope, receive_or_disconnect, send_no_error)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 176, in __call__
with recv_stream, send_stream, collapse_excgroups():
File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
self.gen.throw(typ, value, traceback)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_utils.py", line 82, in collapse_excgroups
raise exc
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 178, in __call__
response = await self.dispatch_func(request, call_next)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/app.py", line 278, in log_http_metrics
response: Response = await call_next(request)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 156, in call_next
raise app_exc
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 141, in coro
await self.app(scope, receive_or_disconnect, send_no_error)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 176, in __call__
with recv_stream, send_stream, collapse_excgroups():
File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
self.gen.throw(typ, value, traceback)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_utils.py", line 82, in collapse_excgroups
raise exc
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 178, in __call__
response = await self.dispatch_func(request, call_next)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/app.py", line 235, in log_request
response: Response = await call_next(request)
^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 156, in call_next
raise app_exc
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/base.py", line 141, in coro
await self.app(scope, receive_or_disconnect, send_no_error)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/middleware/exceptions.py", line 62, in __call__
await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/routing.py", line 714, in __call__
await self.middleware_stack(scope, receive, send)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/routing.py", line 734, in app
await route.handle(scope, receive, send)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/routing.py", line 288, in handle
await self.app(scope, receive, send)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/routing.py", line 76, in app
await wrap_app_handling_exceptions(app, request)(scope, receive, send)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
raise exc
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
await app(scope, receive, sender)
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/starlette/routing.py", line 73, in app
response = await f(request)
^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/fastapi/routing.py", line 301, in app
raw_response = await run_endpoint_function(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/fastapi/routing.py", line 212, in run_endpoint_function
return await dependant.call(**values)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/routers/logs.py", line 30, in poll_logs
return await logs.poll_logs_async(project=project, request=body)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/services/logs/__init__.py", line 78, in poll_logs_async
return await run_async(get_log_storage().poll_logs, project=project, request=request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/utils/common.py", line 21, in run_async
return await asyncio.get_running_loop().run_in_executor(None, func_with_args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/concurrent/futures/thread.py", line 58, in run
result = self.fn(*self.args, **self.kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/.local/share/uv/tools/dstack/lib/python3.11/site-packages/dstack/_internal/server/services/logs/filelog.py", line 87, in poll_logs
raise LogStorageError(f"Failed to read log file {log_file_path}: {e}")
dstack._internal.server.services.logs.base.LogStorageError: Failed to read log file /root/.dstack/server/projects/main/logs/tasty-cheetah-6/7b0433a3-9344-46f5-a9df-e6473acb1458/job.log: [Errno 2] No such file or directory: '/root/.dstack/server/projects/main/logs/tasty-cheetah-6/7b0433a3-9344-46f5-a9df-e6473acb1458/job.log'Additional information
This appears to be a new issue since 0.19.9, but I cannot speak to more accuracy than that.