Skip to content

Fix fleet provisioning error message when fleet retried #3109

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
r4victor merged 1 commit into from
Sep 17, 2025
Merged

Fix fleet provisioning error message when fleet retried #3109

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 20 additions & 6 deletions src/dstack/_internal/cli/services/configurators/fleet.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,12 +159,19 @@ def _apply_plan(self, plan: FleetPlan, command_args: argparse.Namespace):
console.print(
get_fleets_table(
[fleet],
verbose=_failed_provisioning(fleet),
verbose=_fleet_has_failed_instances(fleet),
format_date=local_time,
)
)
if _failed_provisioning(fleet):
console.print("\n[error]Some instances failed. Check the table above for errors.[/]")
if _fleet_has_failed_instances(fleet):
if _fleet_retrying(fleet):
console.print(
"\n[error]Some instances failed. Provisioning will be retried in the background.[/]"
)
else:
console.print(
"\n[error]Some instances failed. Check the table above for errors.[/]"
)
exit(1)

def _apply_plan_on_old_server(self, plan: FleetPlan, command_args: argparse.Namespace):
Expand Down Expand Up @@ -253,11 +260,11 @@ def _apply_plan_on_old_server(self, plan: FleetPlan, command_args: argparse.Name
console.print(
get_fleets_table(
[fleet],
verbose=_failed_provisioning(fleet),
verbose=_fleet_has_failed_instances(fleet),
format_date=local_time,
)
)
if _failed_provisioning(fleet):
if _fleet_has_failed_instances(fleet):
console.print("\n[error]Some instances failed. Check the table above for errors.[/]")
exit(1)

Expand Down Expand Up @@ -462,13 +469,20 @@ def _finished_provisioning(fleet: Fleet) -> bool:
return True


def _failed_provisioning(fleet: Fleet) -> bool:
def _fleet_has_failed_instances(fleet: Fleet) -> bool:
for instance in fleet.instances:
if instance.status == InstanceStatus.TERMINATED:
return True
return False


def _fleet_retrying(fleet: Fleet) -> bool:
if fleet.spec.configuration.nodes is None:
return False
active_instances = [i for i in fleet.instances if i.status.is_active()]
return len(active_instances) < fleet.spec.configuration.nodes.min


def _apply_plan(api: Client, plan: FleetPlan) -> Fleet:
try:
return api.client.fleets.apply_plan(
Expand Down
Loading