Replace enums with strings in the DB, JobSubmission.termination_reason, and Run.termination_reason (#2949)

* Store all enums as strings in DB

* Make run and job termination_reason a string

* Start using JobTerminationReason.WAITING_INSTANCE_LIMIT_EXCEEDED

* Rebase migrations

Author: r4victor

src/dstack/_internal/core/models/runs.py

@@ -350,15 +350,17 @@ class JobSubmission(CoreModel):
     deployment_num: int = 0  # default for compatibility with pre-0.19.14 servers
     submitted_at: datetime
     last_processed_at: datetime
-    finished_at: Optional[datetime]
-    inactivity_secs: Optional[int]
+    finished_at: Optional[datetime] = None
+    inactivity_secs: Optional[int] = None
     status: JobStatus
     status_message: str = ""  # default for backward compatibility
-    termination_reason: Optional[JobTerminationReason]
-    termination_reason_message: Optional[str]
-    exit_status: Optional[int]
-    job_provisioning_data: Optional[JobProvisioningData]
-    job_runtime_data: Optional[JobRuntimeData]
+    # termination_reason stores JobTerminationReason.
+    # str allows adding new enum members without breaking compatibility with old clients.
+    termination_reason: Optional[str] = None
+    termination_reason_message: Optional[str] = None
+    exit_status: Optional[int] = None
+    job_provisioning_data: Optional[JobProvisioningData] = None
+    job_runtime_data: Optional[JobRuntimeData] = None
     error: Optional[str] = None
     probes: list[Probe] = []
 
@@ -508,7 +510,9 @@ class Run(CoreModel):
     last_processed_at: datetime
     status: RunStatus
     status_message: str = ""  # default for backward compatibility
-    termination_reason: Optional[RunTerminationReason] = None
+    # termination_reason stores RunTerminationReason.
+    # str allows adding new enum members without breaking compatibility with old clients.
+    termination_reason: Optional[str] = None
     run_spec: RunSpec
     jobs: List[Job]
     latest_job_submission: Optional[JobSubmission] = None

src/dstack/_internal/server/background/tasks/process_running_jobs.py

@@ -455,8 +455,7 @@ async def _wait_for_instance_provisioning_data(job_model: JobModel):
 
     if job_model.instance.status == InstanceStatus.TERMINATED:
         job_model.status = JobStatus.TERMINATING
-        # TODO use WAITING_INSTANCE_LIMIT_EXCEEDED after 0.19.x
-        job_model.termination_reason = JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY
+        job_model.termination_reason = JobTerminationReason.WAITING_INSTANCE_LIMIT_EXCEEDED
         return
 
     job_model.job_provisioning_data = job_model.instance.job_provisioning_data

src/dstack/_internal/server/background/tasks/process_runs.py

@@ -574,7 +574,7 @@ def _should_retry_job(run: Run, job: Job, job_model: JobModel) -> Optional[datet
 
     if (
         last_provisioned_submission.termination_reason is not None
-        and last_provisioned_submission.termination_reason.to_retry_event()
+        and JobTerminationReason(last_provisioned_submission.termination_reason).to_retry_event()
         in job.job_spec.retry.on_events
     ):
         return common.get_current_datetime() - last_provisioned_submission.last_processed_at