Merge branch 'master' into issue_2802_runpod_instant_clusters

Author: r4victor

CONTRIBUTING.md

@@ -35,7 +35,7 @@ If you make a non-trivial change to `dstack`, we recommend you learn about `dsta
 * Bug fixes that address a clearly defined bug. Include steps to reproduce in the linked issue or the PR.
 * New features. Before submitting a feature PR, create an issue with a proposal to discuss it with the core team and other interested parties.
 * Minor fixes such as typos.
-* [Examples](examples/README.md).
+* [Examples](examples).
 
 ### Before pushing your changes
 

docs/docs/guides/server-deployment.md

@@ -400,6 +400,28 @@ export DSTACK_DB_MAX_OVERFLOW=80
 You have to ensure your Postgres installation supports that many connections by
 configuring [`max_connections`](https://www.postgresql.org/docs/current/runtime-config-connection.html#GUC-MAX-CONNECTIONS) and/or using connection pooler.
 
+## Server upgrades
+
+When upgrading the `dstack` server, follow these guidelines to ensure a smooth transition and minimize downtime.
+
+### Before upgrading
+
+1. **Check the changelog**: Review the [release notes :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/releases){:target="_blank"} for breaking changes, new features, and migration notes.
+2. **Review backward compatibility**: Understand the [backward compatibility](#backward-compatibility) policy.
+3. **Back up your data**: Ensure you always create a backup before upgrading.
+
+### Best practices
+
+- **Test in staging**: Always test upgrades in a non-production environment first.
+- **Monitor logs**: Watch server logs during and after the upgrade for any errors or warnings.
+- **Keep backups**: Retain backups for at least a few days after a successful upgrade.
+
+### Troubleshooting
+
+**Deadlock when upgrading a multi-replica PostgreSQL deployment**
+
+If a deployment is stuck due to a deadlock when applying DB migrations, try scaling server replicas to 1 and retry the deployment multiple times. Some releases may not support rolling deployments, which is always noted in the release notes. If you think there is a bug, please [file an issue](https://github.com/dstackai/dstack/issues).
+
 ## FAQs
 
 ??? info "Can I run multiple replicas of dstack server?"

frontend/src/pages/Runs/Details/Jobs/Details/JobDetails/index.tsx

@@ -9,6 +9,7 @@ import { useGetRunQuery } from 'services/run';
 
 import {
     getJobError,
+    getJobFinishedAt,
     getJobListItemBackend,
     getJobListItemInstance,
     getJobListItemPrice,
@@ -59,6 +60,11 @@ export const JobDetails = () => {
                         <div>{getJobSubmittedAt(jobData)}</div>
                     </div>
 
+                    <div>
+                        <Box variant="awsui-key-label">{t('projects.run.finished_at')}</Box>
+                        <div>{getJobFinishedAt(jobData)}</div>
+                    </div>
+
                     <div>
                         <Box variant="awsui-key-label">{t('projects.run.status')}</Box>
                         <div>

frontend/src/pages/Runs/Details/Jobs/List/helpers.ts

@@ -39,6 +39,11 @@ export const getJobSubmittedAt = (job: IJob) => {
         : '';
 };
 
+export const getJobFinishedAt = (job: IJob) => {
+    const finished_at = job.job_submissions?.[job.job_submissions.length - 1].finished_at;
+    return finished_at ? format(new Date(finished_at), DATE_TIME_FORMAT) : '';
+};
+
 export const getJobStatus = (job: IJob) => {
     return job.job_submissions?.[job.job_submissions.length - 1].status;
 };

frontend/src/pages/Runs/Details/Jobs/List/hooks.tsx

@@ -8,6 +8,7 @@ import { ROUTES } from 'routes';
 
 import {
     getJobError,
+    getJobFinishedAt,
     getJobListItemBackend,
     getJobListItemInstance,
     getJobListItemPrice,
@@ -48,6 +49,11 @@ export const useColumnsDefinitions = ({
             header: t('projects.run.submitted_at'),
             cell: getJobSubmittedAt,
         },
+        {
+            id: 'finished_at',
+            header: t('projects.run.finished_at'),
+            cell: getJobFinishedAt,
+        },
         {
             id: 'status',
             header: t('projects.run.status'),

frontend/src/pages/Runs/Details/RunDetails/index.tsx

@@ -15,6 +15,7 @@ import { finishedRunStatuses } from 'pages/Runs/constants';
 import { runIsStopped } from 'pages/Runs/utils';
 
 import {
+    getRunListFinishedAt,
     getRunListItemBackend,
     getRunListItemInstanceId,
     getRunListItemPrice,
@@ -59,6 +60,8 @@ export const RunDetails = () => {
         ? runData.latest_job_submission?.termination_reason
         : null;
 
+    const finishedAt = getRunListFinishedAt(runData);
+
     return (
         <>
             <Container header={<Header variant="h2">{t('common.general')}</Header>}>
@@ -101,7 +104,7 @@ export const RunDetails = () => {
 
                     <div>
                         <Box variant="awsui-key-label">{t('projects.run.finished_at')}</Box>
-                        <div>{runData.terminated_at ? format(new Date(runData.terminated_at), DATE_TIME_FORMAT) : '-'}</div>
+                        <div>{finishedAt ? format(new Date(finishedAt), DATE_TIME_FORMAT) : '-'}</div>
                     </div>
 
                     <div>

frontend/src/pages/Runs/List/Preferences/consts.ts

@@ -9,12 +9,12 @@ export const DEFAULT_PREFERENCES: CollectionPreferencesProps.Preferences = {
         { id: 'hub_user_name', visible: true },
         { id: 'price', visible: true },
         { id: 'submitted_at', visible: true },
+        { id: 'finished_at', visible: true },
         { id: 'status', visible: true },
         { id: 'error', visible: true },
         { id: 'cost', visible: true },
         // hidden by default
         { id: 'priority', visible: false },
-        { id: 'finished_at', visible: false },
         { id: 'project', visible: false },
         { id: 'repo', visible: false },
         { id: 'instance', visible: false },

frontend/src/pages/Runs/List/helpers.ts

@@ -2,7 +2,7 @@ import { groupBy as _groupBy } from 'lodash';
 
 import { getBaseUrl } from 'App/helpers';
 
-import { finishedJobs } from '../constants';
+import { finishedJobs, finishedRunStatuses } from '../constants';
 import { getJobStatus } from '../Details/Jobs/List/helpers';
 
 export const getGroupedRunsByProjectAndRepoID = (runs: IRun[]) => {
@@ -98,3 +98,10 @@ export const getRunListItemSchedule = (run: IRun) => {
 
     return run.run_spec.configuration.schedule.cron.join(', ');
 };
+
+export const getRunListFinishedAt = (run: IRun) => {
+    if (!run.latest_job_submission || !run.latest_job_submission.finished_at || !finishedRunStatuses.includes(run.status)) {
+        return null;
+    }
+    return run.latest_job_submission.finished_at;
+};

frontend/src/pages/Runs/List/hooks/useColumnsDefinitions.tsx

@@ -18,6 +18,7 @@ import { ROUTES } from 'routes';
 import { finishedRunStatuses } from 'pages/Runs/constants';
 
 import {
+    getRunListFinishedAt,
     getRunListItemBackend,
     getRunListItemInstance,
     getRunListItemPrice,
@@ -68,7 +69,10 @@ export const useColumnsDefinitions = () => {
         {
             id: 'finished_at',
             header: t('projects.run.finished_at'),
-            cell: (item: IRun) => (item.terminated_at ? format(new Date(item.terminated_at), DATE_TIME_FORMAT) : null),
+            cell: (item: IRun) => {
+                const finishedAt = getRunListFinishedAt(item);
+                return finishedAt ? format(new Date(finishedAt), DATE_TIME_FORMAT) : '-';
+            },
         },
         {
             id: 'status',

frontend/src/types/run.d.ts

@@ -228,6 +228,7 @@ declare interface IJobSubmission {
     submission_num: number;
     status: TJobStatus;
     submitted_at: number;
+    finished_at: string | null;
     termination_reason?: string | null;
     termination_reason_message?: string | null;
     exit_status?: number | null;
@@ -283,7 +284,6 @@ declare interface IRun {
     project_name: string;
     user: string;
     submitted_at: string;
-    terminated_at: string | null;
     status: TJobStatus;
     error?: string | null;
     jobs: IJob[];