Add repos[].path property (#3041)

- If not set, the legacy `/workflow` path is used (only during
  the transitional period)
- Relative paths are resolved relative to `working_dir`
- `~[/path]` is supported, but `~username[/path]` is not, as with files
- Available inside the container as `DSTACK_REPO_RER` env variable

`working_dir`

- Must be absolute
- If not set, the image default value is used for tasks and services
  without `commands` and the legacy `/workflow` path for other
  configurations (replicating pre-0.19.27 `JobConfigurator` logic in CLI
  only during the transitional period)
- `~[/path]` is supported, but `~username[/path]` is not, as with files

runner

- `/workflow/.venv` moved to `/dstack/venv`
- `/tmp/dstack_profile` moved to `/dstack/profile`
- `--working-dir` is deprecated and ignored

Closes: #2851

Author: un-def

docs/docs/reference/dstack.yml/dev-environment.md

@@ -100,6 +100,28 @@ The `dev-environment` configuration type allows running [dev environments](../..
     * `volume-name:/container/path` for network volumes
     * `/instance/path:/container/path` for instance volumes
 
+### `repos[n]` { #_repos data-toc-label="repos" }
+
+> Currently, a maximum of one repo is supported.
+
+> Either `local_path` or `url` must be specified.
+
+#SCHEMA# dstack._internal.core.models.configurations.RepoSpec
+    overrides:
+      show_root_heading: false
+      type:
+        required: true
+
+??? info "Short syntax"
+
+    The short syntax for repos is a colon-separated string in the form of `local_path_or_url:path`.
+
+    * `.:/repo`
+    * `..:repo`
+    * `~/repos/demo:~/repo`
+    * `https://github.com/org/repo:~/data/repo`
+    * `git@github.com:org/repo.git:data/repo`
+
 ### `files[n]` { #_files data-toc-label="files" }
 
 #SCHEMA# dstack._internal.core.models.files.FilePathMapping

docs/docs/reference/dstack.yml/service.md

@@ -215,6 +215,28 @@ The `service` configuration type allows running [services](../../concepts/servic
     * `volume-name:/container/path` for network volumes
     * `/instance/path:/container/path` for instance volumes
 
+### `repos[n]` { #_repos data-toc-label="repos" }
+
+> Currently, a maximum of one repo is supported.
+
+> Either `local_path` or `url` must be specified.
+
+#SCHEMA# dstack._internal.core.models.configurations.RepoSpec
+    overrides:
+      show_root_heading: false
+      type:
+        required: true
+
+??? info "Short syntax"
+
+    The short syntax for repos is a colon-separated string in the form of `local_path_or_url:path`.
+
+    * `.:/repo`
+    * `..:repo`
+    * `~/repos/demo:~/repo`
+    * `https://github.com/org/repo:~/data/repo`
+    * `git@github.com:org/repo.git:data/repo`
+
 ### `files[n]` { #_files data-toc-label="files" }
 
 #SCHEMA# dstack._internal.core.models.files.FilePathMapping

docs/docs/reference/dstack.yml/task.md

@@ -100,6 +100,28 @@ The `task` configuration type allows running [tasks](../../concepts/tasks.md).
     * `volume-name:/container/path` for network volumes
     * `/instance/path:/container/path` for instance volumes
 
+### `repos[n]` { #_repos data-toc-label="repos" }
+
+> Currently, a maximum of one repo is supported.
+
+> Either `local_path` or `url` must be specified.
+
+#SCHEMA# dstack._internal.core.models.configurations.RepoSpec
+    overrides:
+      show_root_heading: false
+      type:
+        required: true
+
+??? info "Short syntax"
+
+    The short syntax for repos is a colon-separated string in the form of `local_path_or_url:path`.
+
+    * `.:/repo`
+    * `..:repo`
+    * `~/repos/demo:~/repo`
+    * `https://github.com/org/repo:~/data/repo`
+    * `git@github.com:org/repo.git:data/repo`
+
 ### `files[n]` { #_files data-toc-label="files" }
 
 #SCHEMA# dstack._internal.core.models.files.FilePathMapping

examples/misc/airflow/dags/dstack_tasks.py

@@ -47,7 +47,7 @@ def dstack_cli_apply_venv() -> str:
         dstack is installed into a separate virtual environment available to Airflow.
         """
         return (
-            f"source {DSTACK_VENV_PATH}/bin/activate"
+            f". {DSTACK_VENV_PATH}/bin/activate"
             f" && cd {DSTACK_REPO_PATH}"
             " && dstack apply -y -f task.dstack.yml --repo ."
         )

runner/cmd/runner/cmd.go

@@ -4,14 +4,16 @@ import (
 	"log"
 	"os"
 
+	"github.com/dstackai/dstack/runner/consts"
 	"github.com/urfave/cli/v2"
 )
 
 // Version is a build-time variable. The value is overridden by ldflags.
 var Version string
 
 func App() {
-	var paths struct{ tempDir, homeDir, workingDir string }
+	var tempDir string
+	var homeDir string
 	var httpPort int
 	var sshPort int
 	var logLevel int
@@ -37,36 +39,36 @@ func App() {
 					&cli.PathFlag{
 						Name:        "temp-dir",
 						Usage:       "Temporary directory for logs and other files",
-						Required:    true,
-						Destination: &paths.tempDir,
+						Value:       consts.RunnerTempDir,
+						Destination: &tempDir,
 					},
 					&cli.PathFlag{
 						Name:        "home-dir",
 						Usage:       "HomeDir directory for credentials and $HOME",
-						Required:    true,
-						Destination: &paths.homeDir,
+						Value:       consts.RunnerHomeDir,
+						Destination: &homeDir,
 					},
+					// TODO: Not used, left for compatibility with old servers. Remove eventually.
 					&cli.PathFlag{
 						Name:        "working-dir",
-						Usage:       "Base path for the job",
-						Required:    true,
-						Destination: &paths.workingDir,
+						Hidden:      true,
+						Destination: nil,
 					},
 					&cli.IntFlag{
 						Name:        "http-port",
 						Usage:       "Set a http port",
-						Value:       10999,
+						Value:       consts.RunnerHTTPPort,
 						Destination: &httpPort,
 					},
 					&cli.IntFlag{
 						Name:        "ssh-port",
 						Usage:       "Set the ssh port",
-						Required:    true,
+						Value:       consts.RunnerSSHPort,
 						Destination: &sshPort,
 					},
 				},
 				Action: func(c *cli.Context) error {
-					err := start(paths.tempDir, paths.homeDir, paths.workingDir, httpPort, sshPort, logLevel, Version)
+					err := start(tempDir, homeDir, httpPort, sshPort, logLevel, Version)
 					if err != nil {
 						return cli.Exit(err, 1)
 					}

runner/cmd/runner/main.go

@@ -19,7 +19,7 @@ func main() {
 	App()
 }
 
-func start(tempDir string, homeDir string, workingDir string, httpPort int, sshPort int, logLevel int, version string) error {
+func start(tempDir string, homeDir string, httpPort int, sshPort int, logLevel int, version string) error {
 	if err := os.MkdirAll(tempDir, 0o755); err != nil {
 		return tracerr.Errorf("Failed to create temp directory: %w", err)
 	}
@@ -38,7 +38,7 @@ func start(tempDir string, homeDir string, workingDir string, httpPort int, sshP
 	log.DefaultEntry.Logger.SetOutput(io.MultiWriter(os.Stdout, defaultLogFile))
 	log.DefaultEntry.Logger.SetLevel(logrus.Level(logLevel))
 
-	server, err := api.NewServer(tempDir, homeDir, workingDir, fmt.Sprintf(":%d", httpPort), sshPort, version)
+	server, err := api.NewServer(tempDir, homeDir, fmt.Sprintf(":%d", httpPort), sshPort, version)
 	if err != nil {
 		return tracerr.Errorf("Failed to create server: %w", err)
 	}

runner/cmd/shim/main.go

@@ -79,14 +79,14 @@ func main() {
 			&cli.IntFlag{
 				Name:        "runner-http-port",
 				Usage:       "Set runner's http port",
-				Value:       10999,
+				Value:       consts.RunnerHTTPPort,
 				Destination: &args.Runner.HTTPPort,
 				EnvVars:     []string{"DSTACK_RUNNER_HTTP_PORT"},
 			},
 			&cli.IntFlag{
 				Name:        "runner-ssh-port",
 				Usage:       "Set runner's ssh port",
-				Value:       10022,
+				Value:       consts.RunnerSSHPort,
 				Destination: &args.Runner.SSHPort,
 				EnvVars:     []string{"DSTACK_RUNNER_SSH_PORT"},
 			},

runner/consts/consts.go

@@ -30,4 +30,9 @@ const (
 	RunnerWorkingDir = "/workflow"
 )
 
+const (
+	RunnerHTTPPort = 10999
+	RunnerSSHPort  = 10022
+)
+
 const ShimLogFileName = "shim.log"

runner/internal/common/utils.go

@@ -0,0 +1,54 @@
+package common
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path"
+	"slices"
+
+	"github.com/dstackai/dstack/runner/internal/log"
+)
+
+func ExpandPath(pth string, base string, home string) (string, error) {
+	pth = path.Clean(pth)
+	if pth == "~" {
+		return path.Clean(home), nil
+	}
+	if len(pth) >= 2 && pth[0] == '~' {
+		if pth[1] == '/' {
+			return path.Join(home, pth[2:]), nil
+		}
+		return "", errors.New("~username syntax is not supported")
+	}
+	if base != "" && !path.IsAbs(pth) {
+		return path.Join(base, pth), nil
+	}
+	return pth, nil
+}
+
+func MkdirAll(ctx context.Context, pth string, uid int, gid int) error {
+	paths := []string{pth}
+	for {
+		pth = path.Dir(pth)
+		if pth == "/" {
+			break
+		}
+		paths = append(paths, pth)
+	}
+	for _, p := range slices.Backward(paths) {
+		if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) {
+			if err := os.Mkdir(p, 0o755); err != nil {
+				return err
+			}
+			if uid != -1 || gid != -1 {
+				if err := os.Chown(p, uid, gid); err != nil {
+					log.Warning(ctx, "Failed to chown", "path", p, "err", err)
+				}
+			}
+		} else if err != nil {
+			return err
+		}
+	}
+	return nil
+}

runner/internal/common/utils_test.go

@@ -0,0 +1,109 @@
+package common
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestExpandPath_NoPath_NoBase(t *testing.T) {
+	path, err := ExpandPath("", "", "")
+	require.NoError(t, err)
+	require.Equal(t, ".", path)
+}
+
+func TestExpandPath_NoPath_RelBase(t *testing.T) {
+	path, err := ExpandPath("", "repo", "")
+	require.NoError(t, err)
+	require.Equal(t, "repo", path)
+}
+
+func TestExpandPath_NoPath_AbsBase(t *testing.T) {
+	path, err := ExpandPath("", "/repo", "")
+	require.NoError(t, err)
+	require.Equal(t, "/repo", path)
+}
+
+func TestExpandtPath_RelPath_NoBase(t *testing.T) {
+	path, err := ExpandPath("repo", "", "")
+	require.NoError(t, err)
+	require.Equal(t, "repo", path)
+}
+
+func TestExpandtPath_RelPath_RelBase(t *testing.T) {
+	path, err := ExpandPath("repo", "data", "")
+	require.NoError(t, err)
+	require.Equal(t, "data/repo", path)
+}
+
+func TestExpandtPath_RelPath_AbsBase(t *testing.T) {
+	path, err := ExpandPath("repo", "/data", "")
+	require.NoError(t, err)
+	require.Equal(t, "/data/repo", path)
+}
+
+func TestExpandtPath_AbsPath_NoBase(t *testing.T) {
+	path, err := ExpandPath("/repo", "", "")
+	require.NoError(t, err)
+	require.Equal(t, "/repo", path)
+}
+
+func TestExpandtPath_AbsPath_RelBase(t *testing.T) {
+	path, err := ExpandPath("/repo", "data", "")
+	require.NoError(t, err)
+	require.Equal(t, "/repo", path)
+}
+
+func TestExpandtPath_AbsPath_AbsBase(t *testing.T) {
+	path, err := ExpandPath("/repo", "/data", "")
+	require.NoError(t, err)
+	require.Equal(t, "/repo", path)
+}
+
+func TestExpandPath_BareTilde_NoHome(t *testing.T) {
+	path, err := ExpandPath("~", "", "")
+	require.NoError(t, err)
+	require.Equal(t, ".", path)
+}
+
+func TestExpandPath_BareTilde_RelHome(t *testing.T) {
+	path, err := ExpandPath("~", "", "user")
+	require.NoError(t, err)
+	require.Equal(t, "user", path)
+}
+
+func TestExpandPath_BareTilde_AbsHome(t *testing.T) {
+	path, err := ExpandPath("~", "", "/home/user")
+	require.NoError(t, err)
+	require.Equal(t, "/home/user", path)
+}
+
+func TestExpandtPath_TildeWithPath_NoHome(t *testing.T) {
+	path, err := ExpandPath("~/repo", "", "")
+	require.NoError(t, err)
+	require.Equal(t, "repo", path)
+}
+
+func TestExpandtPath_TildeWithPath_RelHome(t *testing.T) {
+	path, err := ExpandPath("~/repo", "", "user")
+	require.NoError(t, err)
+	require.Equal(t, "user/repo", path)
+}
+
+func TestExpandtPath_TildeWithPath_AbsHome(t *testing.T) {
+	path, err := ExpandPath("~/repo", "", "/home/user")
+	require.NoError(t, err)
+	require.Equal(t, "/home/user/repo", path)
+}
+
+func TestExpandtPath_ErrorTildeUsernameNotSupported_BareTildeUsername(t *testing.T) {
+	path, err := ExpandPath("~username", "", "")
+	require.ErrorContains(t, err, "~username syntax is not supported")
+	require.Equal(t, "", path)
+}
+
+func TestExpandtPath_ErrorTildeUsernameNotSupported_TildeUsernameWithPath(t *testing.T) {
+	path, err := ExpandPath("~username/repo", "", "")
+	require.ErrorContains(t, err, "~username syntax is not supported")
+	require.Equal(t, "", path)
+}