diff --git a/docs/docs/concepts/repos.md b/docs/docs/concepts/repos.md
index 53d4f4d39d..33ebe404e9 100644
--- a/docs/docs/concepts/repos.md
+++ b/docs/docs/concepts/repos.md
@@ -7,13 +7,13 @@ This allows accessing the directory files from within the run.
## Initialize a repo
To use a directory with `dstack apply`, it must first be initialized as a repo by running [`dstack init`](../reference/cli/dstack/init.md).
-The directory can be either a regular local directory or a cloned Git repo.
+The directory must be a cloned Git repo.
[`dstack init`](../reference/cli/dstack/init.md) is not required if you pass `-P` (or `--repo`) to [`dstack apply`](../reference/cli/dstack/apply.md) (see below).
### Git credentials
-If the directory is a cloned Git repo, [`dstack init`](../reference/cli/dstack/init.md) grants the `dstack` server access by uploading the current user's default
+[`dstack init`](../reference/cli/dstack/init.md) grants the `dstack` server access by uploading the current user's default
Git credentials, ensuring that dstack can clone the Git repo when running the container.
To use custom credentials, pass them directly with `--token` (GitHub token) or `--git-identity` (path to a private SSH
@@ -24,21 +24,19 @@ key).
### .gitignore and folder size
-If the directory is cloned Git repo, [`dstack apply`](../reference/cli/dstack/apply.md) uploads to the `dstack` server only local changes.
-If the directory is not a cloned Git repo, it uploads the entire directory.
+[`dstack apply`](../reference/cli/dstack/apply.md) uploads to the `dstack` server only local changes.
Uploads are limited to 2MB. Use `.gitignore` to exclude unnecessary files from being uploaded.
You can set the `DSTACK_SERVER_CODE_UPLOAD_LIMIT` environment variable to increase the default server limit.
Increasing the limit is recommended only if you [configure an object storage](../guides/server-deployment.md).
-### Initialize as a local directory
+### Use a local directory instead of a Git repo
-If the directory is a cloned Git repo but you want to initialize it as a regular local directory,
-use `--local` with [`dstack init`](../reference/cli/dstack/init.md).
+If the directory is not a cloned Git repo, use [`files`](../reference/dstack.yml/task.md#_files).
## Specify the repo
-By default, `dstack apply` uses the current directory as a repo and requires `dstack init`.
+By default, `dstack apply` uses the current directory as a repo if it is already initialized.
You can change this by explicitly specifying the repo to use for `dstack apply`.
### Pass the repo path
@@ -47,8 +45,8 @@ To use a specific directory as the repo, specify its path using `-P` (or `--repo
-```shell
-$ dstack apply -f .dstack.yml -P ../parent_dir
+```shell
+$ dstack apply -f .dstack.yml -P ../parent_dir
```
@@ -73,7 +71,7 @@ If you use a private Git repo, you can pass Git credentials to `dstack apply` us
### Do not use a repo
-To run a configuration without a repo (the `/workflow` directory inside the container will be empty), use `--no-repo`:
+To run a configuration without a repo (the `/workflow` directory inside the container will be empty) if it is already initialized, use `--no-repo`:
diff --git a/docs/docs/reference/cli/dstack/apply.md b/docs/docs/reference/cli/dstack/apply.md
index 3cabc165dd..4cc6215a0e 100644
--- a/docs/docs/reference/cli/dstack/apply.md
+++ b/docs/docs/reference/cli/dstack/apply.md
@@ -3,7 +3,7 @@
This command applies a given configuration. If a resource does not exist, `dstack apply` creates the resource.
If a resource exists, `dstack apply` updates the resource in-place or re-creates the resource if the update is not possible.
-When applying run configurations, `dstack apply` requires that you run `dstack init` first,
+To mount a Git repo to the run's container, `dstack apply` requires that you run `dstack init` first,
or specify a repo to work with via `-P` (or `--repo`), or specify `--no-repo` if you don't need any repo for the run.
## Usage
@@ -17,4 +17,9 @@ $ dstack apply --help
+## User SSH key
+
+By default, `dstack` uses its own SSH key to attach to runs (`~/.dstack/ssh/id_rsa`).
+It is possible to override this key via the `--ssh-identity` argument.
+
[//]: # (TODO: Provide examples)
diff --git a/docs/docs/reference/cli/dstack/attach.md b/docs/docs/reference/cli/dstack/attach.md
index 21aec63225..eae8c7f176 100644
--- a/docs/docs/reference/cli/dstack/attach.md
+++ b/docs/docs/reference/cli/dstack/attach.md
@@ -13,4 +13,9 @@ $ dstack attach --help
+## User SSH key
+
+By default, `dstack` uses its own SSH key to attach to runs (`~/.dstack/ssh/id_rsa`).
+It is possible to override this key via the `--ssh-identity` argument.
+
[//]: # (TODO: Provide examples)
diff --git a/docs/docs/reference/cli/dstack/init.md b/docs/docs/reference/cli/dstack/init.md
index 94aa9c5fce..e70bef6ac0 100644
--- a/docs/docs/reference/cli/dstack/init.md
+++ b/docs/docs/reference/cli/dstack/init.md
@@ -1,11 +1,12 @@
# dstack init
This command initializes the current directory as a `dstack` [repo](../../../concepts/repos.md).
+The directory must be a cloned Git repository.
**Git credentials**
-If the directory is a cloned Git repository, `dstack init` ensures that `dstack` can access it.
-By default, the command uses the user's default Git credentials. These can be overridden with
+`dstack init` ensures that `dstack` can access a remote Git repository.
+By default, the command uses the user's default Git credentials. These can be overridden with
`--git-identity` (private SSH key) or `--token` (OAuth token).
@@ -16,10 +17,3 @@ $ dstack init --help
```
-
-**User SSH key**
-
-By default, `dstack` uses its own SSH key to access instances (`~/.dstack/ssh/id_rsa`).
-It is possible to override this key via the `--ssh-identity` argument.
-
-[//]: # (TODO: Mention that it's optional, provide reference to `dstack apply`)
diff --git a/src/dstack/_internal/cli/commands/apply.py b/src/dstack/_internal/cli/commands/apply.py
index 44e957feb7..caf6bf93e8 100644
--- a/src/dstack/_internal/cli/commands/apply.py
+++ b/src/dstack/_internal/cli/commands/apply.py
@@ -1,4 +1,5 @@
import argparse
+from pathlib import Path
from argcomplete import FilesCompleter
@@ -13,7 +14,7 @@
init_repo,
register_init_repo_args,
)
-from dstack._internal.cli.utils.common import console
+from dstack._internal.cli.utils.common import console, warn
from dstack._internal.core.errors import CLIError
from dstack._internal.core.models.configurations import ApplyConfigurationType
@@ -65,6 +66,13 @@ def _register(self):
help="Exit immediately after submitting configuration",
action="store_true",
)
+ self._parser.add_argument(
+ "--ssh-identity",
+ metavar="SSH_PRIVATE_KEY",
+ help="The private SSH key path for SSH tunneling",
+ type=Path,
+ dest="ssh_identity_file",
+ )
repo_group = self._parser.add_argument_group("Repo Options")
repo_group.add_argument(
"-P",
@@ -111,6 +119,11 @@ def _command(self, args: argparse.Namespace):
raise CLIError("Cannot read configuration from stdin if -y/--yes is not specified")
if args.repo and args.no_repo:
raise CLIError("Either --repo or --no-repo can be specified")
+ if args.local:
+ warn(
+ "Local repos are deprecated since 0.19.25 and will be removed soon."
+ " Consider using `files` instead: https://dstack.ai/docs/concepts/tasks/#files"
+ )
repo = None
if args.repo:
repo = init_repo(
@@ -121,7 +134,6 @@ def _command(self, args: argparse.Namespace):
local=args.local,
git_identity_file=args.git_identity_file,
oauth_token=args.gh_token,
- ssh_identity_file=args.ssh_identity_file,
)
elif args.no_repo:
repo = init_default_virtual_repo(api=self.api)
diff --git a/src/dstack/_internal/cli/commands/init.py b/src/dstack/_internal/cli/commands/init.py
index 0e6a09cc1c..0076fc6ac1 100644
--- a/src/dstack/_internal/cli/commands/init.py
+++ b/src/dstack/_internal/cli/commands/init.py
@@ -4,7 +4,10 @@
from dstack._internal.cli.commands import BaseCommand
from dstack._internal.cli.services.repos import init_repo, register_init_repo_args
-from dstack._internal.cli.utils.common import configure_logging, console
+from dstack._internal.cli.utils.common import configure_logging, confirm_ask, console, warn
+from dstack._internal.core.errors import ConfigurationError
+from dstack._internal.core.models.repos.base import RepoType
+from dstack._internal.core.services.configs import ConfigManager
from dstack.api import Client
@@ -19,12 +22,55 @@ def _register(self):
default=os.getenv("DSTACK_PROJECT"),
)
register_init_repo_args(self._parser)
+ # Deprecated since 0.19.25, ignored
+ self._parser.add_argument(
+ "--ssh-identity",
+ metavar="SSH_PRIVATE_KEY",
+ help=argparse.SUPPRESS,
+ type=Path,
+ dest="ssh_identity_file",
+ )
+ # A hidden mode for transitional period only, remove it with local repos
+ self._parser.add_argument(
+ "--remove",
+ help=argparse.SUPPRESS,
+ action="store_true",
+ )
def _command(self, args: argparse.Namespace):
configure_logging()
+ if args.remove:
+ config_manager = ConfigManager()
+ repo_path = Path.cwd()
+ repo_config = config_manager.get_repo_config(repo_path)
+ if repo_config is None:
+ raise ConfigurationError("The repo is not initialized, nothing to remove")
+ if repo_config.repo_type != RepoType.LOCAL:
+ raise ConfigurationError("`dstack init --remove` is for local repos only")
+ console.print(
+ f"You are about to remove the local repo {repo_path}\n"
+ "Only the record about the repo will be removed,"
+ " the repo files will remain intact\n"
+ )
+ if not confirm_ask("Remove the local repo?"):
+ return
+ config_manager.delete_repo_config(repo_config.repo_id)
+ config_manager.save()
+ console.print("Local repo has been removed")
+ return
api = Client.from_config(
project_name=args.project, ssh_identity_file=args.ssh_identity_file
)
+ if args.local:
+ warn(
+ "Local repos are deprecated since 0.19.25 and will be removed soon."
+ " Consider using `files` instead: https://dstack.ai/docs/concepts/tasks/#files"
+ )
+ if args.ssh_identity_file:
+ warn(
+ "`--ssh-identity` in `dstack init` is deprecated and ignored since 0.19.25."
+ " Use this option with `dstack apply` and `dstack attach` instead"
+ )
init_repo(
api=api,
repo_path=Path.cwd(),
@@ -33,6 +79,5 @@ def _command(self, args: argparse.Namespace):
local=args.local,
git_identity_file=args.git_identity_file,
oauth_token=args.gh_token,
- ssh_identity_file=args.ssh_identity_file,
)
console.print("OK")
diff --git a/src/dstack/_internal/cli/services/configurators/run.py b/src/dstack/_internal/cli/services/configurators/run.py
index 363497e01f..6133999ef6 100644
--- a/src/dstack/_internal/cli/services/configurators/run.py
+++ b/src/dstack/_internal/cli/services/configurators/run.py
@@ -15,9 +15,11 @@
BaseApplyConfigurator,
)
from dstack._internal.cli.services.profile import apply_profile_args, register_profile_args
+from dstack._internal.cli.services.repos import init_default_virtual_repo
from dstack._internal.cli.utils.common import (
confirm_ask,
console,
+ warn,
)
from dstack._internal.cli.utils.rich import MultiItemStatus
from dstack._internal.cli.utils.run import get_runs_table, print_run_plan
@@ -40,6 +42,7 @@
TaskConfiguration,
)
from dstack._internal.core.models.repos.base import Repo
+from dstack._internal.core.models.repos.local import LocalRepo
from dstack._internal.core.models.resources import CPUSpec
from dstack._internal.core.models.runs import JobStatus, JobSubmission, RunSpec, RunStatus
from dstack._internal.core.services.configs import ConfigManager
@@ -76,17 +79,42 @@ def apply_configuration(
self.apply_args(conf, configurator_args, unknown_args)
self.validate_gpu_vendor_and_image(conf)
self.validate_cpu_arch_and_image(conf)
- if repo is None:
- repo = self.api.repos.load(Path.cwd())
config_manager = ConfigManager()
- if repo.repo_dir is not None:
- repo_config = config_manager.get_repo_config_or_error(repo.repo_dir)
- self.api.ssh_identity_file = repo_config.ssh_key_path
- else:
- self.api.ssh_identity_file = get_ssh_keypair(
- command_args.ssh_identity_file,
- config_manager.dstack_key_path,
- )
+ if repo is None:
+ repo_path = Path.cwd()
+ repo_config = config_manager.get_repo_config(repo_path)
+ if repo_config is None:
+ warn(
+ "The repo is not initialized. Starting from 0.19.25, repos are optional\n"
+ "There are three options:\n"
+ " - Run `dstack init` to initialize the current directory as a repo\n"
+ " - Specify `--repo`\n"
+ " - Specify `--no-repo` to not use any repo and supress this warning"
+ " (this will be the default in the future versions)"
+ )
+ if not command_args.yes and not confirm_ask("Continue without the repo?"):
+ console.print("\nExiting...")
+ return
+ repo = init_default_virtual_repo(self.api)
+ else:
+ # Unlikely, but may raise ConfigurationError if the repo does not exist
+ # on the server side (stale entry in `config.yml`)
+ repo = self.api.repos.load(repo_path)
+ if isinstance(repo, LocalRepo):
+ warn(
+ f"{repo.repo_dir} is a local repo.\n"
+ "Local repos are deprecated since 0.19.25"
+ " and will be removed soon\n"
+ "There are two options:\n"
+ " - Migrate to `files`: https://dstack.ai/docs/concepts/tasks/#files\n"
+ " - Specify `--no-repo` if you don't need the repo at all\n"
+ "In either case, you can run `dstack init --remove` to remove the repo"
+ " (only the record about the repo, not its files) and this warning"
+ )
+ self.api.ssh_identity_file = get_ssh_keypair(
+ command_args.ssh_identity_file,
+ config_manager.dstack_key_path,
+ )
profile = load_profile(Path.cwd(), configurator_args.profile)
with console.status("Getting apply plan..."):
run_plan = self.api.runs.get_run_plan(
diff --git a/src/dstack/_internal/cli/services/repos.py b/src/dstack/_internal/cli/services/repos.py
index e9b1d44329..5e7a10589f 100644
--- a/src/dstack/_internal/cli/services/repos.py
+++ b/src/dstack/_internal/cli/services/repos.py
@@ -1,12 +1,12 @@
+import argparse
from pathlib import Path
from typing import Optional
from dstack._internal.cli.services.configurators.base import ArgsParser
from dstack._internal.core.errors import CLIError
-from dstack._internal.core.models.repos.base import Repo, RepoType
+from dstack._internal.core.models.repos.base import Repo
from dstack._internal.core.models.repos.remote import GitRepoURL, RemoteRepo, RepoError
from dstack._internal.core.models.repos.virtual import VirtualRepo
-from dstack._internal.core.services.configs import ConfigManager
from dstack._internal.core.services.repos import get_default_branch
from dstack._internal.utils.path import PathLike
from dstack.api._public import Client
@@ -28,49 +28,31 @@ def register_init_repo_args(parser: ArgsParser):
type=str,
dest="git_identity_file",
)
- parser.add_argument(
- "--ssh-identity",
- metavar="SSH_PRIVATE_KEY",
- help="The private SSH key path for SSH tunneling",
- type=Path,
- dest="ssh_identity_file",
- )
+ # Deprecated since 0.19.25
parser.add_argument(
"--local",
action="store_true",
- help="Do not use Git",
+ help=argparse.SUPPRESS,
)
def init_repo(
api: Client,
- repo_path: Optional[PathLike],
+ repo_path: PathLike,
repo_branch: Optional[str],
repo_hash: Optional[str],
local: bool,
git_identity_file: Optional[PathLike],
oauth_token: Optional[str],
- ssh_identity_file: Optional[PathLike],
) -> Repo:
- init = True
- if repo_path is None:
- init = False
- repo_path = Path.cwd()
if Path(repo_path).exists():
repo = api.repos.load(
repo_dir=repo_path,
local=local,
- init=init,
+ init=True,
git_identity_file=git_identity_file,
oauth_token=oauth_token,
)
- if ssh_identity_file:
- ConfigManager().save_repo_config(
- repo_path=repo.get_repo_dir_or_error(),
- repo_id=repo.repo_id,
- repo_type=RepoType(repo.run_repo_data.repo_type),
- ssh_key_path=ssh_identity_file,
- )
elif isinstance(repo_path, str):
try:
GitRepoURL.parse(repo_path)
diff --git a/src/dstack/_internal/cli/utils/common.py b/src/dstack/_internal/cli/utils/common.py
index 543fea931a..b319b837dc 100644
--- a/src/dstack/_internal/cli/utils/common.py
+++ b/src/dstack/_internal/cli/utils/common.py
@@ -103,3 +103,10 @@ def add_row_from_dict(table: Table, data: Dict[Union[str, int], Any], **kwargs):
else:
row.append("")
table.add_row(*row, **kwargs)
+
+
+def warn(message: str):
+ if not message.endswith("\n"):
+ # Additional blank line for better visibility if there are more than one warning
+ message = f"{message}\n"
+ console.print(f"[warning][bold]{message}[/]")
diff --git a/src/dstack/_internal/core/models/config.py b/src/dstack/_internal/core/models/config.py
index bb17b40b44..67ea5ed2f3 100644
--- a/src/dstack/_internal/core/models/config.py
+++ b/src/dstack/_internal/core/models/config.py
@@ -16,7 +16,9 @@ class RepoConfig(CoreModel):
path: str
repo_id: str
repo_type: RepoType
- ssh_key_path: str
+ # Deprecated since 0.19.25, not used. Can be removed when most users update their `config.yml`
+ # (it's updated each time a project or repo is added)
+ ssh_key_path: Annotated[Optional[str], Field(exclude=True)] = None
class GlobalConfig(CoreModel):
diff --git a/src/dstack/_internal/core/services/configs/__init__.py b/src/dstack/_internal/core/services/configs/__init__.py
index f11983e382..b38ab35710 100644
--- a/src/dstack/_internal/core/services/configs/__init__.py
+++ b/src/dstack/_internal/core/services/configs/__init__.py
@@ -71,19 +71,15 @@ def list_projects(self):
def delete_project(self, name: str):
self.config.projects = [p for p in self.config.projects if p.name != name]
- def save_repo_config(
- self, repo_path: PathLike, repo_id: str, repo_type: RepoType, ssh_key_path: PathLike
- ):
+ def save_repo_config(self, repo_path: PathLike, repo_id: str, repo_type: RepoType):
self.config_filepath.parent.mkdir(parents=True, exist_ok=True)
with filelock.FileLock(str(self.config_filepath) + ".lock"):
self.load()
repo_path = os.path.abspath(repo_path)
- ssh_key_path = os.path.abspath(ssh_key_path)
for repo in self.config.repos:
if repo.path == repo_path:
repo.repo_id = repo_id
repo.repo_type = repo_type
- repo.ssh_key_path = ssh_key_path
break
else:
self.config.repos.append(
@@ -91,7 +87,6 @@ def save_repo_config(
path=repo_path,
repo_id=repo_id,
repo_type=repo_type,
- ssh_key_path=ssh_key_path,
)
)
self.save()
@@ -110,6 +105,9 @@ def get_repo_config_or_error(self, repo_path: PathLike) -> RepoConfig:
return repo_config
raise DstackError("No repo config found")
+ def delete_repo_config(self, repo_id: str):
+ self.config.repos = [p for p in self.config.repos if p.repo_id != repo_id]
+
@property
def dstack_ssh_dir(self) -> Path:
return self.dstack_dir / "ssh"
diff --git a/src/dstack/api/_public/repos.py b/src/dstack/api/_public/repos.py
index ced2bf8c9d..9655e48f91 100644
--- a/src/dstack/api/_public/repos.py
+++ b/src/dstack/api/_public/repos.py
@@ -112,27 +112,28 @@ def load(
" Run `dstack init` to initialize the current directory as a repo or specify `--repo`."
)
repo = load_repo(repo_config)
- try:
- self._api_client.repos.get(self._project, repo.repo_id, include_creds=False)
- except ResourceNotExistsError:
+ if not self.is_initialized(repo):
raise ConfigurationError(
"The repo is not initialized."
" Run `dstack init` to initialize the current directory as a repo or specify `--repo`."
)
else:
logger.debug("Initializing repo")
- repo = LocalRepo(repo_dir=repo_dir) # default
- if not local:
+ if local:
+ repo = LocalRepo(repo_dir=repo_dir)
+ else:
try:
repo = RemoteRepo.from_dir(repo_dir)
except InvalidGitRepositoryError:
- pass # use default
+ raise ConfigurationError(
+ f"Git repo not found: {repo_dir}. Use `files` to mount an arbitrary"
+ " directory: https://dstack.ai/docs/concepts/tasks/#files"
+ )
self.init(repo, git_identity_file, oauth_token)
config.save_repo_config(
repo.get_repo_dir_or_error(),
repo.repo_id,
RepoType(repo.run_repo_data.repo_type),
- get_ssh_keypair(None, config.dstack_key_path),
)
return repo