From 797f84d55e943dd685614664247728ed73216ea4 Mon Sep 17 00:00:00 2001 From: Brian Greunke Date: Fri, 22 Aug 2025 13:51:20 -0500 Subject: [PATCH 1/4] feat: added client endpoint to retrieve creds --- .vscode/settings.json | 4 ++-- dreadnode/api/client.py | 13 +++++++++++++ dreadnode/api/models.py | 7 +++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 1179470a..2ef0b41b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -8,8 +8,8 @@ "editor.defaultFormatter": "charliermarsh.ruff" }, "python.testing.pytestArgs": [ - "dreadnode_cli" + "tests" ], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true -} +} \ No newline at end of file diff --git a/dreadnode/api/client.py b/dreadnode/api/client.py index 12243907..cf20d6b5 100644 --- a/dreadnode/api/client.py +++ b/dreadnode/api/client.py @@ -12,6 +12,7 @@ from dreadnode.api.models import ( AccessRefreshTokenResponse, + ContainerRegistryCredentials, DeviceCodeResponse, GithubTokenResponse, MetricAggregationType, @@ -538,3 +539,15 @@ def get_user_data_credentials(self) -> UserDataCredentials: """ response = self._request("GET", "/user-data/credentials") return UserDataCredentials(**response.json()) + + # Container registry access + + def get_container_registry_credentials(self) -> ContainerRegistryCredentials: + """ + Retrieves container registry credentials for Docker image access. + + Returns: + The container registry credentials object. + """ + response = self._request("GET", "/platform/container-registry/credentials") + return ContainerRegistryCredentials(**response.json()) diff --git a/dreadnode/api/models.py b/dreadnode/api/models.py index 61c52dda..e310058b 100644 --- a/dreadnode/api/models.py +++ b/dreadnode/api/models.py @@ -43,6 +43,13 @@ class UserDataCredentials(BaseModel): endpoint: str | None +class ContainerRegistryCredentials(BaseModel): + registry: str + username: str + password: str + expires_at: datetime + + # Auth From 2b826169e5e4432bb61cf227362a1d7877c03d46 Mon Sep 17 00:00:00 2001 From: Brian Greunke Date: Sat, 23 Aug 2025 07:38:10 -0500 Subject: [PATCH 2/4] wip --- dreadnode/api/client.py | 1 + dreadnode/cli/main.py | 4 +++- pyproject.toml | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/dreadnode/api/client.py b/dreadnode/api/client.py index cf20d6b5..11bbff1d 100644 --- a/dreadnode/api/client.py +++ b/dreadnode/api/client.py @@ -100,6 +100,7 @@ def __init__( headers=headers, base_url=self._base_url, timeout=30, + cookies=_cookies, ) if debug: diff --git a/dreadnode/cli/main.py b/dreadnode/cli/main.py index ccb732e3..5b78cd2f 100644 --- a/dreadnode/cli/main.py +++ b/dreadnode/cli/main.py @@ -18,6 +18,7 @@ download_and_unzip_archive, validate_server_for_clone, ) +from dreadnode.cli.platform import cli as platform_cli from dreadnode.cli.profile import cli as profile_cli from dreadnode.config import ServerConfig, UserConfig from dreadnode.constants import DEBUG, PLATFORM_BASE_URL @@ -26,8 +27,9 @@ cli["--help"].group = "Meta" -cli.command(profile_cli) cli.command(agent_cli) +cli.command(platform_cli) +cli.command(profile_cli) @cli.meta.default diff --git a/pyproject.toml b/pyproject.toml index caaba069..afbbf033 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ presidio-analyzer = "^2.2.359" [tool.poetry.extras] training = ["transformers"] multimodal = ["pillow", "soundfile", "moviepy"] +platform = ["docker"] all = ["multimodal", "training"] [tool.poetry.group.dev.dependencies] From f8eaefc077d0be2e5c7c54537a97641981f320b7 Mon Sep 17 00:00:00 2001 From: Brian Greunke Date: Tue, 26 Aug 2025 23:50:40 -0500 Subject: [PATCH 3/4] feat: added platform command --- dreadnode/api/client.py | 21 ++- dreadnode/api/models.py | 16 ++ dreadnode/cli/platform/__init__.py | 3 + dreadnode/cli/platform/check_for_updates.py | 45 +++++ dreadnode/cli/platform/cli.py | 75 +++++++++ dreadnode/cli/platform/configure.py | 9 + dreadnode/cli/platform/constants.py | 10 ++ dreadnode/cli/platform/docker/__init__.py | 77 +++++++++ dreadnode/cli/platform/docker/download.py | 121 ++++++++++++++ dreadnode/cli/platform/docker/login.py | 22 +++ dreadnode/cli/platform/docker/start.py | 56 +++++++ dreadnode/cli/platform/init.py | 125 ++++++++++++++ dreadnode/cli/platform/templates/.api.env.j2 | 75 +++++++++ dreadnode/cli/platform/templates/.ui.env.j2 | 18 ++ .../platform/templates/docker-compose.yaml.j2 | 154 ++++++++++++++++++ dreadnode/cli/platform/utils.py | 60 +++++++ dreadnode/constants.py | 2 + poetry.lock | 40 ++++- pyproject.toml | 6 +- 19 files changed, 925 insertions(+), 10 deletions(-) create mode 100644 dreadnode/cli/platform/__init__.py create mode 100644 dreadnode/cli/platform/check_for_updates.py create mode 100644 dreadnode/cli/platform/cli.py create mode 100644 dreadnode/cli/platform/configure.py create mode 100644 dreadnode/cli/platform/constants.py create mode 100644 dreadnode/cli/platform/docker/__init__.py create mode 100644 dreadnode/cli/platform/docker/download.py create mode 100644 dreadnode/cli/platform/docker/login.py create mode 100644 dreadnode/cli/platform/docker/start.py create mode 100644 dreadnode/cli/platform/init.py create mode 100644 dreadnode/cli/platform/templates/.api.env.j2 create mode 100644 dreadnode/cli/platform/templates/.ui.env.j2 create mode 100644 dreadnode/cli/platform/templates/docker-compose.yaml.j2 create mode 100644 dreadnode/cli/platform/utils.py diff --git a/dreadnode/api/client.py b/dreadnode/api/client.py index 11bbff1d..1a2f2c3f 100644 --- a/dreadnode/api/client.py +++ b/dreadnode/api/client.py @@ -19,6 +19,7 @@ Project, RawRun, RawTask, + RegistryImageDetails, Run, RunSummary, StatusFilter, @@ -550,5 +551,23 @@ def get_container_registry_credentials(self) -> ContainerRegistryCredentials: Returns: The container registry credentials object. """ - response = self._request("GET", "/platform/container-registry/credentials") + response = self.request("POST", "/platform/registry-token") return ContainerRegistryCredentials(**response.json()) + + def get_platform_releases( + self, arch: str, tag: str, services: list[str], cli_version: str + ) -> RegistryImageDetails: + """ + Resolves the platform releases for the current project. + + Returns: + The resolved platform releases as a ResolveReleasesResponse object. + """ + payload = { + "arch": arch, + "tag": tag, + "services": services, + "cli_version": cli_version, + } + response = self.request("POST", "/platform/get-releases", json_data=payload) + return RegistryImageDetails(**response.json()) diff --git a/dreadnode/api/models.py b/dreadnode/api/models.py index e310058b..250b5fa7 100644 --- a/dreadnode/api/models.py +++ b/dreadnode/api/models.py @@ -50,6 +50,22 @@ class ContainerRegistryCredentials(BaseModel): expires_at: datetime +class PlatformImage(BaseModel): + service: str + uri: str + digest: str + version: str + + @property + def full_uri(self) -> str: + return f"{self.uri}@{self.digest}" + + +class RegistryImageDetails(BaseModel): + version: str + images: list[PlatformImage] + + # Auth diff --git a/dreadnode/cli/platform/__init__.py b/dreadnode/cli/platform/__init__.py new file mode 100644 index 00000000..7a874c7c --- /dev/null +++ b/dreadnode/cli/platform/__init__.py @@ -0,0 +1,3 @@ +from dreadnode.cli.platform.cli import cli + +__all__ = ["cli"] diff --git a/dreadnode/cli/platform/check_for_updates.py b/dreadnode/cli/platform/check_for_updates.py new file mode 100644 index 00000000..608ecbe6 --- /dev/null +++ b/dreadnode/cli/platform/check_for_updates.py @@ -0,0 +1,45 @@ +import rich + +from dreadnode.cli.api import create_api_client +from dreadnode.cli.platform.constants import SERVICES +from dreadnode.cli.platform.utils import get_local_arch, get_local_cache_dir, get_local_version + + +def check_for_updates() -> None: + import importlib.metadata # noqa: PLC0415 + + local_cache_dir = get_local_cache_dir() + rich.print(f"Checking local cache directory: {local_cache_dir}") + + if not local_cache_dir.exists(): + rich.print( + "Local cache directory does not exist. Please run \n[dim]$[/dim] [bold green]dreadnode platform init[/bold green]" + ) + return + + arch = get_local_arch() + api_client = create_api_client() + registry_image_details = api_client.get_platform_releases( + arch=arch, + tag="latest", + services=SERVICES, + cli_version=importlib.metadata.version("dreadnode"), + ) + + local_image_details = get_local_version() + + for image_detail in local_image_details.images: + for remote_image_detail in registry_image_details.images: + if image_detail.service == remote_image_detail.service: + if image_detail.version != remote_image_detail.version: + rich.print( + f"[yellow]Update available for {image_detail.service}: " + f"{image_detail.version} -> {remote_image_detail.version}[/yellow]" + ) + else: + rich.print( + f"[green]{image_detail.service} is up to date: {image_detail.version}[/green]" + ) + rich.print( + "[blue]You can update with:[/blue]\n[dim]$[/dim] [bold green]dreadnode platform update[/bold green]" + ) diff --git a/dreadnode/cli/platform/cli.py b/dreadnode/cli/platform/cli.py new file mode 100644 index 00000000..bb43dd64 --- /dev/null +++ b/dreadnode/cli/platform/cli.py @@ -0,0 +1,75 @@ +import cyclopts + +from dreadnode.cli.platform.check_for_updates import check_for_updates as check_for_updates_ +from dreadnode.cli.platform.configure import configure_platform +from dreadnode.cli.platform.docker.download import download as download_platform +from dreadnode.cli.platform.docker.login import docker_login +from dreadnode.cli.platform.docker.start import start as start_platform +from dreadnode.cli.platform.docker.start import stop as stop_platform +from dreadnode.cli.platform.init import init as init_platform +from dreadnode.cli.platform.init import initialized as platform_initilized + +cli = cyclopts.App("platform", help="Run and manage the platform.", help_flags=[]) + + +@cli.command() +def init(tag: str = "latest", arch: str | None = None) -> None: + """ + Initialize the platform. + """ + init_platform(tag=tag, arch=arch) + + +@cli.command() +def download(tag: str = "latest", arch: str | None = None) -> None: + """ + Download the platform files. + """ + docker_login() + + if not platform_initilized() or tag != "latest" or arch: + init_platform(tag=tag, arch=arch) + + download_platform() + + +@cli.command() +def configure() -> None: + """ + Configure the platform. + """ + configure_platform() + + +@cli.command() +def start() -> None: + """ + Start the platform services. + """ + start_platform() + + +@cli.command() +def stop() -> None: + """ + Stop the platform services. + """ + stop_platform() + + +@cli.command() +def check_for_updates() -> None: + """ + Check for platform updates. + """ + check_for_updates_() + + +@cli.command() +def update() -> None: + """ + Update the platform. + """ + stop_platform() + download_platform() + start_platform() diff --git a/dreadnode/cli/platform/configure.py b/dreadnode/cli/platform/configure.py new file mode 100644 index 00000000..2f2e32c1 --- /dev/null +++ b/dreadnode/cli/platform/configure.py @@ -0,0 +1,9 @@ +import rich + +from dreadnode.cli.platform.utils import get_local_cache_dir + + +def configure_platform() -> None: + rich.print(f"Configure the API by modifying {get_local_cache_dir()}/.api.env") + rich.print(f"Configure the UI by modifying {get_local_cache_dir()}/.ui.env") + rich.print("See https://docs.dreadnode.io/platform/manage for more details.") diff --git a/dreadnode/cli/platform/constants.py b/dreadnode/cli/platform/constants.py new file mode 100644 index 00000000..231e427e --- /dev/null +++ b/dreadnode/cli/platform/constants.py @@ -0,0 +1,10 @@ +from pathlib import Path + +API_SERVICE = "api" +UI_SERVICE = "ui" +SERVICES = [API_SERVICE, UI_SERVICE] + +TEMPLATE_DIR = Path(__file__).parent / "templates" +DOCKER_COMPOSE_TEMPLATE = TEMPLATE_DIR / "docker-compose.yaml.j2" +API_ENV_TEMPLATE = TEMPLATE_DIR / ".api.env.j2" +UI_ENV_TEMPLATE = TEMPLATE_DIR / ".ui.env.j2" diff --git a/dreadnode/cli/platform/docker/__init__.py b/dreadnode/cli/platform/docker/__init__.py new file mode 100644 index 00000000..c74d5b8f --- /dev/null +++ b/dreadnode/cli/platform/docker/__init__.py @@ -0,0 +1,77 @@ +import subprocess +import sys + +import rich + +from dreadnode.cli.platform.utils import get_compose_file_path + + +def run_docker_compose_command( + args: list[str], + compose_file: str | None = None, + project_name: str | None = None, + timeout: int = 300, + command_name: str = "docker compose", + stdin_input: str | None = None, +) -> subprocess.CompletedProcess[str]: + """ + Execute a docker compose command with common error handling and configuration. + + Args: + args: Additional arguments for the docker compose command + compose_file: Path to docker-compose file (optional) + project_name: Docker compose project name (optional) + timeout: Command timeout in seconds + command_name: Name of the command for error messages + stdin_input: Input to pass to stdin (for commands like docker login) + + Returns: + CompletedProcess object with command results + + Raises: + subprocess.CalledProcessError: If command fails + subprocess.TimeoutExpired: If command times out + FileNotFoundError: If docker/docker-compose not found + """ + cmd = ["docker", "compose"] + + # Add compose file + compose_file = compose_file or get_compose_file_path() + cmd.extend(["-f", compose_file]) + + # Add project name if specified + if project_name: + cmd.extend(["-p", project_name]) + + # Add the specific command arguments + cmd.extend(args) + + try: + # Remove capture_output=True to allow real-time streaming + # stdout and stderr will go directly to the terminal + result = subprocess.run( # noqa: S603 + cmd, + check=True, + text=True, + timeout=timeout, + encoding="utf-8", + errors="replace", + input=stdin_input, + ) + + except subprocess.CalledProcessError as e: + rich.print(f"{command_name} failed with exit code {e.returncode}", file=sys.stderr) + raise + + except subprocess.TimeoutExpired: + rich.print(f"{command_name} timed out after {timeout} seconds", file=sys.stderr) + raise + + except FileNotFoundError: + rich.print( + "Docker or docker compose not found. Please ensure Docker is installed.", + file=sys.stderr, + ) + raise + + return result diff --git a/dreadnode/cli/platform/docker/download.py b/dreadnode/cli/platform/docker/download.py new file mode 100644 index 00000000..dc977fcc --- /dev/null +++ b/dreadnode/cli/platform/docker/download.py @@ -0,0 +1,121 @@ +import subprocess + +from dreadnode.cli.platform.docker import run_docker_compose_command + +# def download_platform( +# registry: str, username: str, password: str, image_name: str, tag: str +# ) -> Image: +# try: +# import docker # type: ignore[import-untyped,unused-ignore] +# except ImportError as e: +# raise ImportError( +# "Running a local platform requires `docker`. Install with: pip install dreadnode\\[platform]" +# ) from e + +# # Initialize Docker client +# client = docker.from_env() + +# # # Method 1: Login first, then pull +# # client.login( +# # username=username, +# # password=password, +# # registry=registry, +# # ) + +# # # Pull the private image +# # image = client.images.pull(f"{registry}/{image}:{tag}") + +# # Method 2: Pull with auth parameter +# return client.images.pull( +# f"{registry}/{image_name}:{tag}", +# auth_config={"username": username, "password": password}, +# ) + + +# def parse_compose_file(compose_file_path: str) -> dict[str, Any]: +# """Parse Docker Compose file with proper error handling.""" +# try: +# with Path(compose_file_path).open("r", encoding="utf-8") as f: +# compose_config = yaml.safe_load(f) + +# if not compose_config: +# raise ValueError("Empty or invalid compose file") + +# # Validate basic structure +# if not isinstance(compose_config, dict): +# raise TypeError("Compose file must contain a YAML mapping") + +# except yaml.YAMLError as e: +# raise ValueError(f"Invalid YAML syntax: {e}") from e +# except FileNotFoundError as e: +# raise FileNotFoundError(f"Compose file not found: {compose_file_path}") from e + +# return compose_config + + +# def pull_images_from_compose(compose_file_path: str) -> None: +# """Pull all images defined in a Docker Compose file.""" + +# # Initialize Docker client +# try: +# import docker # type: ignore[import-untyped,unused-ignore] +# except ImportError as e: +# raise ImportError( +# "Running a local platform requires `docker`. Install with: pip install dreadnode\\[platform]" +# ) from e + +# # Initialize Docker client +# client = docker.from_env() + +# compose_config = parse_compose_file(compose_file_path) + +# # Handle different compose file versions +# services = compose_config.get("services", {}) + +# if not services: +# logger.error("No services found in compose file") +# return + +# for service_name, service_config in services.items(): +# if not isinstance(service_config, dict): +# logger.warning(f"⚠ Skipping invalid service config for '{service_name}'") +# continue + +# image = service_config.get("image") +# if image: +# try: +# logger.info(f"Pulling {image}...") +# client.images.pull(image) +# logger.success(f"✓ Pulled {image}") +# except DockerApiError as e: +# logger.error(f"✗ Failed to pull {image}: {e}") +# else: +# # Handle services with 'build' context instead of 'image' +# build_config = service_config.get("build") +# if build_config: +# logger.warning(f"⚠ Service '{service_name}' uses build context, skipping pull") +# else: +# logger.warning(f"⚠ Service '{service_name}' has no image or build config") + + +def download( + compose_file: str | None = None, project_name: str | None = None, timeout: int = 300 +) -> subprocess.CompletedProcess[str]: + """ + Pull docker images for the platform. + + Args: + compose_file: Path to docker-compose file (optional) + project_name: Docker compose project name (optional) + timeout: Command timeout in seconds + + Returns: + CompletedProcess object with command results + + Raises: + subprocess.CalledProcessError: If command fails + subprocess.TimeoutExpired: If command times out + """ + return run_docker_compose_command( + ["--profile", "run", "pull"], compose_file, project_name, timeout, "Docker compose pull" + ) diff --git a/dreadnode/cli/platform/docker/login.py b/dreadnode/cli/platform/docker/login.py new file mode 100644 index 00000000..5b3d1473 --- /dev/null +++ b/dreadnode/cli/platform/docker/login.py @@ -0,0 +1,22 @@ +import subprocess +import sys + +import rich + +from dreadnode.cli.api import create_api_client + + +def docker_login(): + client = create_api_client() + container_registry_creds = client.get_container_registry_credentials() + + cmd = ["docker", "login", container_registry_creds.registry] + cmd.extend(["--username", container_registry_creds.username]) + cmd.extend(["--password-stdin"]) + + try: + subprocess.run(cmd, input=container_registry_creds.password, text=True, check=True) # noqa: S603 + rich.print(f"Logged in to Docker registry: {container_registry_creds.registry}") + except subprocess.CalledProcessError as e: + rich.print(f"Failed to log in to Docker registry: {e}", file=sys.stderr) + raise diff --git a/dreadnode/cli/platform/docker/start.py b/dreadnode/cli/platform/docker/start.py new file mode 100644 index 00000000..d7f6c867 --- /dev/null +++ b/dreadnode/cli/platform/docker/start.py @@ -0,0 +1,56 @@ +import rich + +from dreadnode.cli.platform.docker import run_docker_compose_command + + +def _start_infra( + compose_file: str | None = None, project_name: str | None = None, timeout: int = 300 +) -> None: + """Start infrastructure services.""" + run_docker_compose_command( + ["up", "-d"], compose_file, project_name, timeout, "Docker compose up (infra)" + ) + + +def _create_storage( + compose_file: str | None = None, project_name: str | None = None, timeout: int = 300 +) -> None: + """Create S3 buckets.""" + run_docker_compose_command( + ["--profile", "create-s3-buckets", "up", "-d"], + compose_file, + project_name, + timeout, + "Docker compose up (storage)", + ) + + +def _start_services( + compose_file: str | None = None, project_name: str | None = None, timeout: int = 300 +) -> None: + """Start application services.""" + run_docker_compose_command( + ["--profile", "run", "up", "-d"], + compose_file, + project_name, + timeout, + "Docker compose up (services)", + ) + + +def start( + compose_file: str | None = None, project_name: str | None = None, timeout: int = 300 +) -> None: + """Start all platform services.""" + rich.print("Starting platform services...") + _start_infra(compose_file, project_name, timeout) + _create_storage(compose_file, project_name, timeout) + _start_services(compose_file, project_name, timeout) + + +def stop( + compose_file: str | None = None, project_name: str | None = None, timeout: int = 300 +) -> None: + """Stop platform services.""" + rich.print("Stopping platform services...") + run_docker_compose_command(["stop"], compose_file, project_name, timeout, "Docker compose stop") diff --git a/dreadnode/cli/platform/init.py b/dreadnode/cli/platform/init.py new file mode 100644 index 00000000..229eed8a --- /dev/null +++ b/dreadnode/cli/platform/init.py @@ -0,0 +1,125 @@ +import json +from pathlib import Path + +import rich +from rich.prompt import Confirm + +from dreadnode.api.models import PlatformImage, RegistryImageDetails +from dreadnode.cli.api import create_api_client +from dreadnode.cli.platform.constants import ( + API_ENV_TEMPLATE, + API_SERVICE, + DOCKER_COMPOSE_TEMPLATE, + SERVICES, + UI_ENV_TEMPLATE, + UI_SERVICE, +) +from dreadnode.cli.platform.utils import ( + get_compose_file_path, + get_local_arch, + get_local_cache_dir, + render_with_string_replace, +) + + +def _write_version_manifest( + local_cache_dir: Path, resolution_response: RegistryImageDetails +) -> None: + rich.print(f"Writing version file for {resolution_response.version} ...") + version_file = local_cache_dir / ".version" + version_file.write_text(json.dumps(resolution_response.model_dump())) + rich.print(f"Version file written to {version_file}") + + +def _create_docker_compose_file(images: list[PlatformImage]) -> None: + rich.print("Updating Compose template ...") + for image in images: + if image.service == API_SERVICE: + api_image_digest = image.full_uri + elif image.service == UI_SERVICE: + ui_image_digest = image.full_uri + else: + raise ValueError(f"Unknown image service: {image.service}") + render_with_string_replace( + api_image_digest=api_image_digest, + ui_image_digest=ui_image_digest, + template_path=DOCKER_COMPOSE_TEMPLATE, + output_path=get_compose_file_path(), + ) + rich.print(f"Compose file written to {get_compose_file_path()}") + + +def _create_env_files(local_cache_dir: Path) -> None: + rich.print("Updating environment files ...") + + for env_file in [API_ENV_TEMPLATE, UI_ENV_TEMPLATE]: + dest = local_cache_dir / env_file.name + dest.write_text(env_file.read_text()) + rich.print(f"Environment file written to {dest}") + + # concatenate environment variables + api_env = local_cache_dir / API_ENV_TEMPLATE.name + ui_env = local_cache_dir / UI_ENV_TEMPLATE.name + dest = local_cache_dir / ".env" + dest.write_text(f"{api_env.read_text()}\n{ui_env.read_text()}") + rich.print(f"Combined environment file written to {dest}") + + +def _confirm_with_context(action: str, details: str | None = None) -> bool: + """Confirmation with additional context in a panel.""" + return Confirm.ask( + f"[bold red]Are you sure you want to {action}? {details}[/bold red]", default=False + ) + + +def init(tag: str, arch: str | None = None) -> None: + if initialized() and not _confirm_with_context( + "re-initialize the platform", "This will overwrite existing files." + ): + return + + import importlib.metadata # noqa: PLC0415 + + local_cache_dir = get_local_cache_dir() + rich.print(f"Using local cache directory: {local_cache_dir}") + + if not local_cache_dir.exists(): + local_cache_dir.mkdir(parents=True, exist_ok=True) + rich.print(f"Local cache directory created at {local_cache_dir}") + else: + rich.print("Local cache directory already exists.") + + if not arch: + arch = get_local_arch() + api_client = create_api_client() + registry_image_details = api_client.get_platform_releases( + arch=arch, + tag=tag, + services=SERVICES, + cli_version=importlib.metadata.version("dreadnode"), + ) + + _write_version_manifest(local_cache_dir, registry_image_details) + _create_docker_compose_file(registry_image_details.images) + _create_env_files(local_cache_dir) + + rich.print("Initialization complete.") + + +def initialized() -> bool: + rich.print("Checking initialization ...") + local_cache_dir = get_local_cache_dir() + if not local_cache_dir.exists(): + rich.print("Local cache directory does not exist.") + return False + + if not (local_cache_dir / "docker-compose.yaml").exists(): + rich.print("Docker Compose file is missing.") + return False + + if not (local_cache_dir / ".env").exists(): + rich.print("Environment file is missing.") + return False + + rich.print("All required files are present.") + return True diff --git a/dreadnode/cli/platform/templates/.api.env.j2 b/dreadnode/cli/platform/templates/.api.env.j2 new file mode 100644 index 00000000..95be9063 --- /dev/null +++ b/dreadnode/cli/platform/templates/.api.env.j2 @@ -0,0 +1,75 @@ +# API Environment Variables + +# Features +### The double underscore (__) is used to denote nested properties. +## E.g. Turn off OAuth login +FEATURES__OAUTH__ENABLED=False +## E.g. Turn off Crucible +FEATURES__COMPETITIVE_LEARNING__ENABLED=${CRUCIBLE_ENABLED:-False} +## E.g. Turn off Spyglass +FEATURES__SPYGLASS__ENABLED=${SPYGLASS_ENABLED:-False} + +# Database +## Used by API and Docker Compose +DATABASE_USER=admin +DATABASE_PASSWORD=dreadnode +DATABASE_NAME=platform +DATABASE_PORT=5432 +DATABASE_HOST=localhost + +# DynamoDB + +# Used by API tests to allow mocking +DYNAMODB_PORT=8085 +DYNAMODB_AWS_ACCESS_KEY_ID=mock +DYNAMODB_AWS_SECRET_ACCESS_KEY=mock +DYNAMODB_AWS_DEFAULT_REGION=us-west-2 +DYNAMODB_URL=http://${PROXY_HOST:-localhost}:${DYNAMODB_PORT} + +# Used by API +AUTHORIZATION_KEY_TABLE="auth-keys" +FLAG_ATTEMPTS_TABLE="flag-attempts" + +# Clickhouse + +# Used by Docker Compose +CLICKHOUSE_TCP_PORT=9009 + +# Perspective +PERSPECTIVE_API_KEY=mock-perspective-api-key + +# API + +API_PORT=8000 + +SECRET_KEY=mock-secret +JWT_SECRET_KEY=mock-jwt-secret +REFRESH_SECRET_KEY=mock-refresh + +GITHUB_CLIENT_SECRET="mock-github-client-secret" +GOOGLE_CLIENT_SECRET="mock-google-client-secret" +BYPASS_GOOGLE_DRIVE="True" +GOOGLE_SERVICE_ACCOUNT='{"mock": "mock"}' + +STRIKES_CLICKHOUSE_HOST=localhost +STRIKES_CLICKHOUSE_USER=admin +STRIKES_CLICKHOUSE_PASSWORD=dreadnode +STRIKES_CLICKHOUSE_DATABASE=platform + +# Used by API to allow mocking and testing and by Docker Compose +S3_AWS_ENDPOINT_URL=http://localhost:9000 + +# Used by API to allow mocking and testing +S3_AWS_DEFAULT_REGION=us-east-1 +S3_AWS_ACCESS_KEY_ID=mock-user +S3_AWS_SECRET_ACCESS_KEY=mock-password +S3_AWS_EXTERNAL_ENDPOINT_URL=http://localhost:9000 + +SPYGLASS_BUCKET_NAME=spyglass +PYTHON_PACKAGE_BUCKET_NAME=python-packages +USER_DATA_BUCKET_NAME=user-data +REQUIRED_BUCKETS="${SPYGLASS_BUCKET_NAME} ${PYTHON_PACKAGE_BUCKET_NAME} ${USER_DATA_BUCKET_NAME}" + +# Slack (Used by API) +SLACK_SIGNING_SECRET=mock-slack-signing-secret +SLACK_BOT_TOKEN=mock-slack-bot-token diff --git a/dreadnode/cli/platform/templates/.ui.env.j2 b/dreadnode/cli/platform/templates/.ui.env.j2 new file mode 100644 index 00000000..d84f4dab --- /dev/null +++ b/dreadnode/cli/platform/templates/.ui.env.j2 @@ -0,0 +1,18 @@ +# UI Environment Variables + +# Proxy + +# PROXY_HOST=dreadnode-laptop +# PROXY_PORT=80 +# PROXY_ORIGIN=http://${PROXY_HOST}:80 +# PROXY_DASHBOARD_PORT=8118 + +# CSP (comma separated lists) +# If PROXY_HOST is set, ALLOWED_HOSTS must include match it. +# ALLOWED_HOSTS="laptop" +CSP_CONNECT_SRC="https://www.google.com,https://www.gstatic.com" +CSP_FONT_SRC="https://cdn.jsdelivr.net,https://fonts.gstatic.com" +CSP_FRAME_SRC="https://www.google.com,https://www.gstatic.com" +CSP_IMG_SRC="https://www.gstatic.com" +CSP_SCRIPT_SRC="https://www.google.com,https://www.gstatic.com" +CSP_STYLE_SRC="https://cdn.jsdelivr.net,https://fonts.googleapis.com" diff --git a/dreadnode/cli/platform/templates/docker-compose.yaml.j2 b/dreadnode/cli/platform/templates/docker-compose.yaml.j2 new file mode 100644 index 00000000..aea4c9c5 --- /dev/null +++ b/dreadnode/cli/platform/templates/docker-compose.yaml.j2 @@ -0,0 +1,154 @@ +--- +services: + postgres: + image: docker.io/library/postgres:16 + environment: + - POSTGRES_USER=${DATABASE_USER?Variable not set} + - POSTGRES_PASSWORD=${DATABASE_PASSWORD?Variable not set} + - POSTGRES_DB=${DATABASE_NAME?Variable not set} + ports: + - ${DATABASE_PORT:-5432}:5432 + volumes: + - ${DATABASE_VOLUME:-postgres-data}:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DATABASE_USER} -d ${DATABASE_NAME}"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + + clickhouse: + image: clickhouse/clickhouse-server:latest + ports: + - ${CLICKHOUSE_HTTP_PORT:-8123}:8123 + - ${CLICKHOUSE_TCP_PORT:-9000}:9000 + volumes: + - ${CLICKHOUSE_VOLUME:-clickhouse-data}:/var/lib/clickhouse + ulimits: + nofile: + soft: 262144 + hard: 262144 + cap_add: + - SYS_NICE + - NET_ADMIN + - IPC_LOCK + - SYS_PTRACE + environment: + - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 + - CLICKHOUSE_USER=${STRIKES_CLICKHOUSE_USER?Variable not set} + - CLICKHOUSE_PASSWORD=${STRIKES_CLICKHOUSE_PASSWORD?Variable not set} + - CLICKHOUSE_DB=${STRIKES_CLICKHOUSE_DATABASE?Variable not set} + + minio: + image: minio/minio:latest + ports: + - ${MINIO_PORT:-9000}:9000 + - ${MINIO_CONSOLE_PORT:-9001}:9001 + volumes: + - ${MINIO_VOLUME:-minio-data}:/data + environment: + - MINIO_ROOT_USER=${S3_AWS_ACCESS_KEY_ID?Variable not set} + - MINIO_ROOT_PASSWORD=${S3_AWS_SECRET_ACCESS_KEY?Variable not set} + command: server /data --console-address ":9001" + + create-s3-buckets: + image: minio/mc:latest + profiles: [create-s3-buckets] + environment: + MC_HOST_minio: http://${S3_AWS_ACCESS_KEY_ID}:${S3_AWS_SECRET_ACCESS_KEY}@minio:${MINIO_PORT:-9000} + BUCKETS: ${REQUIRED_BUCKETS?Variable not set} + entrypoint: + - sh + - -c + - | + until mc ls minio > /dev/null 2>&1; do + sleep 0.5 + done + + for bucket in $$BUCKETS; do + if mc ls minio/$$bucket > /dev/null 2>&1; then + echo "Bucket $$bucket already exists, skipping..." + else + mc mb minio/$$bucket && echo "Created bucket: $$bucket" + fi + done + + echo "All buckets processed successfully" + exit 0 + + traefik: + image: traefik:v3.4 + profiles: [run, run-api, run-ui] + command: + - "--api.insecure=true" + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.web.address=:80" + ports: + - "${PROXY_PORT:-80}:80" + - "${PROXY_DASHBOARD_PORT:-8118}:8080" + volumes: + - "/var/run/docker.sock:/var/run/docker.sock:ro" + restart: unless-stopped + + platform-api: + profiles: [run, run-api] + container_name: api + image: {{ api_image_digest }} + env_file: .api.env + environment: + - SKIP_DB_UPGRADE=0 + # User the Docker service name as the host for inter-service communication + - DATABASE_HOST=postgres + - STRIKES_CLICKHOUSE_HOST=clickhouse + - DYNAMODB_URL=http://dynamodb:8000 + - S3_AWS_ENDPOINT_URL=http://minio:9000 + - S3_AWS_ACCESS_KEY_ID=${S3_AWS_ACCESS_KEY_ID?Variable not set} + - S3_AWS_SECRET_ACCESS_KEY=${S3_AWS_SECRET_ACCESS_KEY?Variable not set} + - S3_AWS_EXTERNAL_ENDPOINT_URL=http://${PROXY_HOST:-localhost}:9000 + + ports: + - ${API_PORT:-8000}:8000 + depends_on: + postgres: + condition: service_healthy + labels: + - "traefik.enable=true" + - "traefik.http.routers.api.rule=PathPrefix(`/api`)" + - "traefik.http.routers.api.priority=10" + - "traefik.http.routers.api.entrypoints=web" + - "traefik.http.services.api.loadbalancer.server.port=8000" + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8000/api/health"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + + platform-ui: + depends_on: + platform-api: + condition: service_healthy + profiles: [run, run-ui] + container_name: ui + image: {{ ui_image_digest }} + pull_policy: never + ports: + - ${UI_PORT:-5173}:3000 + env_file: .ui.env + environment: + - NODE_ENV=production + - API_BASE_URL=http://api:8000 + - ORIGIN=${PROXY_PROTOCOL:-http}://${PROXY_HOST:-localhost:80} + - ALLOWED_HOSTS=${PROXY_HOST:-localhost} + labels: + - "traefik.enable=true" + - "traefik.http.routers.ui.rule=PathPrefix(`/`) && !PathPrefix(`/api`)" + - "traefik.http.routers.ui.priority=1" + - "traefik.http.routers.ui.entrypoints=web" + - "traefik.http.services.ui.loadbalancer.server.port=3000" + +volumes: + postgres-data: + clickhouse-data: + minio-data: diff --git a/dreadnode/cli/platform/utils.py b/dreadnode/cli/platform/utils.py new file mode 100644 index 00000000..b6ff4c52 --- /dev/null +++ b/dreadnode/cli/platform/utils.py @@ -0,0 +1,60 @@ +import json +import platform +import typing as t +from pathlib import Path + +from dreadnode.api.models import RegistryImageDetails + +archs = t.Literal["amd64", "arm64"] + + +def get_local_arch() -> archs: + arch = platform.machine() + + # Check for specific architectures + if arch in ["x86_64", "AMD64"]: + return "amd64" + if arch in ["arm64", "aarch64", "ARM64"]: + return "arm64" + raise ValueError(f"Unsupported architecture: {arch}") + + +def get_local_cache_dir() -> Path: + return Path.home() / ".dreadnode" / "platform" + + +def get_local_version() -> RegistryImageDetails | None: + local_cache_dir = get_local_cache_dir() + version_file = local_cache_dir / ".version" + if version_file.exists(): + return RegistryImageDetails(**json.loads(version_file.read_text())) + return None + + +def get_compose_file_path() -> Path: + return get_local_cache_dir() / "docker-compose.yaml" + + +def render_with_string_replace( + api_image_digest: str, + ui_image_digest: str, + template_path: str, + output_path: str, +) -> str: + """ + Simple string replacement - lightest option. + Works for basic {{ variable }} patterns. + """ + + with Path(template_path).open() as file: + content = file.read() + + rendered = content.replace("{{ api_image_digest }}", api_image_digest).replace( + "{{ ui_image_digest }}", ui_image_digest + ) + + if output_path: + with Path(output_path).open("w") as file: + file.write(rendered) + + return rendered diff --git a/dreadnode/constants.py b/dreadnode/constants.py index f2888347..cd633979 100644 --- a/dreadnode/constants.py +++ b/dreadnode/constants.py @@ -5,6 +5,8 @@ # Defaults # +# name of the default local storage path +DEFAULT_LOCAL_STORAGE_DIR = ".dreadnode" # name of the default server profile DEFAULT_PROFILE_NAME = "main" # default poll interval for the authentication flow diff --git a/poetry.lock b/poetry.lock index 015a0bd3..39724e40 100644 --- a/poetry.lock +++ b/poetry.lock @@ -385,7 +385,7 @@ version = "2025.8.3" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" -groups = ["main", "dev"] +groups = ["main", "dev", "platform"] files = [ {file = "certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5"}, {file = "certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407"}, @@ -490,7 +490,7 @@ version = "3.4.3" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7" -groups = ["main", "dev"] +groups = ["main", "dev", "platform"] files = [ {file = "charset_normalizer-3.4.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72"}, {file = "charset_normalizer-3.4.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe"}, @@ -866,6 +866,29 @@ files = [ {file = "distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed"}, ] +[[package]] +name = "docker" +version = "7.1.0" +description = "A Python library for the Docker Engine API." +optional = false +python-versions = ">=3.8" +groups = ["platform"] +files = [ + {file = "docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0"}, + {file = "docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c"}, +] + +[package.dependencies] +pywin32 = {version = ">=304", markers = "sys_platform == \"win32\""} +requests = ">=2.26.0" +urllib3 = ">=1.26.0" + +[package.extras] +dev = ["coverage (==7.2.7)", "pytest (==7.4.2)", "pytest-cov (==4.1.0)", "pytest-timeout (==2.1.0)", "ruff (==0.1.8)"] +docs = ["myst-parser (==0.18.0)", "sphinx (==5.1.1)"] +ssh = ["paramiko (>=2.4.3)"] +websockets = ["websocket-client (>=1.3.0)"] + [[package]] name = "docstring-parser" version = "0.17.0" @@ -1303,7 +1326,7 @@ version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.6" -groups = ["main", "dev"] +groups = ["main", "dev", "platform"] files = [ {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, @@ -3950,7 +3973,7 @@ version = "311" description = "Python for Window Extensions" optional = false python-versions = "*" -groups = ["main", "dev"] +groups = ["main", "dev", "platform"] files = [ {file = "pywin32-311-cp310-cp310-win32.whl", hash = "sha256:d03ff496d2a0cd4a5893504789d4a15399133fe82517455e78bad62efbb7f0a3"}, {file = "pywin32-311-cp310-cp310-win_amd64.whl", hash = "sha256:797c2772017851984b97180b0bebe4b620bb86328e8a884bb626156295a63b3b"}, @@ -3973,7 +3996,7 @@ files = [ {file = "pywin32-311-cp39-cp39-win_amd64.whl", hash = "sha256:e0c4cfb0621281fe40387df582097fd796e80430597cb9944f0ae70447bacd91"}, {file = "pywin32-311-cp39-cp39-win_arm64.whl", hash = "sha256:62ea666235135fee79bb154e695f3ff67370afefd71bd7fea7512fc70ef31e3d"}, ] -markers = {main = "sys_platform == \"win32\"", dev = "sys_platform == \"win32\" and platform_python_implementation != \"PyPy\""} +markers = {main = "sys_platform == \"win32\"", dev = "sys_platform == \"win32\" and platform_python_implementation != \"PyPy\"", platform = "sys_platform == \"win32\""} [[package]] name = "pyyaml" @@ -4278,7 +4301,7 @@ version = "2.32.4" description = "Python HTTP for Humans." optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main", "dev", "platform"] files = [ {file = "requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c"}, {file = "requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422"}, @@ -5536,7 +5559,7 @@ version = "2.5.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main", "dev", "platform"] files = [ {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, @@ -6078,9 +6101,10 @@ type = ["pytest-mypy"] [extras] all = [] multimodal = ["moviepy", "pillow", "soundfile"] +platform = [] training = ["transformers"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.14" -content-hash = "608bdd485f2f8fb2d4390f37791f6fdd484c4ca4aa5ef661346c68dd3038f726" +content-hash = "3bba2420a863db24d08eac93beea7dc5d73e04f474073d265f3406336efef0b8" diff --git a/pyproject.toml b/pyproject.toml index afbbf033..f6bf998c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ presidio-analyzer = "^2.2.359" [tool.poetry.extras] training = ["transformers"] multimodal = ["pillow", "soundfile", "moviepy"] -platform = ["docker"] +platform = ["docker", "pyyaml"] all = ["multimodal", "training"] [tool.poetry.group.dev.dependencies] @@ -57,6 +57,10 @@ markdownify = "^1.1.0" mkdocstrings-python = "^1.17.0" ipykernel = "^6.29.5" + +[tool.poetry.group.platform.dependencies] +docker = "^7.1.0" + [build-system] requires = ["poetry-core>=1.0.0", "setuptools>=42", "wheel"] build-backend = "poetry.core.masonry.api" From f9b390414a2a761ee87977ad49c012cb2e0d5d8c Mon Sep 17 00:00:00 2001 From: Brian Greunke Date: Tue, 26 Aug 2025 23:59:38 -0500 Subject: [PATCH 4/4] fix: removed old template files --- dreadnode/cli/platform/templates/.api.env.j2 | 75 --------- dreadnode/cli/platform/templates/.ui.env.j2 | 18 -- .../platform/templates/docker-compose.yaml.j2 | 154 ------------------ 3 files changed, 247 deletions(-) diff --git a/dreadnode/cli/platform/templates/.api.env.j2 b/dreadnode/cli/platform/templates/.api.env.j2 index 95be9063..e69de29b 100644 --- a/dreadnode/cli/platform/templates/.api.env.j2 +++ b/dreadnode/cli/platform/templates/.api.env.j2 @@ -1,75 +0,0 @@ -# API Environment Variables - -# Features -### The double underscore (__) is used to denote nested properties. -## E.g. Turn off OAuth login -FEATURES__OAUTH__ENABLED=False -## E.g. Turn off Crucible -FEATURES__COMPETITIVE_LEARNING__ENABLED=${CRUCIBLE_ENABLED:-False} -## E.g. Turn off Spyglass -FEATURES__SPYGLASS__ENABLED=${SPYGLASS_ENABLED:-False} - -# Database -## Used by API and Docker Compose -DATABASE_USER=admin -DATABASE_PASSWORD=dreadnode -DATABASE_NAME=platform -DATABASE_PORT=5432 -DATABASE_HOST=localhost - -# DynamoDB - -# Used by API tests to allow mocking -DYNAMODB_PORT=8085 -DYNAMODB_AWS_ACCESS_KEY_ID=mock -DYNAMODB_AWS_SECRET_ACCESS_KEY=mock -DYNAMODB_AWS_DEFAULT_REGION=us-west-2 -DYNAMODB_URL=http://${PROXY_HOST:-localhost}:${DYNAMODB_PORT} - -# Used by API -AUTHORIZATION_KEY_TABLE="auth-keys" -FLAG_ATTEMPTS_TABLE="flag-attempts" - -# Clickhouse - -# Used by Docker Compose -CLICKHOUSE_TCP_PORT=9009 - -# Perspective -PERSPECTIVE_API_KEY=mock-perspective-api-key - -# API - -API_PORT=8000 - -SECRET_KEY=mock-secret -JWT_SECRET_KEY=mock-jwt-secret -REFRESH_SECRET_KEY=mock-refresh - -GITHUB_CLIENT_SECRET="mock-github-client-secret" -GOOGLE_CLIENT_SECRET="mock-google-client-secret" -BYPASS_GOOGLE_DRIVE="True" -GOOGLE_SERVICE_ACCOUNT='{"mock": "mock"}' - -STRIKES_CLICKHOUSE_HOST=localhost -STRIKES_CLICKHOUSE_USER=admin -STRIKES_CLICKHOUSE_PASSWORD=dreadnode -STRIKES_CLICKHOUSE_DATABASE=platform - -# Used by API to allow mocking and testing and by Docker Compose -S3_AWS_ENDPOINT_URL=http://localhost:9000 - -# Used by API to allow mocking and testing -S3_AWS_DEFAULT_REGION=us-east-1 -S3_AWS_ACCESS_KEY_ID=mock-user -S3_AWS_SECRET_ACCESS_KEY=mock-password -S3_AWS_EXTERNAL_ENDPOINT_URL=http://localhost:9000 - -SPYGLASS_BUCKET_NAME=spyglass -PYTHON_PACKAGE_BUCKET_NAME=python-packages -USER_DATA_BUCKET_NAME=user-data -REQUIRED_BUCKETS="${SPYGLASS_BUCKET_NAME} ${PYTHON_PACKAGE_BUCKET_NAME} ${USER_DATA_BUCKET_NAME}" - -# Slack (Used by API) -SLACK_SIGNING_SECRET=mock-slack-signing-secret -SLACK_BOT_TOKEN=mock-slack-bot-token diff --git a/dreadnode/cli/platform/templates/.ui.env.j2 b/dreadnode/cli/platform/templates/.ui.env.j2 index d84f4dab..e69de29b 100644 --- a/dreadnode/cli/platform/templates/.ui.env.j2 +++ b/dreadnode/cli/platform/templates/.ui.env.j2 @@ -1,18 +0,0 @@ -# UI Environment Variables - -# Proxy - -# PROXY_HOST=dreadnode-laptop -# PROXY_PORT=80 -# PROXY_ORIGIN=http://${PROXY_HOST}:80 -# PROXY_DASHBOARD_PORT=8118 - -# CSP (comma separated lists) -# If PROXY_HOST is set, ALLOWED_HOSTS must include match it. -# ALLOWED_HOSTS="laptop" -CSP_CONNECT_SRC="https://www.google.com,https://www.gstatic.com" -CSP_FONT_SRC="https://cdn.jsdelivr.net,https://fonts.gstatic.com" -CSP_FRAME_SRC="https://www.google.com,https://www.gstatic.com" -CSP_IMG_SRC="https://www.gstatic.com" -CSP_SCRIPT_SRC="https://www.google.com,https://www.gstatic.com" -CSP_STYLE_SRC="https://cdn.jsdelivr.net,https://fonts.googleapis.com" diff --git a/dreadnode/cli/platform/templates/docker-compose.yaml.j2 b/dreadnode/cli/platform/templates/docker-compose.yaml.j2 index aea4c9c5..e69de29b 100644 --- a/dreadnode/cli/platform/templates/docker-compose.yaml.j2 +++ b/dreadnode/cli/platform/templates/docker-compose.yaml.j2 @@ -1,154 +0,0 @@ ---- -services: - postgres: - image: docker.io/library/postgres:16 - environment: - - POSTGRES_USER=${DATABASE_USER?Variable not set} - - POSTGRES_PASSWORD=${DATABASE_PASSWORD?Variable not set} - - POSTGRES_DB=${DATABASE_NAME?Variable not set} - ports: - - ${DATABASE_PORT:-5432}:5432 - volumes: - - ${DATABASE_VOLUME:-postgres-data}:/var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${DATABASE_USER} -d ${DATABASE_NAME}"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 30s - - clickhouse: - image: clickhouse/clickhouse-server:latest - ports: - - ${CLICKHOUSE_HTTP_PORT:-8123}:8123 - - ${CLICKHOUSE_TCP_PORT:-9000}:9000 - volumes: - - ${CLICKHOUSE_VOLUME:-clickhouse-data}:/var/lib/clickhouse - ulimits: - nofile: - soft: 262144 - hard: 262144 - cap_add: - - SYS_NICE - - NET_ADMIN - - IPC_LOCK - - SYS_PTRACE - environment: - - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 - - CLICKHOUSE_USER=${STRIKES_CLICKHOUSE_USER?Variable not set} - - CLICKHOUSE_PASSWORD=${STRIKES_CLICKHOUSE_PASSWORD?Variable not set} - - CLICKHOUSE_DB=${STRIKES_CLICKHOUSE_DATABASE?Variable not set} - - minio: - image: minio/minio:latest - ports: - - ${MINIO_PORT:-9000}:9000 - - ${MINIO_CONSOLE_PORT:-9001}:9001 - volumes: - - ${MINIO_VOLUME:-minio-data}:/data - environment: - - MINIO_ROOT_USER=${S3_AWS_ACCESS_KEY_ID?Variable not set} - - MINIO_ROOT_PASSWORD=${S3_AWS_SECRET_ACCESS_KEY?Variable not set} - command: server /data --console-address ":9001" - - create-s3-buckets: - image: minio/mc:latest - profiles: [create-s3-buckets] - environment: - MC_HOST_minio: http://${S3_AWS_ACCESS_KEY_ID}:${S3_AWS_SECRET_ACCESS_KEY}@minio:${MINIO_PORT:-9000} - BUCKETS: ${REQUIRED_BUCKETS?Variable not set} - entrypoint: - - sh - - -c - - | - until mc ls minio > /dev/null 2>&1; do - sleep 0.5 - done - - for bucket in $$BUCKETS; do - if mc ls minio/$$bucket > /dev/null 2>&1; then - echo "Bucket $$bucket already exists, skipping..." - else - mc mb minio/$$bucket && echo "Created bucket: $$bucket" - fi - done - - echo "All buckets processed successfully" - exit 0 - - traefik: - image: traefik:v3.4 - profiles: [run, run-api, run-ui] - command: - - "--api.insecure=true" - - "--providers.docker=true" - - "--providers.docker.exposedbydefault=false" - - "--entrypoints.web.address=:80" - ports: - - "${PROXY_PORT:-80}:80" - - "${PROXY_DASHBOARD_PORT:-8118}:8080" - volumes: - - "/var/run/docker.sock:/var/run/docker.sock:ro" - restart: unless-stopped - - platform-api: - profiles: [run, run-api] - container_name: api - image: {{ api_image_digest }} - env_file: .api.env - environment: - - SKIP_DB_UPGRADE=0 - # User the Docker service name as the host for inter-service communication - - DATABASE_HOST=postgres - - STRIKES_CLICKHOUSE_HOST=clickhouse - - DYNAMODB_URL=http://dynamodb:8000 - - S3_AWS_ENDPOINT_URL=http://minio:9000 - - S3_AWS_ACCESS_KEY_ID=${S3_AWS_ACCESS_KEY_ID?Variable not set} - - S3_AWS_SECRET_ACCESS_KEY=${S3_AWS_SECRET_ACCESS_KEY?Variable not set} - - S3_AWS_EXTERNAL_ENDPOINT_URL=http://${PROXY_HOST:-localhost}:9000 - - ports: - - ${API_PORT:-8000}:8000 - depends_on: - postgres: - condition: service_healthy - labels: - - "traefik.enable=true" - - "traefik.http.routers.api.rule=PathPrefix(`/api`)" - - "traefik.http.routers.api.priority=10" - - "traefik.http.routers.api.entrypoints=web" - - "traefik.http.services.api.loadbalancer.server.port=8000" - healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:8000/api/health"] - interval: 10s - timeout: 5s - retries: 5 - start_period: 30s - - platform-ui: - depends_on: - platform-api: - condition: service_healthy - profiles: [run, run-ui] - container_name: ui - image: {{ ui_image_digest }} - pull_policy: never - ports: - - ${UI_PORT:-5173}:3000 - env_file: .ui.env - environment: - - NODE_ENV=production - - API_BASE_URL=http://api:8000 - - ORIGIN=${PROXY_PROTOCOL:-http}://${PROXY_HOST:-localhost:80} - - ALLOWED_HOSTS=${PROXY_HOST:-localhost} - labels: - - "traefik.enable=true" - - "traefik.http.routers.ui.rule=PathPrefix(`/`) && !PathPrefix(`/api`)" - - "traefik.http.routers.ui.priority=1" - - "traefik.http.routers.ui.entrypoints=web" - - "traefik.http.services.ui.loadbalancer.server.port=3000" - -volumes: - postgres-data: - clickhouse-data: - minio-data: