From 10c6e4f5e3bbe6ccccc7c75c58f747eb4cc68965 Mon Sep 17 00:00:00 2001 From: Emma Zhou Date: Sat, 10 May 2025 15:53:48 -0700 Subject: [PATCH 1/3] split out a build command from the synapsectl deploy command --- synapse/cli/__main__.py | 19 +- synapse/cli/build.py | 540 +++++++++++++++++++++++++++++++++++++++ synapse/cli/deploy.py | 543 ++++------------------------------------ 3 files changed, 602 insertions(+), 500 deletions(-) create mode 100644 synapse/cli/build.py diff --git a/synapse/cli/__main__.py b/synapse/cli/__main__.py index 3ac5d2f..1e87963 100755 --- a/synapse/cli/__main__.py +++ b/synapse/cli/__main__.py @@ -1,13 +1,23 @@ #!/usr/bin/env python import argparse -import logging import ipaddress +import logging import sys - from importlib import metadata -from synapse.cli import discover, rpc, streaming, offline_plot, files, deploy, taps -from rich.logging import RichHandler + from rich.console import Console +from rich.logging import RichHandler + +from synapse.cli import ( + build, + deploy, + discover, + files, + offline_plot, + rpc, + streaming, + taps, +) from synapse.utils.discover import find_device_by_name @@ -66,6 +76,7 @@ def main(): files.add_commands(subparsers) taps.add_commands(subparsers) deploy.add_commands(subparsers) + build.add_commands(subparsers) args = parser.parse_args() # If we need to setup the device URI, do that now diff --git a/synapse/cli/build.py b/synapse/cli/build.py new file mode 100644 index 0000000..c37ed23 --- /dev/null +++ b/synapse/cli/build.py @@ -0,0 +1,540 @@ +from __future__ import annotations + +import glob +import json +import os +import shutil +import subprocess +import tempfile +from typing import Any + +from rich import box +from rich.console import Console +from rich.panel import Panel + +console = Console() + + +def validate_manifest(manifest_path: str) -> dict[str, Any] | bool: + """Return the parsed ``manifest.json`` dictionary or ``False`` on error.""" + + try: + with open(manifest_path, "r", encoding="utf-8") as fp: + manifest = json.load(fp) + + if "name" not in manifest: + console.print( + "[bold red]Error:[/bold red] manifest.json is missing required 'name' property" + ) + return False + return manifest + except FileNotFoundError: + console.print( + f"[bold red]Error:[/bold red] manifest.json not found in {manifest_path}" + ) + return False + except json.JSONDecodeError: + console.print("[bold red]Error:[/bold red] manifest.json is not valid JSON") + return False + + +def detect_arch() -> str: + """Return an architecture tag suffix (``arm64`` or ``amd64``).""" + arch = subprocess.check_output(["uname", "-m"]).decode("utf-8").strip() + return "arm64" if arch in ("arm64", "aarch64") else "amd64" + + +def ensure_docker() -> bool: + """Check that *docker* CLI and daemon are available. + + Prints user-friendly errors and returns ``False`` if Docker cannot be used – + allowing the caller to abort early without raising. + """ + if shutil.which("docker") is None: + console.print( + "[bold red]Error:[/bold red] Docker CLI not found. Please install Docker before running this command." + ) + return False + + try: + subprocess.run( + ["docker", "info"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=True, + ) + return True + except subprocess.CalledProcessError: + console.print( + "[bold red]Error:[/bold red] Docker daemon does not appear to be running. Please start Docker and try again." + ) + return False + + +def build_docker_image(app_dir: str, app_name: str | None = None) -> str: + """(Re)build the cross-compile SDK Docker image and return its tag.""" + + if app_name is None: + app_name = os.path.basename(app_dir) + + arch_suffix = detect_arch() # "arm64" or "amd64" + + # Prefer an arch-specific Dockerfile if provided (``ops/docker/Dockerfile.arm64``) + dockerfile_rel = ( + f"ops/docker/Dockerfile.{arch_suffix}" + if arch_suffix == "arm64" + else "ops/docker/Dockerfile" + ) + dockerfile_path = os.path.join(app_dir, dockerfile_rel) + + if not os.path.exists(dockerfile_path): + # Fall back to generic Dockerfile regardless of arch. + dockerfile_path = os.path.join(app_dir, "ops/docker/Dockerfile") + + if not os.path.exists(dockerfile_path): + raise FileNotFoundError( + f"Expected Dockerfile not found at {dockerfile_path}. " + "Ensure your application provides the required build Dockerfile(s)." + ) + + image_tag = f"{app_name}:latest-{arch_suffix}" + + console.print(f"[yellow]Building Docker image [bold]{image_tag}[/bold]...[/yellow]") + subprocess.run( + [ + "docker", + "build", + "-t", + image_tag, + "-f", + dockerfile_path, + ".", + ], + check=True, + cwd=app_dir, + ) + + console.print(f"[green]Successfully built Docker image {image_tag}[/green]") + return image_tag + + +def build_app(app_dir: str, app_name: str) -> bool: + """Cross-compile *app_name* inside its SDK container. + + Returns ``True`` on success, ``False`` otherwise. + """ + + console.print(f"[yellow]Building application: {app_name}...[/yellow]") + + binary_path = os.path.join(app_dir, "build-aarch64", app_name) + + # Skip if binary already exists + if os.path.exists(binary_path): + console.print(f"[green]Binary already exists at: {binary_path}[/green]") + return True + + console.print("[yellow]Binary not found, attempting to build...[/yellow]") + + arch_suffix = detect_arch() + image_tag = f"{os.path.basename(app_dir)}:latest-{arch_suffix}" + + console.print( + f"[yellow]Docker image {image_tag} not found, building it first...[/yellow]" + ) + + # Build (or rebuild) the Docker image – this function is idempotent. + try: + image_tag = build_docker_image(app_dir, app_name) + except (subprocess.CalledProcessError, FileNotFoundError) as exc: + console.print( + f"[bold red]Error:[/bold red] Failed to build Docker image: {exc}" + ) + return False + + console.print("[blue]Installing dependencies...[/blue]") + vcpkg_cmd = [ + "docker", + "run", + "--rm", + "-v", + f"{os.path.abspath(app_dir)}:/home/workspace", + image_tag, + "/bin/bash", + "-c", + "cd /home/workspace && if [ -f vcpkg.json ]; then " + "echo 'Installing dependencies from vcpkg.json...' && " + "${VCPKG_ROOT}/vcpkg install --triplet arm64-linux-dynamic-release; fi", + ] + + try: + subprocess.run(vcpkg_cmd, check=True, cwd=app_dir) + except subprocess.CalledProcessError: + console.print( + "[yellow]Warning: Failed to install dependencies. The build might still succeed.[/yellow]" + ) + + console.print("[blue]Running build command...[/blue]") + + build_cmd = [ + "docker", + "run", + "--rm", + "-v", + f"{os.path.abspath(app_dir)}:/home/workspace", + image_tag, + "/bin/bash", + "-c", + ( + "cd /home/workspace && " + "if [ -f CMakePresets.json ]; then " + "echo 'Using existing CMake presets...' && " + "cmake --preset=dynamic-aarch64 -DVCPKG_TARGET_TRIPLET='arm64-linux-dynamic-release' && " + "cmake --build --preset=cross-release -j$(nproc); " + "else " + "echo 'No CMake presets found, using manual configuration...' && " + "export VCPKG_DEFAULT_TRIPLET=arm64-linux-dynamic-release && " + "cmake -B build-aarch64 -S . " + "-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake " + "-DVCPKG_TARGET_TRIPLET=arm64-linux-dynamic-release " + "-DVCPKG_INSTALLED_DIR=${VCPKG_ROOT}/vcpkg_installed " + "-DBUILD_SHARED_LIBS=ON " + "-DCMAKE_BUILD_TYPE=Release " + "-DBUILD_FOR_ARM64=ON && " + "cmake --build build-aarch64 -j$(nproc); " + "fi" + ), + ] + + try: + subprocess.run(build_cmd, check=True, cwd=app_dir) + except subprocess.CalledProcessError: + console.print( + "[bold red]Error:[/bold red] Failed to build application. Check the CMake output above for details." + ) + return False + + if os.path.exists(binary_path): + console.print(f"[green]Successfully built binary at: {binary_path}[/green]") + return True + + # Fallback: try to locate the binary elsewhere in the tree + console.print( + f"[bold yellow]Warning: Build completed but binary not found at expected location: {binary_path}[/bold yellow]" + ) + + try: + binary_found = subprocess.run( + [ + "find", + app_dir, + "-type", + "f", + "-name", + app_name, + "-not", + "-path", + "*/.*", + ], + capture_output=True, + text=True, + check=False, + ).stdout.strip() + + if binary_found: + located = binary_found.split("\n")[0] + build_dir = os.path.dirname(binary_path) + os.makedirs(build_dir, exist_ok=True) + shutil.copy(located, binary_path) + console.print( + f"[green]Copied binary from {located} to standard location {binary_path}[/green]" + ) + return True + except Exception: + pass + + return False + + +def build_deb_package(app_dir: str, app_name: str, version: str = "0.1.0") -> bool: + """Stage *app_name* and produce a ``.deb`` file within *app_dir*.""" + + try: + staging_dir = tempfile.mkdtemp(prefix="synapse-package-") + binary_path = os.path.join(app_dir, "build-aarch64", app_name) + + if not os.path.exists(binary_path): + console.print( + f"[bold red]Error:[/bold red] Compiled binary '{app_name}' not found at {binary_path}." + ) + return False + + bin_dst_dir = os.path.join(staging_dir, "opt", "scifi", "bin") + os.makedirs(bin_dst_dir, exist_ok=True) + shutil.copy2(binary_path, os.path.join(bin_dst_dir, app_name)) + + svc_content = f"""[Unit] +Description=Synapse Application +After=network-online.target +Wants=network-online.target +Requires=systemd-udevd.service +After=systemd-udevd.service + +[Service] +Type=simple +User=root +Restart=no +ExecStartPre=/sbin/sysctl -w net.core.wmem_max=4194304 +ExecStartPre=/sbin/sysctl -w net.core.wmem_default=4194304 +Environment=LD_LIBRARY_PATH=/opt/scifi/usr-libs:/opt/scifi/lib +Environment=SCIFI_ROOT=/opt/scifi +ExecStart=/opt/scifi/bin/{app_name} +WorkingDirectory=/opt/scifi + +[Install] +WantedBy=multi-user.target +""" + svc_dst = os.path.join( + staging_dir, "etc", "systemd", "system", f"{app_name}.service" + ) + os.makedirs(os.path.dirname(svc_dst), exist_ok=True) + with open(svc_dst, "w", encoding="utf-8") as fp: + fp.write(svc_content) + + lifecycle_scripts_tmp: list[str] = [] + + postinstall_path = os.path.join(staging_dir, "postinstall.sh") + with open(postinstall_path, "w", encoding="utf-8") as fp: + fp.write("#!/bin/bash\nset -e\nsystemctl daemon-reload\n") + os.chmod(postinstall_path, 0o755) + lifecycle_scripts_tmp.append(postinstall_path) + + preremove_path = os.path.join(staging_dir, "preremove.sh") + with open(preremove_path, "w", encoding="utf-8") as fp: + fp.write( + f"#!/bin/bash\nset -e\nsystemctl stop {app_name} || true\nsystemctl disable {app_name} || true\n" + ) + os.chmod(preremove_path, 0o755) + lifecycle_scripts_tmp.append(preremove_path) + + postremove_path = os.path.join(staging_dir, "postremove.sh") + with open(postremove_path, "w", encoding="utf-8") as fp: + fp.write("#!/bin/bash\nset -e\nsystemctl daemon-reload\n") + os.chmod(postremove_path, 0o755) + lifecycle_scripts_tmp.append(postremove_path) + + lib_dst_dir = os.path.join(staging_dir, "opt", "scifi", "lib") + os.makedirs(lib_dst_dir, exist_ok=True) + + try: + arch_suffix = detect_arch() + image_tag = f"{app_name}:latest-{arch_suffix}" + platform_opt = "linux/arm64" if arch_suffix == "arm64" else "linux/amd64" + + console.print( + f"[yellow]Extracting SDK libraries from Docker image [bold]{image_tag}[/bold]...[/yellow]" + ) + + docker_cmd = [ + "docker", + "run", + "--rm", + "--platform", + platform_opt, + "-v", + f"{lib_dst_dir}:/out", + image_tag, + "/bin/bash", + "-c", + "find /usr/lib -name 'libsynapse*.so*' -exec cp -av {} /out/ \\;", + ] + + subprocess.run(docker_cmd, check=True) + + except subprocess.CalledProcessError as exc: + console.print( + f"[bold red]Error:[/bold red] Failed to copy SDK libraries from Docker image: {exc}" + ) + console.print( + "[yellow]Falling back to host /usr/lib lookup for libsynapse*.so* (results may be incomplete).[/yellow]" + ) + + for lib in glob.glob("/usr/lib/**/libsynapse*.so*", recursive=True): + try: + shutil.copy2(lib, lib_dst_dir) + except PermissionError: + console.print(f"[yellow]Skipping lib copy (perm): {lib}[/yellow]") + + fpm_cmd = [ + "fpm", + "-s", + "dir", + "-t", + "deb", + "-n", + app_name, + "-f", + "-v", + version, + "-C", + staging_dir, + "--deb-no-default-config-files", + "--depends", + "systemd", + "--vendor", + "Science Corporation", + "--description", + "Synapse Application", + "--architecture", + "arm64", + ] + + # Attach lifecycle scripts + script_map = { + "postinstall.sh": "--after-install", + "preremove.sh": "--before-remove", + "postremove.sh": "--after-remove", + } + for path in lifecycle_scripts_tmp: + opt = script_map.get(os.path.basename(path)) + if opt: + container_path = f"/pkg/{os.path.basename(path)}" + fpm_cmd.extend([opt, container_path]) + + fpm_cmd.append(".") + + fpm_image = "cdrx/fpm-ubuntu:latest" + console.print(f"[yellow]Running FPM (Docker image: {fpm_image}) ...[/yellow]") + + # Replace host-specific staging dir with container mount path + fpm_args = fpm_cmd[1:] + try: + c_index = fpm_args.index("-C") + 1 + fpm_args[c_index] = "/pkg" + except ValueError: + pass + + docker_fpm_cmd = [ + "docker", + "run", + "--rm", + "--platform", + "linux/amd64", + "-v", + f"{staging_dir}:/pkg", + "-v", + f"{app_dir}:/out", + "-w", + "/out", + fpm_image, + "fpm", + ] + fpm_args + + subprocess.run(docker_fpm_cmd, check=True) + + # Verify that a .deb was produced + deb_files = [ + f for f in os.listdir(app_dir) if f.endswith(".deb") and "arm64" in f + ] + if not deb_files: + console.print( + f"[bold red]Error:[/bold red] FPM completed but no .deb found in {app_dir}." + ) + return False + + console.print("[green]Package created successfully![/green]") + return True + + except subprocess.CalledProcessError as exc: + console.print(f"[bold red]Error:[/bold red] FPM failed: {exc}") + return False + + finally: + # "staging_dir" is intentionally *not* deleted so that users can inspect + # its contents when troubleshooting. The host's temp directory will + # eventually clean it up automatically. + pass + + +def package_app(app_dir: str, app_name: str) -> bool: + """Thin wrapper used by callers (e.g., CLI) to build the .deb.""" + return build_deb_package(app_dir, app_name) + + +def find_deb_package(app_dir: str) -> str | None: + """Return the path to the .deb generated in *app_dir* or *None*.""" + for file in os.listdir(app_dir): + if file.endswith(".deb"): + return os.path.join(app_dir, file) + + console.print( + f"[bold red]Error:[/bold red] Could not find .deb package in {app_dir}" + ) + return None + + +def build_cmd(args) -> None: + """Handle the ``synapsectl build`` sub-command.""" + + if not ensure_docker(): + return + + app_dir = os.path.abspath(args.app_dir) + + manifest = validate_manifest(os.path.join(app_dir, "manifest.json")) + if not manifest: + return + + app_name = manifest["name"] + console.print(f"[bold]Building application:[/bold] [yellow]{app_name}[/yellow]") + + # 1. Build phase (unless explicitly skipped) + if not args.skip_build: + if not build_app(app_dir, app_name): + console.print( + "[bold red]Error:[/bold red] Failed to build the application." + ) + return + else: + console.print( + "[yellow]Skipping compile phase as requested (--skip-build).[/yellow]" + ) + + # 2. Package (.deb) + if not package_app(app_dir, app_name): + return + + # 3. Locate artefact and present summary panel + deb_path = find_deb_package(app_dir) + if not deb_path: + return + + console.print( + Panel( + f"[green]Build complete![/green]\n\nGenerated package: [bold]{deb_path}[/bold]", + title="Build Successful", + border_style="green", + box=box.DOUBLE, + ) + ) + + +def add_commands(subparsers) -> None: + """Register the *build* command with the top-level CLI parser.""" + + build_parser = subparsers.add_parser( + "build", + help="Cross-compile and package an application into a .deb without deploying", + ) + build_parser.add_argument( + "app_dir", + nargs="?", + default=".", + help="Path to the application directory (defaults to current working directory)", + ) + build_parser.add_argument( + "--skip-build", + action="store_true", + default=False, + help="Skip compilation phase; assume the binary already exists and only build the .deb package.", + ) + build_parser.set_defaults(func=build_cmd) diff --git a/synapse/cli/deploy.py b/synapse/cli/deploy.py index 1071198..f94e0c0 100644 --- a/synapse/cli/deploy.py +++ b/synapse/cli/deploy.py @@ -1,274 +1,18 @@ -import os -import subprocess -import shutil import json -import logging -import tempfile -import glob +import os + +from rich import box from rich.console import Console from rich.panel import Panel -from rich.progress import ( - Progress, - SpinnerColumn, - TextColumn, - TimeElapsedColumn, -) +from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn from rich.prompt import Prompt -from rich import box import synapse.client.sftp as sftp +from synapse.cli import build as builder -# Set up console for normal output and a separate one for logs console = Console() log_console = Console(stderr=True) -# Configure logging for paramiko to be less verbose -logging.getLogger("paramiko").setLevel(logging.WARNING) - - -def validate_manifest(manifest_path): - """Validate the manifest file exists and has required properties""" - try: - with open(manifest_path, "r") as f: - manifest = json.load(f) - - # Basic validation - if "name" not in manifest: - console.print( - "[bold red]Error:[/bold red] manifest.json is missing required 'name' property" - ) - return False - - return manifest - except FileNotFoundError: - console.print( - f"[bold red]Error:[/bold red] manifest.json not found in {manifest_path}" - ) - return False - except json.JSONDecodeError: - console.print("[bold red]Error:[/bold red] manifest.json is not valid JSON") - return False - - -def build_deb_package(app_dir: str, app_name: str, version: str = "0.1.0") -> bool: - """Create a *.deb* package for *app_name* and place it in *app_dir*. - - Returns ``True`` on success, ``False`` otherwise. - """ - - try: - staging_dir = tempfile.mkdtemp(prefix="synapse-package-") - binary_path = os.path.join(app_dir, "build-aarch64", app_name) - - if not os.path.exists(binary_path): - console.print( - f"[bold red]Error:[/bold red] Compiled binary '{app_name}' not found at {binary_path}." - ) - return False - - bin_dst_dir = os.path.join(staging_dir, "opt", "scifi", "bin") - os.makedirs(bin_dst_dir, exist_ok=True) - shutil.copy2(binary_path, os.path.join(bin_dst_dir, app_name)) - - # Generate systemd unit - svc_content = f"""[Unit] -Description=Synapse Application -After=network-online.target -Wants=network-online.target -Requires=systemd-udevd.service -After=systemd-udevd.service - -[Service] -Type=simple -User=root -Restart=no -ExecStartPre=/sbin/sysctl -w net.core.wmem_max=4194304 -ExecStartPre=/sbin/sysctl -w net.core.wmem_default=4194304 -Environment=LD_LIBRARY_PATH=/opt/scifi/usr-libs:/opt/scifi/lib -Environment=SCIFI_ROOT=/opt/scifi -ExecStart=/opt/scifi/bin/{app_name} -WorkingDirectory=/opt/scifi - -[Install] -WantedBy=multi-user.target -""" - - svc_dst = os.path.join( - staging_dir, "etc", "systemd", "system", f"{app_name}.service" - ) - os.makedirs(os.path.dirname(svc_dst), exist_ok=True) - with open(svc_dst, "w", encoding="utf-8") as f: - f.write(svc_content) - - lifecycle_scripts_tmp = [] - - postinstall_path = os.path.join(staging_dir, "postinstall.sh") - with open(postinstall_path, "w", encoding="utf-8") as f: - f.write("#!/bin/bash\nset -e\nsystemctl daemon-reload\n") - os.chmod(postinstall_path, 0o755) - lifecycle_scripts_tmp.append(postinstall_path) - - preremove_path = os.path.join(staging_dir, "preremove.sh") - with open(preremove_path, "w", encoding="utf-8") as f: - f.write( - f"#!/bin/bash\nset -e\nsystemctl stop {app_name} || true\nsystemctl disable {app_name} || true\n" - ) - os.chmod(preremove_path, 0o755) - lifecycle_scripts_tmp.append(preremove_path) - - postremove_path = os.path.join(staging_dir, "postremove.sh") - with open(postremove_path, "w", encoding="utf-8") as f: - f.write("#!/bin/bash\nset -e\nsystemctl daemon-reload\n") - os.chmod(postremove_path, 0o755) - lifecycle_scripts_tmp.append(postremove_path) - - lib_dst_dir = os.path.join(staging_dir, "opt", "scifi", "lib") - os.makedirs(lib_dst_dir, exist_ok=True) - - try: - arch_suffix = detect_arch() # "arm64" or "amd64" - image_tag = f"{app_name}:latest-{arch_suffix}" - platform_opt = "linux/arm64" if arch_suffix == "arm64" else "linux/amd64" - - console.print( - f"[yellow]Extracting SDK libraries from Docker image [bold]{image_tag}[/bold]...[/yellow]" - ) - - docker_cmd = [ - "docker", - "run", - "--rm", - "--platform", - platform_opt, - "-v", - f"{lib_dst_dir}:/out", - image_tag, - "/bin/bash", - "-c", - "find /usr/lib -name 'libsynapse*.so*' -exec cp -av {} /out/ \\;", - ] - - subprocess.run(docker_cmd, check=True) - - except subprocess.CalledProcessError as e: - console.print( - f"[bold red]Error:[/bold red] Failed to copy SDK libraries from Docker image: {e}" - ) - console.print( - "[yellow]Falling back to host /usr/lib lookup for libsynapse*.so* (results may be incomplete).[/yellow]" - ) - - for lib in glob.glob("/usr/lib/**/libsynapse*.so*", recursive=True): - try: - shutil.copy2(lib, lib_dst_dir) - except PermissionError: - console.print(f"[yellow]Skipping lib copy (perm): {lib}[/yellow]") - - fpm_cmd = [ - "fpm", - "-s", - "dir", - "-t", - "deb", - "-n", - app_name, - "-f", - "-v", - version, - "-C", - staging_dir, - "--deb-no-default-config-files", - "--depends", - "systemd", - "--vendor", - "Science Corporation", - "--description", - "Synapse Application", - "--architecture", - "arm64", - ] - - # Attach lifecycle scripts (referenced relative to /pkg inside container) - script_map = { - "postinstall.sh": "--after-install", - "preremove.sh": "--before-remove", - "postremove.sh": "--after-remove", - } - for path in lifecycle_scripts_tmp: - opt = script_map.get(os.path.basename(path)) - if opt: - container_path = f"/pkg/{os.path.basename(path)}" - fpm_cmd.extend([opt, container_path]) - - fpm_cmd.append(".") - - fpm_image = "cdrx/fpm-ubuntu:latest" - console.print(f"[yellow]Running FPM (Docker image: {fpm_image}) ...[/yellow]") - - # Replace the host-specific staging dir with the container mount path - fpm_args = fpm_cmd[1:] - try: - c_index = fpm_args.index("-C") + 1 - fpm_args[c_index] = "/pkg" - except ValueError: - pass - - docker_fpm_cmd = [ - "docker", - "run", - "--rm", - "--platform", - "linux/amd64", - "-v", - f"{staging_dir}:/pkg", - "-v", - f"{app_dir}:/out", - "-w", - "/out", - fpm_image, - "fpm", - ] + fpm_args - - subprocess.run(docker_fpm_cmd, check=True) - - # Verify that a .deb was produced - deb_files = [ - f for f in os.listdir(app_dir) if f.endswith(".deb") and "arm64" in f - ] - if not deb_files: - console.print( - f"[bold red]Error:[/bold red] FPM completed but no .deb found in {app_dir}." - ) - return False - - console.print("[green]Package created successfully![/green]") - return True - - except subprocess.CalledProcessError as exc: - console.print(f"[bold red]Error:[/bold red] FPM failed: {exc}") - return False - - finally: - pass - - -def package_app(app_dir, app_name): - """Package *app_name* into a .deb using the pure-Python builder.""" - - return build_deb_package(app_dir, app_name) - - -def find_deb_package(app_dir): - """Find the generated .deb package in the app directory""" - for file in os.listdir(app_dir): - if file.endswith(".deb"): - return os.path.join(app_dir, file) - - console.print( - f"[bold red]Error:[/bold red] Could not find .deb package in {app_dir}" - ) - return None - def get_device_credentials(ip_address): """Get user credentials with clear prompts""" @@ -570,171 +314,54 @@ def save_credentials(ip_address, username, login_password, root_password): console.print(f"[yellow]Warning: Failed to save credentials: {e}[/yellow]") -def build_app(app_dir, app_name): - """Build the application binary before packaging""" - console.print(f"[yellow]Building application: {app_name}...[/yellow]") - - # Check if binary already exists - binary_path = os.path.join(app_dir, "build-aarch64", app_name) - - if os.path.exists(binary_path): - console.print(f"[green]Binary already exists at: {binary_path}[/green]") - return True - - # Binary doesn't exist, build it - console.print("[yellow]Binary not found, attempting to build...[/yellow]") - - # Detect architecture - tag_suffix = detect_arch() - - # Image name - image = f"{os.path.basename(app_dir)}:latest-{tag_suffix}" - - # Docker image doesn't exist, build it - console.print( - f"[yellow]Docker image {image} not found, building it first...[/yellow]" - ) - - # Build the Docker image directly via Python helper - try: - image = build_docker_image(app_dir, app_name) - except (subprocess.CalledProcessError, FileNotFoundError) as e: - console.print(f"[bold red]Error:[/bold red] Failed to build Docker image: {e}") - return False - - # Now build the app in Docker - console.print("[yellow]Building application in Docker container...[/yellow]") - console.print( - "[dim]This may take a few minutes. You'll see output during the build process.[/dim]" - ) - - # First, try to run vcpkg to install dependencies - vcpkg_cmd = [ - "docker", - "run", - "--rm", - "-v", - f"{os.path.abspath(app_dir)}:/home/workspace", - image, - "/bin/bash", - "-c", - "cd /home/workspace && if [ -f vcpkg.json ]; then echo 'Installing dependencies from vcpkg.json...'; ${VCPKG_ROOT}/vcpkg install --triplet arm64-linux-dynamic-release; fi", - ] +def deploy_cmd(args): + """Handle the deploy command""" + # If user supplied a pre-built package, skip local build/pkg steps. + if args.package: + deb_package = os.path.abspath(args.package) + if not os.path.exists(deb_package): + console.print( + f"[bold red]Error:[/bold red] Provided package not found: {deb_package}" + ) + return - try: - console.print("[blue]Installing dependencies...[/blue]") - subprocess.run(vcpkg_cmd, check=True, cwd=app_dir) - except subprocess.CalledProcessError: console.print( - "[yellow]Warning: Failed to install dependencies. The build might still succeed.[/yellow]" + f"[bold]Deploying pre-built package:[/bold] [yellow]{os.path.basename(deb_package)}[/yellow]" ) - # Now run the actual build command with a proper CMake preset - build_cmd = [ - "docker", - "run", - "--rm", - "-v", - f"{os.path.abspath(app_dir)}:/home/workspace", - image, - "/bin/bash", - "-c", - """cd /home/workspace && - if [ -f CMakePresets.json ]; then - # Use the existing presets if available - echo 'Using existing CMake presets...' && - cmake --preset=dynamic-aarch64 -DVCPKG_TARGET_TRIPLET="arm64-linux-dynamic-release" && - cmake --build --preset=cross-release -j$(nproc); - else - # Fall back to manual configuration - echo 'No CMake presets found, using manual configuration...' && - export VCPKG_DEFAULT_TRIPLET=arm64-linux-dynamic-release && - cmake -B build-aarch64 -S . \ - -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake \ - -DVCPKG_TARGET_TRIPLET=arm64-linux-dynamic-release \ - -DVCPKG_INSTALLED_DIR=${VCPKG_ROOT}/vcpkg_installed \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_FOR_ARM64=ON && - cmake --build build-aarch64 -j$(nproc); - fi""", - ] + else: + # Ensure Docker is available and running only when we need to build + if not builder.ensure_docker(): + return - try: - # Run without capturing output so the user can see progress - console.print("[blue]Running build command...[/blue]") - subprocess.run(build_cmd, check=True, cwd=app_dir) + # Get absolute path of app directory + app_dir = os.path.abspath(args.app_dir) - # Check if build succeeded - if os.path.exists(binary_path): - console.print(f"[green]Successfully built binary at: {binary_path}[/green]") - return True + # Validate manifest.json + manifest_path = os.path.join(app_dir, "manifest.json") + manifest = builder.validate_manifest(manifest_path) + if not manifest: + return - # If we get here, the build might have succeeded but we can't find the binary + # Get app name from manifest + app_name = manifest["name"] console.print( - f"[bold yellow]Warning: Build completed but binary not found at expected location: {binary_path}[/bold yellow]" + f"[bold]Deploying application:[/bold] [yellow]{app_name}[/yellow]" ) - # Try to find it manually - binary_found = subprocess.run( - ["find", app_dir, "-type", "f", "-name", app_name, "-not", "-path", "*/.*"], - capture_output=True, - text=True, - check=False, - ).stdout.strip() - - if binary_found: - binary_found_path = binary_found.split("\n")[0] # Take the first match if multiple - console.print(f"[green]Found binary at: {binary_found_path}[/green]") - - # Try to copy it to the standard location - build_dir = os.path.dirname(binary_path) - os.makedirs(build_dir, exist_ok=True) - shutil.copy(binary_found_path, binary_path) + + # Build & package locally + if not builder.build_app(app_dir, app_name): console.print( - f"[green]Copied binary to: {binary_path}[/green]" + "[bold red]Error:[/bold red] Failed to build the application." ) - return True - - return False - except subprocess.CalledProcessError: - console.print( - "[bold red]Error:[/bold red] Failed to build application. Check the CMake output above for details." - ) - return False - - -def deploy_cmd(args): - """Handle the deploy command""" - # Ensure Docker is available and running - if not ensure_docker(): - return + return - # Get absolute path of app directory - app_dir = os.path.abspath(args.app_dir) + if not builder.package_app(app_dir, app_name): + return - # Validate manifest.json - manifest_path = os.path.join(app_dir, "manifest.json") - manifest = validate_manifest(manifest_path) - if not manifest: - return - - # Get app name from manifest - app_name = manifest["name"] - console.print(f"[bold]Deploying application:[/bold] [yellow]{app_name}[/yellow]") - - # First, build the app - if not build_app(app_dir, app_name): - console.print("[bold red]Error:[/bold red] Failed to build the application.") - return - - # Package the app - if not package_app(app_dir, app_name): - return - - # Find the generated .deb package - deb_package = find_deb_package(app_dir) - if not deb_package: - return + deb_package = builder.find_deb_package(app_dir) + if not deb_package: + return # Deploy the package to the device uri = args.uri @@ -756,87 +383,11 @@ def add_commands(subparsers): deploy_parser.add_argument( "app_dir", nargs="?", default=".", help="Path to the application directory" ) - deploy_parser.set_defaults(func=deploy_cmd) - - -def detect_arch() -> str: - """Return an architecture tag suffix (``arm64`` or ``amd64``).""" - arch = subprocess.check_output(["uname", "-m"]).decode("utf-8").strip() - return "arm64" if arch in ("arm64", "aarch64") else "amd64" - - -def ensure_docker() -> bool: - """Return True if the *docker* CLI is available and the daemon responds. - - Prints a clear, user-friendly message and returns ``False`` otherwise so the - caller can abort early. - """ - if shutil.which("docker") is None: - console.print( - "[bold red]Error:[/bold red] Docker CLI not found. Please install Docker before running this command." - ) - return False - - try: - subprocess.run( - ["docker", "info"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=True, - ) - return True - except subprocess.CalledProcessError: - console.print( - "[bold red]Error:[/bold red] Docker daemon does not appear to be running. Please start Docker and try again." - ) - return False - - -def build_docker_image(app_dir: str, app_name: str | None = None) -> str: - """Build (or rebuild) the SDK Docker image used for cross-compiling *app_name*. - - Returns the fully-qualified image tag (``:latest-``) or raises - ``subprocess.CalledProcessError`` if the build fails. - """ - if app_name is None: - app_name = os.path.basename(app_dir) - - arch_suffix = detect_arch() # "arm64" or "amd64" - - # Pick an arch-specific Dockerfile if it exists, otherwise fall back to the - # generic one. - dockerfile_rel = ( - f"ops/docker/Dockerfile.{arch_suffix}" - if arch_suffix == "arm64" - else "ops/docker/Dockerfile" - ) - dockerfile_path = os.path.join(app_dir, dockerfile_rel) - if not os.path.exists(dockerfile_path): - # Last chance: fall back to the generic Dockerfile regardless of arch. - dockerfile_path = os.path.join(app_dir, "ops/docker/Dockerfile") - - if not os.path.exists(dockerfile_path): - raise FileNotFoundError( - f"Expected Dockerfile not found at {dockerfile_path}. " - "Ensure your application provides the required build Dockerfile(s)." - ) - - image_tag = f"{app_name}:latest-{arch_suffix}" - - console.print(f"[yellow]Building Docker image [bold]{image_tag}[/bold]...[/yellow]") - subprocess.run( - [ - "docker", - "build", - "-t", - image_tag, - "-f", - dockerfile_path, - ".", - ], - check=True, - cwd=app_dir, + deploy_parser.add_argument( + "--package", + "-p", + help="Path to a pre-built .deb to deploy (skips local build and package steps)", + type=str, + default=None, ) - - console.print(f"[green]Successfully built Docker image {image_tag}[/green]") - return image_tag + deploy_parser.set_defaults(func=deploy_cmd) From ad1e7599e4cf1aa77676eb0a89c4a5d2b45f7f45 Mon Sep 17 00:00:00 2001 From: Emma Zhou Date: Sat, 10 May 2025 16:31:01 -0700 Subject: [PATCH 2/3] build by default, only skip if --skip-build is passed --- synapse/cli/build.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/synapse/cli/build.py b/synapse/cli/build.py index c37ed23..f626305 100644 --- a/synapse/cli/build.py +++ b/synapse/cli/build.py @@ -118,19 +118,18 @@ def build_docker_image(app_dir: str, app_name: str | None = None) -> str: return image_tag -def build_app(app_dir: str, app_name: str) -> bool: - """Cross-compile *app_name* inside its SDK container. - - Returns ``True`` on success, ``False`` otherwise. - """ +def build_app(app_dir: str, app_name: str, force_rebuild: bool = False) -> bool: + """Cross-compile *app_name* inside its SDK container.""" console.print(f"[yellow]Building application: {app_name}...[/yellow]") binary_path = os.path.join(app_dir, "build-aarch64", app_name) - # Skip if binary already exists - if os.path.exists(binary_path): - console.print(f"[green]Binary already exists at: {binary_path}[/green]") + # Skip if binary already exists unless a rebuild was requested + if (not force_rebuild) and os.path.exists(binary_path): + console.print( + f"[green]Binary already exists at: {binary_path} (skipping rebuild) [/green]" + ) return True console.print("[yellow]Binary not found, attempting to build...[/yellow]") @@ -138,10 +137,6 @@ def build_app(app_dir: str, app_name: str) -> bool: arch_suffix = detect_arch() image_tag = f"{os.path.basename(app_dir)}:latest-{arch_suffix}" - console.print( - f"[yellow]Docker image {image_tag} not found, building it first...[/yellow]" - ) - # Build (or rebuild) the Docker image – this function is idempotent. try: image_tag = build_docker_image(app_dir, app_name) @@ -489,7 +484,7 @@ def build_cmd(args) -> None: # 1. Build phase (unless explicitly skipped) if not args.skip_build: - if not build_app(app_dir, app_name): + if not build_app(app_dir, app_name, force_rebuild=True): console.print( "[bold red]Error:[/bold red] Failed to build the application." ) From 7b53ce9f8ca972385d15661f212c5e38ad07757c Mon Sep 17 00:00:00 2001 From: Max Hodak Date: Sat, 10 May 2025 19:39:36 -0700 Subject: [PATCH 3/3] some very minor formatting fixes --- synapse/cli/rpc.py | 14 ++++++++------ synapse/server/nodes/base.py | 2 +- synapse/server/rpc.py | 11 ++++++----- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/synapse/cli/rpc.py b/synapse/cli/rpc.py index 2a41dc6..a5223d7 100644 --- a/synapse/cli/rpc.py +++ b/synapse/cli/rpc.py @@ -208,6 +208,8 @@ def start(args): device = syn.Device(args.uri, args.verbose) + device_name = device.get_name() + # If we have a configuration, apply it first. if config_obj is not None: with console.status("Configuring device...", spinner="bouncingBall"): @@ -217,10 +219,10 @@ def start(args): return if cfg_ret.code != StatusCode.kOk: console.print( - f"[bold red]Error configuring device[/bold red]\n{cfg_ret.message}" + f"[bold red]Error configuring device[/bold red]\nResponse from {device_name}:\n{cfg_ret.message}" ) return - console.print("[green]Device Configured") + console.print("[green]Device configured") with console.status("Starting device...", spinner="bouncingBall"): start_ret = device.start_with_status() @@ -229,11 +231,11 @@ def start(args): return if start_ret.code != StatusCode.kOk: console.print( - f"[bold red]Error starting device[/bold red]\n{start_ret.message}" + f"[bold red]Error starting device[/bold red]\nResponse from {device_name}:\n{start_ret.message}" ) return - console.print("[green]Device Started") + console.print("[green]Device started") def stop(args): @@ -257,7 +259,7 @@ def stop(args): if stop_ret.code != StatusCode.kOk: console.print(f"[bold red]Error stopping\n{stop_ret.message}") return - console.print("[green]Device Stopped") + console.print("[green]Device stopped") def configure(args): @@ -279,7 +281,7 @@ def configure(args): if config_ret.code != StatusCode.kOk: console.print(f"[bold red]Error configuring\n{config_ret.message}") return - console.print("[green]Device Configured") + console.print("[green]Device configured") def get_logs(args): diff --git a/synapse/server/nodes/base.py b/synapse/server/nodes/base.py index 2031103..413c493 100644 --- a/synapse/server/nodes/base.py +++ b/synapse/server/nodes/base.py @@ -51,7 +51,7 @@ def stop(self): for task in self.tasks: task.cancel() self.tasks = [] - self.logger.info("stopped") + self.logger.info("Stopped") return Status() async def on_data_received(self, data: SynapseData): diff --git a/synapse/server/rpc.py b/synapse/server/rpc.py index c7a7b6d..ff72ec2 100644 --- a/synapse/server/rpc.py +++ b/synapse/server/rpc.py @@ -24,6 +24,7 @@ LOG_FILEPATH = str(Path.home() / ".science" / "synapse" / "logs" / "server.log") + def _read_api_version(): try: with open(str(Path(__file__).parent.parent / "api" / "version.txt")) as f: @@ -362,21 +363,21 @@ def _reconfigure(self, configuration): return True async def _start_streaming(self): - self.logger.info("starting streaming...") + self.logger.info("Starting streaming...") for node in self.nodes: node.start() self.state = DeviceState.kRunning - self.logger.info("streaming started.") + self.logger.info("Streaming started.") return True async def _stop_streaming(self): if self.state != DeviceState.kRunning: return False - self.logger.info("stopping streaming...") + self.logger.info("Stopping streaming...") for node in self.nodes: node.stop() self.state = DeviceState.kStopped - self.logger.info("streaming stopped.") + self.logger.info("Streaming stopped.") return True def _sockets_status_info(self): @@ -386,7 +387,7 @@ def _synapse_api_version(self): if self.synapse_api_version is None: return 0 try: - major, minor, patch = map(int, self.synapse_api_version.split('.')) + major, minor, patch = map(int, self.synapse_api_version.split(".")) return (major & 0x3FF) << 20 | (minor & 0x3FF) << 10 | (patch & 0x3FF) except Exception: return 0