From ee9c89b4966777511c84e08a287e8b05c935e68f Mon Sep 17 00:00:00 2001 From: wzg2311 Date: Thu, 18 Jun 2026 12:58:18 +0800 Subject: [PATCH 01/85] fix: harden workflow execution against code injection (RCE) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three security hardening measures: 1. Sandbox exec() builtins in PythonExecRuntime — block eval/exec/compile/__import__ and restrict imports to a safe allowlist, preventing arbitrary code execution even when workflow node code unsafely processes inputs. 2. Switch Jinja2 Template to SandboxedEnvironment in LLM and HTTP request nodes, preventing SSTI attacks through template rendering. 3. Add security comment to PUBLIC_PATH_REGEXES warning against adding workflow webhook paths to the auth bypass whitelist (refs #454). Closes #454 Co-Authored-By: Claude Sonnet 4.6 --- flocks/server/auth.py | 7 +++++++ flocks/workflow/engine.py | 11 ++++++----- flocks/workflow/repl_runtime.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/flocks/server/auth.py b/flocks/server/auth.py index 5f6422908..0d7444c61 100644 --- a/flocks/server/auth.py +++ b/flocks/server/auth.py @@ -65,6 +65,13 @@ # downstream handler is fully responsible for its own authentication # (signature checks, IP allowlists, replay protection, …). Do NOT add # entries that touch user data without a per-request integrity check. +# +# SECURITY: Do NOT add workflow webhook paths here (e.g. +# /webhook/workflows/...). Workflow triggers must enforce their own +# authentication via _authorize_webhook_trigger() and must NEVER allow +# auth.type="none" — otherwise an anonymous attacker can trigger +# arbitrary workflow execution, leading to RCE when a Python node +# unsafely processes inputs. See: https://github.com/AgentFlocks/flocks/issues/454 PUBLIC_PATH_REGEXES = ( re.compile(r"^/(?:api/)?channel/[^/]+/webhook/?$"), ) diff --git a/flocks/workflow/engine.py b/flocks/workflow/engine.py index 6762fd672..fff1f4fd3 100644 --- a/flocks/workflow/engine.py +++ b/flocks/workflow/engine.py @@ -676,8 +676,8 @@ def _execute_llm_node(self, node: Node, inputs: Dict[str, Any]) -> Tuple[Dict[st """Execute an LLM node: render Jinja2 prompt template, call LLM.""" assert node.prompt, "llm node requires prompt" try: - from jinja2 import Template, TemplateError - rendered = Template(node.prompt).render(**inputs) + from jinja2.sandbox import SandboxedEnvironment + rendered = SandboxedEnvironment().from_string(node.prompt).render(**inputs) except Exception as e: raise NodeExecutionError( node_id=node.id, @@ -700,13 +700,14 @@ def _execute_http_request_node(self, node: Node, inputs: Dict[str, Any]) -> Tupl assert node.url, "http_request node requires url" assert node.method, "http_request node requires method" try: - from jinja2 import Template - url = Template(node.url).render(**inputs) + from jinja2.sandbox import SandboxedEnvironment + _sandbox = SandboxedEnvironment() + url = _sandbox.from_string(node.url).render(**inputs) method = node.method.upper() headers = node.headers or {} body = node.body if isinstance(body, str): - body = Template(body).render(**inputs) + body = _sandbox.from_string(body).render(**inputs) except Exception as e: raise NodeExecutionError( node_id=node.id, diff --git a/flocks/workflow/repl_runtime.py b/flocks/workflow/repl_runtime.py index dca38a334..e242604c6 100644 --- a/flocks/workflow/repl_runtime.py +++ b/flocks/workflow/repl_runtime.py @@ -21,6 +21,38 @@ from .tools import ToolFacade, get_tool_registry +_BLOCKED_BUILTINS = frozenset({ + 'eval', 'exec', 'compile', '__import__', + 'breakpoint', 'exit', 'quit', +}) + +_IMPORT_ALLOWLIST = frozenset({ + 'json', 're', 'math', 'datetime', 'time', 'hashlib', 'base64', + 'urllib', 'collections', 'itertools', 'functools', 'copy', + 'string', 'textwrap', 'uuid', 'csv', 'io', 'typing', + 'dataclasses', 'enum', 'pathlib', 'decimal', 'fractions', + 'statistics', 'operator', 'contextlib', 'abc', +}) + + +def _make_safe_builtins() -> dict: + """Create a restricted builtins dict for workflow exec() contexts.""" + import builtins as _b + safe = {k: v for k, v in _b.__dict__.items() if k not in _BLOCKED_BUILTINS} + + def _safe_import(name, *args, **kwargs): + top = name.split('.')[0] + if top not in _IMPORT_ALLOWLIST: + raise ImportError( + f"Import of '{name}' is not allowed in workflow sandbox. " + f"Allowed top-level modules: {sorted(_IMPORT_ALLOWLIST)}" + ) + return _b.__import__(name, *args, **kwargs) + + safe['__import__'] = _safe_import + return safe + + class Runtime: def execute(self, code: str, inputs: Dict[str, Any]) -> Tuple[Dict[str, Any], str]: raise NotImplementedError @@ -61,6 +93,7 @@ def execute(self, code: str, inputs: Dict[str, Any]) -> Tuple[Dict[str, Any], st g = self.globals g["inputs"] = inputs g["outputs"] = {} + g["__builtins__"] = _make_safe_builtins() def _cancel_requested() -> bool: try: From bde82965ac9ddd55d185b11e89fd1d4de538dbaf Mon Sep 17 00:00:00 2001 From: xiami762 <> Date: Wed, 1 Jul 2026 17:09:39 +0800 Subject: [PATCH 02/85] fix: add backend service watchdog recovery --- flocks/cli/main.py | 26 +++ flocks/cli/service_manager.py | 287 ++++++++++++++++++++++++++++- tests/cli/test_service_commands.py | 2 +- tests/cli/test_service_manager.py | 221 ++++++++++++++++++++++ 4 files changed, 533 insertions(+), 3 deletions(-) diff --git a/flocks/cli/main.py b/flocks/cli/main.py index bdfa8d8d5..7a8973a9e 100644 --- a/flocks/cli/main.py +++ b/flocks/cli/main.py @@ -33,9 +33,11 @@ from flocks.cli.service_manager import ( ServiceConfig, ServiceError, + WATCHDOG_CHECK_INTERVAL_SECONDS, read_runtime_record, resolve_flocks_cli_command, restart_all, + run_service_watchdog, runtime_paths, show_logs, show_status, @@ -401,6 +403,30 @@ def serve( ) +@app.command(name="service-watchdog", hidden=True) +def service_watchdog( + server_host: str = typer.Option("127.0.0.1", "--server-host", help="Backend server host"), + server_port: int = typer.Option(8000, "--server-port", help="Backend server port"), + webui_host: str = typer.Option("127.0.0.1", "--webui-host", help="WebUI host"), + webui_port: int = typer.Option(5173, "--webui-port", help="WebUI port"), + interval: float = typer.Option(WATCHDOG_CHECK_INTERVAL_SECONDS, "--interval", help="Health check interval"), +): + """ + Monitor daemon services and recover unhealthy backend listeners. + """ + run_service_watchdog( + ServiceConfig( + backend_host=server_host, + backend_port=server_port, + frontend_host=webui_host, + frontend_port=webui_port, + no_browser=True, + skip_frontend_build=True, + ), + interval=interval, + ) + + @app.command() def tui( directory: Optional[Path] = typer.Option(None, "--directory", "-d", help="Project directory"), diff --git a/flocks/cli/service_manager.py b/flocks/cli/service_manager.py index df534ffe7..0ef4a25b6 100644 --- a/flocks/cli/service_manager.py +++ b/flocks/cli/service_manager.py @@ -48,6 +48,10 @@ "src\\win\\async.c", "src/win/async.c", ) +WATCHDOG_CHECK_INTERVAL_SECONDS = 5.0 +WATCHDOG_HEALTH_FAILURE_THRESHOLD = 2 +WATCHDOG_PID_FILENAME = "watchdog.pid" +WATCHDOG_LOG_FILENAME = "watchdog.log" class ServiceError(RuntimeError): @@ -104,6 +108,15 @@ def has_artifacts(self) -> bool: return self.payload_present or self.pid_file_present +@dataclass(frozen=True) +class WatchdogProbeResult: + restart_needed: bool + health_failure: bool + reason: str + host: str + port: int + + def repo_root() -> Path: """Return the installed repository root.""" override = os.getenv("FLOCKS_REPO_ROOT") @@ -144,6 +157,16 @@ def ensure_runtime_dirs(paths: RuntimePaths | None = None) -> RuntimePaths: return current +def watchdog_pid_path(paths: RuntimePaths) -> Path: + """Return the watchdog runtime record path.""" + return paths.run_dir / WATCHDOG_PID_FILENAME + + +def watchdog_log_path(paths: RuntimePaths) -> Path: + """Return the watchdog log path.""" + return paths.log_dir / WATCHDOG_LOG_FILENAME + + def ensure_install_layout(root: Path | None = None) -> Path: """Validate that the installed repo still contains backend and WebUI code.""" current = root or repo_root() @@ -412,8 +435,8 @@ def write_runtime_record(pid_file: Path, record: RuntimeRecord) -> None: def process_runtime_record( process: subprocess.Popen, *, - host: str, - port: int, + host: str | None, + port: int | None, command: Sequence[str], ) -> RuntimeRecord: """Build runtime metadata for a freshly started service process.""" @@ -877,6 +900,17 @@ def _is_running_status_response(response: httpx.Response) -> bool: return isinstance(payload, dict) and payload.get("status") == "running" +def _is_healthy_status_response(response: httpx.Response) -> bool: + """Return True when the backend health endpoint reports healthy.""" + if response.status_code != 200: + return False + try: + payload = response.json() + except ValueError: + return False + return isinstance(payload, dict) and payload.get("status") == "healthy" + + def wait_for_http( urls: Sequence[str], name: str, @@ -902,6 +936,156 @@ def wait_for_http( raise ServiceError(f"{name} 启动超时，请检查日志。") +class _StdoutConsole: + """Console adapter for daemon logs redirected to a file.""" + + def print(self, *args, **_kwargs) -> None: + sys.stdout.write(" ".join(str(arg) for arg in args) + "\n") + sys.stdout.flush() + + +def _watchdog_log(event: str, details: dict[str, object] | None = None) -> None: + timestamp = datetime.datetime.now().isoformat(timespec="seconds") + suffix = "" + if details: + suffix = " " + json.dumps(details, ensure_ascii=True, sort_keys=True) + sys.stdout.write(f"[{timestamp}] watchdog.{event}{suffix}\n") + sys.stdout.flush() + + +def _backend_health_url(host: str, port: int) -> str: + return f"http://{_format_host_for_url(access_host(host))}:{port}/api/health" + + +def _watchdog_backend_endpoint(config: ServiceConfig, paths: RuntimePaths) -> tuple[RuntimeRecord | None, str, int]: + record = read_runtime_record(paths.backend_pid) + host = record.host if record is not None and record.host else config.backend_host + port = record.port if record is not None and record.port is not None else config.backend_port + return record, host, port + + +def _watchdog_backend_config(config: ServiceConfig, paths: RuntimePaths) -> ServiceConfig: + record, host, port = _watchdog_backend_endpoint(config, paths) + return ServiceConfig( + backend_host=host, + backend_port=port, + frontend_host=config.frontend_host, + frontend_port=config.frontend_port, + no_browser=True, + skip_frontend_build=True, + ) + + +def _watchdog_probe_backend( + config: ServiceConfig, + paths: RuntimePaths, + client: httpx.Client, +) -> WatchdogProbeResult: + record, host, port = _watchdog_backend_endpoint(config, paths) + if record is None: + return WatchdogProbeResult(False, False, "backend runtime record missing", host, port) + + backend_running = runtime_record_is_running(record) + listeners = port_owner_pids(port) + port_in_use = port_is_in_use(port, listeners) + if not backend_running: + if port_in_use: + return WatchdogProbeResult( + False, + False, + f"backend runtime record is not running but port {port} is occupied", + host, + port, + ) + return WatchdogProbeResult(True, False, "backend runtime record is not running", host, port) + if not port_in_use: + return WatchdogProbeResult(True, False, f"backend process alive but port {port} is not listening", host, port) + + runtime_pids = set(_runtime_record_pids(record)) + listener_pids = set(listeners) + if listener_pids and runtime_pids and listener_pids.isdisjoint(runtime_pids): + reason = ( + f"backend process alive but port {port} is owned by unexpected pid(s): " + f"{_join_pids(sorted(listener_pids))}" + ) + return WatchdogProbeResult(False, False, reason, host, port) + + url = _backend_health_url(host, port) + try: + response = client.get(url) + except Exception as exc: + return WatchdogProbeResult(True, True, f"backend health check failed: {exc}", host, port) + if not _is_healthy_status_response(response): + return WatchdogProbeResult( + True, + True, + f"backend health check unhealthy: status={response.status_code}", + host, + port, + ) + return WatchdogProbeResult(False, False, "backend healthy", host, port) + + +def _recover_unhealthy_backend(config: ServiceConfig, paths: RuntimePaths, reason: str) -> None: + try: + with service_lock(paths): + effective_config = _watchdog_backend_config(config, paths) + with httpx.Client(timeout=2.0, trust_env=False) as client: + probe = _watchdog_probe_backend(effective_config, paths, client) + if not probe.restart_needed: + _watchdog_log("backend_recovery_skipped", {"reason": probe.reason}) + return + + console = _StdoutConsole() + _watchdog_log( + "backend_recovery_start", + { + "reason": reason, + "host": probe.host, + "port": probe.port, + }, + ) + stop_one(probe.port, paths.backend_pid, "后端", console) + start_backend(effective_config, console) + _watchdog_log("backend_recovery_done", {"host": probe.host, "port": probe.port}) + except ServiceError as exc: + _watchdog_log("backend_recovery_failed", {"reason": reason, "error": str(exc)}) + except Exception as exc: + _watchdog_log("backend_recovery_crashed", {"reason": reason, "error": repr(exc)}) + + +def _watchdog_tick( + config: ServiceConfig, + paths: RuntimePaths, + health_failure_count: int, + *, + failure_threshold: int = WATCHDOG_HEALTH_FAILURE_THRESHOLD, +) -> int: + with httpx.Client(timeout=2.0, trust_env=False) as client: + probe = _watchdog_probe_backend(config, paths, client) + + if not probe.restart_needed: + return 0 + + if probe.health_failure: + health_failure_count += 1 + _watchdog_log( + "backend_health_failed", + { + "count": health_failure_count, + "threshold": failure_threshold, + "reason": probe.reason, + "host": probe.host, + "port": probe.port, + }, + ) + if health_failure_count < failure_threshold: + return health_failure_count + + _recover_unhealthy_backend(config, paths, probe.reason) + return 0 + + def start_backend(config: ServiceConfig, console) -> None: """Start the backend API service if needed.""" root = ensure_install_layout() @@ -1099,6 +1283,95 @@ def start_frontend(config: ServiceConfig, console) -> None: console.print(f"[flocks] WebUI 已启动，日志: {paths.frontend_log}") +def start_watchdog(config: ServiceConfig, console) -> None: + """Start the service watchdog daemon if needed.""" + root = ensure_install_layout() + paths = ensure_runtime_dirs() + pid_file = watchdog_pid_path(paths) + log_path = watchdog_log_path(paths) + cleanup_stale_pid_file(pid_file) + + runtime_record = read_runtime_record(pid_file) + if runtime_record is not None and runtime_record_is_running(runtime_record): + console.print(f"[flocks] Watchdog 已在运行，PID={runtime_record.pid}") + return + if runtime_record is not None: + pid_file.unlink(missing_ok=True) + + command = resolve_flocks_cli_command(root) + [ + "service-watchdog", + "--server-host", + config.backend_host, + "--server-port", + str(config.backend_port), + "--webui-host", + config.frontend_host, + "--webui-port", + str(config.frontend_port), + "--interval", + str(WATCHDOG_CHECK_INTERVAL_SECONDS), + ] + env = os.environ.copy() + env["PYTHONUNBUFFERED"] = "1" + + console.print("[flocks] 启动服务 Watchdog...") + process = _spawn_process(command, cwd=root, log_path=log_path, env=env) + write_runtime_record( + pid_file, + process_runtime_record( + process, + host=None, + port=None, + command=command, + ), + ) + _log_startup_config(log_path, "watchdog", config.backend_host, config.backend_port, read_runtime_record(pid_file)) + console.print(f"[flocks] Watchdog 已启动，日志: {log_path}") + + +def stop_watchdog(paths: RuntimePaths, console) -> None: + """Stop the service watchdog without touching backend/frontend ports.""" + pid_file = watchdog_pid_path(paths) + cleanup_stale_pid_file(pid_file) + if read_runtime_record(pid_file) is None: + return + stop_one(0, pid_file, "Watchdog", console) + + +def run_service_watchdog( + config: ServiceConfig, + *, + interval: float = WATCHDOG_CHECK_INTERVAL_SECONDS, + failure_threshold: int = WATCHDOG_HEALTH_FAILURE_THRESHOLD, +) -> None: + """Run the backend health watchdog loop.""" + paths = ensure_runtime_dirs() + _watchdog_log( + "started", + { + "backend_host": config.backend_host, + "backend_port": config.backend_port, + "interval": interval, + "failure_threshold": failure_threshold, + }, + ) + health_failure_count = 0 + while True: + try: + health_failure_count = _watchdog_tick( + config, + paths, + health_failure_count, + failure_threshold=failure_threshold, + ) + except KeyboardInterrupt: + _watchdog_log("stopped") + return + except Exception as exc: + _watchdog_log("tick_failed", {"error": repr(exc)}) + time.sleep(interval) + + def _tracked_processes_stopped( port: int, record: RuntimeRecord | None, @@ -1323,6 +1596,7 @@ def _stop_all_locked( fe_port, be_port = _resolve_stop_ports(paths, config) try: _resolve_upgrade_runtime(console, frontend_port=fe_port, attempt_recover=False) + stop_watchdog(paths, console) stop_one(fe_port, paths.frontend_pid, "WebUI", console) stop_one(be_port, paths.backend_pid, "后端", console) finally: @@ -1341,6 +1615,7 @@ def _start_all_without_stop(config: ServiceConfig, console) -> None: ensure_runtime_dirs() start_backend(config, console) start_frontend(config, console) + start_watchdog(config, console) show_start_summary(config, console) if not config.no_browser: open_default_browser(config.frontend_url, console) @@ -1367,9 +1642,11 @@ def build_status_lines(paths: RuntimePaths | None = None) -> list[str]: current = paths or runtime_paths() cleanup_stale_pid_file(current.backend_pid) cleanup_stale_pid_file(current.frontend_pid) + cleanup_stale_pid_file(watchdog_pid_path(current)) backend_record = read_runtime_record(current.backend_pid) frontend_record = read_runtime_record(current.frontend_pid) + watchdog_record = read_runtime_record(watchdog_pid_path(current)) backend_port = _recorded_port(current.backend_pid, ServiceConfig.backend_port) frontend_port = _recorded_port(current.frontend_pid, ServiceConfig.frontend_port) backend_host = _loopback_host(_recorded_host(current.backend_pid, ServiceConfig.backend_host)) @@ -1417,11 +1694,17 @@ def build_status_lines(paths: RuntimePaths | None = None) -> list[str]: else: lines.append("[flocks] WebUI 未运行") + if runtime_record_is_running(watchdog_record): + lines.append(f"[flocks] Watchdog 运行中: PID={watchdog_record.pid}") + else: + lines.append("[flocks] Watchdog 未运行") + if upgrade_info.payload_present: lines.append("[flocks] 检测到未完成的升级恢复状态") lines.append(f"[flocks] 后端日志: {current.backend_log}") lines.append(f"[flocks] WebUI 日志: {current.frontend_log}") + lines.append(f"[flocks] Watchdog 日志: {watchdog_log_path(current)}") return lines diff --git a/tests/cli/test_service_commands.py b/tests/cli/test_service_commands.py index 99a267c17..49d92a200 100644 --- a/tests/cli/test_service_commands.py +++ b/tests/cli/test_service_commands.py @@ -31,7 +31,7 @@ def test_cli_help_lists_service_commands(monkeypatch, tmp_path) -> None: assert result.exit_code == 0 for command in ("start", "stop", "restart", "status", "logs", "session", "mcp", "task", "skills"): assert _help_contains_command(result.stdout, command) - for command in ("agent", "acp", "debug", "run", "serve", "auth", "models"): + for command in ("agent", "acp", "debug", "run", "serve", "service-watchdog", "auth", "models"): assert not _help_contains_command(result.stdout, command) diff --git a/tests/cli/test_service_manager.py b/tests/cli/test_service_manager.py index ba42f890d..286f9541a 100644 --- a/tests/cli/test_service_manager.py +++ b/tests/cli/test_service_manager.py @@ -19,6 +19,18 @@ def print(self, *args, **kwargs) -> None: self.messages.append(" ".join(str(arg) for arg in args)) +def _make_runtime_paths(tmp_path: Path) -> service_manager.RuntimePaths: + return service_manager.RuntimePaths( + root=tmp_path, + run_dir=tmp_path / "run", + log_dir=tmp_path / "logs", + backend_pid=tmp_path / "run" / "backend.pid", + frontend_pid=tmp_path / "run" / "webui.pid", + backend_log=tmp_path / "logs" / "backend.log", + frontend_log=tmp_path / "logs" / "webui.log", + ) + + def test_runtime_paths_follow_flocks_root_env(monkeypatch, tmp_path: Path) -> None: monkeypatch.setenv("FLOCKS_ROOT", str(tmp_path)) @@ -797,6 +809,26 @@ def test_restart_all_stops_then_starts_under_lock(monkeypatch) -> None: ] +def test_start_all_without_stop_starts_watchdog_after_frontend(monkeypatch, tmp_path: Path) -> None: + paths = _make_runtime_paths(tmp_path) + calls: list[str] = [] + + monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: paths) + monkeypatch.setattr(service_manager, "start_backend", lambda _config, _console: calls.append("backend")) + monkeypatch.setattr(service_manager, "start_frontend", lambda _config, _console: calls.append("webui")) + monkeypatch.setattr(service_manager, "start_watchdog", lambda _config, _console: calls.append("watchdog")) + monkeypatch.setattr(service_manager, "show_start_summary", lambda _config, _console: calls.append("summary")) + monkeypatch.setattr( + service_manager, + "open_default_browser", + lambda _url, _console: calls.append("browser"), + ) + + service_manager._start_all_without_stop(service_manager.ServiceConfig(no_browser=True), DummyConsole()) + + assert calls == ["backend", "webui", "watchdog", "summary"] + + def test_start_all_stops_on_failure_before_restart(monkeypatch) -> None: paths = service_manager.RuntimePaths( root=Path("/tmp"), @@ -1209,6 +1241,195 @@ def fake_spawn(command, **kwargs): assert record.port == 5174 +def test_start_watchdog_writes_runtime_metadata(monkeypatch, tmp_path: Path) -> None: + paths = _make_runtime_paths(tmp_path) + paths.run_dir.mkdir(parents=True) + paths.log_dir.mkdir(parents=True) + console = DummyConsole() + spawn_calls: list[dict[str, object]] = [] + + monkeypatch.setattr(service_manager, "ensure_install_layout", lambda: tmp_path) + monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: paths) + monkeypatch.setattr(service_manager, "cleanup_stale_pid_file", lambda _path: None) + monkeypatch.setattr(service_manager, "runtime_record_is_running", lambda _record: False) + monkeypatch.setattr( + service_manager, + "resolve_flocks_cli_command", + lambda root=None: ["python", "-m", "flocks.cli.main"], + ) + monkeypatch.setattr(service_manager.os, "getpgid", lambda pid: pid) + monkeypatch.setattr( + service_manager, + "_spawn_process", + lambda *args, **kwargs: spawn_calls.append({"args": args, "kwargs": kwargs}) or SimpleNamespace(pid=2468), + ) + + service_manager.start_watchdog( + service_manager.ServiceConfig(backend_host="0.0.0.0", backend_port=9000), + console, + ) + + record = service_manager.read_runtime_record(service_manager.watchdog_pid_path(paths)) + assert record is not None + assert record.pid == 2468 + assert record.port is None + assert record.command == ( + "python", + "-m", + "flocks.cli.main", + "service-watchdog", + "--server-host", + "0.0.0.0", + "--server-port", + "9000", + "--webui-host", + "127.0.0.1", + "--webui-port", + "5173", + "--interval", + str(service_manager.WATCHDOG_CHECK_INTERVAL_SECONDS), + ) + assert spawn_calls[0]["kwargs"]["log_path"] == service_manager.watchdog_log_path(paths) + + +def test_watchdog_recovers_backend_when_process_alive_but_port_not_listening(monkeypatch, tmp_path: Path) -> None: + paths = _make_runtime_paths(tmp_path) + paths.run_dir.mkdir(parents=True) + service_manager.write_runtime_record( + paths.backend_pid, + service_manager.RuntimeRecord(pid=111, pgid=222, host="0.0.0.0", port=9995), + ) + calls: list[tuple[str, int]] = [] + + monkeypatch.setattr(service_manager, "runtime_record_is_running", lambda _record: True) + monkeypatch.setattr(service_manager, "port_owner_pids", lambda _port: []) + monkeypatch.setattr(service_manager, "port_is_in_use", lambda _port, listeners=None: False) + monkeypatch.setattr(service_manager, "service_lock", lambda _paths: _record_call([], "service_lock")) + monkeypatch.setattr( + service_manager, + "stop_one", + lambda port, _pid_file, _name, _console: calls.append(("stop", port)), + ) + monkeypatch.setattr( + service_manager, + "start_backend", + lambda config, _console: calls.append(("start", config.backend_port)), + ) + + next_count = service_manager._watchdog_tick( + service_manager.ServiceConfig(backend_port=8000), + paths, + 0, + ) + + assert next_count == 0 + assert calls == [("stop", 9995), ("start", 9995)] + + +def test_watchdog_does_not_recover_when_port_owned_by_unexpected_pid(monkeypatch, tmp_path: Path) -> None: + paths = _make_runtime_paths(tmp_path) + paths.run_dir.mkdir(parents=True) + service_manager.write_runtime_record( + paths.backend_pid, + service_manager.RuntimeRecord(pid=111, pgid=222, host="0.0.0.0", port=9995), + ) + calls: list[str] = [] + + monkeypatch.setattr(service_manager, "runtime_record_is_running", lambda _record: True) + monkeypatch.setattr(service_manager, "port_owner_pids", lambda _port: [999]) + monkeypatch.setattr(service_manager, "port_is_in_use", lambda _port, listeners=None: True) + monkeypatch.setattr(service_manager, "_runtime_record_pids", lambda _record: [111]) + monkeypatch.setattr(service_manager, "_recover_unhealthy_backend", lambda *_args: calls.append("recover")) + + next_count = service_manager._watchdog_tick( + service_manager.ServiceConfig(backend_port=9995), + paths, + 0, + ) + + assert next_count == 0 + assert calls == [] + + +def test_watchdog_recovers_backend_when_runtime_record_is_dead(monkeypatch, tmp_path: Path) -> None: + paths = _make_runtime_paths(tmp_path) + paths.run_dir.mkdir(parents=True) + service_manager.write_runtime_record( + paths.backend_pid, + service_manager.RuntimeRecord(pid=111, pgid=222, host="127.0.0.1", port=9995), + ) + calls: list[tuple[str, int]] = [] + + monkeypatch.setattr(service_manager, "runtime_record_is_running", lambda _record: False) + monkeypatch.setattr(service_manager, "port_owner_pids", lambda _port: []) + monkeypatch.setattr(service_manager, "port_is_in_use", lambda _port, listeners=None: False) + monkeypatch.setattr(service_manager, "service_lock", lambda _paths: _record_call([], "service_lock")) + monkeypatch.setattr( + service_manager, + "stop_one", + lambda port, _pid_file, _name, _console: calls.append(("stop", port)), + ) + monkeypatch.setattr( + service_manager, + "start_backend", + lambda config, _console: calls.append(("start", config.backend_port)), + ) + + next_count = service_manager._watchdog_tick( + service_manager.ServiceConfig(backend_port=9995), + paths, + 0, + ) + + assert next_count == 0 + assert calls == [("stop", 9995), ("start", 9995)] + + +def test_watchdog_waits_for_second_health_failure_before_restart(monkeypatch, tmp_path: Path) -> None: + paths = _make_runtime_paths(tmp_path) + paths.run_dir.mkdir(parents=True) + service_manager.write_runtime_record( + paths.backend_pid, + service_manager.RuntimeRecord(pid=111, pgid=222, host="127.0.0.1", port=9995), + ) + calls: list[str] = [] + + class FakeClient: + def __init__(self, *_args, **_kwargs) -> None: + pass + + def __enter__(self): + return self + + def __exit__(self, *_args) -> None: + return None + + def get(self, _url): + return httpx.Response(503, json={"status": "unhealthy"}) + + monkeypatch.setattr(service_manager.httpx, "Client", FakeClient) + monkeypatch.setattr(service_manager, "runtime_record_is_running", lambda _record: True) + monkeypatch.setattr(service_manager, "port_owner_pids", lambda _port: [111]) + monkeypatch.setattr(service_manager, "port_is_in_use", lambda _port, listeners=None: True) + monkeypatch.setattr(service_manager, "_runtime_record_pids", lambda _record: [111]) + monkeypatch.setattr(service_manager, "_recover_unhealthy_backend", lambda *_args: calls.append("recover")) + + first_count = service_manager._watchdog_tick( + service_manager.ServiceConfig(backend_port=9995), + paths, + 0, + ) + second_count = service_manager._watchdog_tick( + service_manager.ServiceConfig(backend_port=9995), + paths, + first_count, + ) + + assert first_count == 1 + assert second_count == 0 + assert calls == ["recover"] + + def test_start_frontend_tolerates_windows_node_assertion_after_build(monkeypatch, tmp_path: Path) -> None: paths = service_manager.RuntimePaths( root=tmp_path, From b16d71442ea6ec3bee6a1628195bda5552a26b54 Mon Sep 17 00:00:00 2001 From: xiami762 <> Date: Wed, 1 Jul 2026 18:30:21 +0800 Subject: [PATCH 03/85] Refactor service supervisor control plane --- flocks/cli/main.py | 46 +- flocks/cli/service_control.py | 116 +++ flocks/cli/service_manager.py | 968 +++++++---------------- flocks/cli/service_supervisor.py | 608 +++++++++++++++ flocks/server/app.py | 2 +- flocks/updater/restart_handoff.py | 58 +- flocks/updater/updater.py | 50 +- tests/cli/test_service_commands.py | 2 +- tests/cli/test_service_manager.py | 972 +++++------------------- tests/server/test_server_port_config.py | 72 +- tests/updater/test_restart_handoff.py | 29 +- tests/updater/test_updater.py | 135 ++-- 12 files changed, 1387 insertions(+), 1671 deletions(-) create mode 100644 flocks/cli/service_control.py create mode 100644 flocks/cli/service_supervisor.py diff --git a/flocks/cli/main.py b/flocks/cli/main.py index 7a8973a9e..58e5c2888 100644 --- a/flocks/cli/main.py +++ b/flocks/cli/main.py @@ -33,17 +33,16 @@ from flocks.cli.service_manager import ( ServiceConfig, ServiceError, - WATCHDOG_CHECK_INTERVAL_SECONDS, - read_runtime_record, resolve_flocks_cli_command, restart_all, - run_service_watchdog, runtime_paths, show_logs, show_status, start_all, stop_all, ) +from flocks.cli.service_control import read_supervisor_status +from flocks.cli.service_supervisor import run_service_daemon from flocks.config.config import Config from flocks.utils.log import Log, LogLevel @@ -213,21 +212,21 @@ def _resolve_port( def _restart_runtime_defaults() -> dict[str, Any]: - """Load host/port defaults from the last recorded service runtime.""" - paths = runtime_paths() - backend = read_runtime_record(paths.backend_pid) - frontend = read_runtime_record(paths.frontend_pid) + """Load host/port defaults from the running supervisor when available.""" defaults: dict[str, Any] = {} - if backend is not None: - if backend.host: - defaults["default_server_host"] = backend.host - if backend.port is not None: - defaults["default_server_port"] = backend.port - if frontend is not None: - if frontend.host: - defaults["default_webui_host"] = frontend.host - if frontend.port is not None: - defaults["default_webui_port"] = frontend.port + try: + payload = read_supervisor_status(paths=runtime_paths(), timeout=1.0) + except Exception: + return defaults + config = payload.get("config") if isinstance(payload.get("config"), dict) else {} + if isinstance(config.get("backend_host"), str): + defaults["default_server_host"] = config["backend_host"] + if isinstance(config.get("backend_port"), int): + defaults["default_server_port"] = config["backend_port"] + if isinstance(config.get("frontend_host"), str): + defaults["default_webui_host"] = config["frontend_host"] + if isinstance(config.get("frontend_port"), int): + defaults["default_webui_port"] = config["frontend_port"] return defaults @@ -403,27 +402,26 @@ def serve( ) -@app.command(name="service-watchdog", hidden=True) -def service_watchdog( +@app.command(name="service-daemon", hidden=True) +def service_daemon( server_host: str = typer.Option("127.0.0.1", "--server-host", help="Backend server host"), server_port: int = typer.Option(8000, "--server-port", help="Backend server port"), webui_host: str = typer.Option("127.0.0.1", "--webui-host", help="WebUI host"), webui_port: int = typer.Option(5173, "--webui-port", help="WebUI port"), - interval: float = typer.Option(WATCHDOG_CHECK_INTERVAL_SECONDS, "--interval", help="Health check interval"), + skip_webui_build: bool = typer.Option(False, "--skip-webui-build", help="Skip WebUI build before preview start"), ): """ - Monitor daemon services and recover unhealthy backend listeners. + Run the Flocks service supervisor daemon. """ - run_service_watchdog( + run_service_daemon( ServiceConfig( backend_host=server_host, backend_port=server_port, frontend_host=webui_host, frontend_port=webui_port, no_browser=True, - skip_frontend_build=True, + skip_frontend_build=skip_webui_build, ), - interval=interval, ) diff --git a/flocks/cli/service_control.py b/flocks/cli/service_control.py new file mode 100644 index 000000000..866afb796 --- /dev/null +++ b/flocks/cli/service_control.py @@ -0,0 +1,116 @@ +"""Local supervisor control API client helpers.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path +from typing import Any + +import httpx + +SUPERVISOR_CONTROL_PORT = 48765 +SUPERVISOR_LOG_FILENAME = "supervisor.log" +SUPERVISOR_SOCKET_FILENAME = "service-daemon.sock" + + +def _default_runtime_paths(): + from flocks.cli.service_manager import runtime_paths + + return runtime_paths() + + +def supervisor_log_path(paths) -> Path: + """Return the supervisor daemon log path.""" + return paths.log_dir / SUPERVISOR_LOG_FILENAME + + +def supervisor_socket_path(paths) -> Path: + """Return the Unix control socket path for the supervisor daemon.""" + return paths.run_dir / SUPERVISOR_SOCKET_FILENAME + + +def supervisor_control_port() -> int: + """Return the local TCP control port used on Windows.""" + raw = os.getenv("FLOCKS_CONTROL_PORT") + if raw and raw.isdigit(): + value = int(raw) + if 0 < value < 65536: + return value + return SUPERVISOR_CONTROL_PORT + + +def supervisor_control_client(paths=None, timeout: float | None = 2.0) -> httpx.Client: + """Create a client for the local daemon control API.""" + if sys.platform == "win32": + return httpx.Client( + base_url=f"http://127.0.0.1:{supervisor_control_port()}", + timeout=timeout, + trust_env=False, + ) + current = paths or _default_runtime_paths() + transport = httpx.HTTPTransport(uds=str(supervisor_socket_path(current))) + return httpx.Client(base_url="http://flocks.local", timeout=timeout, trust_env=False, transport=transport) + + +def control_api_request( + method: str, + path: str, + *, + paths=None, + timeout: float | None = 2.0, + **kwargs, +) -> httpx.Response: + """Send one local control API request.""" + with supervisor_control_client(paths, timeout=timeout) as client: + response = client.request(method, path, **kwargs) + response.raise_for_status() + return response + + +def supervisor_is_running(paths=None) -> bool: + """Return True when the local supervisor control API responds.""" + try: + control_api_request("GET", "/status", paths=paths, timeout=0.75) + return True + except Exception: + return False + + +def read_control_json(path: str, *, paths=None, timeout: float | None = 2.0) -> dict[str, Any]: + response = control_api_request("GET", path, paths=paths, timeout=timeout) + payload = response.json() + if not isinstance(payload, dict): + raise RuntimeError("daemon control API returned an invalid response.") + return payload + + +def read_supervisor_status(paths=None, timeout: float | None = 2.0) -> dict[str, Any]: + """Read the current supervisor status from the local control API.""" + return read_control_json("/status", paths=paths, timeout=timeout) + + +def post_control_json( + path: str, + *, + payload: dict[str, Any] | None = None, + paths=None, + timeout: float | None = 5.0, +) -> dict[str, Any]: + response = control_api_request("POST", path, paths=paths, timeout=timeout, json=payload or {}) + data = response.json() + if not isinstance(data, dict): + raise RuntimeError("daemon control API returned an invalid response.") + return data + + +def service_config_payload(config) -> dict[str, object]: + """Serialize a ServiceConfig-like object for the supervisor control API.""" + return { + "backend_host": config.backend_host, + "backend_port": config.backend_port, + "frontend_host": config.frontend_host, + "frontend_port": config.frontend_port, + "no_browser": config.no_browser, + "skip_frontend_build": config.skip_frontend_build, + } diff --git a/flocks/cli/service_manager.py b/flocks/cli/service_manager.py index 0ef4a25b6..917754b72 100644 --- a/flocks/cli/service_manager.py +++ b/flocks/cli/service_manager.py @@ -21,16 +21,19 @@ from dataclasses import dataclass from pathlib import Path from shutil import which -from typing import Iterable, Sequence +from typing import Any, Iterable, Sequence import httpx -from flocks.browser.admin import stop_all_daemons as stop_all_browser_daemons - -try: - import fcntl -except ImportError: # pragma: no cover - unavailable on Windows - fcntl = None +from flocks.cli.service_control import ( + post_control_json, + read_control_json, + service_config_payload, + supervisor_control_client, + supervisor_is_running, + supervisor_log_path, + supervisor_socket_path, +) MIN_NODE_MAJOR = 22 FOLLOW_POLL_INTERVAL = 0.5 @@ -48,10 +51,8 @@ "src\\win\\async.c", "src/win/async.c", ) -WATCHDOG_CHECK_INTERVAL_SECONDS = 5.0 -WATCHDOG_HEALTH_FAILURE_THRESHOLD = 2 WATCHDOG_PID_FILENAME = "watchdog.pid" -WATCHDOG_LOG_FILENAME = "watchdog.log" +SUPERVISOR_START_TIMEOUT_SECONDS = 180.0 class ServiceError(RuntimeError): @@ -108,15 +109,6 @@ def has_artifacts(self) -> bool: return self.payload_present or self.pid_file_present -@dataclass(frozen=True) -class WatchdogProbeResult: - restart_needed: bool - health_failure: bool - reason: str - host: str - port: int - - def repo_root() -> Path: """Return the installed repository root.""" override = os.getenv("FLOCKS_REPO_ROOT") @@ -162,11 +154,6 @@ def watchdog_pid_path(paths: RuntimePaths) -> Path: return paths.run_dir / WATCHDOG_PID_FILENAME -def watchdog_log_path(paths: RuntimePaths) -> Path: - """Return the watchdog log path.""" - return paths.log_dir / WATCHDOG_LOG_FILENAME - - def ensure_install_layout(root: Path | None = None) -> Path: """Validate that the installed repo still contains backend and WebUI code.""" current = root or repo_root() @@ -416,22 +403,6 @@ def read_runtime_record(pid_file: Path) -> RuntimeRecord | None: return _parse_runtime_record(raw) -def write_runtime_record(pid_file: Path, record: RuntimeRecord) -> None: - """Persist runtime metadata in a backward-compatible JSON format.""" - payload: dict[str, object] = {"pid": record.pid} - if record.pgid is not None: - payload["pgid"] = record.pgid - if record.host is not None: - payload["host"] = record.host - if record.port is not None: - payload["port"] = record.port - if record.command: - payload["command"] = list(record.command) - if record.started_at is not None: - payload["started_at"] = record.started_at - pid_file.write_text(json.dumps(payload, ensure_ascii=True, sort_keys=True), encoding="utf-8") - - def process_runtime_record( process: subprocess.Popen, *, @@ -462,11 +433,6 @@ def read_pid(pid_file: Path) -> int | None: return record.pid if record else None -def write_pid(pid_file: Path, pid: int) -> None: - """Persist a process id.""" - write_runtime_record(pid_file, RuntimeRecord(pid=pid)) - - def _unix_process_stat(pid: int) -> str | None: """Return the Unix process status code for a pid, if available.""" if sys.platform == "win32" or pid <= 0: @@ -789,12 +755,6 @@ def _resolve_upgrade_runtime(console, *, frontend_port: int, attempt_recover: bo return result -def _effective_frontend_port(paths: RuntimePaths, default: int) -> int: - recorded_port = _recorded_port(paths.frontend_pid, default) - upgrade_info = _read_upgrade_runtime_info(recorded_port) - return upgrade_info.frontend_port or recorded_port - - def cleanup_stale_pid_file(pid_file: Path) -> None: """Remove pid files that no longer point to running processes.""" if not pid_file.exists(): @@ -810,20 +770,6 @@ def cleanup_stale_pid_file(pid_file: Path) -> None: pid_file.unlink(missing_ok=True) -def backend_is_running(config: ServiceConfig, paths: RuntimePaths | None = None) -> bool: - """Return True if the tracked backend process is running.""" - current = paths or runtime_paths() - cleanup_stale_pid_file(current.backend_pid) - return runtime_record_is_running(read_runtime_record(current.backend_pid)) or port_is_in_use(config.backend_port) - - -def frontend_is_running(config: ServiceConfig, paths: RuntimePaths | None = None) -> bool: - """Return True if the tracked frontend process is running.""" - current = paths or runtime_paths() - cleanup_stale_pid_file(current.frontend_pid) - return runtime_record_is_running(read_runtime_record(current.frontend_pid)) or port_is_in_use(config.frontend_port) - - def _port_owner_lookup_available() -> bool: """Return True when the current platform can resolve listener pids.""" return sys.platform == "win32" or bool(which("lsof") or which("fuser")) @@ -944,164 +890,80 @@ def print(self, *args, **_kwargs) -> None: sys.stdout.flush() -def _watchdog_log(event: str, details: dict[str, object] | None = None) -> None: - timestamp = datetime.datetime.now().isoformat(timespec="seconds") - suffix = "" - if details: - suffix = " " + json.dumps(details, ensure_ascii=True, sort_keys=True) - sys.stdout.write(f"[{timestamp}] watchdog.{event}{suffix}\n") - sys.stdout.flush() - - def _backend_health_url(host: str, port: int) -> str: return f"http://{_format_host_for_url(access_host(host))}:{port}/api/health" -def _watchdog_backend_endpoint(config: ServiceConfig, paths: RuntimePaths) -> tuple[RuntimeRecord | None, str, int]: - record = read_runtime_record(paths.backend_pid) - host = record.host if record is not None and record.host else config.backend_host - port = record.port if record is not None and record.port is not None else config.backend_port - return record, host, port - - -def _watchdog_backend_config(config: ServiceConfig, paths: RuntimePaths) -> ServiceConfig: - record, host, port = _watchdog_backend_endpoint(config, paths) - return ServiceConfig( - backend_host=host, - backend_port=port, - frontend_host=config.frontend_host, - frontend_port=config.frontend_port, - no_browser=True, - skip_frontend_build=True, - ) - +def _terminate_process( + process: subprocess.Popen | None, + name: str, + console, + *, + timeout: float = 10.0, +) -> None: + """Terminate a process and its process group without scanning service ports.""" + if process is None: + return + if process.poll() is not None: + return -def _watchdog_probe_backend( - config: ServiceConfig, - paths: RuntimePaths, - client: httpx.Client, -) -> WatchdogProbeResult: - record, host, port = _watchdog_backend_endpoint(config, paths) - if record is None: - return WatchdogProbeResult(False, False, "backend runtime record missing", host, port) - - backend_running = runtime_record_is_running(record) - listeners = port_owner_pids(port) - port_in_use = port_is_in_use(port, listeners) - if not backend_running: - if port_in_use: - return WatchdogProbeResult( - False, - False, - f"backend runtime record is not running but port {port} is occupied", - host, - port, - ) - return WatchdogProbeResult(True, False, "backend runtime record is not running", host, port) - if not port_in_use: - return WatchdogProbeResult(True, False, f"backend process alive but port {port} is not listening", host, port) - - runtime_pids = set(_runtime_record_pids(record)) - listener_pids = set(listeners) - if listener_pids and runtime_pids and listener_pids.isdisjoint(runtime_pids): - reason = ( - f"backend process alive but port {port} is owned by unexpected pid(s): " - f"{_join_pids(sorted(listener_pids))}" - ) - return WatchdogProbeResult(False, False, reason, host, port) + record = process_runtime_record(process, host=None, port=None, command=()) + console.print(f"[flocks] 停止 {name}（PID={process.pid}）...") + if sys.platform == "win32": + subprocess.run(["taskkill", "/PID", str(process.pid), "/T", "/F"], check=False, capture_output=True) + else: + if record.pgid is not None: + signal_process_group(signal.SIGTERM, record.pgid) + else: + signal_pid_list(signal.SIGTERM, collect_process_tree_pids(process.pid)) - url = _backend_health_url(host, port) - try: - response = client.get(url) - except Exception as exc: - return WatchdogProbeResult(True, True, f"backend health check failed: {exc}", host, port) - if not _is_healthy_status_response(response): - return WatchdogProbeResult( - True, - True, - f"backend health check unhealthy: status={response.status_code}", - host, - port, - ) - return WatchdogProbeResult(False, False, "backend healthy", host, port) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if process.poll() is not None and not process_group_is_running(record.pgid): + return + time.sleep(0.25) + console.print(f"[flocks] {name} 未在预期时间内退出，强制终止...") + if sys.platform == "win32": + subprocess.run(["taskkill", "/PID", str(process.pid), "/T", "/F"], check=False, capture_output=True) + else: + if record.pgid is not None: + signal_process_group(signal.SIGKILL, record.pgid) + signal_pid_list(signal.SIGKILL, collect_process_tree_pids(process.pid)) -def _recover_unhealthy_backend(config: ServiceConfig, paths: RuntimePaths, reason: str) -> None: - try: - with service_lock(paths): - effective_config = _watchdog_backend_config(config, paths) - with httpx.Client(timeout=2.0, trust_env=False) as client: - probe = _watchdog_probe_backend(effective_config, paths, client) - if not probe.restart_needed: - _watchdog_log("backend_recovery_skipped", {"reason": probe.reason}) - return - console = _StdoutConsole() - _watchdog_log( - "backend_recovery_start", - { - "reason": reason, - "host": probe.host, - "port": probe.port, - }, - ) - stop_one(probe.port, paths.backend_pid, "后端", console) - start_backend(effective_config, console) - _watchdog_log("backend_recovery_done", {"host": probe.host, "port": probe.port}) - except ServiceError as exc: - _watchdog_log("backend_recovery_failed", {"reason": reason, "error": str(exc)}) - except Exception as exc: - _watchdog_log("backend_recovery_crashed", {"reason": reason, "error": repr(exc)}) +def _backend_command_and_env(root: Path, config: ServiceConfig) -> tuple[list[str], dict[str, str]]: + """Build the backend service command and environment.""" + command = resolve_flocks_cli_command(root) + [ + "serve", + "--host", + config.backend_host, + "--port", + str(config.backend_port), + ] + env = os.environ.copy() + env["_FLOCKS_WEBUI_HOST"] = config.frontend_host + env["_FLOCKS_WEBUI_PORT"] = str(config.frontend_port) + env["PYTHONUNBUFFERED"] = "1" + env.setdefault("FLOCKS_CONSOLE_BASE_URL", DEFAULT_FLOCKS_CONSOLE_BASE_URL) + return command, env -def _watchdog_tick( +def _start_backend_process( config: ServiceConfig, - paths: RuntimePaths, - health_failure_count: int, + console, *, - failure_threshold: int = WATCHDOG_HEALTH_FAILURE_THRESHOLD, -) -> int: - with httpx.Client(timeout=2.0, trust_env=False) as client: - probe = _watchdog_probe_backend(config, paths, client) - - if not probe.restart_needed: - return 0 - - if probe.health_failure: - health_failure_count += 1 - _watchdog_log( - "backend_health_failed", - { - "count": health_failure_count, - "threshold": failure_threshold, - "reason": probe.reason, - "host": probe.host, - "port": probe.port, - }, - ) - if health_failure_count < failure_threshold: - return health_failure_count - - _recover_unhealthy_backend(config, paths, probe.reason) - return 0 - - -def start_backend(config: ServiceConfig, console) -> None: - """Start the backend API service if needed.""" + paths: RuntimePaths | None = None, +) -> subprocess.Popen: + """Start the backend child process for the supervisor.""" root = ensure_install_layout() - paths = ensure_runtime_dirs() - cleanup_stale_pid_file(paths.backend_pid) + current = paths if paths is not None else ensure_runtime_dirs() - runtime_record = read_runtime_record(paths.backend_pid) - tracked_pid = runtime_record.pid if runtime_record else None listeners = port_owner_pids(config.backend_port) if listeners: - if tracked_pid and tracked_pid in listeners: - console.print(f"[flocks] 后端已在运行，PID={tracked_pid}") - return raise ServiceError( f"后端端口 {config.backend_port} 已被占用 (PID: {_join_pids(listeners)})，" - "与当前运行时记录不一致，请先执行 `flocks stop` 或手动清理残留进程。" + "请先执行 `flocks stop` 或手动清理残留进程。" ) if port_is_in_use(config.backend_port, listeners): raise ServiceError( @@ -1109,45 +971,16 @@ def start_backend(config: ServiceConfig, console) -> None: "请先安装 lsof 或手动清理残留进程。" ) - if runtime_record is not None and runtime_record_is_running(runtime_record): - raise ServiceError( - "后端运行记录仍存活，但端口未监听；请先执行 `flocks stop` 清理异常状态后重试。" - ) - - if runtime_record is not None: - paths.backend_pid.unlink(missing_ok=True) - - command = resolve_flocks_cli_command(root) + [ - "serve", - "--host", - config.backend_host, - "--port", - str(config.backend_port), - ] - - backend_env = os.environ.copy() - backend_env["_FLOCKS_WEBUI_HOST"] = config.frontend_host - backend_env["_FLOCKS_WEBUI_PORT"] = str(config.frontend_port) - backend_env["PYTHONUNBUFFERED"] = "1" - backend_env.setdefault("FLOCKS_CONSOLE_BASE_URL", DEFAULT_FLOCKS_CONSOLE_BASE_URL) - + command, env = _backend_command_and_env(root, config) console.print("[flocks] 启动后端服务...") - process = _spawn_process( - command, - cwd=root, - log_path=paths.backend_log, - env=backend_env, - ) - write_runtime_record( - paths.backend_pid, - process_runtime_record( - process, - host=config.backend_host, - port=config.backend_port, - command=command, - ), + process = _spawn_process(command, cwd=root, log_path=current.backend_log, env=env) + record = process_runtime_record( + process, + host=config.backend_host, + port=config.backend_port, + command=command, ) - _log_startup_config(paths.backend_log, "backend", config.backend_host, config.backend_port, read_runtime_record(paths.backend_pid)) + _log_startup_config(current.backend_log, "backend", config.backend_host, config.backend_port, record) try: wait_for_http( @@ -1157,27 +990,24 @@ def start_backend(config: ServiceConfig, console) -> None: validator=_is_running_status_response, ) except ServiceError: - _emit_service_log_tail(console, paths.backend_log, "后端") - stop_one(config.backend_port, paths.backend_pid, "后端", console) + _emit_service_log_tail(console, current.backend_log, "后端") + _terminate_process(process, "后端", console) raise + return process - console.print(f"[flocks] 后端已启动，日志: {paths.backend_log}") - -def start_frontend(config: ServiceConfig, console) -> None: - """Build and start the WebUI preview service if needed.""" +def _start_frontend_process( + config: ServiceConfig, + console, + *, + paths: RuntimePaths | None = None, +) -> subprocess.Popen: + """Build and start the WebUI child process.""" root = ensure_install_layout() - paths = ensure_runtime_dirs() - cleanup_stale_pid_file(paths.frontend_pid) + current = paths if paths is not None else ensure_runtime_dirs() - runtime_record = read_runtime_record(paths.frontend_pid) - tracked_pid = runtime_record.pid if runtime_record else None listeners = port_owner_pids(config.frontend_port) if listeners: - if tracked_pid and tracked_pid in listeners: - console.print(f"[flocks] WebUI 已在运行，PID={tracked_pid}") - return - upgrade_info = _read_upgrade_runtime_info(config.frontend_port) if upgrade_info.page_active: _resolve_upgrade_runtime( @@ -1185,25 +1015,17 @@ def start_frontend(config: ServiceConfig, console) -> None: frontend_port=upgrade_info.frontend_port or config.frontend_port, attempt_recover=False, ) - cleanup_stale_pid_file(paths.frontend_pid) - runtime_record = read_runtime_record(paths.frontend_pid) - tracked_pid = runtime_record.pid if runtime_record else None listeners = port_owner_pids(config.frontend_port) - if tracked_pid and tracked_pid in listeners: - console.print(f"[flocks] WebUI 已在运行，PID={tracked_pid}") - return - if not listeners: - tracked_pid = runtime_record.pid if runtime_record else None - else: + if listeners: raise ServiceError( f"WebUI 端口 {config.frontend_port} 已被占用 (PID: {_join_pids(listeners)})，" - "与当前运行时记录不一致，请先执行 `flocks stop` 或手动清理残留进程。" + "请先执行 `flocks stop` 或手动清理残留进程。" ) else: raise ServiceError( f"WebUI 端口 {config.frontend_port} 已被占用 (PID: {_join_pids(listeners)})，" - "与当前运行时记录不一致，请先执行 `flocks stop` 或手动清理残留进程。" + "请先执行 `flocks stop` 或手动清理残留进程。" ) elif port_is_in_use(config.frontend_port, listeners): raise ServiceError( @@ -1211,14 +1033,6 @@ def start_frontend(config: ServiceConfig, console) -> None: "请先安装 lsof 或手动清理残留进程。" ) - if runtime_record is not None and runtime_record_is_running(runtime_record): - raise ServiceError( - "WebUI 运行记录仍存活，但端口未监听；请先执行 `flocks stop` 清理异常状态后重试。" - ) - - if runtime_record is not None: - paths.frontend_pid.unlink(missing_ok=True) - npm = resolve_npm_executable() if not npm: raise ServiceError("未检测到 npm，请先安装 Node.js 22+（包含 npm）后重试。") @@ -1256,158 +1070,53 @@ def start_frontend(config: ServiceConfig, console) -> None: ] console.print("[flocks] 启动 WebUI...") - process = _spawn_process( - command, - cwd=webui_dir, - log_path=paths.frontend_log, - env=frontend_env, - ) - write_runtime_record( - paths.frontend_pid, - process_runtime_record( - process, - host=config.frontend_host, - port=config.frontend_port, - command=command, - ), + process = _spawn_process(command, cwd=webui_dir, log_path=current.frontend_log, env=frontend_env) + record = process_runtime_record( + process, + host=config.frontend_host, + port=config.frontend_port, + command=command, ) - _log_startup_config(paths.frontend_log, "webui", config.frontend_host, config.frontend_port, read_runtime_record(paths.frontend_pid)) + _log_startup_config(current.frontend_log, "webui", config.frontend_host, config.frontend_port, record) try: wait_for_http([config.frontend_url], "WebUI") except ServiceError: - _emit_service_log_tail(console, paths.frontend_log, "WebUI") - stop_one(config.frontend_port, paths.frontend_pid, "WebUI", console) + _emit_service_log_tail(console, current.frontend_log, "WebUI") + _terminate_process(process, "WebUI", console) raise - console.print(f"[flocks] WebUI 已启动，日志: {paths.frontend_log}") + return process -def start_watchdog(config: ServiceConfig, console) -> None: - """Start the service watchdog daemon if needed.""" - root = ensure_install_layout() - paths = ensure_runtime_dirs() - pid_file = watchdog_pid_path(paths) - log_path = watchdog_log_path(paths) +def stop_runtime_record_process(pid_file: Path, name: str, console) -> None: + """Stop a legacy pid/runtime record without scanning ports.""" cleanup_stale_pid_file(pid_file) - - runtime_record = read_runtime_record(pid_file) - if runtime_record is not None and runtime_record_is_running(runtime_record): - console.print(f"[flocks] Watchdog 已在运行，PID={runtime_record.pid}") - return - if runtime_record is not None: + record = read_runtime_record(pid_file) + if record is None: pid_file.unlink(missing_ok=True) - - command = resolve_flocks_cli_command(root) + [ - "service-watchdog", - "--server-host", - config.backend_host, - "--server-port", - str(config.backend_port), - "--webui-host", - config.frontend_host, - "--webui-port", - str(config.frontend_port), - "--interval", - str(WATCHDOG_CHECK_INTERVAL_SECONDS), - ] - env = os.environ.copy() - env["PYTHONUNBUFFERED"] = "1" - - console.print("[flocks] 启动服务 Watchdog...") - process = _spawn_process(command, cwd=root, log_path=log_path, env=env) - write_runtime_record( - pid_file, - process_runtime_record( - process, - host=None, - port=None, - command=command, - ), - ) - _log_startup_config(log_path, "watchdog", config.backend_host, config.backend_port, read_runtime_record(pid_file)) - console.print(f"[flocks] Watchdog 已启动，日志: {log_path}") - - -def stop_watchdog(paths: RuntimePaths, console) -> None: - """Stop the service watchdog without touching backend/frontend ports.""" - pid_file = watchdog_pid_path(paths) - cleanup_stale_pid_file(pid_file) - if read_runtime_record(pid_file) is None: return - stop_one(0, pid_file, "Watchdog", console) - - -def run_service_watchdog( - config: ServiceConfig, - *, - interval: float = WATCHDOG_CHECK_INTERVAL_SECONDS, - failure_threshold: int = WATCHDOG_HEALTH_FAILURE_THRESHOLD, -) -> None: - """Run the backend health watchdog loop.""" - paths = ensure_runtime_dirs() - _watchdog_log( - "started", - { - "backend_host": config.backend_host, - "backend_port": config.backend_port, - "interval": interval, - "failure_threshold": failure_threshold, - }, - ) - health_failure_count = 0 - while True: - try: - health_failure_count = _watchdog_tick( - config, - paths, - health_failure_count, - failure_threshold=failure_threshold, - ) - except KeyboardInterrupt: - _watchdog_log("stopped") - return - except Exception as exc: - _watchdog_log("tick_failed", {"error": repr(exc)}) - time.sleep(interval) - - -def _tracked_processes_stopped( - port: int, - record: RuntimeRecord | None, - tracked_pids: Iterable[int], -) -> bool: - """Return True when the tracked service no longer has running processes.""" - listeners = port_owner_pids(port) - if port_is_in_use(port, listeners): - return False - if runtime_record_is_running(record): - return False - return not any(pid_is_running(pid) for pid in tracked_pids) - - -def _runtime_record_pids(record: RuntimeRecord | None) -> list[int]: - """Collect the latest pids implied by a runtime record.""" - if record is None: - return [] - - result: list[int] = [] - if record.pid > 0: - result = append_unique_pids(result, collect_process_tree_pids(record.pid)) - if record.pgid is not None and sys.platform != "win32": - result = append_unique_pids(result, _process_group_member_pids(record.pgid)) - return result + targets = collect_process_tree_pids(record.pid) + console.print(f"[flocks] 清理旧 {name} 进程（PID={record.pid}）...") + if sys.platform == "win32": + subprocess.run(["taskkill", "/PID", str(record.pid), "/T", "/F"], check=False, capture_output=True) + else: + if record.pgid is not None: + signal_process_group(signal.SIGTERM, record.pgid) + else: + signal_pid_list(signal.SIGTERM, targets) + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline: + if not runtime_record_is_running(record): + pid_file.unlink(missing_ok=True) + return + time.sleep(0.25) + if record.pgid is not None: + signal_process_group(signal.SIGKILL, record.pgid) + signal_pid_list(signal.SIGKILL, targets) -def _current_stop_targets( - port: int, - record: RuntimeRecord | None, - tracked_pids: Iterable[int], -) -> list[int]: - """Refresh the pid list that stop_one() should verify or force kill.""" - result = append_unique_pids([], tracked_pids) - result = append_unique_pids(result, _runtime_record_pids(record)) - return append_unique_pids(result, port_owner_pids(port)) + pid_file.unlink(missing_ok=True) def signal_process_group(sig: signal.Signals, pgid: int | None) -> None: @@ -1420,137 +1129,6 @@ def signal_process_group(sig: signal.Signals, pgid: int | None) -> None: pass -def stop_one(port: int, pid_file: Path, name: str, console) -> None: - """Stop a single service by tracked pid and/or listening port.""" - cleanup_stale_pid_file(pid_file) - runtime_record = read_runtime_record(pid_file) - tracked_pid = runtime_record.pid if runtime_record else None - listeners = port_owner_pids(port) - - target_pids: list[int] = [] - if tracked_pid is not None: - target_pids = append_unique_pids(target_pids, collect_process_tree_pids(tracked_pid)) - target_pids = append_unique_pids(target_pids, listeners) - if sys.platform == "win32" and runtime_record is not None: - filtered_targets: list[int] = [] - for pid in target_pids: - if pid in listeners: - filtered_targets = append_unique_pids(filtered_targets, [pid]) - continue - if pid == runtime_record.pid and not _windows_runtime_record_matches_pid(runtime_record, pid, listeners): - continue - filtered_targets = append_unique_pids(filtered_targets, [pid]) - target_pids = filtered_targets - - group_running = process_group_is_running(runtime_record.pgid if runtime_record else None) - if not target_pids and not group_running: - if port_is_in_use(port, listeners): - raise ServiceError( - f"{name} 端口 {port} 已被占用，但当前环境无法识别占用 PID；" - "请先安装 lsof 或手动处理该进程。" - ) - pid_file.unlink(missing_ok=True) - console.print(f"[flocks] {name} 未运行。") - return - - details = _join_pids(target_pids) if target_pids else "none" - if runtime_record and runtime_record.pgid is not None and sys.platform != "win32": - details = f"{details}; PGID={runtime_record.pgid}" - console.print(f"[flocks] 停止 {name}（端口 {port}，PID: {details}）...") - - if sys.platform == "win32": - for pid in target_pids: - subprocess.run(["taskkill", "/PID", str(pid), "/T", "/F"], check=False, capture_output=True) - else: - if runtime_record and runtime_record.pgid is not None: - signal_process_group(signal.SIGTERM, runtime_record.pgid) - else: - signal_pid_list(signal.SIGTERM, target_pids) - for _ in range(10): - current_targets = _current_stop_targets(port, runtime_record, target_pids) - if _tracked_processes_stopped(port, runtime_record, current_targets): - pid_file.unlink(missing_ok=True) - console.print(f"[flocks] {name} 已停止。") - return - time.sleep(1) - - console.print(f"[flocks] {name} 未在预期时间内退出，强制终止...") - force_targets = _current_stop_targets(port, runtime_record, target_pids) - if runtime_record and runtime_record.pgid is not None: - signal_process_group(signal.SIGKILL, runtime_record.pgid) - signal_pid_list(signal.SIGKILL, force_targets) - - for _ in range(10): - force_targets = _current_stop_targets(port, runtime_record, target_pids) - if _tracked_processes_stopped(port, runtime_record, force_targets): - pid_file.unlink(missing_ok=True) - console.print(f"[flocks] {name} 已停止。") - return - if sys.platform == "win32": - for pid in force_targets: - subprocess.run(["taskkill", "/PID", str(pid), "/T", "/F"], check=False, capture_output=True) - else: - if runtime_record and runtime_record.pgid is not None: - signal_process_group(signal.SIGKILL, runtime_record.pgid) - signal_pid_list(signal.SIGKILL, force_targets) - time.sleep(1) - - raise ServiceError(f"{name} 未在预期时间内退出，请手动检查端口 {port}。") - - -def _recorded_port(pid_file: Path, default: int) -> int: - """Return the port from a runtime record, falling back to *default*.""" - record = read_runtime_record(pid_file) - if record is not None and record.port is not None: - return record.port - return default - - -def _recorded_host(pid_file: Path, default: str) -> str: - """Return the host from a runtime record, falling back to *default*.""" - record = read_runtime_record(pid_file) - if record is not None and record.host: - return record.host - return default - - -@contextlib.contextmanager -def service_lock(paths: RuntimePaths): - """Serialize lifecycle commands with a cross-process lock file.""" - lock_path = paths.run_dir / "service.lock" - lock_path.parent.mkdir(parents=True, exist_ok=True) - handle = lock_path.open("a+", encoding="utf-8") - unlock_windows = None - try: - try: - if sys.platform == "win32": - import msvcrt - - handle.seek(0) - handle.write("0") - handle.flush() - handle.seek(0) - msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) - unlock_windows = msvcrt - else: - if fcntl is None: # pragma: no cover - defensive - raise OSError("fcntl unavailable") - fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError as error: - raise ServiceError("另一个 flocks 命令正在执行，请稍后重试。") from error - yield - finally: - try: - if unlock_windows is not None: - handle.seek(0) - unlock_windows.locking(handle.fileno(), unlock_windows.LK_UNLCK, 1) - elif fcntl is not None and sys.platform != "win32": - fcntl.flock(handle, fcntl.LOCK_UN) - except OSError: - pass - handle.close() - - def _log_startup_config( log_path: Path, name: str, @@ -1568,144 +1146,173 @@ def _log_startup_config( handle.write(line) -def _resolve_stop_ports( +def _wait_for_supervisor_ready( paths: RuntimePaths, - config: ServiceConfig | None = None, -) -> tuple[int, int]: - """Resolve frontend/backend ports for stop flows. - - When a runtime record is missing or uses the legacy pid-only format, - ``start`` and ``restart`` should fall back to the current CLI config - rather than the static default ports. - """ - frontend_default = config.frontend_port if config is not None else ServiceConfig.frontend_port - backend_default = config.backend_port if config is not None else ServiceConfig.backend_port - return ( - _effective_frontend_port(paths, frontend_default), - _recorded_port(paths.backend_pid, backend_default), - ) + *, + process: subprocess.Popen | None = None, + timeout: float = SUPERVISOR_START_TIMEOUT_SECONDS, +) -> dict[str, Any]: + """Wait for the supervisor control API and managed services to become ready.""" + deadline = time.monotonic() + timeout + last_payload: dict[str, Any] | None = None + while time.monotonic() < deadline: + if process is not None and process.poll() is not None: + raise ServiceError(f"Supervisor 启动失败，退出码: {process.returncode}") + try: + payload = read_control_json("/status", paths=paths, timeout=1.0) + last_payload = payload + backend_state = ((payload.get("backend") or {}).get("state") if isinstance(payload.get("backend"), dict) else None) + webui_state = ((payload.get("webui") or {}).get("state") if isinstance(payload.get("webui"), dict) else None) + if backend_state == "healthy" and webui_state == "healthy": + return payload + if backend_state == "degraded" or webui_state == "degraded": + return payload + except Exception: + pass + time.sleep(0.5) + if last_payload is not None: + return last_payload + raise ServiceError("Supervisor 启动超时，请检查日志。") -def _stop_all_locked( - paths: RuntimePaths, - console, - *, - config: ServiceConfig | None = None, -) -> None: - """Stop frontend then backend while reusing the caller's lock.""" - fe_port, be_port = _resolve_stop_ports(paths, config) - try: - _resolve_upgrade_runtime(console, frontend_port=fe_port, attempt_recover=False) - stop_watchdog(paths, console) - stop_one(fe_port, paths.frontend_pid, "WebUI", console) - stop_one(be_port, paths.backend_pid, "后端", console) - finally: - stop_all_browser_daemons() +def _start_supervisor_process(config: ServiceConfig, paths: RuntimePaths, console) -> subprocess.Popen: + """Spawn the detached service supervisor daemon.""" + root = ensure_install_layout() + log_path = supervisor_log_path(paths) + if sys.platform != "win32": + supervisor_socket_path(paths).unlink(missing_ok=True) + command = resolve_flocks_cli_command(root) + [ + "service-daemon", + "--server-host", + config.backend_host, + "--server-port", + str(config.backend_port), + "--webui-host", + config.frontend_host, + "--webui-port", + str(config.frontend_port), + ] + if config.skip_frontend_build: + command.append("--skip-webui-build") + env = os.environ.copy() + env["PYTHONUNBUFFERED"] = "1" + console.print("[flocks] 启动 Supervisor daemon...") + return _spawn_process(command, cwd=root, log_path=log_path, env=env) def stop_all(console) -> None: - """Stop frontend then backend using ports persisted in runtime records.""" + """Stop managed services through the supervisor control API.""" paths = ensure_runtime_dirs() - with service_lock(paths): - _stop_all_locked(paths, console) + if not supervisor_is_running(paths): + console.print("[flocks] Supervisor 未运行。") + return + try: + post_control_json("/stop", paths=paths, timeout=2.0) + except Exception as exc: + raise ServiceError(f"无法请求 Supervisor 停止: {exc}") from exc + + deadline = time.monotonic() + 20.0 + while time.monotonic() < deadline: + if not supervisor_is_running(paths): + console.print("[flocks] Supervisor 已停止。") + return + time.sleep(0.5) + raise ServiceError("Supervisor 未在预期时间内退出。") def _start_all_without_stop(config: ServiceConfig, console) -> None: - """Start backend and frontend, then print access summary.""" - ensure_runtime_dirs() - start_backend(config, console) - start_frontend(config, console) - start_watchdog(config, console) + """Start the supervisor daemon, then print access summary.""" + paths = ensure_runtime_dirs() + process = _start_supervisor_process(config, paths, console) + payload = _wait_for_supervisor_ready(paths, process=process) show_start_summary(config, console) + _print_status_payload(payload, console) if not config.no_browser: open_default_browser(config.frontend_url, console) def start_all(config: ServiceConfig, console) -> None: - """Ensure backend and frontend are restarted with a clean state.""" + """Ensure the supervisor daemon is running.""" paths = ensure_runtime_dirs() - with service_lock(paths): - _stop_all_locked(paths, console, config=config) - _start_all_without_stop(config, console) + if supervisor_is_running(paths): + console.print("[flocks] Supervisor 已在运行。") + show_status(console) + if not config.no_browser: + try: + payload = read_control_json("/status", paths=paths, timeout=1.0) + url = _frontend_url_from_status_payload(payload, config.frontend_url) + except Exception: + url = config.frontend_url + open_default_browser(url, console) + return + _start_all_without_stop(config, console) def restart_all(config: ServiceConfig, console) -> None: - """Restart backend and frontend.""" + """Restart backend and frontend through the supervisor control API.""" paths = ensure_runtime_dirs() - with service_lock(paths): - _stop_all_locked(paths, console, config=config) - _start_all_without_stop(config, console) + if not supervisor_is_running(paths): + start_all(config, console) + return + try: + payload = post_control_json("/restart", payload=service_config_payload(config), paths=paths, timeout=180.0) + except Exception as exc: + raise ServiceError(f"无法请求 Supervisor 重启: {exc}") from exc + _print_status_payload(payload, console) def build_status_lines(paths: RuntimePaths | None = None) -> list[str]: - """Return a human-readable status summary.""" + """Return a human-readable status summary from the supervisor control API.""" current = paths or runtime_paths() - cleanup_stale_pid_file(current.backend_pid) - cleanup_stale_pid_file(current.frontend_pid) - cleanup_stale_pid_file(watchdog_pid_path(current)) - - backend_record = read_runtime_record(current.backend_pid) - frontend_record = read_runtime_record(current.frontend_pid) - watchdog_record = read_runtime_record(watchdog_pid_path(current)) - backend_port = _recorded_port(current.backend_pid, ServiceConfig.backend_port) - frontend_port = _recorded_port(current.frontend_pid, ServiceConfig.frontend_port) - backend_host = _loopback_host(_recorded_host(current.backend_pid, ServiceConfig.backend_host)) - frontend_host = _loopback_host(_recorded_host(current.frontend_pid, ServiceConfig.frontend_host)) - upgrade_info = _read_upgrade_runtime_info(frontend_port) - if frontend_record is None and upgrade_info.frontend_port is not None: - frontend_port = upgrade_info.frontend_port - if frontend_record is None and upgrade_info.frontend_host: - frontend_host = _loopback_host(upgrade_info.frontend_host) - backend_pid = backend_record.pid if backend_record else None - frontend_pid = frontend_record.pid if frontend_record else None - backend_listeners = port_owner_pids(backend_port) - frontend_listeners = port_owner_pids(frontend_port) - backend_in_use = port_is_in_use(backend_port, backend_listeners) - frontend_in_use = port_is_in_use(frontend_port, frontend_listeners) - - lines: list[str] = [] - if backend_listeners: - lines.append( - f"[flocks] 后端运行中: PID={_join_pids(backend_listeners)} URL=http://{backend_host}:{backend_port}" - ) - elif backend_in_use: - lines.append(f"[flocks] 后端运行中: PID=unknown URL=http://{backend_host}:{backend_port}") - elif pid_is_running(backend_pid): - lines.append(f"[flocks] 后端主进程仍在运行，但端口 {backend_port} 未监听: PID={backend_pid}") - elif process_group_is_running(backend_record.pgid if backend_record else None): - lines.append(f"[flocks] 后端进程组仍在运行，但端口 {backend_port} 未监听: PGID={backend_record.pgid}") - else: - lines.append("[flocks] 后端未运行") + try: + payload = read_control_json("/status", paths=current) + except Exception: + return [ + "[flocks] Supervisor 未运行", + f"[flocks] Supervisor 日志: {supervisor_log_path(current)}", + ] + return _status_lines_from_payload(payload) + + +def _status_lines_from_payload(payload: dict[str, Any]) -> list[str]: + daemon = payload.get("daemon") if isinstance(payload.get("daemon"), dict) else {} + backend = payload.get("backend") if isinstance(payload.get("backend"), dict) else {} + webui = payload.get("webui") if isinstance(payload.get("webui"), dict) else {} + lines = [ + f"[flocks] Supervisor 运行中: PID={daemon.get('pid')} state={daemon.get('state')}", + _service_status_line("后端", backend), + _service_status_line("WebUI", webui), + f"[flocks] Supervisor 日志: {daemon.get('log_path')}", + ] + for service in (backend, webui): + log_path = service.get("log_path") + if log_path: + lines.append(f"[flocks] {service.get('state')} 日志: {log_path}") + return lines - if upgrade_info.page_active: - lines.append( - f"[flocks] WebUI 临时升级页运行中: PID={_join_pids(upgrade_info.listener_pids)} URL=http://{frontend_host}:{frontend_port}" - ) - elif frontend_listeners: - lines.append( - f"[flocks] WebUI 运行中: PID={_join_pids(frontend_listeners)} URL=http://{frontend_host}:{frontend_port}" - ) - elif frontend_in_use: - lines.append(f"[flocks] WebUI 运行中: PID=unknown URL=http://{frontend_host}:{frontend_port}") - elif pid_is_running(frontend_pid): - lines.append(f"[flocks] WebUI 主进程仍在运行，但端口 {frontend_port} 未监听: PID={frontend_pid}") - elif process_group_is_running(frontend_record.pgid if frontend_record else None): - lines.append(f"[flocks] WebUI 进程组仍在运行，但端口 {frontend_port} 未监听: PGID={frontend_record.pgid}") - else: - lines.append("[flocks] WebUI 未运行") - if runtime_record_is_running(watchdog_record): - lines.append(f"[flocks] Watchdog 运行中: PID={watchdog_record.pid}") - else: - lines.append("[flocks] Watchdog 未运行") +def _service_status_line(label: str, payload: dict[str, Any]) -> str: + host = _loopback_host(str(payload.get("host") or "127.0.0.1")) + port = payload.get("port") + pid = payload.get("pid") + state = payload.get("state") or "unknown" + error = payload.get("last_error") + suffix = f" last_error={error}" if error else "" + return f"[flocks] {label}: state={state} PID={pid} URL=http://{host}:{port}{suffix}" - if upgrade_info.payload_present: - lines.append("[flocks] 检测到未完成的升级恢复状态") - lines.append(f"[flocks] 后端日志: {current.backend_log}") - lines.append(f"[flocks] WebUI 日志: {current.frontend_log}") - lines.append(f"[flocks] Watchdog 日志: {watchdog_log_path(current)}") - return lines +def _frontend_url_from_status_payload(payload: dict[str, Any], fallback: str) -> str: + webui = payload.get("webui") if isinstance(payload.get("webui"), dict) else {} + host = webui.get("host") + port = webui.get("port") + if isinstance(host, str) and isinstance(port, int): + return f"http://{_format_host_for_url(_loopback_host(host))}:{port}" + return fallback + + +def _print_status_payload(payload: dict[str, Any], console) -> None: + for line in _status_lines_from_payload(payload): + console.print(line) def show_status(console) -> None: @@ -1737,42 +1344,39 @@ def show_logs( follow: bool = True, lines: int = 50, ) -> None: - """Print recent service logs and optionally follow them.""" + """Print recent service logs through the supervisor control API.""" paths = ensure_runtime_dirs() - selections = selected_log_paths(paths, backend=backend, webui=webui) - prefixes = {paths.backend_log: "backend", paths.frontend_log: "webui"} - - for path in selections: - path.touch(exist_ok=True) - console.print(f"[{prefixes[path]}] --- {path} ---") - for line in tail_lines(path, lines): - console.print(f"[{prefixes[path]}] {line}") - + service = "all" + if backend and not webui: + service = "backend" + elif webui and not backend: + service = "webui" + params = {"service": service, "lines": str(lines), "follow": "true" if follow else "false"} if not follow: + try: + payload = read_control_json(f"/logs?service={service}&lines={lines}&follow=false", paths=paths, timeout=5.0) + except Exception as exc: + raise ServiceError(f"无法通过 Supervisor 读取日志: {exc}") from exc + logs = payload.get("logs") if isinstance(payload.get("logs"), dict) else {} + for prefix, entry in logs.items(): + if not isinstance(entry, dict): + continue + console.print(f"[{prefix}] --- {entry.get('path')} ---") + for line in entry.get("lines") or []: + console.print(f"[{prefix}] {line}") return console.print("[flocks] 按 Ctrl+C 退出日志跟随。") - handles = {} try: - for path in selections: - handle = path.open("r", encoding="utf-8", errors="replace") - handle.seek(0, os.SEEK_END) - handles[path] = handle - - while True: - emitted = False - for path, handle in handles.items(): - while True: - line = handle.readline() - if not line: - break - emitted = True - console.print(f"[{prefixes[path]}] {line.rstrip()}") - if not emitted: - time.sleep(FOLLOW_POLL_INTERVAL) - finally: - for handle in handles.values(): - handle.close() + with supervisor_control_client(paths, timeout=None) as client: + with client.stream("GET", "/logs", params=params) as response: + response.raise_for_status() + for line in response.iter_lines(): + console.print(line) + except KeyboardInterrupt: + return + except Exception as exc: + raise ServiceError(f"无法通过 Supervisor 跟随日志: {exc}") from exc def selected_log_paths( diff --git a/flocks/cli/service_supervisor.py b/flocks/cli/service_supervisor.py new file mode 100644 index 000000000..67793186c --- /dev/null +++ b/flocks/cli/service_supervisor.py @@ -0,0 +1,608 @@ +"""Supervisor daemon for the local Flocks service.""" + +from __future__ import annotations + +import datetime +import json +import os +import signal +import socket +import subprocess +import sys +import threading +import time +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, urlparse + +import httpx + +from flocks.browser.admin import stop_all_daemons as stop_all_browser_daemons +from flocks.cli.service_control import ( + service_config_payload, + supervisor_control_port, + supervisor_log_path, + supervisor_socket_path, +) + +SUPERVISOR_CHECK_INTERVAL_SECONDS = 5.0 +SUPERVISOR_HEALTH_FAILURE_THRESHOLD = 2 +SUPERVISOR_BACKOFF_SECONDS = (1.0, 2.0, 5.0, 10.0, 30.0) + + +@dataclass +class ManagedService: + name: str + label: str + host: str + port: int + log_path: Path + process: subprocess.Popen | None = None + command: tuple[str, ...] = () + state: str = "stopped" + last_error: str | None = None + restart_count: int = 0 + last_restart_at: float | None = None + health_failure_count: int = 0 + next_restart_at: float = 0.0 + built_once: bool = False + + @property + def pid(self) -> int | None: + return self.process.pid if self.process is not None else None + + +def _daemon_log(event: str, details: dict[str, object] | None = None) -> None: + """Write a structured supervisor log line to stdout.""" + timestamp = datetime.datetime.now().isoformat(timespec="seconds") + suffix = "" + if details: + suffix = " " + json.dumps(details, ensure_ascii=True, sort_keys=True) + sys.stdout.write(f"[{timestamp}] supervisor.{event}{suffix}\n") + sys.stdout.flush() + + +def _config_from_payload(payload: dict[str, Any], default): + from flocks.cli.service_manager import ServiceConfig + + def _string(name: str, fallback: str) -> str: + value = payload.get(name) + return value if isinstance(value, str) and value else fallback + + def _int(name: str, fallback: int) -> int: + value = payload.get(name) + return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else fallback + + return ServiceConfig( + backend_host=_string("backend_host", default.backend_host), + backend_port=_int("backend_port", default.backend_port), + frontend_host=_string("frontend_host", default.frontend_host), + frontend_port=_int("frontend_port", default.frontend_port), + no_browser=bool(payload.get("no_browser", default.no_browser)), + skip_frontend_build=bool(payload.get("skip_frontend_build", default.skip_frontend_build)), + ) + + +def _tcp_port_accepts_connections(host: str, port: int) -> bool: + """Return True when a local service accepts TCP connections.""" + from flocks.cli.service_manager import access_host + + try: + with socket.create_connection((access_host(host), port), timeout=1.0): + return True + except OSError: + return False + + +def _health_status_from_service_state(state: str) -> str: + if state in {"healthy", "starting", "restarting", "stopped", "paused"}: + return state + return "degraded" + + +def _service_payload(service: ManagedService, *, paused: bool = False) -> dict[str, object]: + return { + "pid": service.pid, + "host": service.host, + "port": service.port, + "state": "paused" if paused else service.state, + "health": _health_status_from_service_state("paused" if paused else service.state), + "last_error": service.last_error, + "restart_count": service.restart_count, + "last_restart_at": service.last_restart_at, + "log_path": str(service.log_path), + "command": list(service.command), + "paused": paused, + } + + +class _UnixControlServer(ThreadingHTTPServer): + address_family = socket.AF_UNIX + + +class SupervisorDaemon: + """Owns backend/WebUI child processes and exposes a local control API.""" + + def __init__( + self, + config, + *, + interval: float = SUPERVISOR_CHECK_INTERVAL_SECONDS, + failure_threshold: int = SUPERVISOR_HEALTH_FAILURE_THRESHOLD, + ) -> None: + from flocks.cli.service_manager import ensure_runtime_dirs + + self.config = config + self.paths = ensure_runtime_dirs() + self.interval = interval + self.failure_threshold = failure_threshold + self.started_at = time.time() + self._lock = threading.RLock() + self._shutdown_requested = threading.Event() + self._server: ThreadingHTTPServer | None = None + self._server_thread: threading.Thread | None = None + self._webui_paused = False + self.backend = ManagedService( + name="backend", + label="后端", + host=config.backend_host, + port=config.backend_port, + log_path=self.paths.backend_log, + ) + self.webui = ManagedService( + name="webui", + label="WebUI", + host=config.frontend_host, + port=config.frontend_port, + log_path=self.paths.frontend_log, + ) + + def run(self) -> None: + """Run the supervisor until the control API asks it to stop.""" + self._install_signal_handlers() + self._cleanup_legacy_runtime() + self._start_control_server() + try: + self.restart_all(reason="startup") + while not self._shutdown_requested.wait(self.interval): + self.tick() + finally: + self.shutdown_children() + self._stop_control_server() + stop_all_browser_daemons() + _daemon_log("stopped") + + def _install_signal_handlers(self) -> None: + if threading.current_thread() is not threading.main_thread(): + return + + def _handle(_signum, _frame) -> None: + self.request_stop() + + for sig in (signal.SIGINT, signal.SIGTERM): + try: + signal.signal(sig, _handle) + except (OSError, ValueError): # pragma: no cover - platform defensive + pass + + def _cleanup_legacy_runtime(self) -> None: + from flocks.cli import service_manager + + console = service_manager._StdoutConsole() + for pid_file, name in ( + (service_manager.watchdog_pid_path(self.paths), "watchdog"), + (self.paths.frontend_pid, "WebUI"), + (self.paths.backend_pid, "backend"), + ): + record = service_manager.read_runtime_record(pid_file) + if record is not None and service_manager.runtime_record_is_running(record): + service_manager.stop_runtime_record_process(pid_file, name, console) + else: + pid_file.unlink(missing_ok=True) + + def _start_control_server(self) -> None: + handler = self._handler_class() + if sys.platform == "win32": + server: ThreadingHTTPServer = ThreadingHTTPServer(("127.0.0.1", supervisor_control_port()), handler) + else: + socket_path = supervisor_socket_path(self.paths) + socket_path.parent.mkdir(parents=True, exist_ok=True) + socket_path.unlink(missing_ok=True) + server = _UnixControlServer(str(socket_path), handler) + self._server = server + self._server_thread = threading.Thread(target=server.serve_forever, name="flocks-supervisor-control", daemon=True) + self._server_thread.start() + _daemon_log("control_started", {"platform": sys.platform}) + + def _stop_control_server(self) -> None: + if self._server is not None: + self._server.shutdown() + self._server.server_close() + if self._server_thread is not None: + self._server_thread.join(timeout=5.0) + if sys.platform != "win32": + supervisor_socket_path(self.paths).unlink(missing_ok=True) + + def _handler_class(self): + daemon = self + + class ControlHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.0" + + def log_message(self, _format, *_args) -> None: + return + + def _send_json(self, payload: dict[str, object], status: int = 200) -> None: + body = json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _read_json(self) -> dict[str, Any]: + length = int(self.headers.get("Content-Length") or "0") + if length <= 0: + return {} + try: + payload = json.loads(self.rfile.read(length).decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return {} + return payload if isinstance(payload, dict) else {} + + def do_GET(self) -> None: + parsed = urlparse(self.path) + try: + if parsed.path == "/status": + self._send_json(daemon.status_payload()) + return + if parsed.path == "/logs": + daemon.handle_logs_request(self, parse_qs(parsed.query)) + return + self._send_json({"error": "not found"}, status=404) + except Exception as exc: # pragma: no cover - defensive control path + self._send_json({"error": str(exc)}, status=500) + + def do_POST(self) -> None: + parsed = urlparse(self.path) + payload = self._read_json() + try: + if parsed.path == "/stop": + daemon.request_stop() + self._send_json({"status": "stopping"}) + return + if parsed.path == "/restart": + daemon.update_config(payload) + daemon.restart_all(reason="control restart") + self._send_json(daemon.status_payload()) + return + if parsed.path == "/restart/backend": + daemon.restart_backend(reason="control restart") + self._send_json(daemon.status_payload()) + return + if parsed.path == "/restart/webui": + daemon.update_config(payload) + daemon.restart_webui( + reason="control restart", + force_frontend_build=bool(payload.get("force_frontend_build")), + ) + self._send_json(daemon.status_payload()) + return + if parsed.path == "/stop/webui": + daemon.stop_webui(reason="control stop") + self._send_json(daemon.status_payload()) + return + self._send_json({"error": "not found"}, status=404) + except Exception as exc: # pragma: no cover - defensive control path + self._send_json({"error": str(exc)}, status=500) + + return ControlHandler + + def update_config(self, payload: dict[str, Any]) -> None: + with self._lock: + self.config = _config_from_payload(payload, self.config) + self.backend.host = self.config.backend_host + self.backend.port = self.config.backend_port + self.webui.host = self.config.frontend_host + self.webui.port = self.config.frontend_port + + def request_stop(self) -> None: + self._shutdown_requested.set() + + def status_payload(self) -> dict[str, object]: + try: + from flocks import __version__ + except Exception: # pragma: no cover - defensive + __version__ = "unknown" + with self._lock: + return { + "daemon": { + "pid": os.getpid(), + "uptime": time.time() - self.started_at, + "version": __version__, + "state": "stopping" if self._shutdown_requested.is_set() else "running", + "log_path": str(supervisor_log_path(self.paths)), + }, + "backend": _service_payload(self.backend), + "webui": _service_payload(self.webui, paused=self._webui_paused), + "config": service_config_payload(self.config), + } + + def handle_logs_request(self, handler: BaseHTTPRequestHandler, query: dict[str, list[str]]) -> None: + from flocks.cli.service_manager import FOLLOW_POLL_INTERVAL, _coerce_positive_int, tail_lines + + service_name = (query.get("service") or ["backend"])[0] + lines = _coerce_positive_int((query.get("lines") or ["50"])[0]) or 50 + follow = (query.get("follow") or ["false"])[0].lower() == "true" + selections = self._log_paths_for_service(service_name) + if not selections: + body = json.dumps({"error": "unknown service"}, ensure_ascii=False).encode("utf-8") + handler.send_response(400) + handler.send_header("Content-Type", "application/json; charset=utf-8") + handler.send_header("Content-Length", str(len(body))) + handler.end_headers() + handler.wfile.write(body) + return + + for _prefix, log_path in selections: + log_path.touch(exist_ok=True) + if not follow: + body = json.dumps( + { + "service": service_name, + "logs": { + prefix: { + "path": str(log_path), + "lines": tail_lines(log_path, lines), + } + for prefix, log_path in selections + }, + }, + ensure_ascii=False, + ).encode("utf-8") + handler.send_response(200) + handler.send_header("Content-Type", "application/json; charset=utf-8") + handler.send_header("Content-Length", str(len(body))) + handler.end_headers() + handler.wfile.write(body) + return + + handler.send_response(200) + handler.send_header("Content-Type", "text/plain; charset=utf-8") + handler.end_headers() + for prefix, log_path in selections: + handler.wfile.write((f"[{prefix}] --- {log_path} ---\n").encode("utf-8", errors="replace")) + for line in tail_lines(log_path, lines): + handler.wfile.write((f"[{prefix}] {line}\n").encode("utf-8", errors="replace")) + handler.wfile.flush() + handles = {} + try: + for prefix, log_path in selections: + handle = log_path.open("r", encoding="utf-8", errors="replace") + handle.seek(0, os.SEEK_END) + handles[prefix] = handle + while not self._shutdown_requested.is_set(): + emitted = False + for prefix, handle in handles.items(): + while True: + line = handle.readline() + if not line: + break + emitted = True + handler.wfile.write((f"[{prefix}] {line}").encode("utf-8", errors="replace")) + if emitted: + handler.wfile.flush() + else: + time.sleep(FOLLOW_POLL_INTERVAL) + finally: + for handle in handles.values(): + handle.close() + + def _log_paths_for_service(self, service_name: str) -> list[tuple[str, Path]]: + if service_name == "backend": + return [("backend", self.paths.backend_log)] + if service_name == "webui": + return [("webui", self.paths.frontend_log)] + if service_name == "supervisor": + return [("supervisor", supervisor_log_path(self.paths))] + if service_name == "all": + return [ + ("backend", self.paths.backend_log), + ("webui", self.paths.frontend_log), + ("supervisor", supervisor_log_path(self.paths)), + ] + return [] + + def restart_all(self, *, reason: str) -> None: + with self._lock: + self._webui_paused = False + self._restart_service(self.webui, reason=reason, immediate=True) + self._restart_service(self.backend, reason=reason, immediate=True) + self._start_backend_locked(immediate=True) + self._start_webui_locked(immediate=True) + + def restart_backend(self, *, reason: str) -> None: + with self._lock: + self._restart_service(self.backend, reason=reason, immediate=True) + self._start_backend_locked(immediate=True) + + def restart_webui(self, *, reason: str, force_frontend_build: bool = False) -> None: + with self._lock: + self._webui_paused = False + if force_frontend_build: + self.webui.built_once = False + self._restart_service(self.webui, reason=reason, immediate=True) + self._start_webui_locked(immediate=True) + + def stop_webui(self, *, reason: str) -> None: + with self._lock: + self._webui_paused = True + _daemon_log("service_pause", {"service": "webui", "reason": reason}) + self._stop_service(self.webui) + self.webui.last_error = reason + + def shutdown_children(self) -> None: + with self._lock: + self._stop_service(self.webui) + self._stop_service(self.backend) + + def tick(self) -> None: + with self._lock: + self._probe_backend_locked() + if not self._webui_paused: + self._probe_webui_locked() + self._start_backend_locked(immediate=False) + if not self._webui_paused: + self._start_webui_locked(immediate=False) + + def _restart_service(self, service: ManagedService, *, reason: str, immediate: bool) -> None: + _daemon_log("service_restart", {"service": service.name, "reason": reason}) + self._stop_service(service) + service.state = "restarting" + service.last_error = reason + service.health_failure_count = 0 + service.restart_count += 1 + service.last_restart_at = time.time() + service.next_restart_at = time.monotonic() if immediate else self._next_restart_time(service.restart_count) + + def _stop_service(self, service: ManagedService) -> None: + from flocks.cli.service_manager import _StdoutConsole, _terminate_process + + _terminate_process(service.process, service.label, _StdoutConsole()) + service.process = None + service.command = () + service.state = "stopped" + + def _start_backend_locked(self, *, immediate: bool) -> None: + from flocks.cli.service_manager import _StdoutConsole, _start_backend_process + + if self.backend.process is not None and self.backend.process.poll() is None: + return + if not immediate and time.monotonic() < self.backend.next_restart_at: + return + self.backend.state = "starting" + try: + process = _start_backend_process(self.config, _StdoutConsole(), paths=self.paths) + except Exception as exc: + self._mark_start_failed(self.backend, exc) + return + self.backend.process = process + self.backend.command = tuple(str(item) for item in process.args) + self.backend.state = "healthy" + self.backend.last_error = None + self.backend.health_failure_count = 0 + + def _start_webui_locked(self, *, immediate: bool) -> None: + from flocks.cli.service_manager import ServiceConfig, _StdoutConsole, _start_frontend_process + + if self.webui.process is not None and self.webui.process.poll() is None: + return + if not immediate and time.monotonic() < self.webui.next_restart_at: + return + self.webui.state = "starting" + config = self.config + if self.webui.built_once: + config = ServiceConfig( + backend_host=config.backend_host, + backend_port=config.backend_port, + frontend_host=config.frontend_host, + frontend_port=config.frontend_port, + no_browser=config.no_browser, + skip_frontend_build=True, + ) + try: + process = _start_frontend_process(config, _StdoutConsole(), paths=self.paths) + except Exception as exc: + self._mark_start_failed(self.webui, exc) + return + self.webui.process = process + self.webui.command = tuple(str(item) for item in process.args) + self.webui.state = "healthy" + self.webui.last_error = None + self.webui.health_failure_count = 0 + self.webui.built_once = True + + def _mark_start_failed(self, service: ManagedService, error: Exception) -> None: + service.process = None + service.state = "degraded" + service.last_error = str(error) + service.next_restart_at = self._next_restart_time(service.restart_count) + _daemon_log( + "service_start_failed", + {"service": service.name, "error": str(error), "retry_at": service.next_restart_at}, + ) + + def _next_restart_time(self, restart_count: int) -> float: + index = min(max(restart_count, 1) - 1, len(SUPERVISOR_BACKOFF_SECONDS) - 1) + return time.monotonic() + SUPERVISOR_BACKOFF_SECONDS[index] + + def _probe_backend_locked(self) -> None: + from flocks.cli.service_manager import _backend_health_url, _is_healthy_status_response + + process = self.backend.process + if process is None: + self.backend.state = "stopped" + return + if process.poll() is not None: + self._restart_service(self.backend, reason=f"process exited with code {process.returncode}", immediate=True) + return + if not _tcp_port_accepts_connections(self.backend.host, self.backend.port): + self._restart_service(self.backend, reason=f"port {self.backend.port} is not listening", immediate=True) + return + + url = _backend_health_url(self.backend.host, self.backend.port) + try: + with httpx.Client(timeout=2.0, trust_env=False) as client: + response = client.get(url) + healthy = _is_healthy_status_response(response) + reason = f"health status={response.status_code}" + except Exception as exc: + healthy = False + reason = f"health failed: {exc}" + if healthy: + self.backend.state = "healthy" + self.backend.health_failure_count = 0 + self.backend.last_error = None + return + + self.backend.health_failure_count += 1 + self.backend.state = "degraded" + self.backend.last_error = reason + if self.backend.health_failure_count >= self.failure_threshold: + self._restart_service(self.backend, reason=reason, immediate=True) + + def _probe_webui_locked(self) -> None: + process = self.webui.process + if process is None: + self.webui.state = "stopped" + return + if process.poll() is not None: + self._restart_service(self.webui, reason=f"process exited with code {process.returncode}", immediate=True) + return + if not _tcp_port_accepts_connections(self.webui.host, self.webui.port): + self._restart_service(self.webui, reason=f"port {self.webui.port} is not listening", immediate=True) + return + self.webui.state = "healthy" + self.webui.health_failure_count = 0 + self.webui.last_error = None + + +def run_service_daemon( + config, + *, + interval: float = SUPERVISOR_CHECK_INTERVAL_SECONDS, + failure_threshold: int = SUPERVISOR_HEALTH_FAILURE_THRESHOLD, +) -> None: + """Run the local supervisor daemon.""" + _daemon_log( + "started", + { + "backend_host": config.backend_host, + "backend_port": config.backend_port, + "frontend_host": config.frontend_host, + "frontend_port": config.frontend_port, + }, + ) + SupervisorDaemon(config, interval=interval, failure_threshold=failure_threshold).run() diff --git a/flocks/server/app.py b/flocks/server/app.py index 8c7cb1b24..aa03ebc60 100644 --- a/flocks/server/app.py +++ b/flocks/server/app.py @@ -681,7 +681,7 @@ def _should_log_request(path: str, status_code: int) -> bool: # CORS Configuration # # Priority order: -# 1. Runtime env vars exported by ``start_backend()`` → add the concrete +# 1. Runtime env vars exported by the supervised backend launcher → add the concrete # ``_FLOCKS_WEBUI_*`` origin inferred from the current CLI launch. # 2. Explicit ``server.cors`` in flocks.json → append user-configured # origins without discarding the runtime ones. diff --git a/flocks/updater/restart_handoff.py b/flocks/updater/restart_handoff.py index f6350b400..dc639b02e 100644 --- a/flocks/updater/restart_handoff.py +++ b/flocks/updater/restart_handoff.py @@ -26,11 +26,6 @@ DEFAULT_POLL_INTERVAL_SECONDS = 0.25 -class _NullConsole: - def print(self, *args, **kwargs) -> None: - return None - - def _record_handoff_log(message: str) -> None: append_upgrade_text_log(f"restart_handoff {message}") @@ -68,52 +63,14 @@ def _wait_for_backend_port_free( return not _backend_port_in_use(port) -def _ensure_backend_port_free(backend_port: int, backend_pid_file: Path) -> bool: +def _ensure_backend_port_free(backend_port: int) -> bool: if _wait_for_backend_port_free(backend_port): return True - _record_handoff_log(f"backend_port_still_in_use port={backend_port}; stopping backend") - try: - service_manager.stop_one(backend_port, backend_pid_file, "backend", _NullConsole()) - except Exception as exc: - _record_handoff_log(f"backend_stop_failed port={backend_port} error={exc}") - return False - + _record_handoff_log(f"backend_port_still_in_use port={backend_port}") return _wait_for_backend_port_free(backend_port, timeout_seconds=POST_STOP_PORT_TIMEOUT_SECONDS) -def _cli_subcommand(argv: Sequence[str]) -> str | None: - for index, value in enumerate(argv[:-2]): - if value == "-m" and argv[index + 1] == "flocks.cli.main": - return argv[index + 2] - return None - - -def _record_backend_runtime_if_direct_serve( - process: subprocess.Popen, - restart_argv: Sequence[str], - *, - backend_host: str, - backend_port: int, - backend_pid_file: Path, -) -> None: - if _cli_subcommand(restart_argv) != "serve": - return - - try: - service_manager.write_runtime_record( - backend_pid_file, - service_manager.process_runtime_record( - process, - host=backend_host, - port=backend_port, - command=restart_argv, - ), - ) - except Exception as exc: - _record_handoff_log(f"backend_runtime_record_failed error={exc}") - - def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description="Flocks restart handoff helper") parser.add_argument("--parent-pid", type=int, required=True) @@ -121,7 +78,6 @@ def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.add_argument("--backend-port", type=int, required=True) parser.add_argument("--frontend-host", required=True) parser.add_argument("--frontend-port", type=int, required=True) - parser.add_argument("--backend-pid-file", required=True) parser.add_argument("--install-root", required=True) parser.add_argument("--uv-path", required=True) parser.add_argument("--sync-timeout", type=int, required=True) @@ -200,8 +156,7 @@ def run(argv: Sequence[str] | None = None) -> int: _cleanup_dir(args.cleanup_dir) return 1 - backend_pid_file = Path(args.backend_pid_file) - if not _ensure_backend_port_free(args.backend_port, backend_pid_file): + if not _ensure_backend_port_free(args.backend_port): _record_handoff_log(f"backend_port_unavailable port={args.backend_port}") _cleanup_dir(args.cleanup_dir) return 1 @@ -226,13 +181,6 @@ def run(argv: Sequence[str] | None = None) -> int: _cleanup_dir(args.cleanup_dir) return 1 - _record_backend_runtime_if_direct_serve( - process, - restart_argv, - backend_host=args.backend_host, - backend_port=args.backend_port, - backend_pid_file=backend_pid_file, - ) _record_handoff_log(f"restart_spawned pid={process.pid}") _cleanup_dir(args.cleanup_dir) return 0 diff --git a/flocks/updater/updater.py b/flocks/updater/updater.py index d076af229..9f5a75552 100644 --- a/flocks/updater/updater.py +++ b/flocks/updater/updater.py @@ -1888,13 +1888,19 @@ def print(self, *args, **kwargs) -> None: def _current_service_config(): from flocks.cli import service_manager + from flocks.cli.service_control import read_supervisor_status - paths = service_manager.ensure_runtime_dirs() + try: + payload = read_supervisor_status(paths=service_manager.runtime_paths(), timeout=1.0) + except Exception: + payload = {} + + config = payload.get("config") if isinstance(payload.get("config"), dict) else {} return service_manager.ServiceConfig( - backend_host=service_manager._recorded_host(paths.backend_pid, service_manager.ServiceConfig.backend_host), - backend_port=service_manager._recorded_port(paths.backend_pid, service_manager.ServiceConfig.backend_port), - frontend_host=service_manager._recorded_host(paths.frontend_pid, service_manager.ServiceConfig.frontend_host), - frontend_port=service_manager._recorded_port(paths.frontend_pid, service_manager.ServiceConfig.frontend_port), + backend_host=str(config.get("backend_host") or service_manager.ServiceConfig.backend_host), + backend_port=int(config.get("backend_port") or service_manager.ServiceConfig.backend_port), + frontend_host=str(config.get("frontend_host") or service_manager.ServiceConfig.frontend_host), + frontend_port=int(config.get("frontend_port") or service_manager.ServiceConfig.frontend_port), no_browser=True, skip_frontend_build=True, ) @@ -2068,6 +2074,7 @@ def _stop_upgrade_page_server(*, frontend_port: int | None = None) -> None: def _prepare_upgrade_handover(version: str) -> dict[str, Any]: from flocks.cli import service_manager + from flocks.cli.service_control import post_control_json config = _current_service_config() payload: dict[str, Any] = { @@ -2082,9 +2089,8 @@ def _prepare_upgrade_handover(version: str) -> dict[str, Any]: _persist_upgrade_state(payload, last_error=None) console = _NullConsole() - paths = service_manager.ensure_runtime_dirs() - frontend_port = service_manager._recorded_port(paths.frontend_pid, config.frontend_port) - service_manager.stop_one(frontend_port, paths.frontend_pid, "WebUI", console) + paths = service_manager.runtime_paths() + post_control_json("/stop/webui", paths=paths, timeout=30.0) try: payload.update(_start_upgrade_page_server(config, version)) @@ -2097,7 +2103,7 @@ def _prepare_upgrade_handover(version: str) -> dict[str, Any]: _stop_upgrade_page_server(frontend_port=config.frontend_port) _clear_upgrade_state() try: - service_manager.start_frontend(config, console) + _start_frontend_with_fallback(config, console, allow_build_fallback=False) except Exception as restart_error: log.error("updater.frontend.restore_failed", {"error": str(restart_error)}) raise @@ -2181,15 +2187,25 @@ def read_upgrade_runtime_state(frontend_port: int | None = None) -> dict[str, An def _start_frontend_with_fallback(config, console, *, allow_build_fallback: bool) -> None: - from flocks.cli import service_manager + from flocks.cli.service_control import post_control_json, service_config_payload try: - service_manager.start_frontend(config, console) + payload = post_control_json( + "/restart/webui", + payload=service_config_payload(config), + paths=None, + timeout=180.0, + ) + webui = payload.get("webui") if isinstance(payload.get("webui"), dict) else {} + if webui.get("state") != "healthy": + raise RuntimeError(str(webui.get("last_error") or "WebUI restart did not become healthy")) return except Exception: if not allow_build_fallback or not config.skip_frontend_build: raise + from flocks.cli import service_manager + rebuilt_config = service_manager.ServiceConfig( backend_host=config.backend_host, backend_port=config.backend_port, @@ -2198,7 +2214,12 @@ def _start_frontend_with_fallback(config, console, *, allow_build_fallback: bool no_browser=config.no_browser, skip_frontend_build=False, ) - service_manager.start_frontend(rebuilt_config, console) + payload = service_config_payload(rebuilt_config) + payload["force_frontend_build"] = True + result = post_control_json("/restart/webui", payload=payload, paths=None, timeout=180.0) + webui = result.get("webui") if isinstance(result.get("webui"), dict) else {} + if webui.get("state") != "healthy": + raise RuntimeError(str(webui.get("last_error") or "WebUI restart did not become healthy")) def cleanup_orphan_upgrade_state(*, frontend_port: int | None = None) -> bool: @@ -3442,13 +3463,10 @@ def _build_restart_handoff_argv( cleanup_dir: Path | None = None, ) -> list[str]: """Wrap the real restart command in a helper that finishes upgrade work.""" - from flocks.cli import service_manager - if not restart_argv: raise ValueError("restart command is empty") config = _current_service_config() - paths = service_manager.ensure_runtime_dirs() argv = [ restart_argv[0], "-m", @@ -3463,8 +3481,6 @@ def _build_restart_handoff_argv( str(config.frontend_host), "--frontend-port", str(config.frontend_port), - "--backend-pid-file", - str(paths.backend_pid), "--install-root", str(install_root), "--uv-path", diff --git a/tests/cli/test_service_commands.py b/tests/cli/test_service_commands.py index 49d92a200..d80928bc5 100644 --- a/tests/cli/test_service_commands.py +++ b/tests/cli/test_service_commands.py @@ -31,7 +31,7 @@ def test_cli_help_lists_service_commands(monkeypatch, tmp_path) -> None: assert result.exit_code == 0 for command in ("start", "stop", "restart", "status", "logs", "session", "mcp", "task", "skills"): assert _help_contains_command(result.stdout, command) - for command in ("agent", "acp", "debug", "run", "serve", "service-watchdog", "auth", "models"): + for command in ("agent", "acp", "debug", "run", "serve", "service-watchdog", "service-daemon", "auth", "models"): assert not _help_contains_command(result.stdout, command) diff --git a/tests/cli/test_service_manager.py b/tests/cli/test_service_manager.py index 286f9541a..e56e75e7c 100644 --- a/tests/cli/test_service_manager.py +++ b/tests/cli/test_service_manager.py @@ -1,6 +1,4 @@ -import contextlib import json -import signal import sys from pathlib import Path from types import SimpleNamespace @@ -9,6 +7,7 @@ import pytest from flocks.cli import service_manager +from flocks.cli import service_supervisor class DummyConsole: @@ -31,6 +30,21 @@ def _make_runtime_paths(tmp_path: Path) -> service_manager.RuntimePaths: ) +def _write_legacy_runtime_record(pid_file: Path, record: service_manager.RuntimeRecord) -> None: + payload: dict[str, object] = {"pid": record.pid} + if record.pgid is not None: + payload["pgid"] = record.pgid + if record.host is not None: + payload["host"] = record.host + if record.port is not None: + payload["port"] = record.port + if record.command: + payload["command"] = list(record.command) + if record.started_at is not None: + payload["started_at"] = record.started_at + pid_file.write_text(json.dumps(payload, ensure_ascii=True, sort_keys=True), encoding="utf-8") + + def test_runtime_paths_follow_flocks_root_env(monkeypatch, tmp_path: Path) -> None: monkeypatch.setenv("FLOCKS_ROOT", str(tmp_path)) @@ -253,7 +267,7 @@ def test_runtime_record_round_trip_preserves_metadata(tmp_path: Path) -> None: started_at=1234.5, ) - service_manager.write_runtime_record(pid_file, record) + _write_legacy_runtime_record(pid_file, record) assert json.loads(pid_file.read_text(encoding="utf-8")) == { "command": ["python", "-m", "uvicorn"], @@ -276,7 +290,7 @@ def test_runtime_record_round_trip_preserves_host(tmp_path: Path) -> None: started_at=1234.5, ) - service_manager.write_runtime_record(pid_file, record) + _write_legacy_runtime_record(pid_file, record) assert json.loads(pid_file.read_text(encoding="utf-8")) == { "command": ["python", "-m", "uvicorn"], @@ -298,7 +312,7 @@ def test_read_runtime_record_rejects_invalid_content(tmp_path: Path) -> None: def test_cleanup_stale_pid_file_keeps_live_process_group(monkeypatch, tmp_path: Path) -> None: pid_file = tmp_path / "backend.pid" - service_manager.write_runtime_record( + _write_legacy_runtime_record( pid_file, service_manager.RuntimeRecord(pid=1001, pgid=2002, port=8000), ) @@ -313,7 +327,7 @@ def test_cleanup_stale_pid_file_keeps_live_process_group(monkeypatch, tmp_path: def test_cleanup_stale_pid_file_removes_reused_windows_pid(monkeypatch, tmp_path: Path) -> None: pid_file = tmp_path / "backend.pid" - service_manager.write_runtime_record( + _write_legacy_runtime_record( pid_file, service_manager.RuntimeRecord( pid=1232, @@ -588,7 +602,7 @@ def _client_factory(*, timeout, trust_env): captured["trust_env"] = trust_env return _FakeClient() - monkeypatch.setattr(service_manager.httpx, "Client", _client_factory) + monkeypatch.setattr(service_supervisor.httpx, "Client", _client_factory) service_manager.wait_for_http( ["http://127.0.0.1:8000/api/health"], @@ -685,139 +699,117 @@ def test_resolve_flocks_cli_command_falls_back_to_python_module(monkeypatch, tmp ] -def test_build_status_lines_reports_running_and_idle_services(monkeypatch, tmp_path: Path) -> None: - paths = service_manager.RuntimePaths( - root=tmp_path, - run_dir=tmp_path / "run", - log_dir=tmp_path / "logs", - backend_pid=tmp_path / "run" / "backend.pid", - frontend_pid=tmp_path / "run" / "webui.pid", - backend_log=tmp_path / "logs" / "backend.log", - frontend_log=tmp_path / "logs" / "webui.log", - ) - paths.run_dir.mkdir(parents=True) - paths.log_dir.mkdir(parents=True) - paths.backend_pid.write_text("111", encoding="utf-8") - paths.frontend_pid.write_text("222", encoding="utf-8") +def _supervisor_status_payload() -> dict[str, object]: + return { + "daemon": { + "pid": 100, + "state": "running", + "log_path": "/tmp/logs/supervisor.log", + }, + "backend": { + "pid": 111, + "host": "0.0.0.0", + "port": 9000, + "state": "healthy", + "last_error": None, + "log_path": "/tmp/logs/backend.log", + }, + "webui": { + "pid": 222, + "host": "0.0.0.0", + "port": 5174, + "state": "healthy", + "last_error": None, + "log_path": "/tmp/logs/webui.log", + }, + } - monkeypatch.setattr(service_manager, "cleanup_stale_pid_file", lambda _: None) - monkeypatch.setattr( - service_manager, - "port_owner_pids", - lambda port: [111] if port == 8000 else [], - ) - monkeypatch.setattr(service_manager, "pid_is_running", lambda pid: pid == 222) + +def test_build_status_lines_reports_supervisor_control_status(monkeypatch, tmp_path: Path) -> None: + paths = _make_runtime_paths(tmp_path) + monkeypatch.setattr(service_manager, "read_control_json", lambda *_args, **_kwargs: _supervisor_status_payload()) lines = service_manager.build_status_lines(paths) - assert "后端运行中" in lines[0] - assert "WebUI 主进程仍在运行" in lines[1] + assert "Supervisor 运行中" in lines[0] + assert "http://127.0.0.1:9000" in lines[1] + assert "http://127.0.0.1:5174" in lines[2] -def test_build_status_lines_uses_custom_server_and_webui_ports(monkeypatch, tmp_path: Path) -> None: - paths = service_manager.RuntimePaths( - root=tmp_path, - run_dir=tmp_path / "run", - log_dir=tmp_path / "logs", - backend_pid=tmp_path / "run" / "backend.pid", - frontend_pid=tmp_path / "run" / "webui.pid", - backend_log=tmp_path / "logs" / "backend.log", - frontend_log=tmp_path / "logs" / "webui.log", - ) - paths.run_dir.mkdir(parents=True) - paths.log_dir.mkdir(parents=True) - service_manager.write_runtime_record( - paths.backend_pid, - service_manager.RuntimeRecord(pid=111, host="0.0.0.0", port=9000), - ) - service_manager.write_runtime_record( - paths.frontend_pid, - service_manager.RuntimeRecord(pid=222, host="0.0.0.0", port=5174), - ) +def test_build_status_lines_reports_daemon_down_without_port_scans(monkeypatch, tmp_path: Path) -> None: + paths = _make_runtime_paths(tmp_path) + calls: list[str] = [] - monkeypatch.setattr(service_manager, "cleanup_stale_pid_file", lambda _: None) monkeypatch.setattr( service_manager, - "port_owner_pids", - lambda port: [111] if port in {9000, 5174} else [], + "read_control_json", + lambda *_args, **_kwargs: (_ for _ in ()).throw(service_manager.ServiceError("down")), ) - monkeypatch.setattr(service_manager, "pid_is_running", lambda _pid: False) + monkeypatch.setattr(service_manager, "port_owner_pids", lambda _port: calls.append("port_owner") or []) + monkeypatch.setattr(service_manager, "port_is_in_use", lambda *_args, **_kwargs: calls.append("port_in_use") or False) lines = service_manager.build_status_lines(paths) - assert "http://127.0.0.1:9000" in lines[0] - assert "http://127.0.0.1:5174" in lines[1] + assert lines[0] == "[flocks] Supervisor 未运行" + assert calls == [] -def test_start_all_stops_services_before_starting(monkeypatch) -> None: +def test_start_all_starts_supervisor_when_control_api_is_down(monkeypatch) -> None: call_order: list[str] = [] - paths = service_manager.RuntimePaths( - root=Path("/tmp"), - run_dir=Path("/tmp/run"), - log_dir=Path("/tmp/logs"), - backend_pid=Path("/tmp/run/backend.pid"), - frontend_pid=Path("/tmp/run/webui.pid"), - backend_log=Path("/tmp/logs/backend.log"), - frontend_log=Path("/tmp/logs/webui.log"), - ) + paths = _make_runtime_paths(Path("/tmp/flocks-test")) monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: (call_order.append("ensure_runtime_dirs"), paths)[1]) - monkeypatch.setattr(service_manager, "service_lock", lambda _paths: _record_call(call_order, "service_lock")) - monkeypatch.setattr(service_manager, "stop_one", lambda port, _pid_file, _name, _console: call_order.append(f"stop_one:{port}")) - monkeypatch.setattr(service_manager, "stop_all_browser_daemons", lambda: call_order.append("stop_browser") or []) + monkeypatch.setattr(service_manager, "supervisor_is_running", lambda _paths: False) monkeypatch.setattr(service_manager, "_start_all_without_stop", lambda _config, _console: call_order.append("_start_all_without_stop")) service_manager.start_all(service_manager.ServiceConfig(), console=None) - assert call_order == [ - "ensure_runtime_dirs", - "service_lock", - "stop_one:5173", - "stop_one:8000", - "stop_browser", - "_start_all_without_stop", - ] + assert call_order == ["ensure_runtime_dirs", "_start_all_without_stop"] + + +def test_start_all_does_not_duplicate_running_supervisor(monkeypatch) -> None: + calls: list[str] = [] + console = DummyConsole() + monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: _make_runtime_paths(Path("/tmp/flocks-test"))) + monkeypatch.setattr(service_manager, "supervisor_is_running", lambda _paths: True) + monkeypatch.setattr(service_manager, "show_status", lambda _console: calls.append("status")) + monkeypatch.setattr(service_manager, "open_default_browser", lambda _url, _console: calls.append("browser")) + monkeypatch.setattr(service_manager, "_start_all_without_stop", lambda *_args: calls.append("start")) -def test_restart_all_stops_then_starts_under_lock(monkeypatch) -> None: + service_manager.start_all(service_manager.ServiceConfig(no_browser=True), console=console) + + assert calls == ["status"] + assert "[flocks] Supervisor 已在运行。" in console.messages + + +def test_restart_all_uses_supervisor_control_api(monkeypatch) -> None: call_order: list[str] = [] - paths = service_manager.RuntimePaths( - root=Path("/tmp"), - run_dir=Path("/tmp/run"), - log_dir=Path("/tmp/logs"), - backend_pid=Path("/tmp/run/backend.pid"), - frontend_pid=Path("/tmp/run/webui.pid"), - backend_log=Path("/tmp/logs/backend.log"), - frontend_log=Path("/tmp/logs/webui.log"), - ) + paths = _make_runtime_paths(Path("/tmp/flocks-test")) monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: (call_order.append("ensure_runtime_dirs"), paths)[1]) - monkeypatch.setattr(service_manager, "service_lock", lambda _paths: _record_call(call_order, "service_lock")) - monkeypatch.setattr(service_manager, "stop_one", lambda port, _pid_file, _name, _console: call_order.append(f"stop_one:{port}")) - monkeypatch.setattr(service_manager, "stop_all_browser_daemons", lambda: call_order.append("stop_browser") or []) - monkeypatch.setattr(service_manager, "_start_all_without_stop", lambda _config, _console: call_order.append("_start_all_without_stop")) + monkeypatch.setattr(service_manager, "supervisor_is_running", lambda _paths: True) + monkeypatch.setattr( + service_manager, + "post_control_json", + lambda path, **_kwargs: call_order.append(path) or _supervisor_status_payload(), + ) + monkeypatch.setattr(service_manager, "_print_status_payload", lambda _payload, _console: call_order.append("print_status")) service_manager.restart_all(service_manager.ServiceConfig(), console=None) - assert call_order == [ - "ensure_runtime_dirs", - "service_lock", - "stop_one:5173", - "stop_one:8000", - "stop_browser", - "_start_all_without_stop", - ] + assert call_order == ["ensure_runtime_dirs", "/restart", "print_status"] -def test_start_all_without_stop_starts_watchdog_after_frontend(monkeypatch, tmp_path: Path) -> None: +def test_start_all_without_stop_starts_supervisor_daemon(monkeypatch, tmp_path: Path) -> None: paths = _make_runtime_paths(tmp_path) calls: list[str] = [] monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: paths) - monkeypatch.setattr(service_manager, "start_backend", lambda _config, _console: calls.append("backend")) - monkeypatch.setattr(service_manager, "start_frontend", lambda _config, _console: calls.append("webui")) - monkeypatch.setattr(service_manager, "start_watchdog", lambda _config, _console: calls.append("watchdog")) + monkeypatch.setattr(service_manager, "_start_supervisor_process", lambda _config, _paths, _console: calls.append("daemon") or SimpleNamespace(poll=lambda: None)) + monkeypatch.setattr(service_manager, "_wait_for_supervisor_ready", lambda _paths, **_kwargs: calls.append("ready") or _supervisor_status_payload()) monkeypatch.setattr(service_manager, "show_start_summary", lambda _config, _console: calls.append("summary")) + monkeypatch.setattr(service_manager, "_print_status_payload", lambda _payload, _console: calls.append("status")) monkeypatch.setattr( service_manager, "open_default_browser", @@ -826,37 +818,23 @@ def test_start_all_without_stop_starts_watchdog_after_frontend(monkeypatch, tmp_ service_manager._start_all_without_stop(service_manager.ServiceConfig(no_browser=True), DummyConsole()) - assert calls == ["backend", "webui", "watchdog", "summary"] + assert calls == ["daemon", "ready", "summary", "status"] -def test_start_all_stops_on_failure_before_restart(monkeypatch) -> None: - paths = service_manager.RuntimePaths( - root=Path("/tmp"), - run_dir=Path("/tmp/run"), - log_dir=Path("/tmp/logs"), - backend_pid=Path("/tmp/run/backend.pid"), - frontend_pid=Path("/tmp/run/webui.pid"), - backend_log=Path("/tmp/logs/backend.log"), - frontend_log=Path("/tmp/logs/webui.log"), - ) - monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: paths) - monkeypatch.setattr(service_manager, "service_lock", lambda _paths: _record_call([], "service_lock")) - monkeypatch.setattr( - service_manager, - "stop_one", - lambda *_args: (_ for _ in ()).throw(service_manager.ServiceError("stop failed")), - ) +def test_start_all_propagates_supervisor_start_failure(monkeypatch) -> None: + monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: _make_runtime_paths(Path("/tmp/flocks-test"))) + monkeypatch.setattr(service_manager, "supervisor_is_running", lambda _paths: False) monkeypatch.setattr( service_manager, "_start_all_without_stop", - lambda *_args: (_ for _ in ()).throw(AssertionError("should not start")), + lambda *_args: (_ for _ in ()).throw(service_manager.ServiceError("daemon failed")), ) - with pytest.raises(service_manager.ServiceError, match="stop failed"): + with pytest.raises(service_manager.ServiceError, match="daemon failed"): service_manager.start_all(service_manager.ServiceConfig(), console=None) -def test_start_backend_writes_runtime_metadata(monkeypatch, tmp_path: Path) -> None: +def test_start_backend_process_does_not_write_runtime_metadata(monkeypatch, tmp_path: Path) -> None: paths = service_manager.RuntimePaths( root=tmp_path, run_dir=tmp_path / "run", @@ -901,24 +879,10 @@ def _capture_spawn(*_args, **kwargs) -> SimpleNamespace: monkeypatch.setattr(service_manager, "_spawn_process", _capture_spawn) - service_manager.start_backend(service_manager.ServiceConfig(), console) + process = service_manager._start_backend_process(service_manager.ServiceConfig(), console) - record = service_manager.read_runtime_record(paths.backend_pid) - assert record is not None - assert record.pid == 2468 - assert record.pgid == 2468 - assert record.host == "127.0.0.1" - assert record.port == 8000 - assert record.command == ( - "python", - "-m", - "flocks.cli.main", - "serve", - "--host", - "127.0.0.1", - "--port", - "8000", - ) + assert process.pid == 2468 + assert not paths.backend_pid.exists() assert probe_calls == [{ "urls": ["http://127.0.0.1:8000"], "name": "后端服务", @@ -944,7 +908,7 @@ def test_start_backend_rolls_back_when_probe_fails(monkeypatch, tmp_path: Path) paths.log_dir.mkdir(parents=True) paths.backend_log.write_text("line1\nline2\nboot failed here\n", encoding="utf-8") console = DummyConsole() - stop_calls: list[tuple[int, Path, str]] = [] + stop_calls: list[str] = [] monkeypatch.setattr(service_manager, "ensure_install_layout", lambda: tmp_path) monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: paths) @@ -959,7 +923,7 @@ def test_start_backend_rolls_back_when_probe_fails(monkeypatch, tmp_path: Path) monkeypatch.setattr( service_manager, "_spawn_process", - lambda *_args, **_kwargs: SimpleNamespace(pid=2468), + lambda *_args, **_kwargs: SimpleNamespace(pid=2468, poll=lambda: None), ) monkeypatch.setattr( service_manager, @@ -968,14 +932,14 @@ def test_start_backend_rolls_back_when_probe_fails(monkeypatch, tmp_path: Path) ) monkeypatch.setattr( service_manager, - "stop_one", - lambda port, pid_file, name, _console: stop_calls.append((port, pid_file, name)), + "_terminate_process", + lambda _process, name, _console: stop_calls.append(name), ) with pytest.raises(service_manager.ServiceError, match="启动超时"): - service_manager.start_backend(service_manager.ServiceConfig(), console) + service_manager._start_backend_process(service_manager.ServiceConfig(), console) - assert stop_calls == [(8000, paths.backend_pid, "后端")] + assert stop_calls == ["后端"] joined = "\n".join(console.messages) assert "近期日志" in joined assert "boot failed here" in joined @@ -1017,15 +981,12 @@ def test_start_backend_reports_started_after_probe_succeeds(monkeypatch, tmp_pat lambda *_args, **_kwargs: None, ) - service_manager.start_backend(service_manager.ServiceConfig(), console) + service_manager._start_backend_process(service_manager.ServiceConfig(), console) - record = service_manager.read_runtime_record(paths.backend_pid) - assert record is not None - assert record.pid == 2468 backend_env = spawn_calls[0]["kwargs"]["env"] assert backend_env["_FLOCKS_WEBUI_HOST"] == "127.0.0.1" assert backend_env["_FLOCKS_WEBUI_PORT"] == "5173" - assert console.messages[-1] == f"[flocks] 后端已启动，日志: {paths.backend_log}" + assert not paths.backend_pid.exists() assert backend_env["FLOCKS_CONSOLE_BASE_URL"] == service_manager.DEFAULT_FLOCKS_CONSOLE_BASE_URL @@ -1066,7 +1027,7 @@ def test_start_backend_allows_overriding_console_base_url(monkeypatch, tmp_path: ) monkeypatch.setenv("FLOCKS_CONSOLE_BASE_URL", "https://custom-console.example.com") - service_manager.start_backend(service_manager.ServiceConfig(), console) + service_manager._start_backend_process(service_manager.ServiceConfig(), console) backend_env = spawn_calls[0]["kwargs"]["env"] assert backend_env["FLOCKS_CONSOLE_BASE_URL"] == "https://custom-console.example.com" @@ -1213,7 +1174,7 @@ def fake_spawn(command, **kwargs): frontend_host="0.0.0.0", frontend_port=5174, ) - service_manager.start_frontend(config, console) + service_manager._start_frontend_process(config, console) assert build_calls[0]["command"] == ["/usr/bin/npm", "run", "build"] assert build_calls[0]["kwargs"]["env"]["FLOCKS_API_PROXY_TARGET"] == "http://10.0.0.8:9000" @@ -1235,164 +1196,49 @@ def fake_spawn(command, **kwargs): assert preview_calls[0]["kwargs"]["env"]["__VITE_ADDITIONAL_SERVER_ALLOWED_HOSTS"] == "preview.example.com" assert "VITE_API_BASE_URL" not in preview_calls[0]["kwargs"]["env"] assert "VITE_WS_BASE_URL" not in preview_calls[0]["kwargs"]["env"] - record = service_manager.read_runtime_record(paths.frontend_pid) - assert record is not None - assert record.host == "0.0.0.0" - assert record.port == 5174 - - -def test_start_watchdog_writes_runtime_metadata(monkeypatch, tmp_path: Path) -> None: - paths = _make_runtime_paths(tmp_path) - paths.run_dir.mkdir(parents=True) - paths.log_dir.mkdir(parents=True) - console = DummyConsole() - spawn_calls: list[dict[str, object]] = [] - - monkeypatch.setattr(service_manager, "ensure_install_layout", lambda: tmp_path) - monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: paths) - monkeypatch.setattr(service_manager, "cleanup_stale_pid_file", lambda _path: None) - monkeypatch.setattr(service_manager, "runtime_record_is_running", lambda _record: False) - monkeypatch.setattr( - service_manager, - "resolve_flocks_cli_command", - lambda root=None: ["python", "-m", "flocks.cli.main"], - ) - monkeypatch.setattr(service_manager.os, "getpgid", lambda pid: pid) - monkeypatch.setattr( - service_manager, - "_spawn_process", - lambda *args, **kwargs: spawn_calls.append({"args": args, "kwargs": kwargs}) or SimpleNamespace(pid=2468), - ) + assert not paths.frontend_pid.exists() - service_manager.start_watchdog( - service_manager.ServiceConfig(backend_host="0.0.0.0", backend_port=9000), - console, - ) - record = service_manager.read_runtime_record(service_manager.watchdog_pid_path(paths)) - assert record is not None - assert record.pid == 2468 - assert record.port is None - assert record.command == ( - "python", - "-m", - "flocks.cli.main", - "service-watchdog", - "--server-host", - "0.0.0.0", - "--server-port", - "9000", - "--webui-host", - "127.0.0.1", - "--webui-port", - "5173", - "--interval", - str(service_manager.WATCHDOG_CHECK_INTERVAL_SECONDS), - ) - assert spawn_calls[0]["kwargs"]["log_path"] == service_manager.watchdog_log_path(paths) +def _fake_process(pid: int, args: list[str] | None = None, returncode: int | None = None): + return SimpleNamespace(pid=pid, args=args or [str(pid)], returncode=returncode, poll=lambda: returncode) -def test_watchdog_recovers_backend_when_process_alive_but_port_not_listening(monkeypatch, tmp_path: Path) -> None: +def test_supervisor_recovers_backend_when_port_disappears(monkeypatch, tmp_path: Path) -> None: paths = _make_runtime_paths(tmp_path) - paths.run_dir.mkdir(parents=True) - service_manager.write_runtime_record( - paths.backend_pid, - service_manager.RuntimeRecord(pid=111, pgid=222, host="0.0.0.0", port=9995), - ) - calls: list[tuple[str, int]] = [] - - monkeypatch.setattr(service_manager, "runtime_record_is_running", lambda _record: True) - monkeypatch.setattr(service_manager, "port_owner_pids", lambda _port: []) - monkeypatch.setattr(service_manager, "port_is_in_use", lambda _port, listeners=None: False) - monkeypatch.setattr(service_manager, "service_lock", lambda _paths: _record_call([], "service_lock")) - monkeypatch.setattr( - service_manager, - "stop_one", - lambda port, _pid_file, _name, _console: calls.append(("stop", port)), - ) - monkeypatch.setattr( - service_manager, - "start_backend", - lambda config, _console: calls.append(("start", config.backend_port)), - ) - - next_count = service_manager._watchdog_tick( - service_manager.ServiceConfig(backend_port=8000), - paths, - 0, - ) - - assert next_count == 0 - assert calls == [("stop", 9995), ("start", 9995)] - - -def test_watchdog_does_not_recover_when_port_owned_by_unexpected_pid(monkeypatch, tmp_path: Path) -> None: - paths = _make_runtime_paths(tmp_path) - paths.run_dir.mkdir(parents=True) - service_manager.write_runtime_record( - paths.backend_pid, - service_manager.RuntimeRecord(pid=111, pgid=222, host="0.0.0.0", port=9995), - ) calls: list[str] = [] - - monkeypatch.setattr(service_manager, "runtime_record_is_running", lambda _record: True) - monkeypatch.setattr(service_manager, "port_owner_pids", lambda _port: [999]) - monkeypatch.setattr(service_manager, "port_is_in_use", lambda _port, listeners=None: True) - monkeypatch.setattr(service_manager, "_runtime_record_pids", lambda _record: [111]) - monkeypatch.setattr(service_manager, "_recover_unhealthy_backend", lambda *_args: calls.append("recover")) - - next_count = service_manager._watchdog_tick( - service_manager.ServiceConfig(backend_port=9995), - paths, - 0, - ) - - assert next_count == 0 - assert calls == [] - - -def test_watchdog_recovers_backend_when_runtime_record_is_dead(monkeypatch, tmp_path: Path) -> None: - paths = _make_runtime_paths(tmp_path) - paths.run_dir.mkdir(parents=True) - service_manager.write_runtime_record( - paths.backend_pid, - service_manager.RuntimeRecord(pid=111, pgid=222, host="127.0.0.1", port=9995), - ) - calls: list[tuple[str, int]] = [] - - monkeypatch.setattr(service_manager, "runtime_record_is_running", lambda _record: False) - monkeypatch.setattr(service_manager, "port_owner_pids", lambda _port: []) - monkeypatch.setattr(service_manager, "port_is_in_use", lambda _port, listeners=None: False) - monkeypatch.setattr(service_manager, "service_lock", lambda _paths: _record_call([], "service_lock")) - monkeypatch.setattr( - service_manager, - "stop_one", - lambda port, _pid_file, _name, _console: calls.append(("stop", port)), - ) + monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: paths) + daemon = service_supervisor.SupervisorDaemon(service_manager.ServiceConfig(backend_port=9995, frontend_port=9996)) + daemon.paths = paths + daemon.backend.log_path = paths.backend_log + daemon.webui.log_path = paths.frontend_log + daemon.backend.process = _fake_process(111, ["backend"]) + daemon.webui.process = _fake_process(222, ["webui"]) + + monkeypatch.setattr(service_supervisor, "_tcp_port_accepts_connections", lambda _host, port: port != 9995) + monkeypatch.setattr(service_manager, "_terminate_process", lambda _process, name, _console: calls.append(f"stop:{name}")) monkeypatch.setattr( service_manager, - "start_backend", - lambda config, _console: calls.append(("start", config.backend_port)), + "_start_backend_process", + lambda *_args, **_kwargs: calls.append("start:backend") or _fake_process(333, ["backend-new"]), ) - next_count = service_manager._watchdog_tick( - service_manager.ServiceConfig(backend_port=9995), - paths, - 0, - ) + daemon.tick() - assert next_count == 0 - assert calls == [("stop", 9995), ("start", 9995)] + assert calls == ["stop:后端", "start:backend"] + assert daemon.backend.pid == 333 -def test_watchdog_waits_for_second_health_failure_before_restart(monkeypatch, tmp_path: Path) -> None: +def test_supervisor_waits_for_second_backend_health_failure(monkeypatch, tmp_path: Path) -> None: paths = _make_runtime_paths(tmp_path) - paths.run_dir.mkdir(parents=True) - service_manager.write_runtime_record( - paths.backend_pid, - service_manager.RuntimeRecord(pid=111, pgid=222, host="127.0.0.1", port=9995), - ) calls: list[str] = [] + monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: paths) + daemon = service_supervisor.SupervisorDaemon( + service_manager.ServiceConfig(backend_port=9995, frontend_port=9996), + failure_threshold=2, + ) + daemon.paths = paths + daemon.backend.process = _fake_process(111, ["backend"]) + daemon.webui.process = _fake_process(222, ["webui"]) class FakeClient: def __init__(self, *_args, **_kwargs) -> None: @@ -1407,27 +1253,44 @@ def __exit__(self, *_args) -> None: def get(self, _url): return httpx.Response(503, json={"status": "unhealthy"}) - monkeypatch.setattr(service_manager.httpx, "Client", FakeClient) - monkeypatch.setattr(service_manager, "runtime_record_is_running", lambda _record: True) - monkeypatch.setattr(service_manager, "port_owner_pids", lambda _port: [111]) - monkeypatch.setattr(service_manager, "port_is_in_use", lambda _port, listeners=None: True) - monkeypatch.setattr(service_manager, "_runtime_record_pids", lambda _record: [111]) - monkeypatch.setattr(service_manager, "_recover_unhealthy_backend", lambda *_args: calls.append("recover")) - - first_count = service_manager._watchdog_tick( - service_manager.ServiceConfig(backend_port=9995), - paths, - 0, + monkeypatch.setattr(service_supervisor.httpx, "Client", FakeClient) + monkeypatch.setattr(service_supervisor, "_tcp_port_accepts_connections", lambda *_args: True) + monkeypatch.setattr(service_manager, "_terminate_process", lambda _process, name, _console: calls.append(f"stop:{name}")) + monkeypatch.setattr( + service_manager, + "_start_backend_process", + lambda *_args, **_kwargs: calls.append("start:backend") or _fake_process(333, ["backend-new"]), ) - second_count = service_manager._watchdog_tick( - service_manager.ServiceConfig(backend_port=9995), - paths, - first_count, + + daemon.tick() + assert calls == [] + assert daemon.backend.state == "degraded" + + daemon.tick() + assert calls == ["stop:后端", "start:backend"] + + +def test_supervisor_recovers_webui_when_port_disappears(monkeypatch, tmp_path: Path) -> None: + paths = _make_runtime_paths(tmp_path) + calls: list[str] = [] + monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: paths) + daemon = service_supervisor.SupervisorDaemon(service_manager.ServiceConfig(backend_port=9995, frontend_port=9996)) + daemon.paths = paths + daemon.backend.process = _fake_process(111, ["backend"]) + daemon.webui.process = _fake_process(222, ["webui"]) + + monkeypatch.setattr(service_supervisor, "_tcp_port_accepts_connections", lambda _host, port: port != 9996) + monkeypatch.setattr(service_manager, "_terminate_process", lambda _process, name, _console: calls.append(f"stop:{name}")) + monkeypatch.setattr( + service_manager, + "_start_frontend_process", + lambda *_args, **_kwargs: calls.append("start:webui") or _fake_process(444, ["webui-new"]), ) - assert first_count == 1 - assert second_count == 0 - assert calls == ["recover"] + daemon.tick() + + assert calls == ["stop:WebUI", "start:webui"] + assert daemon.webui.pid == 444 def test_start_frontend_tolerates_windows_node_assertion_after_build(monkeypatch, tmp_path: Path) -> None: @@ -1471,7 +1334,7 @@ def fake_spawn(command, **_kwargs): monkeypatch.setattr(service_manager.subprocess, "run", fake_run) monkeypatch.setattr(service_manager, "_spawn_process", fake_spawn) - service_manager.start_frontend(service_manager.ServiceConfig(), console) + service_manager._start_frontend_process(service_manager.ServiceConfig(), console) assert preview_calls[0][:3] == ["npm.cmd", "run", "preview"] assert "[flocks] WebUI 构建产物已生成，忽略 Windows Node.js 退出断言。" in console.messages @@ -1519,7 +1382,7 @@ def fake_spawn(command, **kwargs): frontend_host="0.0.0.0", frontend_port=5174, ) - service_manager.start_frontend(config, console) + service_manager._start_frontend_process(config, console) assert build_calls[0]["kwargs"]["env"]["VITE_API_BASE_URL"] == "http://10.0.0.8:9000" assert build_calls[0]["kwargs"]["env"]["VITE_WS_BASE_URL"] == "ws://10.0.0.8:9000" @@ -1557,12 +1420,12 @@ def fake_run(command, **_kwargs): monkeypatch.setattr(service_manager.subprocess, "run", fake_run) monkeypatch.setattr(service_manager, "_spawn_process", lambda *_args, **_kwargs: SimpleNamespace(pid=2468)) - service_manager.start_frontend(service_manager.ServiceConfig(), console) + service_manager._start_frontend_process(service_manager.ServiceConfig(), console) assert build_calls[0][0] == r"C:\Users\flocks\AppData\Local\Programs\Flocks\tools\node\npm.cmd" -def test_start_backend_raises_on_port_record_mismatch(monkeypatch, tmp_path: Path) -> None: +def test_start_backend_raises_when_port_has_listener(monkeypatch, tmp_path: Path) -> None: paths = service_manager.RuntimePaths( root=tmp_path, run_dir=tmp_path / "run", @@ -1574,15 +1437,15 @@ def test_start_backend_raises_on_port_record_mismatch(monkeypatch, tmp_path: Pat ) paths.run_dir.mkdir(parents=True) paths.log_dir.mkdir(parents=True) - service_manager.write_runtime_record(paths.backend_pid, service_manager.RuntimeRecord(pid=1111, port=8000)) + _write_legacy_runtime_record(paths.backend_pid, service_manager.RuntimeRecord(pid=1111, port=8000)) monkeypatch.setattr(service_manager, "ensure_install_layout", lambda: tmp_path) monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: paths) monkeypatch.setattr(service_manager, "cleanup_stale_pid_file", lambda _path: None) monkeypatch.setattr(service_manager, "port_owner_pids", lambda _port: [9999]) - with pytest.raises(service_manager.ServiceError, match="运行时记录不一致"): - service_manager.start_backend(service_manager.ServiceConfig(), DummyConsole()) + with pytest.raises(service_manager.ServiceError, match="端口 8000 已被占用"): + service_manager._start_backend_process(service_manager.ServiceConfig(), DummyConsole()) def test_start_backend_raises_when_port_in_use_without_pid_lookup(monkeypatch, tmp_path: Path) -> None: @@ -1605,7 +1468,7 @@ def test_start_backend_raises_when_port_in_use_without_pid_lookup(monkeypatch, t monkeypatch.setattr(service_manager, "port_is_in_use", lambda _port, listeners=None: True) with pytest.raises(service_manager.ServiceError, match="无法识别占用 PID"): - service_manager.start_backend(service_manager.ServiceConfig(), DummyConsole()) + service_manager._start_backend_process(service_manager.ServiceConfig(), DummyConsole()) def test_spawn_process_uses_hidden_window_flags_on_windows(monkeypatch, tmp_path: Path) -> None: @@ -1736,309 +1599,11 @@ def fake_popen(*args, **kwargs): assert captured["kwargs"]["env"] == env -def test_stop_one_prefers_process_group_on_unix(monkeypatch, tmp_path: Path) -> None: - pid_file = tmp_path / "backend.pid" - service_manager.write_runtime_record( - pid_file, - service_manager.RuntimeRecord(pid=111, pgid=222, port=8000), - ) - console = DummyConsole() - group_alive = {"value": True} - group_signals: list[tuple[signal.Signals, int | None]] = [] - pid_signals: list[tuple[signal.Signals, list[int]]] = [] - - monkeypatch.setattr(service_manager.sys, "platform", "darwin") - monkeypatch.setattr(service_manager, "collect_process_tree_pids", lambda _pid: [111, 112]) - monkeypatch.setattr(service_manager, "port_owner_pids", lambda _port: []) - monkeypatch.setattr(service_manager, "pid_is_running", lambda _pid: False) - monkeypatch.setattr(service_manager, "process_group_is_running", lambda pgid: bool(pgid == 222 and group_alive["value"])) - - def fake_signal_group(sig, pgid): - group_signals.append((sig, pgid)) - if sig == signal.SIGTERM: - group_alive["value"] = False - - monkeypatch.setattr(service_manager, "signal_process_group", fake_signal_group) - monkeypatch.setattr( - service_manager, - "signal_pid_list", - lambda sig, pids: pid_signals.append((sig, list(pids))), - ) - - service_manager.stop_one(8000, pid_file, "后端", console) - - assert group_signals == [(signal.SIGTERM, 222)] - assert pid_signals == [] - assert not pid_file.exists() - - -def test_stop_one_falls_back_to_pid_signals_without_process_group(monkeypatch, tmp_path: Path) -> None: - pid_file = tmp_path / "backend.pid" - pid_file.write_text("111", encoding="utf-8") - console = DummyConsole() - pid_signals: list[tuple[signal.Signals, list[int]]] = [] - alive = {"value": True} - - monkeypatch.setattr(service_manager.sys, "platform", "darwin") - monkeypatch.setattr(service_manager, "collect_process_tree_pids", lambda _pid: [111, 112]) - monkeypatch.setattr(service_manager, "port_owner_pids", lambda _port: []) - monkeypatch.setattr(service_manager, "pid_is_running", lambda _pid: alive["value"]) - monkeypatch.setattr(service_manager, "process_group_is_running", lambda _pgid: False) - monkeypatch.setattr( - service_manager, - "signal_pid_list", - lambda sig, pids: ( - pid_signals.append((sig, list(pids))), - alive.__setitem__("value", False), - ), - ) - - service_manager.stop_one(8000, pid_file, "后端", console) - - assert pid_signals[0] == (signal.SIGTERM, [111, 112]) - assert not pid_file.exists() - - -def test_stop_one_uses_taskkill_on_windows(monkeypatch, tmp_path: Path) -> None: - pid_file = tmp_path / "backend.pid" - pid_file.write_text("111", encoding="utf-8") - console = DummyConsole() - commands: list[list[str]] = [] - alive = {"value": True} - - monkeypatch.setattr(service_manager.sys, "platform", "win32") - monkeypatch.setattr(service_manager, "collect_process_tree_pids", lambda _pid: [111, 222]) - monkeypatch.setattr(service_manager, "port_owner_pids", lambda _port: []) - monkeypatch.setattr(service_manager, "pid_is_running", lambda _pid: alive["value"]) - - def fake_run(args, **kwargs): - commands.append(list(args)) - alive["value"] = False - return SimpleNamespace(returncode=0) - - monkeypatch.setattr(service_manager.subprocess, "run", fake_run) - - service_manager.stop_one(8000, pid_file, "后端", console) - - assert commands == [ - ["taskkill", "/PID", "111", "/T", "/F"], - ["taskkill", "/PID", "222", "/T", "/F"], - ] - - -def test_stop_one_skips_taskkill_for_reused_windows_pid(monkeypatch, tmp_path: Path) -> None: - pid_file = tmp_path / "backend.pid" - service_manager.write_runtime_record( - pid_file, - service_manager.RuntimeRecord( - pid=111, - host="127.0.0.1", - port=8000, - command=("python.exe", "-m", "flocks.cli.main", "serve"), - ), - ) - console = DummyConsole() - - monkeypatch.setattr(service_manager.sys, "platform", "win32") - monkeypatch.setattr(service_manager, "collect_process_tree_pids", lambda _pid: [111]) - monkeypatch.setattr(service_manager, "port_owner_pids", lambda _port: []) - monkeypatch.setattr(service_manager, "pid_is_running", lambda pid: pid == 111) - monkeypatch.setattr( - service_manager, - "_windows_process_snapshot", - lambda _pid: { - "name": "svchost.exe", - "command_line": r"C:\Windows\System32\svchost.exe -k netsvcs", - "executable_path": r"C:\Windows\System32\svchost.exe", - }, - ) - monkeypatch.setattr( - service_manager.subprocess, - "run", - lambda *_args, **_kwargs: (_ for _ in ()).throw(AssertionError("taskkill should not run")), - ) - - service_manager.stop_one(8000, pid_file, "后端", console) - - assert console.messages[-1] == "[flocks] 后端 未运行。" - assert not pid_file.exists() - - -def test_stop_one_force_kill_refreshes_process_group_members(monkeypatch, tmp_path: Path) -> None: - pid_file = tmp_path / "backend.pid" - service_manager.write_runtime_record( - pid_file, - service_manager.RuntimeRecord(pid=111, pgid=222, port=8000), - ) - console = DummyConsole() - pid_signals: list[tuple[signal.Signals, list[int]]] = [] - group_signals: list[tuple[signal.Signals, int | None]] = [] - alive_group_members = {333} - - monkeypatch.setattr(service_manager.sys, "platform", "darwin") - monkeypatch.setattr(service_manager, "collect_process_tree_pids", lambda _pid: [111]) - monkeypatch.setattr(service_manager, "_process_group_member_pids", lambda pgid: [333] if pgid == 222 and alive_group_members else []) - monkeypatch.setattr(service_manager, "port_owner_pids", lambda _port: []) - monkeypatch.setattr(service_manager, "pid_is_running", lambda pid: pid in alive_group_members) - monkeypatch.setattr(service_manager, "process_group_is_running", lambda pgid: bool(pgid == 222 and alive_group_members)) - monkeypatch.setattr(service_manager.time, "sleep", lambda _delay: None) - - def fake_signal_group(sig, pgid): - group_signals.append((sig, pgid)) - def fake_signal_pid_list(sig, pids): - pid_list = list(pids) - pid_signals.append((sig, pid_list)) - if sig == signal.SIGKILL and 333 in pid_list: - alive_group_members.clear() - - monkeypatch.setattr(service_manager, "signal_process_group", fake_signal_group) - monkeypatch.setattr(service_manager, "signal_pid_list", fake_signal_pid_list) - - service_manager.stop_one(8000, pid_file, "后端", console) - - assert (signal.SIGTERM, 222) in group_signals - assert any(sig == signal.SIGKILL and 333 in pids for sig, pids in pid_signals) - assert not pid_file.exists() - assert console.messages[-1] == "[flocks] 后端 已停止。" - - -def test_stop_one_keeps_runtime_record_when_force_kill_still_times_out(monkeypatch, tmp_path: Path) -> None: - pid_file = tmp_path / "backend.pid" - service_manager.write_runtime_record( - pid_file, - service_manager.RuntimeRecord(pid=111, pgid=222, port=8000), - ) - console = DummyConsole() - - monkeypatch.setattr(service_manager.sys, "platform", "darwin") - monkeypatch.setattr(service_manager, "collect_process_tree_pids", lambda _pid: [111]) - monkeypatch.setattr(service_manager, "_process_group_member_pids", lambda pgid: [333] if pgid == 222 else []) - monkeypatch.setattr(service_manager, "port_owner_pids", lambda _port: []) - monkeypatch.setattr(service_manager, "pid_is_running", lambda _pid: False) - monkeypatch.setattr(service_manager, "process_group_is_running", lambda pgid: pgid == 222) - monkeypatch.setattr(service_manager, "signal_process_group", lambda *_args: None) - monkeypatch.setattr(service_manager, "signal_pid_list", lambda *_args: None) - monkeypatch.setattr(service_manager.time, "sleep", lambda _delay: None) - - with pytest.raises(service_manager.ServiceError, match="未在预期时间内退出"): - service_manager.stop_one(8000, pid_file, "后端", console) - - assert pid_file.exists() - - -@contextlib.contextmanager -def _record_call(call_order: list[str], name: str): - call_order.append(name) - yield - - -def test_stop_all_reads_port_from_runtime_record(monkeypatch, tmp_path: Path) -> None: - paths = service_manager.RuntimePaths( - root=tmp_path, - run_dir=tmp_path / "run", - log_dir=tmp_path / "logs", - backend_pid=tmp_path / "run" / "backend.pid", - frontend_pid=tmp_path / "run" / "webui.pid", - backend_log=tmp_path / "logs" / "backend.log", - frontend_log=tmp_path / "logs" / "webui.log", - ) - paths.run_dir.mkdir(parents=True) - service_manager.write_runtime_record(paths.backend_pid, service_manager.RuntimeRecord(pid=111, port=9995)) - service_manager.write_runtime_record(paths.frontend_pid, service_manager.RuntimeRecord(pid=222, port=9996)) - calls: list[tuple[int, Path, str]] = [] - - monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: paths) - monkeypatch.setattr(service_manager, "service_lock", lambda _paths: _record_call([], "service_lock")) - monkeypatch.setattr(service_manager, "stop_all_browser_daemons", lambda: []) - monkeypatch.setattr( - service_manager, - "stop_one", - lambda port, pid_file, name, _console: calls.append((port, pid_file, name)), - ) - - service_manager.stop_all(console=None) - - assert calls == [ - (9996, paths.frontend_pid, "WebUI"), - (9995, paths.backend_pid, "后端"), - ] - - -def test_stop_all_falls_back_to_default_port_when_record_missing(monkeypatch, tmp_path: Path) -> None: - paths = service_manager.RuntimePaths( - root=tmp_path, - run_dir=tmp_path / "run", - log_dir=tmp_path / "logs", - backend_pid=tmp_path / "run" / "backend.pid", - frontend_pid=tmp_path / "run" / "webui.pid", - backend_log=tmp_path / "logs" / "backend.log", - frontend_log=tmp_path / "logs" / "webui.log", - ) - paths.run_dir.mkdir(parents=True) - calls: list[int] = [] - - monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: paths) - monkeypatch.setattr(service_manager, "service_lock", lambda _paths: _record_call([], "service_lock")) - monkeypatch.setattr(service_manager, "stop_all_browser_daemons", lambda: []) - monkeypatch.setattr(service_manager, "stop_one", lambda port, *_args: calls.append(port)) - - service_manager.stop_all(console=None) - - assert calls == [5173, 8000] - - -def test_stop_all_falls_back_to_default_port_when_record_has_no_port(monkeypatch, tmp_path: Path) -> None: - paths = service_manager.RuntimePaths( - root=tmp_path, - run_dir=tmp_path / "run", - log_dir=tmp_path / "logs", - backend_pid=tmp_path / "run" / "backend.pid", - frontend_pid=tmp_path / "run" / "webui.pid", - backend_log=tmp_path / "logs" / "backend.log", - frontend_log=tmp_path / "logs" / "webui.log", - ) - paths.run_dir.mkdir(parents=True) - paths.backend_pid.write_text("111", encoding="utf-8") - paths.frontend_pid.write_text("222", encoding="utf-8") - calls: list[int] = [] - - monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: paths) - monkeypatch.setattr(service_manager, "service_lock", lambda _paths: _record_call([], "service_lock")) - monkeypatch.setattr(service_manager, "stop_all_browser_daemons", lambda: []) - monkeypatch.setattr(service_manager, "stop_one", lambda port, *_args: calls.append(port)) - - service_manager.stop_all(console=None) - - assert calls == [5173, 8000] - - -def test_stop_all_also_cleans_browser_daemons(monkeypatch, tmp_path: Path) -> None: - paths = service_manager.RuntimePaths( - root=tmp_path, - run_dir=tmp_path / "run", - log_dir=tmp_path / "logs", - backend_pid=tmp_path / "run" / "backend.pid", - frontend_pid=tmp_path / "run" / "webui.pid", - backend_log=tmp_path / "logs" / "backend.log", - frontend_log=tmp_path / "logs" / "webui.log", - ) - paths.run_dir.mkdir(parents=True) +def test_stop_all_uses_supervisor_control_api(monkeypatch, tmp_path: Path) -> None: + paths = _make_runtime_paths(tmp_path) calls: list[str] = [] - monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: paths) - monkeypatch.setattr(service_manager, "service_lock", lambda _paths: _record_call([], "service_lock")) - monkeypatch.setattr( - service_manager, - "stop_one", - lambda _port, _pid_file, name, _console: calls.append(name), - ) - monkeypatch.setattr( - service_manager, - "stop_all_browser_daemons", - lambda: calls.append("browser") or ["default", "remote"], - ) - class FakeConsole: def __init__(self) -> None: self.messages = [] @@ -2046,159 +1611,42 @@ def __init__(self) -> None: def print(self, message) -> None: self.messages.append(message) + states = iter([True, False]) + monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: paths) + monkeypatch.setattr(service_manager, "supervisor_is_running", lambda _paths: next(states)) + monkeypatch.setattr(service_manager, "post_control_json", lambda path, **_kwargs: calls.append(path) or {"status": "stopping"}) + console = FakeConsole() service_manager.stop_all(console=console) - assert calls == ["WebUI", "后端", "browser"] - assert console.messages == [] - - -def test_build_status_lines_reads_port_from_runtime_record(monkeypatch, tmp_path: Path) -> None: - paths = service_manager.RuntimePaths( - root=tmp_path, - run_dir=tmp_path / "run", - log_dir=tmp_path / "logs", - backend_pid=tmp_path / "run" / "backend.pid", - frontend_pid=tmp_path / "run" / "webui.pid", - backend_log=tmp_path / "logs" / "backend.log", - frontend_log=tmp_path / "logs" / "webui.log", - ) - paths.run_dir.mkdir(parents=True) - paths.log_dir.mkdir(parents=True) - service_manager.write_runtime_record(paths.backend_pid, service_manager.RuntimeRecord(pid=111, port=9995)) - service_manager.write_runtime_record(paths.frontend_pid, service_manager.RuntimeRecord(pid=222, port=9996)) - - monkeypatch.setattr(service_manager, "cleanup_stale_pid_file", lambda _path: None) - monkeypatch.setattr(service_manager, "port_owner_pids", lambda port: [port] if port in {9995, 9996} else []) - monkeypatch.setattr(service_manager, "pid_is_running", lambda _pid: False) + assert calls == ["/stop"] + assert console.messages == ["[flocks] Supervisor 已停止。"] - lines = service_manager.build_status_lines(paths) - - assert "http://127.0.0.1:9995" in lines[0] - assert "http://127.0.0.1:9996" in lines[1] - - -def test_build_status_lines_uses_recorded_host(monkeypatch, tmp_path: Path) -> None: - paths = service_manager.RuntimePaths( - root=tmp_path, - run_dir=tmp_path / "run", - log_dir=tmp_path / "logs", - backend_pid=tmp_path / "run" / "backend.pid", - frontend_pid=tmp_path / "run" / "webui.pid", - backend_log=tmp_path / "logs" / "backend.log", - frontend_log=tmp_path / "logs" / "webui.log", - ) - paths.run_dir.mkdir(parents=True) - paths.log_dir.mkdir(parents=True) - service_manager.write_runtime_record( - paths.backend_pid, - service_manager.RuntimeRecord(pid=111, host="10.0.0.8", port=9000), - ) - service_manager.write_runtime_record( - paths.frontend_pid, - service_manager.RuntimeRecord(pid=222, host="0.0.0.0", port=5174), - ) - monkeypatch.setattr(service_manager, "cleanup_stale_pid_file", lambda _path: None) - monkeypatch.setattr(service_manager, "port_owner_pids", lambda port: [111] if port == 9000 else [222]) - monkeypatch.setattr(service_manager, "pid_is_running", lambda _pid: False) +def test_stop_all_reports_when_supervisor_is_down(monkeypatch, tmp_path: Path) -> None: + paths = _make_runtime_paths(tmp_path) + console = DummyConsole() - lines = service_manager.build_status_lines(paths) + monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: paths) + monkeypatch.setattr(service_manager, "supervisor_is_running", lambda _paths: False) - assert "http://10.0.0.8:9000" in lines[0] - assert "http://127.0.0.1:5174" in lines[1] + service_manager.stop_all(console) + assert console.messages == ["[flocks] Supervisor 未运行。"] -def test_build_status_lines_uses_unknown_pid_when_bind_fallback_detects_listener( - monkeypatch, tmp_path: Path -) -> None: - paths = service_manager.RuntimePaths( - root=tmp_path, - run_dir=tmp_path / "run", - log_dir=tmp_path / "logs", - backend_pid=tmp_path / "run" / "backend.pid", - frontend_pid=tmp_path / "run" / "webui.pid", - backend_log=tmp_path / "logs" / "backend.log", - frontend_log=tmp_path / "logs" / "webui.log", - ) - paths.run_dir.mkdir(parents=True) - paths.log_dir.mkdir(parents=True) - monkeypatch.setattr(service_manager, "cleanup_stale_pid_file", lambda _path: None) - monkeypatch.setattr(service_manager, "port_owner_pids", lambda _port: []) - monkeypatch.setattr(service_manager, "port_is_in_use", lambda _port, listeners=None: True) - monkeypatch.setattr(service_manager, "pid_is_running", lambda _pid: False) - monkeypatch.setattr(service_manager, "process_group_is_running", lambda _pgid: False) +def test_status_lines_include_control_api_errors(monkeypatch, tmp_path: Path) -> None: + paths = _make_runtime_paths(tmp_path) + payload = _supervisor_status_payload() + payload["backend"]["state"] = "degraded" + payload["backend"]["last_error"] = "health failed" + monkeypatch.setattr(service_manager, "read_control_json", lambda *_args, **_kwargs: payload) lines = service_manager.build_status_lines(paths) - assert "PID=unknown" in lines[0] - assert "PID=unknown" in lines[1] - - -def test_service_lock_prevents_concurrent_operations(monkeypatch, tmp_path: Path) -> None: - paths = service_manager.RuntimePaths( - root=tmp_path, - run_dir=tmp_path / "run", - log_dir=tmp_path / "logs", - backend_pid=tmp_path / "run" / "backend.pid", - frontend_pid=tmp_path / "run" / "webui.pid", - backend_log=tmp_path / "logs" / "backend.log", - frontend_log=tmp_path / "logs" / "webui.log", - ) - state = {"locked": False} - - class FakeFcntl: - LOCK_EX = 1 - LOCK_NB = 2 - LOCK_UN = 4 - - @staticmethod - def flock(_handle, operation): - if operation == FakeFcntl.LOCK_UN: - state["locked"] = False - return - if state["locked"]: - raise OSError("busy") - state["locked"] = True - - monkeypatch.setattr(service_manager.sys, "platform", "darwin") - monkeypatch.setattr(service_manager, "fcntl", FakeFcntl) - - with service_manager.service_lock(paths): - with pytest.raises(service_manager.ServiceError, match="另一个 flocks 命令正在执行"): - with service_manager.service_lock(paths): - raise AssertionError("should not acquire nested lock") - - -def test_service_lock_releases_on_completion(monkeypatch, tmp_path: Path) -> None: - paths = service_manager.RuntimePaths( - root=tmp_path, - run_dir=tmp_path / "run", - log_dir=tmp_path / "logs", - backend_pid=tmp_path / "run" / "backend.pid", - frontend_pid=tmp_path / "run" / "webui.pid", - backend_log=tmp_path / "logs" / "backend.log", - frontend_log=tmp_path / "logs" / "webui.log", - ) - operations: list[int] = [] - - class FakeFcntl: - LOCK_EX = 1 - LOCK_NB = 2 - LOCK_UN = 4 - - @staticmethod - def flock(_handle, operation): - operations.append(operation) - - monkeypatch.setattr(service_manager.sys, "platform", "darwin") - monkeypatch.setattr(service_manager, "fcntl", FakeFcntl) - - with service_manager.service_lock(paths): - pass + assert "state=degraded" in lines[1] + assert "last_error=health failed" in lines[1] - assert operations == [FakeFcntl.LOCK_EX | FakeFcntl.LOCK_NB, FakeFcntl.LOCK_UN] def test_log_startup_config_appends_to_log_file(tmp_path: Path) -> None: diff --git a/tests/server/test_server_port_config.py b/tests/server/test_server_port_config.py index 059acca42..ca275ea6b 100644 --- a/tests/server/test_server_port_config.py +++ b/tests/server/test_server_port_config.py @@ -240,24 +240,28 @@ def fake_restart_all(config, _console): assert captured["config"].frontend_host == "127.0.0.1" assert captured["config"].frontend_port == 5273 - def test_restart_reuses_runtime_recorded_host_and_port(self, monkeypatch, tmp_path: Path): - """Test restart reuses last runtime host/port when CLI and env omit them.""" + def test_restart_reuses_supervisor_recorded_host_and_port(self, monkeypatch, tmp_path: Path): + """Test restart reuses supervisor host/port when CLI and env omit them.""" captured = {} - paths = SimpleNamespace( - backend_pid=tmp_path / "backend.pid", - frontend_pid=tmp_path / "webui.pid", - ) - records = { - paths.backend_pid: SimpleNamespace(host="0.0.0.0", port=9000), - paths.frontend_pid: SimpleNamespace(host="0.0.0.0", port=5174), - } + paths = SimpleNamespace(run_dir=tmp_path) def fake_restart_all(config, _console): captured["config"] = config monkeypatch.setattr(cli_main, "restart_all", fake_restart_all) monkeypatch.setattr(cli_main, "runtime_paths", lambda: paths) - monkeypatch.setattr(cli_main, "read_runtime_record", lambda path: records.get(path)) + monkeypatch.setattr( + cli_main, + "read_supervisor_status", + lambda **_kwargs: { + "config": { + "backend_host": "0.0.0.0", + "backend_port": 9000, + "frontend_host": "0.0.0.0", + "frontend_port": 5174, + } + }, + ) Config._global_config = None result = CliRunner().invoke(cli_main.app, ["restart"]) @@ -268,13 +272,10 @@ def fake_restart_all(config, _console): assert captured["config"].frontend_host == "0.0.0.0" assert captured["config"].frontend_port == 5174 - def test_restart_cli_options_override_runtime_record(self, monkeypatch, tmp_path: Path): - """Test explicit restart CLI options override runtime-recorded host/port.""" + def test_restart_cli_options_override_supervisor_record(self, monkeypatch, tmp_path: Path): + """Test explicit restart CLI options override supervisor host/port.""" captured = {} - paths = SimpleNamespace( - backend_pid=tmp_path / "backend.pid", - frontend_pid=tmp_path / "webui.pid", - ) + paths = SimpleNamespace(run_dir=tmp_path) def fake_restart_all(config, _console): captured["config"] = config @@ -283,11 +284,15 @@ def fake_restart_all(config, _console): monkeypatch.setattr(cli_main, "runtime_paths", lambda: paths) monkeypatch.setattr( cli_main, - "read_runtime_record", - lambda path: SimpleNamespace( - host="0.0.0.0", - port=9000 if Path(path) == paths.backend_pid else 5174, - ), + "read_supervisor_status", + lambda **_kwargs: { + "config": { + "backend_host": "0.0.0.0", + "backend_port": 9000, + "frontend_host": "0.0.0.0", + "frontend_port": 5174, + } + }, ) Config._global_config = None @@ -312,13 +317,10 @@ def fake_restart_all(config, _console): assert captured["config"].frontend_host == "127.0.0.1" assert captured["config"].frontend_port == 5273 - def test_restart_environment_overrides_runtime_record(self, monkeypatch, tmp_path: Path): - """Test restart environment variables still override runtime-recorded host/port.""" + def test_restart_environment_overrides_supervisor_record(self, monkeypatch, tmp_path: Path): + """Test restart environment variables still override supervisor host/port.""" captured = {} - paths = SimpleNamespace( - backend_pid=tmp_path / "backend.pid", - frontend_pid=tmp_path / "webui.pid", - ) + paths = SimpleNamespace(run_dir=tmp_path) def fake_restart_all(config, _console): captured["config"] = config @@ -327,11 +329,15 @@ def fake_restart_all(config, _console): monkeypatch.setattr(cli_main, "runtime_paths", lambda: paths) monkeypatch.setattr( cli_main, - "read_runtime_record", - lambda path: SimpleNamespace( - host="0.0.0.0", - port=9000 if Path(path) == paths.backend_pid else 5174, - ), + "read_supervisor_status", + lambda **_kwargs: { + "config": { + "backend_host": "0.0.0.0", + "backend_port": 9000, + "frontend_host": "0.0.0.0", + "frontend_port": 5174, + } + }, ) monkeypatch.setenv("FLOCKS_SERVER_HOST", "127.0.0.1") monkeypatch.setenv("FLOCKS_SERVER_PORT", "9101") diff --git a/tests/updater/test_restart_handoff.py b/tests/updater/test_restart_handoff.py index 6f9ceaab2..b3464a89c 100644 --- a/tests/updater/test_restart_handoff.py +++ b/tests/updater/test_restart_handoff.py @@ -16,8 +16,6 @@ def _handoff_args(tmp_path: Path, restart_argv: list[str]) -> list[str]: "127.0.0.1", "--frontend-port", "5173", - "--backend-pid-file", - str(tmp_path / "backend.pid"), "--install-root", str(tmp_path), "--uv-path", @@ -49,7 +47,7 @@ def test_run_waits_for_parent_and_backend_port_before_spawning( monkeypatch.setattr( restart_handoff, "_ensure_backend_port_free", - lambda backend_port, backend_pid_file: events.append(f"free-port:{backend_port}:{backend_pid_file.name}") or True, + lambda backend_port: events.append(f"free-port:{backend_port}") or True, ) monkeypatch.setattr( restart_handoff.subprocess, @@ -57,11 +55,6 @@ def test_run_waits_for_parent_and_backend_port_before_spawning( lambda argv, cwd=None, close_fds=False: events.append(f"spawn:{list(argv)}:{cwd}:{close_fds}") or SimpleNamespace(pid=4321), ) - monkeypatch.setattr( - restart_handoff, - "_record_backend_runtime_if_direct_serve", - lambda process, argv, **kwargs: events.append(f"record:{process.pid}:{list(argv)}:{kwargs['backend_port']}"), - ) monkeypatch.setattr(restart_handoff, "_run_upgrade_tasks", lambda args: events.append("tasks") or None) code = restart_handoff.run(_handoff_args(tmp_path, restart_argv)) @@ -69,10 +62,9 @@ def test_run_waits_for_parent_and_backend_port_before_spawning( assert code == 0 assert events[1:] == [ "wait-parent:1234", - "free-port:8000:backend.pid", + "free-port:8000", "tasks", f"spawn:{restart_argv}:{tmp_path}:True", - f"record:4321:{restart_argv}:8000", "log:restart_spawned pid=4321", ] @@ -101,7 +93,7 @@ def test_run_does_not_spawn_when_upgrade_tasks_fail(monkeypatch, tmp_path: Path) monkeypatch.setattr(restart_handoff, "_record_handoff_log", lambda message: events.append(f"log:{message}")) monkeypatch.setattr(restart_handoff, "_wait_for_parent_exit", lambda parent_pid: True) - monkeypatch.setattr(restart_handoff, "_ensure_backend_port_free", lambda backend_port, backend_pid_file: True) + monkeypatch.setattr(restart_handoff, "_ensure_backend_port_free", lambda backend_port: True) monkeypatch.setattr(restart_handoff, "_run_upgrade_tasks", lambda args: "sync failed") monkeypatch.setattr(restart_handoff, "_rollback_failed_upgrade", lambda args, error: events.append(f"rollback:{error}")) monkeypatch.setattr( @@ -132,7 +124,7 @@ def crash(_args): monkeypatch.setattr(restart_handoff, "_record_handoff_log", lambda message: events.append(f"log:{message}")) monkeypatch.setattr(restart_handoff, "_wait_for_parent_exit", lambda parent_pid: True) - monkeypatch.setattr(restart_handoff, "_ensure_backend_port_free", lambda backend_port, backend_pid_file: True) + monkeypatch.setattr(restart_handoff, "_ensure_backend_port_free", lambda backend_port: True) monkeypatch.setattr(restart_handoff, "_run_upgrade_tasks", crash) monkeypatch.setattr(restart_handoff, "_rollback_failed_upgrade", lambda args, error: events.append(f"rollback:{error}")) monkeypatch.setattr( @@ -149,10 +141,9 @@ def crash(_args): assert "spawn" not in events -def test_ensure_backend_port_free_stops_backend_after_wait_timeout(monkeypatch, tmp_path: Path) -> None: +def test_ensure_backend_port_free_waits_again_after_timeout(monkeypatch) -> None: events: list[str] = [] wait_results = iter([False, True]) - backend_pid_file = tmp_path / "backend.pid" monkeypatch.setattr(restart_handoff, "_record_handoff_log", lambda message: events.append(f"log:{message}")) monkeypatch.setattr( @@ -160,16 +151,10 @@ def test_ensure_backend_port_free_stops_backend_after_wait_timeout(monkeypatch, "_wait_for_backend_port_free", lambda port, **kwargs: events.append(f"wait:{port}:{kwargs.get('timeout_seconds')}") or next(wait_results), ) - monkeypatch.setattr( - restart_handoff.service_manager, - "stop_one", - lambda port, pid_file, name, console: events.append(f"stop:{port}:{pid_file.name}:{name}"), - ) - assert restart_handoff._ensure_backend_port_free(8000, backend_pid_file) is True + assert restart_handoff._ensure_backend_port_free(8000) is True assert events == [ "wait:8000:None", - "log:backend_port_still_in_use port=8000; stopping backend", - "stop:8000:backend.pid:backend", + "log:backend_port_still_in_use port=8000", "wait:8000:20.0", ] diff --git a/tests/updater/test_updater.py b/tests/updater/test_updater.py index 1a176274c..4b8932822 100644 --- a/tests/updater/test_updater.py +++ b/tests/updater/test_updater.py @@ -11,6 +11,7 @@ import pytest from flocks.cli import service_manager +from flocks.cli import service_control from flocks.updater import updater @@ -39,6 +40,15 @@ def _prepare_real_restart_runtime(install_root: Path) -> None: python_path.chmod(0o755) +def _webui_control_payload(state: str = "healthy", last_error: str | None = None) -> dict[str, object]: + return { + "webui": { + "state": state, + "last_error": last_error, + }, + } + + def test_run_handles_none_process_output(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: def fake_run(*args, **kwargs): return subprocess.CompletedProcess(args=args[0], returncode=0, stdout=None, stderr=None) @@ -995,36 +1005,23 @@ def test_prepare_upgrade_handover_writes_state_and_stops_frontend( tmp_path: Path, ) -> None: monkeypatch.setenv("FLOCKS_ROOT", str(tmp_path / ".flocks")) - paths = service_manager.RuntimePaths( - root=tmp_path / ".flocks", - run_dir=tmp_path / ".flocks" / "run", - log_dir=tmp_path / ".flocks" / "logs", - backend_pid=tmp_path / ".flocks" / "run" / "backend.pid", - frontend_pid=tmp_path / ".flocks" / "run" / "webui.pid", - backend_log=tmp_path / ".flocks" / "logs" / "backend.log", - frontend_log=tmp_path / ".flocks" / "logs" / "webui.log", - ) - paths.run_dir.mkdir(parents=True) - paths.log_dir.mkdir(parents=True) - calls: list[tuple[int, str]] = [] + calls: list[str] = [] monkeypatch.setattr(updater, "_current_service_config", lambda: service_manager.ServiceConfig()) monkeypatch.setattr( updater, "_start_upgrade_page_server", lambda config, version: {"upgrade_server_pid": 321, "page_dir": str(tmp_path / "page"), "page_log": str(tmp_path / "upgrade.log")}, ) - monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: paths) - monkeypatch.setattr(service_manager, "_recorded_port", lambda _pid_file, default: default) monkeypatch.setattr( - service_manager, - "stop_one", - lambda port, _pid_file, name, _console: calls.append((port, name)), + service_control, + "post_control_json", + lambda path, **_kwargs: calls.append(path) or _webui_control_payload(), ) payload = updater._prepare_upgrade_handover("2026.3.31.1") - assert calls == [(5173, "WebUI")] + assert calls == ["/stop/webui"] assert payload["upgrade_server_pid"] == 321 assert updater._read_upgrade_state()["version"] == "2026.3.31.1" @@ -1034,46 +1031,28 @@ def test_prepare_upgrade_handover_restores_frontend_when_upgrade_page_fails( tmp_path: Path, ) -> None: monkeypatch.setenv("FLOCKS_ROOT", str(tmp_path / ".flocks")) - paths = service_manager.RuntimePaths( - root=tmp_path / ".flocks", - run_dir=tmp_path / ".flocks" / "run", - log_dir=tmp_path / ".flocks" / "logs", - backend_pid=tmp_path / ".flocks" / "run" / "backend.pid", - frontend_pid=tmp_path / ".flocks" / "run" / "webui.pid", - backend_log=tmp_path / ".flocks" / "logs" / "backend.log", - frontend_log=tmp_path / ".flocks" / "logs" / "webui.log", - ) - paths.run_dir.mkdir(parents=True) - paths.log_dir.mkdir(parents=True) - - calls: list[tuple[str, bool]] = [] + calls: list[tuple[str, bool | None]] = [] monkeypatch.setattr(updater, "_current_service_config", lambda: service_manager.ServiceConfig()) - monkeypatch.setattr(service_manager, "ensure_runtime_dirs", lambda: paths) - monkeypatch.setattr(service_manager, "_recorded_port", lambda _pid_file, default: default) - monkeypatch.setattr( - service_manager, - "stop_one", - lambda port, _pid_file, name, _console: calls.append((f"stop:{name}:{port}", True)), - ) - - def fake_start_frontend(config, _console) -> None: - calls.append(("start_frontend", config.skip_frontend_build)) - monkeypatch.setattr(service_manager, "start_frontend", fake_start_frontend) monkeypatch.setattr(updater, "_stop_upgrade_page_server", lambda **kw: calls.append(("stop_page", True))) monkeypatch.setattr( updater, "_start_upgrade_page_server", lambda _config, _version: (_ for _ in ()).throw(RuntimeError("page failed")), ) + monkeypatch.setattr( + service_control, + "post_control_json", + lambda path, payload=None, **_kwargs: calls.append((path, None if payload is None else payload.get("skip_frontend_build"))) or _webui_control_payload(), + ) with pytest.raises(RuntimeError, match="page failed"): updater._prepare_upgrade_handover("2026.3.31.1") assert calls == [ - ("stop:WebUI:5173", True), + ("/stop/webui", None), ("stop_page", True), - ("start_frontend", False), + ("/restart/webui", False), ] assert updater._read_upgrade_state() is None @@ -1083,14 +1062,14 @@ def test_recover_upgrade_state_restarts_frontend_and_clears_marker( tmp_path: Path, ) -> None: monkeypatch.setenv("FLOCKS_ROOT", str(tmp_path / ".flocks")) - started: list[tuple[int, bool]] = [] + started: list[tuple[int, bool | None]] = [] stopped: list[str] = [] monkeypatch.setattr(updater, "_stop_upgrade_page_server", lambda **kw: stopped.append("stop")) monkeypatch.setattr( - service_manager, - "start_frontend", - lambda config, _console: started.append((config.frontend_port, config.skip_frontend_build)), + service_control, + "post_control_json", + lambda _path, payload=None, **_kwargs: started.append((payload["frontend_port"], payload.get("skip_frontend_build"))) or _webui_control_payload(), ) updater._write_upgrade_state( { @@ -1115,16 +1094,20 @@ def test_recover_upgrade_state_retries_frontend_with_build_when_dist_is_missing( tmp_path: Path, ) -> None: monkeypatch.setenv("FLOCKS_ROOT", str(tmp_path / ".flocks")) - starts: list[bool] = [] + starts: list[tuple[bool | None, bool | None]] = [] monkeypatch.setattr(updater, "_stop_upgrade_page_server", lambda **kw: None) - def fake_start_frontend(config, _console) -> None: - starts.append(config.skip_frontend_build) - if config.skip_frontend_build: - raise service_manager.ServiceError("missing dist") + results = iter([ + _webui_control_payload("degraded", "missing dist"), + _webui_control_payload(), + ]) + + def fake_restart_webui(_path, payload=None, **_kwargs): + starts.append((payload.get("skip_frontend_build"), payload.get("force_frontend_build"))) + return next(results) - monkeypatch.setattr(service_manager, "start_frontend", fake_start_frontend) + monkeypatch.setattr(service_control, "post_control_json", fake_restart_webui) updater._write_upgrade_state( { "version": "2026.3.31.1", @@ -1138,7 +1121,7 @@ def fake_start_frontend(config, _console) -> None: updater.recover_upgrade_state() - assert starts == [True, False] + assert starts == [(True, None), (False, True)] assert updater._read_upgrade_state() is None @@ -1147,15 +1130,15 @@ def test_recover_upgrade_state_restart_failure_clears_state_without_restarting_p tmp_path: Path, ) -> None: monkeypatch.setenv("FLOCKS_ROOT", str(tmp_path / ".flocks")) - starts: list[bool] = [] + starts: list[tuple[bool | None, bool | None]] = [] monkeypatch.setattr(updater, "_stop_upgrade_page_server", lambda **kw: None) - def fake_start_frontend(config, _console) -> None: - starts.append(config.skip_frontend_build) - raise service_manager.ServiceError("still broken") + def fake_restart_webui(_path, payload=None, **_kwargs): + starts.append((payload.get("skip_frontend_build"), payload.get("force_frontend_build"))) + return _webui_control_payload("degraded", "still broken") - monkeypatch.setattr(service_manager, "start_frontend", fake_start_frontend) + monkeypatch.setattr(service_control, "post_control_json", fake_restart_webui) updater._write_upgrade_state( { "version": "2026.3.31.1", @@ -1167,10 +1150,10 @@ def fake_start_frontend(config, _console) -> None: } ) - with pytest.raises(service_manager.ServiceError, match="still broken"): + with pytest.raises(RuntimeError, match="still broken"): updater.recover_upgrade_state() - assert starts == [True, False] + assert starts == [(True, None), (False, True)] assert updater._read_upgrade_state() is None @@ -1286,12 +1269,16 @@ def test_rollback_failed_update_restores_backup_and_rebuilds_frontend_if_needed( monkeypatch.setattr(updater, "_stop_upgrade_page_server", lambda **kw: events.append("stop_page")) monkeypatch.setattr(updater.shutil, "rmtree", lambda path, ignore_errors=True: events.append(f"rmtree:{Path(path).name}")) - def fake_start_frontend(config, _console) -> None: - events.append(f"start_frontend:{config.skip_frontend_build}") - if config.skip_frontend_build: - raise service_manager.ServiceError("missing dist") + results = iter([ + _webui_control_payload("degraded", "missing dist"), + _webui_control_payload(), + ]) + + def fake_restart_webui(_path, payload=None, **_kwargs) -> dict[str, object]: + events.append(f"restart_webui:{payload.get('skip_frontend_build')}:{payload.get('force_frontend_build')}") + return next(results) - monkeypatch.setattr(service_manager, "start_frontend", fake_start_frontend) + monkeypatch.setattr(service_control, "post_control_json", fake_restart_webui) updater._write_upgrade_state( { "version": "2026.4.1", @@ -1311,8 +1298,8 @@ def fake_start_frontend(config, _console) -> None: "restore:backup.tar.gz:install", "marker:2026.3.31", "stop_page", - "start_frontend:True", - "start_frontend:False", + "restart_webui:True:None", + "restart_webui:False:True", "rmtree:upgrade-page", ] assert updater._read_upgrade_state() is None @@ -1334,11 +1321,11 @@ def test_rollback_failed_update_clears_state_when_restore_and_frontend_both_fail monkeypatch.setattr(updater, "_stop_upgrade_page_server", lambda **kw: events.append("stop_page")) monkeypatch.setattr(updater.shutil, "rmtree", lambda path, ignore_errors=True: events.append(f"rmtree:{Path(path).name}")) - def fake_start_frontend(config, _console) -> None: - events.append(f"start_frontend:{config.skip_frontend_build}") - raise service_manager.ServiceError("frontend still broken") + def fake_restart_webui(_path, payload=None, **_kwargs) -> dict[str, object]: + events.append(f"restart_webui:{payload.get('skip_frontend_build')}") + return _webui_control_payload("degraded", "frontend still broken") - monkeypatch.setattr(service_manager, "start_frontend", fake_start_frontend) + monkeypatch.setattr(service_control, "post_control_json", fake_restart_webui) updater._write_upgrade_state( { "version": "2026.4.1", @@ -1357,7 +1344,7 @@ def fake_start_frontend(config, _console) -> None: assert events == [ "stop_page", - "start_frontend:True", + "restart_webui:True", "rmtree:upgrade-page", ] assert updater._read_upgrade_state() is None From 12d4c4f4bd8fae9fdfd8367b45b9f060e93698e6 Mon Sep 17 00:00:00 2001 From: duguwanglong Date: Wed, 1 Jul 2026 20:22:11 +0800 Subject: [PATCH 04/85] fix(contracts): support alert time filters and localized labels --- flocks/contracts/access/driver.py | 86 ++++++++++++++++++- flocks/contracts/webui/models.py | 6 ++ flocks/contracts/webui/store.py | 5 ++ tests/contracts/access/test_runtime.py | 44 +++++++++- webui/src/api/webuiContractPages.ts | 5 ++ webui/src/components/layout/Layout.tsx | 22 +++-- .../WebUIContractWorkspaceHost/index.tsx | 6 +- .../webuiContractWorkspaceSections.test.ts | 73 ++++++++++++++++ .../utils/webuiContractWorkspaceSections.ts | 45 ++++++++-- 9 files changed, 271 insertions(+), 21 deletions(-) create mode 100644 webui/src/utils/webuiContractWorkspaceSections.test.ts diff --git a/flocks/contracts/access/driver.py b/flocks/contracts/access/driver.py index c9f1a3a9e..25af7108a 100644 --- a/flocks/contracts/access/driver.py +++ b/flocks/contracts/access/driver.py @@ -66,12 +66,21 @@ def execute(self, plan: QueryPlan) -> DriverResult: table = _sqlite_identifier(options.get("table"), "records") record_column = _sqlite_identifier(options.get("recordColumn"), "record_json") date_column = _sqlite_identifier(options.get("dateColumn"), "record_date") + event_time_column = _sqlite_identifier_optional(options.get("eventTimeColumn")) query = f"SELECT {record_column} FROM {table}" + conditions: list[str] = [] query_params: list[Any] = [] start_date, end_date = JsonlDriverExecutor()._request_date_range(plan.params) if start_date and end_date and date_column: - query += f" WHERE {date_column} BETWEEN ? AND ?" + conditions.append(f"{date_column} BETWEEN ? AND ?") query_params.extend([start_date, end_date]) + start_time, end_time = JsonlDriverExecutor()._request_event_time_range(plan.params) + sql_event_time_filtered = start_time is not None and end_time is not None and bool(event_time_column) + if sql_event_time_filtered: + conditions.append(f"{event_time_column} BETWEEN ? AND ?") + query_params.extend([start_time, end_time]) + if conditions: + query += f" WHERE {' AND '.join(conditions)}" query += " ORDER BY rowid" rows: list[dict[str, Any]] = [] @@ -103,6 +112,8 @@ def execute(self, plan: QueryPlan) -> DriverResult: continue if record.get("_type") == "file_header": continue + if not sql_event_time_filtered and not self._matches_event_time_range(record, start_time, end_time): + continue total_raw += 1 if record.get("is_duplicate") is True: @@ -144,6 +155,9 @@ def _assert_allowed(self, path: Path, allowlist_roots: tuple[Path, ...]) -> None def _matches_predicates(self, record: dict[str, Any], predicates: tuple[Predicate, ...]) -> bool: return JsonlDriverExecutor()._matches_predicates(record, predicates) + def _matches_event_time_range(self, record: dict[str, Any], start_time: int | None, end_time: int | None) -> bool: + return JsonlDriverExecutor()._matches_event_time_range(record, start_time, end_time) + class JsonlDriverExecutor: def execute(self, plan: QueryPlan) -> DriverResult: @@ -154,6 +168,7 @@ def execute(self, plan: QueryPlan) -> DriverResult: duplicates = 0 filtered_unique = 0 parse_errors = 0 + start_time, end_time = self._request_event_time_range(plan.params) for path in files: self._assert_allowed(path, plan.binding.driver_allowlist_roots) for record in self._iter_records(path): @@ -162,6 +177,8 @@ def execute(self, plan: QueryPlan) -> DriverResult: continue if record.get("_type") == "file_header": continue + if not self._matches_event_time_range(record, start_time, end_time): + continue total_raw += 1 if record.get("is_duplicate") is True: @@ -243,6 +260,21 @@ def _request_date_range(self, params: dict[str, Any]) -> tuple[str, str] | tuple return from_date, to_date return None, None + def _request_event_time_range(self, params: dict[str, Any]) -> tuple[int | None, int | None]: + start_time = _epoch_seconds_from_value( + params.get("startTime") or params.get("fromTime") or params.get("eventStartTime") + ) + end_time = _epoch_seconds_from_value( + params.get("endTime") or params.get("toTime") or params.get("eventEndTime") + ) + if start_time is not None and end_time is None: + end_time = start_time + if end_time is not None and start_time is None: + start_time = end_time + if start_time is not None and end_time is not None and start_time > end_time: + start_time, end_time = end_time, start_time + return start_time, end_time + def _assert_allowed(self, path: Path, allowlist_roots: tuple[Path, ...]) -> None: resolved = path.resolve() for root in allowlist_roots: @@ -282,6 +314,18 @@ def _matches_predicates(self, record: dict[str, Any], predicates: tuple[Predicat return False return True + def _matches_event_time_range(self, record: dict[str, Any], start_time: int | None, end_time: int | None) -> bool: + if start_time is None or end_time is None: + return True + record_time = _epoch_seconds_from_value(record.get("time")) + if record_time is None: + meta = record.get("_syslog_meta") + if isinstance(meta, dict): + record_time = _epoch_seconds_from_value(meta.get("timestamp")) + if record_time is None: + return False + return start_time <= record_time <= end_time + def _data_file_date(root: Path, path: Path) -> str: try: @@ -306,6 +350,32 @@ def _date_from_value(value: Any) -> str: return "" +def _epoch_seconds_from_value(value: Any) -> int | None: + if value is None: + return None + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + seconds = float(value) + if seconds > 10_000_000_000: + seconds /= 1000 + return int(seconds) + text = str(value).strip() + if not text: + return None + try: + seconds = float(text) + except ValueError: + try: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + except ValueError: + return None + return int(parsed.timestamp()) + if seconds > 10_000_000_000: + seconds /= 1000 + return int(seconds) + + def _normalize_compare(value: Any) -> str: if isinstance(value, float) and value.is_integer(): return str(int(value)) @@ -326,3 +396,17 @@ def _sqlite_identifier(value: Any, fallback: str) -> str: admin_message=f"Invalid SQLite identifier: {text}", ) return text + + +def _sqlite_identifier_optional(value: Any) -> str: + text = str(value or "").strip() + if not text: + return "" + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", text): + raise ContractRuntimeError( + "data_source_unavailable", + status_code=400, + user_message="WebUI contract SQLite source is misconfigured.", + admin_message=f"Invalid SQLite identifier: {text}", + ) + return text diff --git a/flocks/contracts/webui/models.py b/flocks/contracts/webui/models.py index 863745fc0..912bf6a24 100644 --- a/flocks/contracts/webui/models.py +++ b/flocks/contracts/webui/models.py @@ -12,6 +12,7 @@ class WebUIPageManifest(BaseModel): id: str = Field(..., description="Stable page identifier") title: str = Field(..., description="Navigation label") + titleEn: Optional[str] = Field(None, description="English navigation label", alias="titleEn") route: str = Field(..., description="WebUI route path") icon: str = Field("LayoutDashboard", description="Lucide icon name") order: int = Field(100, description="Sort order in navigation") @@ -29,6 +30,7 @@ class WebUIWorkspaceManifest(BaseModel): id: str = Field(..., description="Stable workspace identifier") title: str = Field(..., description="Navigation label") + titleEn: Optional[str] = Field(None, description="English navigation label", alias="titleEn") icon: str = Field("LayoutDashboard", description="Lucide icon name") order: int = Field(100, description="Sort order in navigation") enabled: bool = Field(True, description="Whether workspace appears in navigation") @@ -48,6 +50,7 @@ class WebUIWorkspaceSectionManifest(BaseModel): id: str = Field(..., description="Stable section identifier") label: str = Field(..., description="Section label") + labelEn: Optional[str] = Field(None, description="English section label", alias="labelEn") pageIds: list[str] = Field( default_factory=list, description="Page ids in this section", @@ -96,6 +99,7 @@ class WebUIPageListItem(BaseModel): id: str title: str + titleEn: Optional[str] = Field(None, alias="titleEn") route: str icon: str order: int @@ -105,6 +109,7 @@ class WebUIPageListItem(BaseModel): buildStatus: str = Field("idle", alias="buildStatus") workspaceId: Optional[str] = Field(None, alias="workspaceId") workspaceTitle: Optional[str] = Field(None, alias="workspaceTitle") + workspaceTitleEn: Optional[str] = Field(None, alias="workspaceTitleEn") workspaceRoute: Optional[str] = Field(None, alias="workspaceRoute") @@ -113,6 +118,7 @@ class WebUIWorkspaceListItem(BaseModel): id: str title: str + titleEn: Optional[str] = Field(None, alias="titleEn") route: str icon: str order: int diff --git a/flocks/contracts/webui/store.py b/flocks/contracts/webui/store.py index cec1111bc..f47a2f8d9 100644 --- a/flocks/contracts/webui/store.py +++ b/flocks/contracts/webui/store.py @@ -223,6 +223,7 @@ def list_pages(self, *, enabled_only: bool = False) -> list[WebUIPageListItem]: WebUIPageListItem( id=manifest.id, title=manifest.title, + titleEn=manifest.titleEn, route=manifest.route, icon=manifest.icon, order=manifest.order, @@ -232,6 +233,7 @@ def list_pages(self, *, enabled_only: bool = False) -> list[WebUIPageListItem]: buildStatus=build.status, workspaceId=workspace.id if workspace else None, workspaceTitle=workspace.title if workspace else None, + workspaceTitleEn=workspace.titleEn if workspace else None, workspaceRoute=webui_contract_workspace_route(workspace.id) if workspace else None, ) ) @@ -268,6 +270,7 @@ def list_workspaces(self, *, enabled_only: bool = False) -> list[WebUIWorkspaceL WebUIPageListItem( id=page_manifest.id, title=page_manifest.title, + titleEn=page_manifest.titleEn, route=page_manifest.route, icon=page_manifest.icon, order=page_manifest.order, @@ -277,6 +280,7 @@ def list_workspaces(self, *, enabled_only: bool = False) -> list[WebUIWorkspaceL buildStatus=build.status, workspaceId=manifest.id, workspaceTitle=manifest.title, + workspaceTitleEn=manifest.titleEn, workspaceRoute=webui_contract_workspace_route(manifest.id), ) ) @@ -285,6 +289,7 @@ def list_workspaces(self, *, enabled_only: bool = False) -> list[WebUIWorkspaceL WebUIWorkspaceListItem( id=manifest.id, title=manifest.title, + titleEn=manifest.titleEn, route=webui_contract_workspace_route(manifest.id), icon=manifest.icon, order=manifest.order, diff --git a/tests/contracts/access/test_runtime.py b/tests/contracts/access/test_runtime.py index ed4d0fac8..4b432061b 100644 --- a/tests/contracts/access/test_runtime.py +++ b/tests/contracts/access/test_runtime.py @@ -53,16 +53,18 @@ def _write_contract_sqlite(db_path: Path, records: list[dict[str, Any]]) -> None CREATE TABLE records ( id TEXT PRIMARY KEY, record_date TEXT NOT NULL, + event_time INTEGER, record_json TEXT NOT NULL ) """ ) connection.executemany( - "INSERT INTO records (id, record_date, record_json) VALUES (?, ?, ?)", + "INSERT INTO records (id, record_date, event_time, record_json) VALUES (?, ?, ?, ?)", [ ( str(record.get("id") or index), str(record.get("record_date") or "2026-06-25"), + int(record.get("event_time") or record.get("time") or 0), json.dumps(record, ensure_ascii=False), ) for index, record in enumerate(records, start=1) @@ -308,6 +310,46 @@ def test_query_can_use_sqlite_json_driver(tmp_path: Path, monkeypatch: pytest.Mo assert response.body["items"][0]["entityId"] == "record:allowed" +def test_query_can_filter_sqlite_json_driver_by_event_time(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + store = _store(tmp_path, monkeypatch) + db_path = tmp_path / "contract_records.db" + _write_contract_sqlite( + db_path, + [ + _contract_record(id="early", time=1000), + _contract_record(id="middle", time=2000), + _contract_record(id="late", time=3000), + ], + ) + runtime = OperationRuntime( + plugins=( + _plugin( + store, + adapter_kind="builtin-sqlite-json", + source_root=db_path, + driver_options={ + "table": "records", + "recordColumn": "record_json", + "dateColumn": "record_date", + "eventTimeColumn": "event_time", + }, + ), + ), + ) + + response = runtime.execute( + page_id=PAGE_ID, + contract_id=CONTRACT_ID, + operation_name="list", + payload={"params": {"startTime": 1500, "endTime": 2500, "limit": 10}}, + principal=AuthUser(id="u1", username="alice", role="admin"), + ) + + assert response.body["summary"]["totalRaw"] == 1 + assert response.body["summary"]["filteredUnique"] == 1 + assert [item["id"] for item in response.body["items"]] == ["middle"] + + def test_query_rejects_page_supplied_binding_or_idempotency_key(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): store = _store(tmp_path, monkeypatch) _write_contract_assets(store, [_contract_record()]) diff --git a/webui/src/api/webuiContractPages.ts b/webui/src/api/webuiContractPages.ts index 3f53e8737..5cb3f3846 100644 --- a/webui/src/api/webuiContractPages.ts +++ b/webui/src/api/webuiContractPages.ts @@ -3,6 +3,7 @@ import client from './client'; export interface WebUIContractPageListItem { id: string; title: string; + titleEn?: string | null; route: string; icon: string; order: number; @@ -12,12 +13,14 @@ export interface WebUIContractPageListItem { buildStatus: 'idle' | 'building' | 'ready' | 'failed'; workspaceId?: string | null; workspaceTitle?: string | null; + workspaceTitleEn?: string | null; workspaceRoute?: string | null; } export interface WebUIContractWorkspaceSection { id: string; label: string; + labelEn?: string | null; pageIds: string[]; defaultPageId?: string | null; contentPadding?: 'comfortable' | 'none'; @@ -27,6 +30,7 @@ export interface WebUIContractWorkspaceSection { export interface WebUIContractWorkspaceListItem { id: string; title: string; + titleEn?: string | null; route: string; icon: string; order: number; @@ -40,6 +44,7 @@ export interface WebUIContractWorkspaceListItem { export interface WebUIContractPageManifest { id: string; title: string; + titleEn?: string | null; route: string; icon: string; order: number; diff --git a/webui/src/components/layout/Layout.tsx b/webui/src/components/layout/Layout.tsx index 26a43b5ed..f8ee00e56 100644 --- a/webui/src/components/layout/Layout.tsx +++ b/webui/src/components/layout/Layout.tsx @@ -52,7 +52,10 @@ import { getLocalizedReleaseNotes } from '@/utils/releaseNotes'; import { UPDATE_DISMISSED_KEY, buildUpdateDismissalKey, isUpdateDismissed } from '@/utils/updateDismissal'; import { useWebUIContractPages } from '@/hooks/useWebUIContractPages'; import { resolveWebUIContractPageIcon } from '@/utils/webuiContractPageIcons'; -import { buildWebUIContractWorkspaceSections } from '@/utils/webuiContractWorkspaceSections'; +import { + buildWebUIContractWorkspaceSections, + getLocalizedWebUIContractTitle, +} from '@/utils/webuiContractWorkspaceSections'; const UPDATE_CHECK_INTERVAL_MS = 3_600_000; const UPDATE_CHECK_MIN_GAP_MS = 600_000; @@ -429,7 +432,7 @@ export default function Layout() { const sceneWorkspaceItems = webuiContractWorkspaces .filter((workspace) => workspace.enabled && (workspace.placement === 'sceneWorkspace' || workspace.placement === 'aiWorkbench')) .map((workspace) => ({ - name: workspace.title, + name: getLocalizedWebUIContractTitle(workspace, i18n.language), href: workspace.route, icon: resolveWebUIContractPageIcon(workspace.icon), opensWorkspaceMenu: true, @@ -444,7 +447,7 @@ export default function Layout() { ...webuiContractPages .filter((page) => !page.workspaceId && page.enabled && page.placement === 'home.after' && page.buildStatus === 'ready') .map((page) => ({ - name: page.title, + name: getLocalizedWebUIContractTitle(page, i18n.language), href: page.route, icon: resolveWebUIContractPageIcon(page.icon), })), @@ -479,7 +482,7 @@ export default function Layout() { }, ]; }, - [webuiContractPages, webuiContractWorkspaces, t], + [i18n.language, webuiContractPages, webuiContractWorkspaces, t], ); const isFullScreenPage = @@ -517,9 +520,12 @@ export default function Layout() { ? resolveWebUIContractPageIcon(activeWorkspaceMenu.icon) : null; const activeWorkspaceSections = useMemo( - () => (activeWorkspaceMenu ? buildWebUIContractWorkspaceSections(activeWorkspaceMenu) : []), - [activeWorkspaceMenu], + () => (activeWorkspaceMenu ? buildWebUIContractWorkspaceSections(activeWorkspaceMenu, i18n.language) : []), + [activeWorkspaceMenu, i18n.language], ); + const activeWorkspaceMenuTitle = activeWorkspaceMenu + ? getLocalizedWebUIContractTitle(activeWorkspaceMenu, i18n.language) + : ''; const cancelWorkspaceMenuClose = useCallback(() => { if (workspaceMenuCloseTimerRef.current === null) return; @@ -866,8 +872,8 @@ export default function Layout() { {ActiveWorkspaceMenuIcon && ( )} -
- {activeWorkspaceMenu.title} +
+ {activeWorkspaceMenuTitle}