FEAT-160: Add gateway server liveness detection and auto-recovery

apps/desktop/package.json

@@ -1,6 +1,6 @@
 {
   "name": "desktop",
-  "version": "0.8.6",
+  "version": "0.8.7",
   "description": "ClosedLoop Desktop",
   "author": "ClosedLoop AI <support@closedloop.ai>",
   "private": true,

apps/desktop/src/main/app.ts

@@ -38,6 +38,7 @@ import type { GatewayApprovalRequest, GatewayApprovalResult } from "../server/ro
 import { normalizeAndValidateOrigin, normalizeWebAppOrigin } from "./origin-policy.js";
 import { LocalSessionStore } from "./local-session-store.js";
 import { enrichJobSnapshot } from "../server/operations/symphony-job-snapshot.js";
+import { GatewayRecoveryManager } from "./gateway-recovery.js";
 import pkg from "electron-updater";
 const { autoUpdater } = pkg;
 import { BUILD_COMMIT_HASH } from "../shared/build-info.js";
@@ -57,6 +58,7 @@ export class DesktopApplication {
   private readonly activityLog: ActivityLogStore;
   private readonly approvalStore: ApprovalStore;
   private readonly jobStore: JobStore;
+  private readonly recovery: GatewayRecoveryManager;
   private readonly gatewayAuthToken: string;
   private readonly sessionStore: LocalSessionStore;
   private shuttingDown = false;
@@ -108,7 +110,8 @@ export class DesktopApplication {
       () => this.settingsStore.getApiOrigin(),
       () => this.settingsStore.getWebAppOrigin(),
       this.isProdOriginsOnly(),
-      this.jobStore
+      this.jobStore,
+      () => this.recovery.onUnexpectedClose()
     );
     this.commandExecutor = new CloudCommandExecutor({
       getGatewayPort: () => this.server.getActivePort(),
@@ -118,7 +121,8 @@ export class DesktopApplication {
       sendCommandEvent: (event) => this.cloudSocket.sendCommandEvent(event),
       onQueueStatsChange: (stats) => {
         const presenceState =
-          this.cloudStatus.state === "online" && !this.cloudCommandsPaused ? "online" : "degraded";
+          this.cloudStatus.state === "online" && !this.cloudCommandsPaused && this.recovery.gatewayHealthy
+            ? "online" : "degraded";
         this.cloudSocket.sendPresence({
           state: presenceState,
           ...(this.cloudCommandsPaused ? { error: "cloud commands paused by user" } : {}),
@@ -182,6 +186,25 @@ export class DesktopApplication {
         this.commandExecutor.acknowledge(event);
       }
     });
+    this.recovery = new GatewayRecoveryManager({
+      probe: () => this.probeGatewayAlive(),
+      restart: () => this.server.restart(),
+      getCloudStatus: () => this.cloudStatus,
+      setConnected: (connected) => this.commandExecutor.setConnected(connected),
+      sendPresence: (state, error) => {
+        const stats = this.commandExecutor.getStats();
+        this.cloudSocket.sendPresence({
+          state,
+          ...(error ? { error } : {}),
+          activeCommands: stats.activeCommands,
+          queueDepth: stats.queueDepth
+        });
+      },
+      refreshTray: (detail) => this.refreshTrayState(detail),
+      log: (level, msg) => gatewayLog[level]("gateway-recovery", msg),
+      isShuttingDown: () => this.shuttingDown,
+      isPaused: () => this.cloudCommandsPaused,
+    });
     this.registerIpcHandlers();
   }
 
@@ -288,6 +311,19 @@ export class DesktopApplication {
     this.tray.dispose();
   }
 
+  private async probeGatewayAlive(): Promise<boolean> {
+    if (!this.server.isAlive()) return false;
+    try {
+      const response = await fetch(
+        `http://127.0.0.1:${this.server.getActivePort()}/health`,
+        { signal: AbortSignal.timeout(2000) }
+      );
+      return response.ok;
+    } catch {
+      return false;
+    }
+  }
+
   private onCloudSocketStatus(status: CloudSocketStatus): void {
     if (!this.cloudConnectionEnabled) {
       this.cloudStatus = { state: "degraded", error: "Cloud connection disabled by user" };
@@ -296,29 +332,23 @@ export class DesktopApplication {
     }
 
     this.cloudStatus = status;
-    const stats = this.commandExecutor.getStats();
-
-    this.commandExecutor.setConnected(status.state === "online");
 
     if (status.state === "online") {
-      this.cloudSocket.sendPresence({
-        state: this.cloudCommandsPaused ? "degraded" : "online",
-        ...(this.cloudCommandsPaused ? { error: "cloud commands paused by user" } : {}),
-        activeCommands: stats.activeCommands,
-        queueDepth: stats.queueDepth
-      });
-      this.refreshTrayState(`Serving on localhost:${this.server.getActivePort()} | cloud: online (${status.targetId})`);
+      void this.recovery.onCloudOnline();
       return;
     }
 
+    this.commandExecutor.setConnected(false);
+
     if (status.state === "degraded") {
       this.cloudSocket.sendPresence({
         state: "degraded",
         error: status.error,
-        activeCommands: stats.activeCommands,
-        queueDepth: stats.queueDepth
+        ...this.commandExecutor.getStats()
       });
-      this.refreshTrayState(`Serving on localhost:${this.server.getActivePort()} | cloud degraded: ${status.error}`);
+      this.refreshTrayState(
+        `Serving on localhost:${this.server.getActivePort()} | cloud degraded: ${status.error}`
+      );
       return;
     }
 
@@ -332,8 +362,11 @@ export class DesktopApplication {
     this.refreshTrayState(paused ? "Gateway paused from tray/menu" : undefined);
 
     const stats = this.commandExecutor.getStats();
+    const presenceState =
+      this.cloudStatus.state === "online" && !paused && this.recovery.gatewayHealthy
+        ? "online" : "degraded";
     this.cloudSocket.sendPresence({
-      state: this.cloudStatus.state === "online" && !paused ? "online" : "degraded",
+      state: presenceState,
       ...(paused ? { error: "cloud commands paused by user" } : {}),
       activeCommands: stats.activeCommands,
       queueDepth: stats.queueDepth
@@ -480,6 +513,14 @@ export class DesktopApplication {
   }
 
   private refreshTrayState(explicitDetails?: string): void {
+    if (!this.recovery.gatewayHealthy) {
+      this.tray.setState(
+        "error",
+        explicitDetails ?? `Gateway down on port ${this.server.getActivePort()}`
+      );
+      return;
+    }
+
     if (this.cloudCommandsPaused) {
       this.tray.setState(
         "degraded",
@@ -818,7 +859,9 @@ export class DesktopApplication {
       apiOrigin: this.settingsStore.getApiOrigin(),
       sandboxBaseDirectory: this.settingsStore.getSandboxBaseDirectory(),
       commandsPaused: this.cloudCommandsPaused,
-      connectionEnabled: this.cloudConnectionEnabled
+      connectionEnabled: this.cloudConnectionEnabled,
+      serverAlive: this.server.isAlive(),
+      gatewayHealthy: this.recovery.gatewayHealthy,
     }));
     ipcMain.handle("desktop:list-running-jobs", async () => {
       const jobs = this.jobStore.listRunning();

apps/desktop/src/main/gateway-recovery.ts

@@ -0,0 +1,81 @@
+export interface GatewayRecoveryDeps {
+  probe: () => Promise<boolean>;
+  restart: () => Promise<void>;
+  getCloudStatus: () => { state: string };
+  setConnected: (connected: boolean) => void;
+  sendPresence: (state: "online" | "degraded", error?: string) => void;
+  refreshTray: (detail?: string) => void;
+  log: (level: "info" | "warn" | "error", msg: string) => void;
+  isShuttingDown: () => boolean;
+  isPaused: () => boolean;
+}
+
+export class GatewayRecoveryManager {
+  private epoch = 0;
+  private recoveryInFlight: Promise<void> | null = null;
+  gatewayHealthy = true;
+
+  constructor(private readonly deps: GatewayRecoveryDeps) {}
+
+  recoverGateway(reason: string): Promise<void> {
+    if (this.recoveryInFlight) {
+      this.deps.log("warn", `Recovery already in flight, deduplicating: ${reason}`);
+      return this.recoveryInFlight;
+    }
+    this.epoch++;
+    this.gatewayHealthy = false;
+    this.deps.setConnected(false);
+    this.deps.sendPresence("degraded", `gateway recovering: ${reason}`);
+
+    this.recoveryInFlight = this.doRecover(reason).finally(() => {
+      this.recoveryInFlight = null;
+    });
+    return this.recoveryInFlight;
+  }
+
+  private async doRecover(reason: string): Promise<void> {
+    this.deps.log("warn", `Gateway recovery started: ${reason}`);
+    try {
+      await this.deps.restart();
+      this.gatewayHealthy = true;
+      this.deps.log("info", "Gateway recovered");
+      if (this.deps.getCloudStatus().state === "online") {
+        this.deps.setConnected(true);
+        this.deps.sendPresence(
+          this.deps.isPaused() ? "degraded" : "online",
+          this.deps.isPaused() ? "cloud commands paused by user" : undefined
+        );
+      }
+      this.deps.refreshTray();
+    } catch (error) {
+      const msg = error instanceof Error ? error.message : "unknown error";
+      this.deps.log("error", `Gateway recovery failed: ${msg}`);
+      this.deps.refreshTray(`Gateway down -- restart failed: ${msg}`);
+    }
+  }
+
+  async onCloudOnline(): Promise<void> {
+    const epoch = ++this.epoch;
+    const alive = await this.deps.probe();
+
+    if (this.epoch !== epoch || this.deps.getCloudStatus().state !== "online") return;
+
+    if (!alive) {
+      await this.recoverGateway("liveness probe failed on cloud reconnect");
+      return;
+    }
+
+    this.gatewayHealthy = true;
+    this.deps.setConnected(true);
+    this.deps.sendPresence(
+      this.deps.isPaused() ? "degraded" : "online",
+      this.deps.isPaused() ? "cloud commands paused by user" : undefined
+    );
+    this.deps.refreshTray();
+  }
+
+  onUnexpectedClose(): void {
+    if (this.deps.isShuttingDown()) return;
+    void this.recoverGateway("unexpected server close");
+  }
+}

apps/desktop/src/server/server.ts

@@ -8,6 +8,7 @@ import {
   type ComputeTargetCapabilities,
   type HealthResponse
 } from "../shared/contracts.js";
+import { gatewayLog } from "../main/gateway-logger.js";
 import type { LocalSessionStore } from "../main/local-session-store.js";
 import type { JobStore } from "../main/job-store.js";
 import {
@@ -40,12 +41,14 @@ export interface DesktopGatewayServerOptions {
   getApiOrigin?: () => string;
   prodOriginsOnly?: boolean;
   jobStore?: JobStore;
+  onUnexpectedClose?: () => void;
 }
 
 export class DesktopGatewayServer {
   private readonly options: DesktopGatewayServerOptions;
   private readonly router: GatewayRouter;
   private server: Server | null = null;
+  private alive = false;
   private activePort: number;
 
   constructor(options: DesktopGatewayServerOptions) {
@@ -93,7 +96,8 @@ export class DesktopGatewayServer {
     getApiOrigin?: () => string,
     getWebAppOrigin?: () => string,
     prodOriginsOnly?: boolean,
-    jobStore?: JobStore
+    jobStore?: JobStore,
+    onUnexpectedClose?: () => void
   ): DesktopGatewayServer {
     return new DesktopGatewayServer({
       host: "127.0.0.1",
@@ -115,6 +119,7 @@ export class DesktopGatewayServer {
       getApiOrigin,
       prodOriginsOnly,
       jobStore,
+      onUnexpectedClose,
     });
   }
 
@@ -152,7 +157,20 @@ export class DesktopGatewayServer {
       try {
         await this.listen(candidateServer, candidate);
         this.server = candidateServer;
-        this.activePort = candidate;
+        this.alive = true;
+        const addr = candidateServer.address();
+        this.activePort = typeof addr === "object" && addr ? addr.port : candidate;
+        candidateServer.on("error", (err) => {
+          gatewayLog.error("gateway-server", `Server error: ${err.message}`);
+        });
+        candidateServer.on("close", () => {
+          if (this.server === candidateServer) {
+            this.alive = false;
+            this.server = null;
+            gatewayLog.warn("gateway-server", "Server closed unexpectedly");
+            this.options.onUnexpectedClose?.();
+          }
+        });
         await this.writeDiscoveryFile();
         return;
       } catch (error) {
@@ -173,6 +191,7 @@ export class DesktopGatewayServer {
   }
 
   async stop(): Promise<void> {
+    this.alive = false;
     if (!this.server) {
       return;
     }
@@ -181,7 +200,7 @@ export class DesktopGatewayServer {
     this.server = null;
     await new Promise<void>((resolve, reject) => {
       runningServer.close((error) => {
-        if (error) {
+        if (error && (error as NodeJS.ErrnoException).code !== "ERR_SERVER_NOT_RUNNING") {
           reject(error);
           return;
         }
@@ -190,6 +209,15 @@ export class DesktopGatewayServer {
     });
   }
 
+  isAlive(): boolean {
+    return this.alive && this.server !== null && this.server.listening;
+  }
+
+  async restart(): Promise<void> {
+    await this.stop();
+    await this.start();
+  }
+
   private async listen(server: Server, port: number): Promise<void> {
     await new Promise<void>((resolve, reject) => {
       server.once("error", reject);

apps/desktop/test/gateway-auth.test.ts

@@ -7,7 +7,7 @@ import path from "node:path";
 import { afterEach, test } from "node:test";
 import { DesktopGatewayServer } from "../src/server/server.js";
 import { LocalSessionStore } from "../src/main/local-session-store.js";
-import { EMPTY_CAPABILITIES, PORT_PROBE_ORDER } from "../src/shared/contracts.js";
+import { EMPTY_CAPABILITIES } from "../src/shared/contracts.js";
 
 const serversToClose: DesktopGatewayServer[] = [];
 const fakeApiServersToClose: http.Server[] = [];
@@ -42,8 +42,8 @@ function makeServer(
 ): DesktopGatewayServer {
   const server = new DesktopGatewayServer({
     host: "127.0.0.1",
-    preferredPort: PORT_PROBE_ORDER[0],
-    fallbackPorts: PORT_PROBE_ORDER.slice(1),
+    preferredPort: 0,
+    fallbackPorts: [0],
     webAppOrigin: "https://app.test.com",
     getAllowedDirectories: () => [tmpDir],
     getGatewayAuthToken: () => "test-gateway-token-hex",
@@ -316,8 +316,8 @@ test("exchange handler passes REST API origin (getApiOrigin) to verifyChallenge,
 
   const server = new DesktopGatewayServer({
     host: "127.0.0.1",
-    preferredPort: PORT_PROBE_ORDER[0],
-    fallbackPorts: PORT_PROBE_ORDER.slice(1),
+    preferredPort: 0,
+    fallbackPorts: [0],
     webAppOrigin: "https://app.test.com",
     getAllowedDirectories: () => [tmpDir],
     getGatewayAuthToken: () => "test-gateway-token-hex",