Cloud deployment add agent health monitoring and auto restart on crash

__tests__/agent-error-store.test.ts

@@ -0,0 +1,55 @@
+import { describe, expect, it, beforeEach } from "vitest"
+import { getAgentHealthSummary, recordAgentExecutionError, recordAgentExecutionSuccess, recordAgentInvocation, resetAgentErrorStoreForTests } from "@/lib/agents/agent-error-store"
+
+describe("agent error store", () => {
+  beforeEach(() => {
+    resetAgentErrorStoreForTests()
+  })
+
+  it("tracks last invocation time and 24h error counts", () => {
+    const now = new Date("2026-06-30T12:00:00.000Z")
+    recordAgentInvocation("cloud-alpha", now)
+    recordAgentExecutionError({ agentId: "cloud-alpha", error: new Error("boom"), taskExcerpt: "run diagnostics", date: now })
+
+    expect(getAgentHealthSummary("cloud-alpha", now.getTime())).toEqual({
+      agentId: "cloud-alpha",
+      status: "active",
+      lastSeen: now.toISOString(),
+      errorCount24h: 1,
+      degraded: false,
+    })
+  })
+
+  it("marks an agent degraded after three consecutive errors", () => {
+    const base = Date.parse("2026-06-30T12:00:00.000Z")
+    for (let i = 0; i < 3; i += 1) {
+      recordAgentExecutionError({
+        agentId: "cloud-beta",
+        error: new Error(`failure ${i + 1}`),
+        taskExcerpt: "sync ledger",
+        date: new Date(base + i * 1000),
+      })
+    }
+
+    expect(getAgentHealthSummary("cloud-beta", base + 3000)).toMatchObject({
+      agentId: "cloud-beta",
+      status: "degraded",
+      errorCount24h: 3,
+      degraded: true,
+    })
+  })
+  it("does not degrade when a successful invocation breaks the error streak", () => {
+    const base = Date.parse("2026-06-30T12:00:00.000Z")
+    recordAgentExecutionError({ agentId: "cloud-gamma", error: new Error("failure 1"), date: new Date(base) })
+    recordAgentExecutionError({ agentId: "cloud-gamma", error: new Error("failure 2"), date: new Date(base + 1000) })
+    recordAgentExecutionSuccess("cloud-gamma", new Date(base + 2000))
+    recordAgentExecutionError({ agentId: "cloud-gamma", error: new Error("failure 3"), date: new Date(base + 3000) })
+
+    expect(getAgentHealthSummary("cloud-gamma", base + 4000)).toMatchObject({
+      status: "active",
+      errorCount24h: 3,
+      degraded: false,
+    })
+  })
+
+})

app/agent-functions/[id]/route.ts

@@ -1,10 +1,11 @@
 import { NextResponse } from "next/server"
 import { getCloudAgentConfig, provisionCloudAgent, updateCloudAgentResult } from "@/lib/agent-runtime/cloud-agents"
 import { recordAgentHeartbeat, HEARTBEAT_INTERVAL_MS } from "@/lib/agents/agent-health-store"
+import { getAgentHealthSummary, recordAgentExecutionError, recordAgentInvocation } from "@/lib/agents/agent-error-store"
 import { publishSystemEvent } from "@/lib/events/system-events"
 import { isAuthorized } from "@/lib/auth"
 
-export const runtime = "edge"
+export const runtime = "nodejs"
 export const dynamic = "force-dynamic"
 
 type RouteContext = { params: Promise<{ id: string }> }
@@ -48,16 +49,27 @@ export async function POST(req: Request, context: RouteContext) {
 
   const taskId = String(body.taskId || `task-${Date.now()}`)
 
-  recordAgentHeartbeat(config.id, { status: "working", cpu: 32, memory: 42, currentTask: task, autoRestart: true })
+  const health = getAgentHealthSummary(config.id)
+  if (health.degraded) console.warn(`[agent-health] Invoking degraded cloud agent ${config.id}`)
+  recordAgentInvocation(config.id)
+  recordAgentHeartbeat(config.id, { status: health.degraded ? "degraded" : "working", cpu: 32, memory: 42, currentTask: task, autoRestart: true })
   publishSystemEvent({ type: "task.started", agentId: config.id, task: { id: taskId, title: task, district: config.district } })
 
   const started = Date.now()
-  const summary = await reasonAboutTask(task, config.model)
-  updateCloudAgentResult(config.id, summary)
-  recordAgentHeartbeat(config.id, { status: "active", cpu: 8, memory: 24, currentTask: summary, autoRestart: true })
-  publishSystemEvent({ type: "task.completed", agentId: config.id, taskId, result: { summary, durationMs: Date.now() - started } })
+  try {
+    const summary = await reasonAboutTask(task, config.model)
+    updateCloudAgentResult(config.id, summary)
+    recordAgentHeartbeat(config.id, { status: getAgentHealthSummary(config.id).degraded ? "degraded" : "active", cpu: 8, memory: 24, currentTask: summary, autoRestart: true })
+    publishSystemEvent({ type: "task.completed", agentId: config.id, taskId, result: { summary, durationMs: Date.now() - started } })
 
-  return NextResponse.json({ ok: true, agentId: config.id, taskId, result: { summary } }, { headers: { "Cache-Control": "no-store" } })
+    return NextResponse.json({ ok: true, agentId: config.id, taskId, result: { summary } }, { headers: { "Cache-Control": "no-store" } })
+  } catch (error) {
+    const failure = recordAgentExecutionError({ agentId: config.id, error, taskExcerpt: task })
+    recordAgentHeartbeat(config.id, { status: failure.degraded ? "degraded" : "error", cpu: 8, memory: 24, currentTask: task, autoRestart: true })
+    const summary = error instanceof Error ? error.message : "Agent execution failed"
+    publishSystemEvent({ type: "task.completed", agentId: config.id, taskId, result: { summary, durationMs: Date.now() - started } })
+    return NextResponse.json({ ok: false, agentId: config.id, taskId, error: summary, degraded: failure.degraded }, { status: 500, headers: { "Cache-Control": "no-store" } })
+  }
 }
 
 export async function GET(req: Request, context: RouteContext) {

app/api/admin/agents/[id]/health/route.ts

@@ -0,0 +1,21 @@
+import { NextResponse } from "next/server"
+import { getAgentHealthSummary } from "@/lib/agents/agent-error-store"
+import { isAuthorized } from "@/lib/auth"
+
+interface RouteContext {
+  params: Promise<{ id: string }>
+}
+
+export const dynamic = "force-dynamic"
+
+export async function GET(req: Request, context: RouteContext) {
+  if (!isAuthorized(req)) {
+    return NextResponse.json({ ok: false, error: "Unauthorized" }, { status: 401 })
+  }
+
+  const { id } = await context.params
+  return NextResponse.json(
+    getAgentHealthSummary(decodeURIComponent(id)),
+    { headers: { "Cache-Control": "no-store" } },
+  )
+}

app/api/agent-functions/[id]/route.ts

@@ -0,0 +1 @@
+export { GET, POST, dynamic, runtime } from "@/app/agent-functions/[id]/route"

app/api/registry/route.ts

@@ -1,12 +1,21 @@
 import { NextResponse } from "next/server"
 import { listRegisteredAgents } from "@/lib/agent-registry"
+import { getAgentHealthSummary } from "@/lib/agents/agent-error-store"
 
 export const dynamic = "force-dynamic"
 
 export async function GET(req: Request) {
   const url = new URL(req.url)
   const agents = listRegisteredAgents({
     capability: url.searchParams.get("capability") ?? undefined,
+  }).map((agent) => {
+    const health = getAgentHealthSummary(agent.agentId)
+    return {
+      ...agent,
+      status: health.degraded ? "degraded" as const : agent.status,
+      errorCount24h: health.errorCount24h,
+      degraded: health.degraded,
+    }
   })
   return NextResponse.json(
     { ok: true, agents },

app/registry/page.tsx

@@ -138,13 +138,23 @@
                           agent.status === "active" ? "bg-emerald-400" :
                           agent.status === "working" ? "bg-cyan-400" :
                           agent.status === "idle" ? "bg-amber-400" :
+                          agent.status === "degraded" ? "bg-rose-400" :
                           "bg-slate-600"
                         }`} />
                         <span className="font-mono text-xs text-slate-400">{agent.status}</span>
+                        {agent.degraded && (
+                          <Badge variant="outline" className="border-rose-500/60 bg-rose-950/40 px-1.5 py-0 text-[10px] uppercase text-rose-300">
+                            Degraded
+                          </Badge>
+                        )}
                         <span className="ml-auto font-mono text-xs text-slate-500">{agent.district}</span>
                       </div>
                     </CardHeader>
                     <CardContent className="pt-0">
+                      <div className="mb-3 rounded-md border border-slate-800 bg-slate-900/60 px-2 py-1 font-mono text-xs text-slate-400">
+                        Errors 24h: <span className={agent.degraded ? "text-rose-300" : "text-slate-200"}>{agent.errorCount24h ?? 0}</span>
+                        {agent.degraded && <span className="ml-2 text-rose-300">Callable with warning</span>}
+                      </div>
                       <div className="flex flex-wrap gap-1">
                         {agent.capabilities.slice(0, 5).map((cap) => {
                           const isMatch = filter && cap.toLowerCase().includes(filter.trim().toLowerCase())

lib/agent-registry.ts

@@ -21,6 +21,8 @@ export interface AgentCapabilityManifest {
   dependencies?: string[]
   x402: AgentX402Manifest
   status: AgentStatus
+  errorCount24h?: number
+  degraded?: boolean
   endpoint: string
   registeredAt: string
   updatedAt: string
@@ -41,7 +43,7 @@ export interface CapabilityCount {
 export type AgentRegistryChangeAction = "registered" | "updated" | "deregistered"
 
 const DISTRICTS: DistrictId[] = ["data-center", "comm-hub", "processing", "defense", "research"]
-const STATUSES: AgentStatus[] = ["active", "idle", "working", "error", "offline"]
+const STATUSES: AgentStatus[] = ["active", "idle", "working", "error", "offline", "degraded"]
 
 interface AgentRegistryState {
   agents: Map<string, AgentCapabilityManifest>

lib/agent-runtime/agent.ts

@@ -1,4 +1,5 @@
 import { recordAgentHeartbeat } from "@/lib/agents/agent-health-store"
+import { getAgentHealthSummary, recordAgentExecutionError, recordAgentExecutionSuccess, recordAgentInvocation } from "@/lib/agents/agent-error-store"
 import { sendAgentMessage } from "@/lib/agent-runtime/messaging"
 import type { AgentConfig, AgentMetrics, AgentRuntimeContext, AgentMessage, MessageHandler, Task, TaskHandler, TaskResult } from "@/lib/agent-runtime/types"
 import { publishSystemEvent } from "@/lib/events/system-events"
@@ -114,7 +115,10 @@ export class Agent implements AgentRuntimeContext {
     const task = normalizeTask(taskInput)
     const startedAt = isoNow()
     const startedMs = Date.now()
-    this.status = "working"
+    const health = getAgentHealthSummary(this.id)
+    if (health.degraded) console.warn(`[agent-health] Executing task for degraded agent ${this.id}`)
+    this.status = health.degraded ? "degraded" : "working"
+    recordAgentInvocation(this.id)
     this.recordHeartbeat(task.title)
     writeTaskRecord(this.id, { task, result: null, status: "running", updatedAt: startedAt })
     publishSystemEvent({ type: "task.started", agentId: this.id, task: { id: task.id, title: task.title, district: task.district } })
@@ -135,6 +139,7 @@ export class Agent implements AgentRuntimeContext {
         durationMs,
       }
       this.metrics.tasksCompleted += 1
+      recordAgentExecutionSuccess(this.id)
       this.taskDurations.push(durationMs)
       this.status = "idle"
       this.recordHeartbeat()
@@ -155,7 +160,8 @@ export class Agent implements AgentRuntimeContext {
         durationMs,
       }
       this.metrics.tasksFailed += 1
-      this.status = "error"
+      const health = recordAgentExecutionError({ agentId: this.id, error, taskExcerpt: task.title })
+      this.status = health.degraded ? "degraded" : "error"
       this.recordHeartbeat(task.title)
       writeTaskRecord(this.id, { task, result, status: "failed", updatedAt: completedAt })
       publishSystemEvent({ type: "task.completed", agentId: this.id, taskId: task.id, result: { summary: result.error ?? result.summary, durationMs } })

lib/agent-runtime/cloud-agents.ts

@@ -2,6 +2,7 @@ import type { DistrictId, MoltbotAgent } from "@/lib/types"
 import { DISTRICTS, SPRITE_COUNT } from "@/lib/data"
 import { publishSystemEvent } from "@/lib/events/system-events"
 import { recordAgentHeartbeat } from "@/lib/agents/agent-health-store"
+import { getAgentHealthSummary, recordAgentExecutionSuccess } from "@/lib/agents/agent-error-store"
 
 export type CloudAgentConfig = {
   id: string
@@ -69,6 +70,7 @@ export function getCloudAgentConfig(id: string): CloudAgentConfig | null {
 export function updateCloudAgentResult(id: string, summary: string): CloudAgentConfig | null {
   const config = configs.get(id)
   if (!config) return null
+  recordAgentExecutionSuccess(id)
   const updated = { ...config, lastTaskAt: new Date().toISOString(), lastResult: summary.slice(0, 240) }
   configs.set(id, updated)
   return updated
@@ -78,17 +80,18 @@ export function cloudConfigToAgent(config: CloudAgentConfig, index = 0): Moltbot
   const district = DISTRICTS.find((d) => d.id === config.district) ?? DISTRICTS[0]
   const x = district.x + 42 + (index * 34) % Math.max(80, district.w - 80)
   const y = district.y + 58 + (index * 29) % Math.max(80, district.h - 80)
+  const health = getAgentHealthSummary(config.id)
   return {
     id: config.id,
     name: config.name,
     model: config.model,
     deployment: "cloud",
-    status: "active",
+    status: health.degraded ? "degraded" : "active",
     district: config.district,
     cpu: 5,
     memory: 18,
     tasksCompleted: config.lastTaskAt ? 1 : 0,
-    currentTask: config.lastResult ?? "Waiting for cloud tasks",
+    currentTask: health.degraded ? `Degraded: ${health.errorCount24h} errors in 24h` : config.lastResult ?? "Waiting for cloud tasks",
     taskProgress: 0,
     color: "#38bdf8",
     pixelX: x,
@@ -100,7 +103,8 @@ export function cloudConfigToAgent(config: CloudAgentConfig, index = 0): Moltbot
     spriteId: (index + 5) % SPRITE_COUNT,
     skills: [{ id: `${config.id}-edge`, name: "Edge Runtime", level: 1, maxLevel: 5, xp: 0, xpToNext: 100 }],
     autoRestart: true,
-    lastHeartbeat: config.createdAt,
+    lastHeartbeat: health.lastSeen ?? config.createdAt,
+    offlineForSeconds: health.errorCount24h,
     appearance: { skin: "neon", accessories: [], customColor: null },
   }
 }