+
+ Errors 24h: {agent.errorCount24h ?? 0}
+ {agent.degraded && Callable with warning}
+
{agent.capabilities.slice(0, 5).map((cap) => {
const isMatch = filter && cap.toLowerCase().includes(filter.trim().toLowerCase())
diff --git a/lib/agent-registry.ts b/lib/agent-registry.ts
index 71ce95b2..1e1d3667 100644
--- a/lib/agent-registry.ts
+++ b/lib/agent-registry.ts
@@ -21,6 +21,8 @@ export interface AgentCapabilityManifest {
dependencies?: string[]
x402: AgentX402Manifest
status: AgentStatus
+ errorCount24h?: number
+ degraded?: boolean
endpoint: string
registeredAt: string
updatedAt: string
@@ -41,7 +43,7 @@ export interface CapabilityCount {
export type AgentRegistryChangeAction = "registered" | "updated" | "deregistered"
const DISTRICTS: DistrictId[] = ["data-center", "comm-hub", "processing", "defense", "research"]
-const STATUSES: AgentStatus[] = ["active", "idle", "working", "error", "offline"]
+const STATUSES: AgentStatus[] = ["active", "idle", "working", "error", "offline", "degraded"]
interface AgentRegistryState {
agents: Map
diff --git a/lib/agent-runtime/agent.ts b/lib/agent-runtime/agent.ts
index cbb896a5..a3d4c410 100644
--- a/lib/agent-runtime/agent.ts
+++ b/lib/agent-runtime/agent.ts
@@ -1,4 +1,5 @@
import { recordAgentHeartbeat } from "@/lib/agents/agent-health-store"
+import { getAgentHealthSummary, recordAgentExecutionError, recordAgentExecutionSuccess, recordAgentInvocation } from "@/lib/agents/agent-error-store"
import { sendAgentMessage } from "@/lib/agent-runtime/messaging"
import type { AgentConfig, AgentMetrics, AgentRuntimeContext, AgentMessage, MessageHandler, Task, TaskHandler, TaskResult } from "@/lib/agent-runtime/types"
import { publishSystemEvent } from "@/lib/events/system-events"
@@ -114,7 +115,10 @@ export class Agent implements AgentRuntimeContext {
const task = normalizeTask(taskInput)
const startedAt = isoNow()
const startedMs = Date.now()
- this.status = "working"
+ const health = getAgentHealthSummary(this.id)
+ if (health.degraded) console.warn(`[agent-health] Executing task for degraded agent ${this.id}`)
+ this.status = health.degraded ? "degraded" : "working"
+ recordAgentInvocation(this.id)
this.recordHeartbeat(task.title)
writeTaskRecord(this.id, { task, result: null, status: "running", updatedAt: startedAt })
publishSystemEvent({ type: "task.started", agentId: this.id, task: { id: task.id, title: task.title, district: task.district } })
@@ -135,6 +139,7 @@ export class Agent implements AgentRuntimeContext {
durationMs,
}
this.metrics.tasksCompleted += 1
+ recordAgentExecutionSuccess(this.id)
this.taskDurations.push(durationMs)
this.status = "idle"
this.recordHeartbeat()
@@ -155,7 +160,8 @@ export class Agent implements AgentRuntimeContext {
durationMs,
}
this.metrics.tasksFailed += 1
- this.status = "error"
+ const health = recordAgentExecutionError({ agentId: this.id, error, taskExcerpt: task.title })
+ this.status = health.degraded ? "degraded" : "error"
this.recordHeartbeat(task.title)
writeTaskRecord(this.id, { task, result, status: "failed", updatedAt: completedAt })
publishSystemEvent({ type: "task.completed", agentId: this.id, taskId: task.id, result: { summary: result.error ?? result.summary, durationMs } })
diff --git a/lib/agent-runtime/cloud-agents.ts b/lib/agent-runtime/cloud-agents.ts
index e01e1fbf..0cc291ff 100644
--- a/lib/agent-runtime/cloud-agents.ts
+++ b/lib/agent-runtime/cloud-agents.ts
@@ -2,6 +2,7 @@ import type { DistrictId, MoltbotAgent } from "@/lib/types"
import { DISTRICTS, SPRITE_COUNT } from "@/lib/data"
import { publishSystemEvent } from "@/lib/events/system-events"
import { recordAgentHeartbeat } from "@/lib/agents/agent-health-store"
+import { getAgentHealthSummary, recordAgentExecutionSuccess } from "@/lib/agents/agent-error-store"
export type CloudAgentConfig = {
id: string
@@ -69,6 +70,7 @@ export function getCloudAgentConfig(id: string): CloudAgentConfig | null {
export function updateCloudAgentResult(id: string, summary: string): CloudAgentConfig | null {
const config = configs.get(id)
if (!config) return null
+ recordAgentExecutionSuccess(id)
const updated = { ...config, lastTaskAt: new Date().toISOString(), lastResult: summary.slice(0, 240) }
configs.set(id, updated)
return updated
@@ -78,17 +80,18 @@ export function cloudConfigToAgent(config: CloudAgentConfig, index = 0): Moltbot
const district = DISTRICTS.find((d) => d.id === config.district) ?? DISTRICTS[0]
const x = district.x + 42 + (index * 34) % Math.max(80, district.w - 80)
const y = district.y + 58 + (index * 29) % Math.max(80, district.h - 80)
+ const health = getAgentHealthSummary(config.id)
return {
id: config.id,
name: config.name,
model: config.model,
deployment: "cloud",
- status: "active",
+ status: health.degraded ? "degraded" : "active",
district: config.district,
cpu: 5,
memory: 18,
tasksCompleted: config.lastTaskAt ? 1 : 0,
- currentTask: config.lastResult ?? "Waiting for cloud tasks",
+ currentTask: health.degraded ? `Degraded: ${health.errorCount24h} errors in 24h` : config.lastResult ?? "Waiting for cloud tasks",
taskProgress: 0,
color: "#38bdf8",
pixelX: x,
@@ -100,7 +103,8 @@ export function cloudConfigToAgent(config: CloudAgentConfig, index = 0): Moltbot
spriteId: (index + 5) % SPRITE_COUNT,
skills: [{ id: `${config.id}-edge`, name: "Edge Runtime", level: 1, maxLevel: 5, xp: 0, xpToNext: 100 }],
autoRestart: true,
- lastHeartbeat: config.createdAt,
+ lastHeartbeat: health.lastSeen ?? config.createdAt,
+ offlineForSeconds: health.errorCount24h,
appearance: { skin: "neon", accessories: [], customColor: null },
}
}
diff --git a/lib/agents/agent-error-store.ts b/lib/agents/agent-error-store.ts
new file mode 100644
index 00000000..558a1e38
--- /dev/null
+++ b/lib/agents/agent-error-store.ts
@@ -0,0 +1,127 @@
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"
+import { dirname, join } from "node:path"
+import { publishSystemEvent } from "@/lib/events/system-events"
+
+export interface AgentErrorLogEntry {
+ date: string
+ agentId: string
+ error: string
+ taskExcerpt: string
+}
+
+export interface AgentHealthSummary {
+ agentId: string
+ status: "active" | "degraded"
+ lastSeen: string | null
+ errorCount24h: number
+ degraded: boolean
+}
+
+const ERROR_LOG_PATH = join(process.cwd(), ".data", "agent-errors.json")
+const MAX_ERROR_LOG_ENTRIES = 1000
+const DAY_MS = 24 * 60 * 60 * 1000
+
+const globalErrors = globalThis as typeof globalThis & {
+ __openStellarAgentErrorState__?: Map
+}
+
+const state = globalErrors.__openStellarAgentErrorState__ ?? new Map()
+if (!globalErrors.__openStellarAgentErrorState__) globalErrors.__openStellarAgentErrorState__ = state
+
+function normalizeAgentId(agentId: string): string {
+ const cleanId = agentId.trim()
+ if (!cleanId) throw new Error("agentId must not be empty")
+ return cleanId.slice(0, 200)
+}
+
+function truncate(value: string, length: number): string {
+ return value.replace(/\s+/g, " ").trim().slice(0, length)
+}
+
+function readErrorLog(): AgentErrorLogEntry[] {
+ if (!existsSync(ERROR_LOG_PATH)) return []
+ try {
+ const parsed = JSON.parse(readFileSync(ERROR_LOG_PATH, "utf8"))
+ if (!Array.isArray(parsed)) return []
+ return parsed.filter((entry): entry is AgentErrorLogEntry => (
+ typeof entry?.date === "string" &&
+ typeof entry.agentId === "string" &&
+ typeof entry.error === "string" &&
+ typeof entry.taskExcerpt === "string"
+ ))
+ } catch {
+ return []
+ }
+}
+
+function writeErrorLogAtomically(entries: AgentErrorLogEntry[]): void {
+ mkdirSync(dirname(ERROR_LOG_PATH), { recursive: true })
+ const tmpPath = `${ERROR_LOG_PATH}.${process.pid}.${Date.now()}.tmp`
+ writeFileSync(tmpPath, `${JSON.stringify(entries, null, 2)}\n`, "utf8")
+ renameSync(tmpPath, ERROR_LOG_PATH)
+}
+
+function recentEntries(agentId: string, nowMs: number): AgentErrorLogEntry[] {
+ const cutoff = nowMs - DAY_MS
+ return readErrorLog().filter((entry) => entry.agentId === agentId && Date.parse(entry.date) >= cutoff)
+}
+
+export function recordAgentInvocation(agentId: string, now = new Date()): void {
+ const cleanId = normalizeAgentId(agentId)
+ const current = state.get(cleanId) ?? { lastSeen: null, degraded: false, consecutiveErrors: 0 }
+ state.set(cleanId, { ...current, lastSeen: now.toISOString() })
+}
+
+export function recordAgentExecutionSuccess(agentId: string, now = new Date()): AgentHealthSummary {
+ const cleanId = normalizeAgentId(agentId)
+ const current = state.get(cleanId) ?? { lastSeen: null, degraded: false, consecutiveErrors: 0 }
+ state.set(cleanId, { ...current, lastSeen: now.toISOString(), consecutiveErrors: 0 })
+ return getAgentHealthSummary(cleanId, now.getTime())
+}
+
+export function recordAgentExecutionError(input: { agentId: string; error: unknown; taskExcerpt?: string; date?: Date }): AgentHealthSummary {
+ const cleanId = normalizeAgentId(input.agentId)
+ const date = input.date ?? new Date()
+ const entry: AgentErrorLogEntry = {
+ date: date.toISOString(),
+ agentId: cleanId,
+ error: truncate(input.error instanceof Error ? input.error.message : String(input.error || "Unknown error"), 500),
+ taskExcerpt: truncate(input.taskExcerpt ?? "", 240),
+ }
+
+ const entries = [...readErrorLog(), entry].slice(-MAX_ERROR_LOG_ENTRIES)
+ writeErrorLogAtomically(entries)
+
+ const current = state.get(cleanId) ?? { lastSeen: null, degraded: false, consecutiveErrors: 0 }
+ const consecutiveErrors = current.consecutiveErrors + 1
+ const degraded = current.degraded || consecutiveErrors >= 3
+ state.set(cleanId, { lastSeen: entry.date, degraded, consecutiveErrors })
+
+ if (!current.degraded && degraded) {
+ console.warn(`[agent-health] ${cleanId} marked degraded after 3 consecutive failed invocations`)
+ publishSystemEvent({ type: "agent.status", agentId: cleanId, status: "degraded", occurredAt: entry.date })
+ }
+
+ return getAgentHealthSummary(cleanId, date.getTime())
+}
+
+export function getAgentErrorCount24h(agentId: string, nowMs = Date.now()): number {
+ return recentEntries(normalizeAgentId(agentId), nowMs).length
+}
+
+export function getAgentHealthSummary(agentId: string, nowMs = Date.now()): AgentHealthSummary {
+ const cleanId = normalizeAgentId(agentId)
+ const current = state.get(cleanId) ?? { lastSeen: null, degraded: false, consecutiveErrors: 0 }
+ return {
+ agentId: cleanId,
+ status: current.degraded ? "degraded" : "active",
+ lastSeen: current.lastSeen,
+ errorCount24h: getAgentErrorCount24h(cleanId, nowMs),
+ degraded: current.degraded,
+ }
+}
+
+export function resetAgentErrorStoreForTests(): void {
+ state.clear()
+ writeErrorLogAtomically([])
+}
diff --git a/lib/agents/agent-health-store.ts b/lib/agents/agent-health-store.ts
index 083dc6af..98522e27 100644
--- a/lib/agents/agent-health-store.ts
+++ b/lib/agents/agent-health-store.ts
@@ -7,7 +7,7 @@ export const HEARTBEAT_INTERVAL_MS = 15_000
export const OFFLINE_AFTER_MS = 30_000
export const ALERT_AFTER_MS = 5 * 60_000
-const VALID_AGENT_STATUSES: AgentStatus[] = ["active", "idle", "working", "error", "offline"]
+const VALID_AGENT_STATUSES: AgentStatus[] = ["active", "idle", "working", "error", "offline", "degraded"]
export interface AgentHeartbeatInput {
status?: unknown
diff --git a/lib/types.ts b/lib/types.ts
index 6c5e895a..15fe4579 100644
--- a/lib/types.ts
+++ b/lib/types.ts
@@ -1,4 +1,4 @@
-export type AgentStatus = "active" | "idle" | "working" | "error" | "offline"
+export type AgentStatus = "active" | "idle" | "working" | "error" | "offline" | "degraded"
export type DistrictId = "data-center" | "comm-hub" | "processing" | "defense" | "research"