feat(loops): change a live loop retry budget in place

openapi/loops.json

@@ -3801,6 +3801,11 @@
             "type": "string",
             "nullable": true
           },
+          "maxAttempts": {
+            "type": "integer",
+            "minimum": 1,
+            "description": "Retry budget for this loop: the scheduler admits a retry while attempt < maxAttempts, so 1 means a single transient failure retires the loop with no retry. Omit to leave the current budget unchanged."
+          },
           "labels": {
             "type": "array",
             "items": {

src/api/index.test.ts

@@ -1579,6 +1579,56 @@ describe("loops-api foundation", () => {
     }
   });
 
+  test("PATCH updates maxAttempts in place and rejects a non-integer budget with a stable 422", async () => {
+    const mod = await import("./index.js");
+    const storage = createSqliteLoopStorage(":memory:");
+    const server = createTestServer(mod, { host: "127.0.0.1", port: 0, storage });
+
+    try {
+      const loop = await storage.createLoop({
+        name: "api-retry-budget",
+        schedule: { type: "once", at: "2027-01-01T00:00:00Z" },
+        target: { type: "command", command: "true" },
+      }, new Date("2026-01-01T00:00:00Z"));
+      expect(loop.maxAttempts).toBe(1);
+
+      const ok = await fetch(apiUrl(server, `/v1/loops/${loop.id}`), {
+        method: "PATCH",
+        headers: jsonHeaders,
+        body: JSON.stringify({ maxAttempts: 3 }),
+      });
+      expect(ok.status).toBe(200);
+      expect((await ok.json()).loop.maxAttempts).toBe(3);
+      expect((await storage.getLoop(loop.id))?.maxAttempts).toBe(3);
+      // The schedule must survive a retry-budget-only PATCH.
+      expect((await storage.getLoop(loop.id))?.nextRunAt).toBe(loop.nextRunAt);
+
+      const before = await storage.getLoop(loop.id);
+      for (const maxAttempts of [0, -1, 1.5, "2", null, {}]) {
+        const response = await fetch(apiUrl(server, `/v1/loops/${loop.id}`), {
+          method: "PATCH",
+          headers: jsonHeaders,
+          body: JSON.stringify({ maxAttempts, labels: ["mutated"] }),
+        });
+        expect(response.status).toBe(422);
+        expect(await response.json()).toEqual({ ok: false, error: "invalid_max_attempts" });
+        expect(await storage.getLoop(loop.id)).toEqual(before);
+      }
+
+      // A PATCH that omits maxAttempts must not reset the budget.
+      const other = await fetch(apiUrl(server, `/v1/loops/${loop.id}`), {
+        method: "PATCH",
+        headers: jsonHeaders,
+        body: JSON.stringify({ status: "paused" }),
+      });
+      expect(other.status).toBe(200);
+      expect((await storage.getLoop(loop.id))?.maxAttempts).toBe(3);
+    } finally {
+      server.stop(true);
+      await storage.close();
+    }
+  });
+
   test("PATCH rejects every invalid loop status atomically with a stable 422", async () => {
     const mod = await import("./index.js");
     const storage = createSqliteLoopStorage(":memory:");

src/api/index.ts

@@ -63,7 +63,7 @@ import {
   type CircuitBreakerThreshold,
 } from "../lib/advancement.js";
 import { normalizeLoopLabels } from "../lib/labels.js";
-import { isLoopStatus, LOOP_STATUSES } from "../lib/loop-status.js";
+import { isLoopStatus, isMaxAttempts, LOOP_STATUSES } from "../lib/loop-status.js";
 import { normalizeRunCompletion } from "../lib/run-completion.js";
 import { scrubSecretsDeep } from "../lib/redact.js";
 import type { LoopStorageContract } from "../lib/storage/contract.js";
@@ -486,17 +486,22 @@ async function handleLoopsRequest(ctx: V1RequestContext, segments: string[]): Pr
       nextRunAt: string | null;
       retryScheduledFor: string | null;
       expiresAt: string | null;
+      maxAttempts: unknown;
     }>;
     // Only forward keys the caller actually sent. Store.updateLoop merges
     // {...current, ...patch}, so a present-but-undefined key overrides the
     // current value: emitting all four keys unconditionally wiped omitted
     // schedule fields (and set status=NULL -> NOT NULL 500). A key set to
     // JSON null is an explicit clear (mapped to undefined -> merged to null).
-    const patch: Partial<{ status: LoopStatus; labels: string[]; nextRunAt: string; retryScheduledFor: string; expiresAt: string }> = {};
+    const patch: Partial<{ status: LoopStatus; labels: string[]; nextRunAt: string; retryScheduledFor: string; expiresAt: string; maxAttempts: number }> = {};
     if ("status" in body) {
       if (!isLoopStatus(body.status)) throw apiError("invalid_loop_status", 422);
       patch.status = body.status;
     }
+    if ("maxAttempts" in body) {
+      if (!isMaxAttempts(body.maxAttempts)) throw apiError("invalid_max_attempts", 422);
+      patch.maxAttempts = body.maxAttempts;
+    }
     if ("labels" in body) patch.labels = normalizedLabels(body.labels);
     if ("nextRunAt" in body) patch.nextRunAt = body.nextRunAt === null ? undefined : body.nextRunAt;
     if ("retryScheduledFor" in body) patch.retryScheduledFor = body.retryScheduledFor === null ? undefined : body.retryScheduledFor;

src/cli/index.test.ts

@@ -1140,6 +1140,58 @@ describe("loops CLI", () => {
     expect(oldName.status).not.toBe(0);
   });
 
+  test("set-max-attempts changes only the retry budget and writes a backup", () => {
+    const dataDir = freshDataDir("loops-cli-max-attempts-");
+    const create = runCli(dataDir, ["--json", "create", "command", "retry-budget-loop", "--at", futureAt(), "--cmd", "true"]);
+    expect(create.status).toBe(0);
+    const created = JSON.parse(create.stdout);
+    expect(created.maxAttempts).toBe(1);
+
+    const set = runCli(dataDir, ["--json", "set-max-attempts", created.id, "3"]);
+
+    expect(set.status).toBe(0);
+    const value = JSON.parse(set.stdout);
+    expect(value).toMatchObject({
+      changed: true,
+      id: created.id,
+      previousMaxAttempts: 1,
+      maxAttempts: 3,
+    });
+    expect(value.backupPath).toContain(join(dataDir, "backups"));
+    expect(existsSync(value.backupPath)).toBe(true);
+
+    // Read it back through a separate process: the loop keeps its id, name,
+    // and schedule, which delete-and-recreate would not have.
+    const after = runCli(dataDir, ["--json", "show", created.id]);
+    expect(after.status).toBe(0);
+    const loop = JSON.parse(after.stdout);
+    expect(loop.id).toBe(created.id);
+    expect(loop.name).toBe("retry-budget-loop");
+    expect(loop.maxAttempts).toBe(3);
+    expect(loop.schedule).toEqual(created.schedule);
+  });
+
+  test("set-max-attempts reports a no-op and rejects a budget below 1", () => {
+    const dataDir = freshDataDir("loops-cli-max-attempts-invalid-");
+    const create = runCli(dataDir, ["--json", "create", "command", "budget-guard", "--at", futureAt(), "--cmd", "true", "--attempts", "2"]);
+    expect(create.status).toBe(0);
+    const created = JSON.parse(create.stdout);
+    expect(created.maxAttempts).toBe(2);
+
+    const noop = runCli(dataDir, ["--json", "set-max-attempts", "budget-guard", "2"]);
+    expect(noop.status).toBe(0);
+    const noopValue = JSON.parse(noop.stdout);
+    expect(noopValue.changed).toBe(false);
+    expect(noopValue.backupPath).toBeUndefined();
+
+    for (const bad of ["0", "-1", "1.5", "abc"]) {
+      const rejected = runCli(dataDir, ["--json", "set-max-attempts", "budget-guard", bad]);
+      expect(rejected.status).not.toBe(0);
+      const still = JSON.parse(runCli(dataDir, ["--json", "show", "budget-guard"]).stdout);
+      expect(still.maxAttempts).toBe(2);
+    }
+  });
+
   test("rename reports no-op without writing a backup", () => {
     const dataDir = freshDataDir("loops-cli-rename-noop-");
     const create = runCli(dataDir, ["create", "command", "stable-name", "--at", futureAt(), "--cmd", "true"]);

src/cli/index.ts

@@ -2653,6 +2653,42 @@ program
     );
   })));
 
+program
+  .command("set-max-attempts <idOrName> <attempts>")
+  .description("change a loop's retry budget in place, without losing its id, schedule, runs, or history")
+  .action(runAction((idOrName, attempts) => withStore(async (store) => {
+    // requireUniqueLoop so an ambiguous name errors instead of mutating the
+    // newest same-named loop -- the same guard rename and pause/resume use.
+    const loop = await store.requireUniqueLoop(idOrName);
+    const previous = loop.maxAttempts;
+    const next = positiveInteger(String(attempts), "<attempts>");
+    if (next === undefined) throw new ValidationError("attempts must be an integer >= 1");
+
+    if (next === previous) {
+      print(
+        { changed: false, id: loop.id, maxAttempts: previous, loop: publicLoop(loop) },
+        `${loop.id} unchanged (maxAttempts=${previous})`,
+      );
+      return;
+    }
+
+    // Backups protect the on-box sqlite file; there is nothing local to snapshot
+    // when the update is routed to the hosted API.
+    const backupPath = store.transport === "local" ? backupLoopsDatabase("set-max-attempts") : undefined;
+    const updated = await store.updateLoop(loop.id, { maxAttempts: next });
+    print(
+      {
+        changed: true,
+        id: updated.id,
+        previousMaxAttempts: previous,
+        maxAttempts: updated.maxAttempts,
+        backupPath,
+        loop: publicLoop(updated),
+      },
+      `${updated.id} maxAttempts ${previous} -> ${updated.maxAttempts}\nbackup=${backupPath ?? "skipped (recent backup exists)"}`,
+    );
+  })));
+
 function updateStatus(idOrName: string, status: "paused" | "active" | "stopped"): Promise<void> {
   return withStore(async (store) => {
     // requireUniqueLoop so an ambiguous name errors instead of mutating the

src/lib/loop-status.ts

@@ -12,3 +12,17 @@ export function assertLoopStatus(value: unknown): asserts value is LoopStatus {
     throw new ValidationError("loop status must be one of active, paused, stopped, expired");
   }
 }
+
+export function isMaxAttempts(value: unknown): value is number {
+  return typeof value === "number" && Number.isSafeInteger(value) && value >= 1;
+}
+
+// maxAttempts is the retry budget the scheduler compares an attempt number
+// against (`attempt < loop.maxAttempts`), so 1 means "no retry" and 0 or a
+// negative value would make every run unretryable AND unadmittable. Validate it
+// in one place because both the sqlite and postgres backends write the column.
+export function assertMaxAttempts(value: unknown): asserts value is number {
+  if (!isMaxAttempts(value)) {
+    throw new ValidationError("loop maxAttempts must be an integer >= 1");
+  }
+}

src/lib/scheduler.test.ts

@@ -444,6 +444,48 @@ describe("scheduler", () => {
     }
   });
 
+  test("catch_up all honors a retry budget raised while the first slot is running", async () => {
+    const store = new Store(":memory:");
+    try {
+      const loop = store.createLoop(
+        {
+          name: "catch-up-live-retry-budget",
+          schedule: { type: "interval", everyMs: 1_000 },
+          target: { type: "command", command: "true" },
+          catchUp: "all",
+          catchUpLimit: 10,
+          maxAttempts: 1,
+          retryDelayMs: 5_000,
+        },
+        new Date("2026-01-01T00:00:00Z"),
+      );
+      const firstSlot = loop.nextRunAt!;
+      let raisedBudget = false;
+
+      const out = await tick({
+        store,
+        runnerId: "test",
+        now: () => new Date("2026-01-01T00:00:05Z"),
+        random: noJitter,
+        beforeFinalize: () => {
+          if (raisedBudget) return;
+          store.updateLoop(loop.id, { maxAttempts: 2 });
+          raisedBudget = true;
+        },
+        execute: async () => result("failed", "2026-01-01T00:00:05.000Z"),
+      });
+
+      expect(out.completed).toHaveLength(1);
+      expect(store.listRuns({ loopId: loop.id })).toHaveLength(1);
+      const updated = store.getLoop(loop.id);
+      expect(updated?.maxAttempts).toBe(2);
+      expect(updated?.retryScheduledFor).toBe(firstSlot);
+      expect(updated?.nextRunAt).toBe("2026-01-01T00:00:10.000Z");
+    } finally {
+      store.close();
+    }
+  });
+
   test("preserves retry intent for recovered abandoned runs before advancing later recovered slots", async () => {
     const store = new Store(":memory:");
     try {

src/lib/scheduler.ts

@@ -601,7 +601,12 @@ export async function tick(deps: SchedulerDeps): Promise<TickResult> {
         loopSkips += 1;
       } else completed.push(run);
       // tick-only retry gate: see recoverAndExpire() doc comment.
-      if (["failed", "timed_out", "abandoned"].includes(run.status) && run.attempt < loop.maxAttempts) break;
+      // The retry budget can change while a run is executing, so decide from
+      // the persisted loop instead of the pre-run snapshot used by dueSlots().
+      if (["failed", "timed_out", "abandoned"].includes(run.status)) {
+        const current = deps.store.getLoop(loop.id);
+        if (current && run.attempt < current.maxAttempts) break;
+      }
     }
   }
 

src/lib/storage/postgres-loop-storage.ts

@@ -108,7 +108,7 @@ import type {
 } from "../../types.js";
 import { normalizeRunReceipt } from "../run-receipts.js";
 import { normalizeLoopLabels } from "../labels.js";
-import { assertLoopStatus } from "../loop-status.js";
+import { assertLoopStatus, assertMaxAttempts } from "../loop-status.js";
 import { normalizeRunCompletion } from "../run-completion.js";
 import type { PoolQueryClient, TypedQueryClient } from "../../generated/storage-kit/query.js";
 import type { LoopStorageContract, LoopStorageMethodName } from "./contract.js";
@@ -618,6 +618,7 @@ export class PostgresLoopStorage implements LoopStorageContract {
   async updateLoop(...args: M<"updateLoop">["args"]): Promise<M<"updateLoop">["result"]> {
     const [id, patch, opts = {}] = args;
     if ("status" in patch && patch.status !== undefined) assertLoopStatus(patch.status);
+    if ("maxAttempts" in patch && patch.maxAttempts !== undefined) assertMaxAttempts(patch.maxAttempts);
     const updated = (opts.now ?? new Date()).toISOString();
     return this.client.transaction(async (c) => {
       const current = await this.loadLoop(c, id);
@@ -630,15 +631,16 @@ export class PostgresLoopStorage implements LoopStorageContract {
         updatedAt: updated,
       };
       const res = await c.query(
-        `UPDATE loops SET status=$1, labels_json=$2::jsonb, next_run_at=$3, retry_scheduled_for=$4, expires_at=$5, updated_at=$6
-         WHERE tenant_id = open_loops_current_tenant_id() AND id=$7
-           AND ($8::text IS NULL OR EXISTS (SELECT 1 FROM daemon_lease WHERE tenant_id = open_loops_current_tenant_id() AND id=$8 AND expires_at > $9))`,
+        `UPDATE loops SET status=$1, labels_json=$2::jsonb, next_run_at=$3, retry_scheduled_for=$4, expires_at=$5, max_attempts=$6, updated_at=$7
+         WHERE tenant_id = open_loops_current_tenant_id() AND id=$8
+           AND ($9::text IS NULL OR EXISTS (SELECT 1 FROM daemon_lease WHERE tenant_id = open_loops_current_tenant_id() AND id=$9 AND expires_at > $10))`,
         [
           merged.status,
           JSON.stringify(merged.labels),
           merged.nextRunAt ?? null,
           merged.retryScheduledFor ?? null,
           merged.expiresAt ?? null,
+          merged.maxAttempts,
           merged.updatedAt,
           id,
           opts.daemonLeaseId ?? null,