JSONbored · JSONbored · Jul 27, 2026 · Jul 27, 2026
@@ -21,6 +21,7 @@ import {
 } from "./transient-locks";
 import { buildPullRequestAdvisory } from "../rules/advisory";
 import { recordAuditEvent, getDecryptedRepositoryAiKey, getRepository, listCheckSummaries, listPullRequestFiles } from "../db/repositories";
+import { registerHeldLock, unregisterHeldLock } from "./held-lock-registry";
 import { recordRoutingShadow } from "../services/reviewer-routing";
 import { createInstallationToken } from "../github/app";
 import type { AgentActionMode } from "../settings/agent-execution";
@@ -120,12 +121,15 @@ export async function claimAiReviewLock(
   mode: string,
   options?: { steal?: boolean },
 ): Promise<TransientLockClaim> {
-  return claimTransientLock(
-    env,
-    aiReviewLockKey(repoFullName, prNumber, headSha, mode),
-    AI_REVIEW_LOCK_TTL_SECONDS,
-    options,
-  );
+  const key = aiReviewLockKey(repoFullName, prNumber, headSha, mode);
+  const claim = await claimTransientLock(env, key, AI_REVIEW_LOCK_TTL_SECONDS, options);
+  // #8998: register a REAL claim (a non-null ownerToken means this call, not a fail-open passthrough or a
+  // no-op adapter, actually owns the key) so a shutdown signal can release it immediately instead of only via
+  // graceful drain or the full 1800s TTL. A fail-open claim (ownerToken null) has nothing to release.
+  if (claim.acquired && claim.ownerToken !== null) {
+    registerHeldLock(key, () => releaseTransientLockIfOwner(env, key, claim.ownerToken));
+  }
+  return claim;
 }
 
 /** Best-effort release, called from a finally block so the lock frees promptly instead of waiting out the TTL. */
@@ -137,7 +141,12 @@ export async function releaseAiReviewLock(
   mode: string,
   ownerToken: string | null,
 ): Promise<void> {
-  await releaseTransientLockIfOwner(env, aiReviewLockKey(repoFullName, prNumber, headSha, mode), ownerToken);
+  const key = aiReviewLockKey(repoFullName, prNumber, headSha, mode);
+  await releaseTransientLockIfOwner(env, key, ownerToken);
+  // #8998: this process no longer holds it -- a later shutdown must not attempt to release a key it already
+  // gave up (harmless either way, since the compare-and-delete release is idempotent, but there is no reason
+  // to carry a stale entry).
+  unregisterHeldLock(key);
 }
 
 /**

@@ -0,0 +1,58 @@
+/**
+ * #8998 — release the transient locks THIS process currently holds the instant a shutdown signal arrives,
+ * rather than only via graceful drain or TTL expiry.
+ *
+ * `queue.stop()` (#9007) already lets an in-flight job finish naturally before the process exits, and a job's
+ * own `finally` block releases whatever lock it holds as part of that. That is the right behavior when the
+ * orchestrator's SIGKILL grace period is long enough for the drain to complete. It is not always long enough:
+ * an AI-review LLM call can legitimately run for tens of seconds to minutes, while a common container-platform
+ * grace period is 10-30s. When the hard kill lands before the drain finishes, the lock this process claimed —
+ * `ai-review-lock` most consequentially, at a 1800s TTL — outlives the process that claimed it by up to that
+ * TTL, starving every subsequent pass (scheduled or an explicit maintainer re-run) with the "already in
+ * progress" placeholder for real time nobody is actually reviewing anything.
+ *
+ * This is the proactive half: a tiny process-local registry of "locks I currently hold and how to release
+ * them", so the shutdown handler can best-effort release every one of them immediately on SIGTERM/SIGINT —
+ * before, and independent of, whether the graceful drain itself has time to finish. Complementary to, not a
+ * replacement for, the boot-time orphaned-lock flush (#9021): that catches whatever this process didn't get a
+ * chance to release (a truly hard `kill -9`, which delivers no signal at all); this catches everything else.
+ */
+
+type HeldLockRelease = () => Promise<void>;
+
+const heldLocks = new Map<string, HeldLockRelease>();
+
+/** Record that this process now holds `key`, with `release` as how to give it up. Call the moment a claim
+ *  actually succeeds with real ownership (a fail-open "acquired but nothing to release" claim has nothing
+ *  worth registering — see the call site's own guard). */
+export function registerHeldLock(key: string, release: HeldLockRelease): void {
+  heldLocks.set(key, release);
+}
+
+/** Record that this process no longer holds `key` (its own release already ran). A shutdown racing the same
+ *  release is harmless either way: `releaseTransientLockIfOwner`'s compare-and-delete makes a second release
+ *  attempt against an already-released key a safe no-op. */
+export function unregisterHeldLock(key: string): void {
+  heldLocks.delete(key);
+}
+
+/** Best-effort release every lock currently on record, clearing the registry as it goes. Never throws: a
+ *  release failure here just means that one lock rides out its own TTL, exactly the pre-#8998 behavior for
+ *  every lock, rather than blocking the rest of shutdown. Returns the count attempted, for one shutdown log
+ *  line — not the count that definitely succeeded, since a released key is removed from the registry either
+ *  way and there is no further signal to distinguish the two once shutdown is already underway. */
+export async function releaseAllHeldLocksAtShutdown(): Promise<number> {
+  const entries = [...heldLocks.entries()];
+  heldLocks.clear();
+  let attempted = 0;
+  for (const [, release] of entries) {
+    attempted += 1;
+    await release().catch(() => undefined);
+  }
+  return attempted;
+}
+
+/** Test-only introspection — never used by production code. */
+export function heldLockCountForTest(): number {
+  return heldLocks.size;
+}
@@ -53,6 +53,7 @@ import { runRetentionPrune } from "./retention";
 import { sweepStaleApprovalQueue } from "../services/agent-approval-queue";
 import { reconcileMissingPrOutcomes } from "../review/pr-outcome-reconciler";
 import { sweepStrandedPendingClosures } from "../review/pending-closure-watchdog";
+import { reconcileSurfaceWithoutDisposition } from "../review/surface-disposition-reconciler";
 // The 15 handlers below have no reason to move -- each is only reachable via this dispatcher (or, for
 // mapWithConcurrency, ALSO used by other still-in-processors.ts code), so they stay put and are exported
 // there purely for this one-directional import-back (processors.ts itself never calls processJob).
@@ -275,7 +276,7 @@ export async function processJob(env: Env, message: JobMessage): Promise<void> {
     }
     case "agent-regate-sweep":
       if (!message.repoFullName && message.requestedBy !== "test") {
-        // Three bounded repair scans ride the sweep's own fan-out tick rather than each earning a job type and
+        // Four bounded repair scans ride the sweep's own fan-out tick rather than each earning a job type and
         // a cron entry. All are best-effort and deliberately BEFORE the fan-out: none may cost the tick its
         // re-gate work, which is the sweep's actual job.
         //
@@ -287,6 +288,11 @@ export async function processJob(env: Env, message: JobMessage): Promise<void> {
         //
         // #9031 re-enqueues a pending-closure Pass 2 whose single delayed job was lost, which otherwise strands
         // the PR permanently: flagged, un-mergeable, un-approvable, and sweep-ineligible.
+        //
+        // #8997 re-enqueues a regate for a PR whose published surface has no matching disposition marker for
+        // that exact head -- the "decisive panel, PR still open" shape a restart killing the pass between
+        // publish and disposition leaves behind, which the ordinary stale-surface repair check cannot see
+        // because the surface itself is not stale.
         const staleness = await sweepStaleApprovalQueue(env).catch(() => null);
         if (staleness && (staleness.reminded > 0 || staleness.expired > 0)) {
           console.log(JSON.stringify({ event: "approval_queue_staleness_swept", ...staleness }));
@@ -299,6 +305,10 @@ export async function processJob(env: Env, message: JobMessage): Promise<void> {
         if (stranded && stranded.requeued > 0) {
           console.log(JSON.stringify({ event: "pending_closure_verifications_requeued", ...stranded }));
         }
+        const orphanedDispositions = await reconcileSurfaceWithoutDisposition(env).catch(() => null);
+        if (orphanedDispositions && orphanedDispositions.requeued > 0) {
+          console.log(JSON.stringify({ event: "surface_without_disposition_reconciled", ...orphanedDispositions }));
+        }
         await fanOutAgentRegateSweepJobs(env, message.requestedBy);
         return;
       }

@@ -682,6 +682,7 @@ import type {
 } from "../types";
 import { sha256Hex } from "../utils/crypto";
 import { errorMessage, nowIso, repoParts } from "../utils/json";
+import { DISPOSITION_CONSIDERED_EVENT_TYPE } from "../review/surface-disposition-reconciler";
 import { maybeSuggestMilestoneMatchForPr } from "../integrations/project-tracker-adapter";
 
 const OFFICIAL_MINER_DETECTION_TTL_MS = 5 * 60 * 1000;
@@ -1150,6 +1151,24 @@ async function isRegateRepairExhausted(env: Env, repoFullName: string, pr: Pick<
   return true;
 }
 
+/** Record that a real disposition attempt began for this exact head (#8997) -- the signal
+ *  reconcileSurfaceWithoutDisposition (surface-disposition-reconciler.ts) reads to tell "a restart killed the
+ *  pass right after publish, before disposition ever started" apart from "disposition genuinely ran (or is
+ *  still running) for this head". Written the moment maintenance CLAIMS its per-PR actuation lock -- i.e. the
+ *  moment a real attempt begins, before anything about WHAT it decides. Best-effort: a failed write here must
+ *  never block the maintenance pass it is only bookkeeping for -- worst case, a missed write costs one extra
+ *  (safe, actuation-lock-deduplicated) repair dispatch on a later sweep tick. */
+async function recordDispositionConsidered(env: Env, repoFullName: string, prNumber: number, headSha: string): Promise<void> {
+  await recordAuditEvent(env, {
+    eventType: DISPOSITION_CONSIDERED_EVENT_TYPE,
+    actor: "loopover",
+    targetKey: `${repoFullName}#${prNumber}#${headSha}`,
+    outcome: "completed",
+    detail: "maintenance actuation lock claimed; disposition attempt begins for this head",
+    metadata: { repoFullName, prNumber, headSha },
+  }).catch(() => undefined);
+}
+
 export async function surfaceRepairPriorityPullNumbers(
   env: Env,
   repoFullName: string,
@@ -2493,6 +2512,11 @@ async function maybeRunAgentMaintenance(
     }).catch(() => undefined);
     throw new PrActuationLockContendedError(repoFullName, pr.number, "agent-maintenance");
   }
+  // #8997: the lock is now genuinely held -- a real disposition attempt for this exact head begins here. See
+  // recordDispositionConsidered's own doc comment for why this specific point is what the boot-time repair
+  // sweep checks for. Absent headSha (a sparse/never-synced PR row) has nothing to key the marker on and is
+  // skipped -- the pre-existing lastPublishedSurfaceSha check already covers a PR with no resolvable head.
+  if (pr.headSha) await recordDispositionConsidered(env, repoFullName, pr.number, pr.headSha);
   try {
     await runAgentMaintenancePlanAndExecute(env, {
       installationId,
@@ -10084,9 +10108,26 @@ async function maybePublishPrPublicSurface(
         // #4889: per-repo admin mode swaps the global-allowlist grant for the live per-repo permission.
         (await isPerTenantAdmin(env, installationId, repoFullName, author)) ||
         isProtectedAutomationAuthor(author, env));
+    // #8999: exempt the freeze when the label's own provenance -- as far as this pass can tell -- is the
+    // #9009 lock-contention marker and NOTHING else. Without this, the freeze deadlocked: a lost AI-review lock
+    // race applies this exact label (aiReviewLockContendedResult's hold), the freeze then reads that SAME label
+    // on the very next pass and refuses a FRESH AI review (only replay), so the pass that could produce the
+    // real verdict needed to auto-clear the label (#9009, below in runAgentMaintenancePlanAndExecute) never
+    // runs -- the label outlives the transient contention that caused it, requiring a human to unstick a PR
+    // that was never substantively held. Reading the SAME marker key #9009 already writes/reads keeps this a
+    // single source of truth for "why is this label here right now" rather than a second, drifting one.
+    //
+    // This does not reopen the gaming surface the freeze exists to close (see #regate-churn doc above): that
+    // surface is a CONTRIBUTOR iterating pushes to game the bot. A lock-contention hold is not a contributor
+    // action -- it is an infra artifact of two ORB passes racing -- so exempting it changes nothing about a
+    // contributor's ability to buy a fresh verdict by pushing. If some OTHER reason also justifies the hold,
+    // that reason re-applies the label on this very pass's own disposition regardless of this exemption.
+    const manualReviewLockContentionMarkerKey = `manual-review-lock-contention:${repoFullName.toLowerCase()}#${pr.number}`;
+    const manualReviewLabelIsPurelyLockContention = manualReviewLabel !== null && (await getTransientKey(env, manualReviewLockContentionMarkerKey)) !== null;
     const isFrozenForManualReview =
       webhook.forceAiReview !== true &&
       !authorIsExemptFromFreeze &&
+      !manualReviewLabelIsPurelyLockContention &&
       manualReviewLabel !== null &&
       pr.labels.some((label) => label.toLowerCase() === manualReviewLabel.toLowerCase());
     let reviewManifestForAutoReview: FocusManifest | null = null;

@@ -0,0 +1,101 @@
+import { recordAuditEvent } from "../db/repositories";
+import { errorMessage } from "../utils/json";
+
+/**
+ * #8997 — rescue a PR left wearing a decisive panel with no matching disposition.
+ *
+ * A deploy restart (SIGTERM, no drain) can kill a pass at the worst possible cut point: after the public-surface
+ * publish commits (panel comment, gate check-run, CI aggregate all reflect the current head) but before
+ * `maybeRunAgentMaintenance` ever claims its per-PR actuation lock for that head. The confirmed live shape
+ * (#8965, 2026-07-26): a red-CI panel published at 14:55:24Z from the dying container, and the matching close
+ * only executed at 15:07:53Z — ~12 minutes later, purely because an UNRELATED later sweep tick happened to
+ * re-run the whole pass from scratch. The regular outage-repair priority check
+ * (`surfaceRepairPriorityPullNumbers`, processors.ts) cannot see this: it flags a STALE surface
+ * (`lastPublishedSurfaceSha !== headSha`), and this incident's surface is not stale — it published successfully
+ * right before the kill. From that check's point of view the PR already looks healthy.
+ *
+ * This scan is the missing half: it looks for the disposition side directly. `maybeRunAgentMaintenance` records
+ * a durable marker (`DISPOSITION_CONSIDERED_EVENT_TYPE`, processors.ts) the moment it claims the actuation lock
+ * for a head — i.e. the moment a real disposition attempt begins. A PR whose surface is current for its head but
+ * carries no such marker never got that far, for whatever reason (a restart being the confirmed one; a thrown
+ * error before the claim is another). Re-enqueuing a regate is cheap and safe either way: `agent-regate-pr`
+ * re-derives everything from live state, and a PR that in fact already has a real disposition in flight is
+ * simply denied by the SAME actuation lock this scan is checking for, exactly as any other contending pass would
+ * be (#9013's per-PR mutex).
+ *
+ * Deliberately independent of surfaceRepairPriorityPullNumbers rather than folded into it: that function's
+ * return value directly drives the regular sweep's staleness-ordered fan-out and is asserted against by a large
+ * existing test surface (dispatch shape, ordering, backlog-restriction semantics). Riding the SAME sweep tick as
+ * its own bounded scan — mirroring reconcileMissingPrOutcomes/sweepStrandedPendingClosures (#9026/#9031) — closes
+ * the identical gap without touching that already-tested surface at all.
+ */
+
+export const DISPOSITION_CONSIDERED_EVENT_TYPE = "agent.maintenance.disposition_considered";
+
+/** How far back to look. Bounded so the scan stays cheap on every tick; a surface-without-disposition PR older
+ *  than this has almost certainly been superseded by a later commit or resolved by a human already. */
+export const SURFACE_DISPOSITION_RECONCILE_LOOKBACK_MS = 24 * 60 * 60 * 1000;
+
+/** Cap per run, mirroring the other bounded repair scans on this same sweep tick. */
+export const SURFACE_DISPOSITION_RECONCILE_LIMIT = 200;
+
+export type SurfaceDispositionReconcileResult = { scanned: number; requeued: number };
+
+/**
+ * Find open PRs with a current published surface but no disposition marker for that exact head, and re-enqueue
+ * an `agent-regate-pr` job for each. Best-effort throughout: a repair pass that can itself break the tick it
+ * rides on is worse than the gap it exists to close.
+ */
+export async function reconcileSurfaceWithoutDisposition(env: Env, nowMs: number = Date.now()): Promise<SurfaceDispositionReconcileResult> {
+  const since = new Date(nowMs - SURFACE_DISPOSITION_RECONCILE_LOOKBACK_MS).toISOString();
+  let rows: Array<{ repoFullName: string; number: number; installationId: number; headSha: string }> = [];
+  try {
+    const result = await env.DB.prepare(
+      `SELECT pr.repo_full_name AS repoFullName, pr.number AS number, repo.installation_id AS installationId, pr.head_sha AS headSha
+         FROM pull_requests AS pr
+         JOIN repositories AS repo ON repo.full_name = pr.repo_full_name
+        WHERE pr.state = 'open'
+          AND pr.head_sha IS NOT NULL
+          AND pr.last_published_surface_sha = pr.head_sha
+          AND pr.updated_at >= ?1
+          AND repo.installation_id IS NOT NULL
+          AND NOT EXISTS (
+            SELECT 1 FROM audit_events AS disposition
+             WHERE disposition.target_key = pr.repo_full_name || '#' || pr.number || '#' || pr.head_sha
+               AND disposition.event_type = ?2
+          )
+        ORDER BY pr.updated_at
+        LIMIT ?3`,
+    )
+      .bind(since, DISPOSITION_CONSIDERED_EVENT_TYPE, SURFACE_DISPOSITION_RECONCILE_LIMIT)
+      .all<{ repoFullName: string; number: number; installationId: number; headSha: string }>();
+    rows = result.results ?? [];
+  } catch (error) {
+    console.warn(JSON.stringify({ level: "warn", event: "surface_disposition_reconcile_scan_failed", message: errorMessage(error).slice(0, 160) }));
+    return { scanned: 0, requeued: 0 };
+  }
+
+  let requeued = 0;
+  for (const row of rows) {
+    const sent = await env.JOBS.send({
+      type: "agent-regate-pr",
+      deliveryId: `surface-without-disposition:${row.repoFullName}#${row.number}#${row.headSha}`,
+      repoFullName: row.repoFullName,
+      prNumber: row.number,
+      installationId: row.installationId,
+    })
+      .then(() => true)
+      .catch(() => false);
+    if (!sent) continue;
+    requeued += 1;
+    await recordAuditEvent(env, {
+      eventType: "agent.sweep.surface_without_disposition",
+      actor: "loopover",
+      targetKey: `${row.repoFullName}#${row.number}`,
+      outcome: "queued",
+      detail: "published surface has no matching disposition for this head; re-gating to close the gap",
+      metadata: { repoFullName: row.repoFullName, pullNumber: row.number, headSha: row.headSha },
+    }).catch(() => undefined);
+  }
+  return { scanned: rows.length, requeued };
+}