Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions src/db/repositories.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6560,6 +6560,32 @@ export async function listStaleActiveReviewTracking(
.where(and(eq(activeReviewTracking.status, "active"), lt(activeReviewTracking.startedAt, olderThanIso)));
}

/**
* Boot-time orphan sweep (#deploy-orphaned-reviews): terminalize every 'active' row whose review started
* BEFORE this process booted. No in-flight review survives a restart -- the pass, its provider subprocess,
* and its locks all died with the old process -- so any such row is definitionally an orphan, and while it
* exists every new pass for that head skips with "AI review already in progress" (aiReviewLockContendedResult).
*
* WHY NOT LEAVE IT TO runActiveReviewReconciliation: that sweep only touches rows older than
* STALE_ACTIVE_REVIEW_MIN_AGE_MS (15 min) and runs on a 10-minute cron, so a container recreation wedged
* every mid-review PR head for 15-25 minutes -- observed live on the ORB (2026-07-29): five deploys in one
* day each orphaned the in-flight review, and even the maintainer's forced re-runs bounced off the stale
* row. At boot the age heuristic is unnecessary: started_at < process boot time is EXACT, not a guess, so
* this neither waits nor risks killing a genuinely live pass (there are none yet).
*
* Returns the terminalized (repo, PR) pairs so the caller can log one line per healed row.
*/
export async function terminalizeActiveReviewsFromBeforeBoot(
env: Env,
bootIso: string,
): Promise<Array<{ repoFullName: string; pullNumber: number }>> {
return getDb(env.DB)
.update(activeReviewTracking)
.set({ status: "terminal", updatedAt: nowIso() })
.where(and(eq(activeReviewTracking.status, "active"), lt(activeReviewTracking.startedAt, bootIso)))
.returning({ repoFullName: activeReviewTracking.repoFullName, pullNumber: activeReviewTracking.pullNumber });
}

// Review memory (#2178, data-model slice of #1964). Hard per-repo cap on stored suppression signals — mirrors
// rag.ts's MAX_CHUNKS_PER_REPO discipline (bound a repo-controlled, unboundedly-growable store). A repo that
// keeps dismissing NEW finding shapes evicts its OLDEST suppression first rather than growing forever.
Expand Down
20 changes: 20 additions & 0 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -887,6 +887,26 @@ async function main(): Promise<void> {
} as unknown as Env;
markEnvReady();

// #deploy-orphaned-reviews: heal rows this very restart orphaned, BEFORE any webhook can bounce off them.
// Fail-safe: a failure here must never block boot -- the 10-minute reconciliation sweep remains the backstop.
try {
const { terminalizeActiveReviewsFromBeforeBoot } = await import("./db/repositories");
const healed = await terminalizeActiveReviewsFromBeforeBoot(env, new Date().toISOString());
for (const row of healed) {
console.warn(
JSON.stringify({
level: "warn",
event: "active_review_boot_orphan_terminalized",
repo: row.repoFullName,
pr: row.pullNumber,
}),
);
}
} catch (error) {
console.error(JSON.stringify({ level: "error", event: "active_review_boot_sweep_failed", message: String(error).slice(0, 200) }));
}


// Fleet-mode credential resolution (#9543): registered HERE, after `env` is fully constructed, because the
// lookup needs the DB binding -- src/selfhost/ai.ts must never import the DB layer itself, so the closure is
// injected instead. Safe to register after createSelfHostAi() above: the resolver is only ever invoked at
Expand Down
36 changes: 36 additions & 0 deletions test/unit/db-persistence.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
persistRepoGithubTotalsSnapshot,
persistSignalSnapshot,
startActiveReviewTracking,
terminalizeActiveReviewsFromBeforeBoot,
terminalizeActiveReviewTracking,
updateUpstreamDriftReportIssue,
upsertBounty,
Expand Down Expand Up @@ -600,3 +601,38 @@ describe("active-review tracking (#review-evasion-protection)", () => {
});
});
});

describe("terminalizeActiveReviewsFromBeforeBoot (#deploy-orphaned-reviews)", () => {
it("REGRESSION: terminalizes a row orphaned by a restart, so the head is not wedged for 15-25 minutes", async () => {
// Observed live on the ORB (2026-07-29): a container recreation 3 seconds after a review pass started
// left its tracking row 'active'; every later pass for that head -- including the maintainer's forced
// re-runs -- skipped with "AI review already in progress" until the 10-minute sweep's 15-minute age
// floor finally cleared it. At boot, started_at < boot time is exact: no review survives a restart.
const env = createTestEnv();
await startActiveReviewTracking(env, { repoFullName: "owner/repo", pullNumber: 7, headSha: "sha7", authorLogin: "alice", deliveryId: "d-7" });
// The process "boots" strictly after the row was written.
const bootIso = new Date(Date.now() + 1000).toISOString();
const healed = await terminalizeActiveReviewsFromBeforeBoot(env, bootIso);
expect(healed).toEqual([{ repoFullName: "owner/repo", pullNumber: 7 }]);
const row = await env.DB.prepare("select status from active_review_tracking where repo_full_name = ? and pull_number = ?").bind("owner/repo", 7).first<{ status: string }>();
expect(row?.status).toBe("terminal");
});

it("INVARIANT: never touches a review that started AFTER boot — a live pass is not an orphan", async () => {
const env = createTestEnv();
const bootIso = new Date(Date.now() - 60_000).toISOString();
await startActiveReviewTracking(env, { repoFullName: "owner/repo", pullNumber: 8, headSha: "sha8", authorLogin: "bob", deliveryId: "d-8" });
expect(await terminalizeActiveReviewsFromBeforeBoot(env, bootIso)).toEqual([]);
const row = await env.DB.prepare("select status from active_review_tracking where repo_full_name = ? and pull_number = ?").bind("owner/repo", 8).first<{ status: string }>();
expect(row?.status).toBe("active");
});

it("INVARIANT: already-terminal rows are untouched (no resurrect, no double-report)", async () => {
const env = createTestEnv();
await startActiveReviewTracking(env, { repoFullName: "owner/repo", pullNumber: 9, headSha: "sha9", authorLogin: "carol", deliveryId: "d-9" });
const boot = new Date(Date.now() + 1000).toISOString();
expect(await terminalizeActiveReviewsFromBeforeBoot(env, boot)).toHaveLength(1);
// Second boot sweep: nothing left to heal, and nothing re-reported.
expect(await terminalizeActiveReviewsFromBeforeBoot(env, boot)).toEqual([]);
});
});