feat(orb): randomized close-audit holdout — the selective-labels fix for close precision

.loopover.yml.example

@@ -107,6 +107,9 @@ publicNotes:
 #   block    — the finding can become a hard `LoopOver Gate` blocker
 #              (always confirmed-contributor-gated).
 gate:
+  # Percent (0-20) of would-auto-close PRs randomly HELD for human adjudication instead of closing --
+  # the randomized holdout behind the unbiased close-precision estimate (#8831). 0/absent disables.
+  # closeAuditHoldoutPct: 5
   # Legacy check-run publish switch (#5355) — despite the name, this does NOT turn the deterministic
   # gate itself on or off. Gate evaluation, comments, labels, audit records, spend, and autonomous
   # merge/close all run identically whether this is true, false, or unset; the per-dimension modes

apps/loopover-ui/public/openapi.json

@@ -9912,6 +9912,10 @@
               "advisory",
               "block"
             ]
+          },
+          "closeAuditHoldoutPct": {
+            "type": "number",
+            "nullable": true
           }
         },
         "required": [
@@ -10652,6 +10656,10 @@
                   "advisory",
                   "block"
                 ]
+              },
+              "closeAuditHoldoutPct": {
+                "type": "number",
+                "nullable": true
               }
             },
             "required": [

docs/decision-audit-rubric.md

@@ -0,0 +1,45 @@
+# Decision-audit rubric — version 1
+
+This rubric governs human adjudication of the weekly decision-audit sample (#8830, epic #8828).
+Adjudications are comparable **only within a rubric version**: any change to the criteria below requires
+bumping `DECISION_AUDIT_RUBRIC_VERSION` in `src/review/decision-audit.ts` in the same PR, so estimates never
+silently mix incompatible labels.
+
+## What is being judged
+
+One question per sampled PR: **was the gate's decision (merge or close) the decision a careful maintainer
+would have made, given only what was knowable at decision time?**
+
+- Judge the decision, not the outcome. A merge that later broke something the gate could not have seen is
+  still `correct`; a merge that shipped a defect visible in the diff at review time is `incorrect` even if
+  nobody noticed.
+- Ignore reversal state. "Nobody reversed it" is not evidence of correctness — that bias is exactly what
+  this audit exists to measure.
+- Use `uncertain` honestly. A genuine judgment call where reasonable maintainers would split is `uncertain`,
+  not a coin flip. Uncertain labels are excluded from the accuracy numerator AND denominator and tracked as
+  their own rate (a rising uncertain rate is a signal the gate is operating in territory that needs a human).
+
+## Labels
+
+| label | meaning |
+|---|---|
+| `correct` | The decision was right given decision-time information. |
+| `incorrect` | The decision was wrong given decision-time information. |
+| `uncertain` | Reasonable maintainers would disagree; no defensible single answer. |
+
+## Reason categories (for `incorrect`, optional otherwise)
+
+- `missed_defect` — merged with a defect visible in the diff.
+- `false_block` — closed a PR that met the bar.
+- `stale_signal` — decided on out-of-date CI/conflict/issue state.
+- `scope_misread` — misjudged linked-issue scope or requirements.
+- `policy_misapplied` — an enforcement rule fired on a case it should not cover.
+- `other` — anything else; describe in the adjudication notes if used.
+
+## Workflow
+
+1. `GET /v1/internal/audit-labels?status=pending` lists the week's sample.
+2. Review each PR **as of its decision time** (the decision record pins head SHA and inputs).
+3. `POST /v1/internal/audit-labels/adjudicate` with `{ id, adjudication, reasonCategory? }`.
+4. A second adjudication of the same label is rejected (409). Corrections require a new rubric version —
+   deliberate, never silent.

migrations/0178_decision_audit_labels.sql

@@ -0,0 +1,26 @@
+-- #8830 (epic #8828, Phase 2 — labels): human-adjudicated ground truth for the gate's decisions.
+--
+-- The reversal-based confirmation signal is a LOWER BOUND on error, not a label: humans reverse only what
+-- they notice, so a silently-wrong merge scores as correct and every downstream accuracy figure inherits the
+-- bias. This table holds the weekly stratified audit sample and its human adjudications — the calibration
+-- set the risk-control thresholds (#8835) and the audited-accuracy estimate both read. One row per PR ever
+-- (the UNIQUE on target_id): a PR re-drawn in a later week is skipped, never double-labeled.
+CREATE TABLE IF NOT EXISTS decision_audit_labels (
+  id              TEXT PRIMARY KEY,                    -- audit:<target_id>
+  project         TEXT NOT NULL,                       -- owner/repo
+  target_id       TEXT NOT NULL,                       -- owner/repo#N
+  verdict         TEXT NOT NULL CHECK (verdict IN ('merge', 'close')),   -- the gate's decision at sample time
+  -- Nullable: a holdout_close row (#8831) is created while the PR is still HELD -- there is no realized
+  -- outcome yet; the human adjudication IS its ground truth. The weekly sampler always supplies one.
+  outcome         TEXT CHECK (outcome IN ('merged', 'closed')),
+  -- holdout_close (#8831): rows sourced by the randomized close-holdout rather than the weekly draw.
+  stratum         TEXT NOT NULL CHECK (stratum IN ('merge_arm', 'close_arm', 'first_time_author', 'holdout_close')),
+  rubric_version  TEXT NOT NULL,                       -- adjudications are only comparable within a version
+  sampled_at      TEXT NOT NULL,
+  status          TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'adjudicated')),
+  adjudication    TEXT CHECK (adjudication IN ('correct', 'incorrect', 'uncertain')),
+  reason_category TEXT,                                -- free-form-bounded category from the rubric
+  adjudicated_at  TEXT,
+  UNIQUE (target_id)
+);
+CREATE INDEX IF NOT EXISTS decision_audit_labels_status ON decision_audit_labels (status, sampled_at);

packages/loopover-engine/src/focus-manifest.ts

@@ -98,6 +98,11 @@ export type FocusManifestGateConfig = {
    *  separate `mergeReadiness` composite gate above. */
   readinessMode: GateRuleMode | null;
   readinessMinScore: number | null;
+  /** `gate.closeAuditHoldoutPct` (#8831): percent (0-20) of would-auto-close PRs randomly HELD for human
+   *  adjudication instead — the randomized holdout that makes an unbiased close-precision estimate possible
+   *  (selective-labels: outcomes are only observed for the action taken). Bounded at 20 because the holdout
+   *  is a measurement instrument, not a review mode; null/0 disables (the default — byte-identical). */
+  closeAuditHoldoutPct: number | null;
   slopMode: GateRuleMode | null;
   slopMinScore: number | null;
   slopAiAdvisory: boolean | null;
@@ -1296,6 +1301,7 @@ const EMPTY_GATE_CONFIG: FocusManifestGateConfig = {
   enabled: null,
   checkMode: null,
   pack: null,
+  closeAuditHoldoutPct: null,
   linkedIssue: null,
   duplicates: null,
   readinessMode: null,
@@ -1617,6 +1623,15 @@ function normalizeOptionalScore(value: JsonValue | undefined, field: string, war
   return Math.max(0, Math.min(100, Math.round(value)));
 }
 
+function normalizeOptionalHoldoutPct(value: JsonValue | undefined, field: string, warnings: string[]): number | null {
+  if (value === undefined || value === null) return null;
+  if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 20) {
+    warnings.push(`Manifest gate field "${field}" must be a number between 0 and 20; ignoring it.`);
+    return null;
+  }
+  return value;
+}
+
 function normalizeOptionalNonNegativeInt(value: JsonValue | undefined, field: string, warnings: string[]): number | null {
   if (value === undefined || value === null) return null;
   if (typeof value !== "number" || !Number.isFinite(value) || !Number.isInteger(value) || value < 0) {
@@ -1781,6 +1796,7 @@ function parseGateConfig(value: JsonValue | undefined, warnings: string[]): Focu
     duplicates: normalizeOptionalGateMode(record.duplicates, "gate.duplicates", warnings),
     readinessMode: normalizeReadinessGateMode(readinessRecord?.mode, "gate.readiness.mode", warnings),
     readinessMinScore: normalizeOptionalScore(readinessRecord?.minScore, "gate.readiness.minScore", warnings),
+    closeAuditHoldoutPct: normalizeOptionalHoldoutPct(record.closeAuditHoldoutPct, "gate.closeAuditHoldoutPct", warnings),
     slopMode: normalizeOptionalGateMode(slopRecord?.mode, "gate.slop.mode", warnings),
     slopMinScore: normalizeOptionalScore(slopRecord?.minScore, "gate.slop.minScore", warnings),
     slopAiAdvisory: normalizeOptionalBoolean(slopRecord?.aiAdvisory, "gate.slop.aiAdvisory", warnings),
@@ -1844,6 +1860,7 @@ function parseGateConfig(value: JsonValue | undefined, warnings: string[]): Focu
     gate.duplicates !== null ||
     gate.readinessMode !== null ||
     gate.readinessMinScore !== null ||
+    gate.closeAuditHoldoutPct !== null ||
     gate.slopMode !== null ||
     gate.slopMinScore !== null ||
     gate.slopAiAdvisory !== null ||
@@ -1909,6 +1926,7 @@ export function gateConfigToJson(gate: FocusManifestGateConfig): JsonValue {
     out.size = size;
   }
   if (gate.lockfileIntegrityMode !== null) out.lockfileIntegrity = gate.lockfileIntegrityMode;
+  if (gate.closeAuditHoldoutPct !== null) out.closeAuditHoldoutPct = gate.closeAuditHoldoutPct;
   if (gate.slopMode !== null || gate.slopMinScore !== null || gate.slopAiAdvisory !== null) {
     const slop: Record<string, JsonValue> = {};
     if (gate.slopMode !== null) slop.mode = gate.slopMode;

scripts/check-schema-drift.ts

@@ -40,6 +40,7 @@ export const RAW_SQL_ONLY_TABLES: Set<string> = new Set([
   "ams_instances",
   "ams_signals",
   "contributor_gate_history",
+  "decision_audit_labels",
   "global_agent_controls",
   "global_contributor_blacklist",
   "global_moderation_config",

src/api/routes.ts

@@ -4397,6 +4397,49 @@ export function createApp() {
   // but only REGISTERED ones count toward fleet calibration (computeFleetAnalytics). Bearer-gated by the
   // `/v1/internal/*` middleware (INTERNAL_JOB_TOKEN). List shows pending + registered instances with their
   // stored-signal counts so an operator knows what they're opting in before they register it.
+  // Decision-audit adjudication (#8830, epic #8828): the operator surface for the weekly stratified sample.
+  // Bearer-gated by the /v1/internal/* middleware (INTERNAL_JOB_TOKEN). List is filterable by status; the
+  // adjudicate write is idempotent-hostile on purpose — a second adjudication 409s rather than silently
+  // rewriting a label (labels are calibration data; rewrites must be deliberate, via a fresh rubric version).
+  app.get("/v1/internal/audit-labels", async (c) => {
+    const status = c.req.query("status");
+    const where = status === "pending" || status === "adjudicated" ? "WHERE status = ?" : "";
+    const stmt = c.env.DB.prepare(
+      `SELECT id, project, target_id AS targetId, verdict, outcome, stratum, rubric_version AS rubricVersion,
+              sampled_at AS sampledAt, status, adjudication, reason_category AS reasonCategory, adjudicated_at AS adjudicatedAt
+         FROM decision_audit_labels ${where} ORDER BY sampled_at DESC, target_id ASC LIMIT 500`,
+    );
+    const rows = await (where ? stmt.bind(status) : stmt).all();
+    return c.json({ labels: rows.results });
+  });
+
+  app.post("/v1/internal/audit-labels/adjudicate", async (c) => {
+    const payload = (await c.req.json().catch(() => null)) as { id?: unknown; adjudication?: unknown; reasonCategory?: unknown } | null;
+    const id = typeof payload?.id === "string" ? payload.id : "";
+    const adjudication = payload?.adjudication;
+    if (!id || (adjudication !== "correct" && adjudication !== "incorrect" && adjudication !== "uncertain")) {
+      return c.json({ error: "id and adjudication (correct|incorrect|uncertain) required" }, 400);
+    }
+    const reasonCategory = typeof payload?.reasonCategory === "string" ? payload.reasonCategory.slice(0, 100) : null;
+    // Atomic claim: the status predicate rides the UPDATE itself, so two concurrent adjudications can never
+    // both win — a select-then-update here would let both pass the pending check and silently violate the
+    // one-adjudication-per-label guarantee (labels are calibration data; rewrites must be impossible, not
+    // merely discouraged). meta.changes disambiguates afterwards: 0 changes is either "no such label" (404)
+    // or "someone else already adjudicated it" (409), resolved by one read that no longer guards anything.
+    const result = await c.env.DB.prepare(
+      `UPDATE decision_audit_labels SET status = 'adjudicated', adjudication = ?, reason_category = ?, adjudicated_at = ?
+        WHERE id = ? AND status = 'pending'`,
+    )
+      .bind(adjudication, reasonCategory, new Date().toISOString(), id)
+      .run();
+    if (result.meta.changes === 0) {
+      const existing = await c.env.DB.prepare("SELECT status FROM decision_audit_labels WHERE id = ?").bind(id).first<{ status: string }>();
+      if (!existing) return c.json({ error: "label_not_found" }, 404);
+      return c.json({ error: "already_adjudicated" }, 409);
+    }
+    return c.json({ id, adjudication, reasonCategory });
+  });
+
   app.get("/v1/internal/orb/instances", async (c) => {
     const rows = await c.env.DB
       .prepare(

src/env.d.ts

@@ -502,6 +502,8 @@ declare global {
      *  recording are wired, reading a promoted override into the live gate is a noted follow-up that must not
      *  risk loosening the gate. See src/review/selftune-wire.ts. */
     LOOPOVER_REVIEW_SELFTUNE?: string;
+    /** #8830: weekly stratified human-audit sampling of gate decisions (default OFF). */
+    LOOPOVER_DECISION_AUDIT?: string;
     /** Experimental `gittensor` plugin (the `experimental:` manifest block, first key): the operator-level
      *  kill-switch for loopover's original subnet mining-registry/scoring integration, now opt-in rather than
      *  a core dependency. ANDed with the per-repo `.loopover.yml experimental.gittensor` opt-in -- neither

src/index.ts

@@ -14,6 +14,7 @@ import { isAprRepoTransferPollEnabled } from "./orb/apr-repo-transfer";
 import { isPrReconciliationEnabled, resolvePrReconciliationManifestOverride } from "./review/pr-reconciliation";
 import { isActiveReviewReconciliationEnabled, resolveActiveReviewReconciliationManifestOverride } from "./review/active-review-reconciliation";
 import { isRagEnabled } from "./review/rag-wire";
+import { isDecisionAuditEnabled } from "./review/decision-audit";
 import { isSelfTuneEnabled } from "./review/selftune-wire";
 import { isSatisfactionFloorAutotuneEnabled } from "./services/satisfaction-floor-loosening-run";
 import {
@@ -290,6 +291,13 @@ async function enqueueScheduledJobs(env: Env, controller: ScheduledController):
   if (isHourly && scheduledAt.getUTCDay() === 1 && hour === 12) {
     jobs.push({ type: "generate-weekly-value-report", requestedBy: "schedule", variant: "operator", days: 7 });
   }
+  // Decision-audit sampling (#8830, flag LOOPOVER_DECISION_AUDIT): weekly stratified draw of decided PRs for
+  // human adjudication. Tuesday 08:00 UTC — its own slot, clear of the Monday weekly report, the 03:00
+  // retention prune, and the 09:00 repo-doc sweep. Enqueued ONLY when the flag is ON — flag-OFF (default)
+  // this job is never created, so the cron tick does ZERO new work and the enqueued set is byte-identical.
+  if (isHourly && scheduledAt.getUTCDay() === 2 && hour === 8 && selfHostedReviews && isDecisionAuditEnabled(env)) {
+    jobs.push({ type: "decision-audit-sample", requestedBy: "schedule" });
+  }
   // Prune expired log/snapshot rows once a day (03:00 UTC) per the conservative RETENTION_POLICY.
   if (isHourly && hour === 3) {
     jobs.push({ type: "prune-retention", requestedBy: "schedule" });

src/openapi/schemas.ts

@@ -751,6 +751,7 @@ export const RepositorySettingsSchema = z
     duplicatePrGateMode: z.enum(["off", "advisory", "block"]),
     qualityGateMode: z.enum(["off", "advisory", "block"]),
     qualityGateMinScore: z.number().nullable().optional(),
+    closeAuditHoldoutPct: z.number().nullable().optional(),
     slopGateMode: z.enum(["off", "advisory", "block"]),
     sizeGateMode: z.enum(["off", "advisory", "block"]).optional(),
     sizeGateMaxFiles: z.number().optional(),
@@ -1037,6 +1038,7 @@ export const RepoSettingsPreviewSchema = z
       duplicatePrGateMode: z.enum(["off", "advisory", "block"]),
       qualityGateMode: z.enum(["off", "advisory", "block"]),
       qualityGateMinScore: z.number().nullable().optional(),
+      closeAuditHoldoutPct: z.number().nullable().optional(),
       slopGateMode: z.enum(["off", "advisory", "block"]),
       mergeReadinessGateMode: z.enum(["off", "advisory", "block"]),
       manifestPolicyGateMode: z.enum(["off", "advisory", "block"]),

src/queue/job-dispatch.ts

@@ -46,6 +46,7 @@ import {
 import { syncBrokeredInstalledRepos } from "../orb/installed-repos-sync";
 import { incr } from "../selfhost/metrics";
 import { generateSignalSnapshots } from "./signal-snapshot";
+import { isDecisionAuditEnabled, runDecisionAuditSample } from "../review/decision-audit";
 import { runRetentionPrune } from "./retention";
 // The 15 handlers below have no reason to move -- each is only reachable via this dispatcher (or, for
 // mapWithConcurrency, ALSO used by other still-in-processors.ts code), so they stay put and are exported
@@ -235,6 +236,14 @@ export async function processJob(env: Env, message: JobMessage): Promise<void> {
         message.dryRun ?? false,
       );
       return;
+    case "decision-audit-sample": {
+      // #8830: flag re-checked at execution (not only at enqueue) so a stale queued job after a flag flip
+      // does zero work, mirroring every sibling flag-gated job.
+      if (!isDecisionAuditEnabled(env)) return;
+      const inserted = await runDecisionAuditSample(env);
+      console.log(JSON.stringify({ event: "decision_audit_sampled", inserted }));
+      return;
+    }
     case "generate-weekly-value-report":
       await generateWeeklyValueReport(env, {
         variant: message.variant ?? "operator",