diff --git a/apps/loopover-ui/public/openapi.json b/apps/loopover-ui/public/openapi.json index 1aab6d0a7c..84541855da 100644 --- a/apps/loopover-ui/public/openapi.json +++ b/apps/loopover-ui/public/openapi.json @@ -457,13 +457,17 @@ "precision": { "type": "number", "nullable": true + }, + "unrecognized": { + "type": "number" } }, "required": [ "ruleId", "decided", "confirmed", - "precision" + "precision", + "unrecognized" ] } }, diff --git a/packages/loopover-contract/src/public-api.ts b/packages/loopover-contract/src/public-api.ts index f3716097e9..cd5410f8d2 100644 --- a/packages/loopover-contract/src/public-api.ts +++ b/packages/loopover-contract/src/public-api.ts @@ -21,7 +21,9 @@ import { z } from "zod"; */ export const PublicRulePrecisionSchema = z.object({ windowDays: z.number(), - rules: z.array(z.object({ ruleId: z.string(), decided: z.number(), confirmed: z.number(), precision: z.number().nullable() })), + rules: z.array( + z.object({ ruleId: z.string(), decided: z.number(), confirmed: z.number(), precision: z.number().nullable(), unrecognized: z.number() }), + ), reversals: z.object({ reopened: z.number(), reverted: z.number(), superseded: z.number() }), latestBacktestRun: z.object({ corpusChecksum: z.string(), at: z.string() }).nullable(), }); diff --git a/src/review/public-rule-precision.ts b/src/review/public-rule-precision.ts index 8747ee0e88..1b6969faa5 100644 --- a/src/review/public-rule-precision.ts +++ b/src/review/public-rule-precision.ts @@ -41,6 +41,10 @@ export type PublicRulePrecisionRow = { confirmed: number; /** confirmed / decided, rounded to 3 decimals; null below {@link PUBLIC_PRECISION_MIN_DECIDED}. */ precision: number | null; + /** Overrides whose `$.verdict` was missing or neither `'reversed'` nor `'confirmed'` -- excluded from + * `decided` and `confirmed` (and so never inflates `precision` or clears the sample floor), but surfaced + * as a data-quality signal rather than silently dropped. */ + unrecognized: number; }; export type PublicRulePrecision = { @@ -79,10 +83,12 @@ export const NON_ATTRIBUTABLE_OVERRIDE_PROVENANCES = ["slop_replay_backfill_v1"] export async function loadPublicRulePrecision(env: Env, nowMs: number = Date.now()): Promise { const sinceIso = new Date(nowMs - PUBLIC_PRECISION_WINDOW_DAYS * 24 * 60 * 60 * 1000).toISOString(); - const overrideRows = await safeAll<{ rule_id: string; decided: number; reversed: number }>( + const overrideRows = await safeAll<{ rule_id: string; confirmed: number; reversed: number; unrecognized: number }>( env, - `SELECT substr(event_type, ${HUMAN_OVERRIDE_EVENT_TYPE_PREFIX.length + 1}) AS rule_id, COUNT(*) AS decided, - SUM(CASE WHEN json_extract(metadata_json, '$.verdict') = 'reversed' THEN 1 ELSE 0 END) AS reversed + `SELECT substr(event_type, ${HUMAN_OVERRIDE_EVENT_TYPE_PREFIX.length + 1}) AS rule_id, + SUM(CASE WHEN json_extract(metadata_json, '$.verdict') = 'confirmed' THEN 1 ELSE 0 END) AS confirmed, + SUM(CASE WHEN json_extract(metadata_json, '$.verdict') = 'reversed' THEN 1 ELSE 0 END) AS reversed, + SUM(CASE WHEN json_extract(metadata_json, '$.verdict') NOT IN ('confirmed', 'reversed') OR json_extract(metadata_json, '$.verdict') IS NULL THEN 1 ELSE 0 END) AS unrecognized FROM audit_events WHERE event_type LIKE '${HUMAN_OVERRIDE_EVENT_TYPE_PREFIX}%' AND created_at >= ? AND COALESCE(json_extract(metadata_json, '$.provenance'), '') NOT IN (${NON_ATTRIBUTABLE_OVERRIDE_PROVENANCES.map((tag) => `'${tag}'`).join(", ")}) @@ -91,15 +97,19 @@ export async function loadPublicRulePrecision(env: Env, nowMs: number = Date.now ); const rules: PublicRulePrecisionRow[] = overrideRows .map((row) => { - /* v8 ignore next 2 -- SUM(CASE) over a GROUP BY always yields a defined integer; the ?? guards a + /* v8 ignore next -- SUM(CASE) over a GROUP BY always yields a defined integer; the ?? guards a * future query-shape change, mirroring loadOverrideDayRows' identical note. */ + const confirmed = row.confirmed ?? 0; + /* v8 ignore next */ const reversed = row.reversed ?? 0; - const decided = row.decided; - const confirmed = decided - reversed; + /* v8 ignore next */ + const unrecognized = row.unrecognized ?? 0; + const decided = confirmed + reversed; return { ruleId: row.rule_id, decided, confirmed, + unrecognized, precision: decided >= PUBLIC_PRECISION_MIN_DECIDED ? Math.round((confirmed / decided) * 1000) / 1000 : null, }; }) diff --git a/src/services/rule-calibration-trend.ts b/src/services/rule-calibration-trend.ts index e1c346f38d..c01c942129 100644 --- a/src/services/rule-calibration-trend.ts +++ b/src/services/rule-calibration-trend.ts @@ -33,6 +33,9 @@ export type CalibrationRuleTrendWeek = { confirmed: number | null; reversed: number | null; precisionPct: number | null; + /** Overrides whose `$.verdict` was missing or neither `'reversed'` nor `'confirmed'` -- a data-quality + * signal, always visible regardless of the publication floor (never folded into `confirmed`). */ + unrecognized: number; }; export type CalibrationRuleTrend = { ruleId: string; weeks: CalibrationRuleTrendWeek[] }; @@ -51,7 +54,7 @@ export type CalibrationTrendReport = { }; export type FiredDayRow = { ruleId: string; day: string; fired: number }; -export type OverrideDayRow = { ruleId: string; day: string; confirmed: number; reversed: number }; +export type OverrideDayRow = { ruleId: string; day: string; confirmed: number; reversed: number; unrecognized: number }; export type BacktestRunDayRow = { day: string; regressed: number; improved: number; unchanged: number }; const MS_PER_WEEK = 7 * 86_400_000; @@ -84,11 +87,11 @@ export function buildCalibrationTrend( const currentStartMs = Date.parse(isoWeekStart(nowMs)); const oldestStartMs = currentStartMs - (weeks - 1) * MS_PER_WEEK; - const ruleBuckets = new Map>(); + const ruleBuckets = new Map>(); const bucketsFor = (ruleId: string) => { const existing = ruleBuckets.get(ruleId); if (existing) return existing; - const created = Array.from({ length: weeks }, () => ({ fired: 0, confirmed: 0, reversed: 0 })); + const created = Array.from({ length: weeks }, () => ({ fired: 0, confirmed: 0, reversed: 0, unrecognized: 0 })); ruleBuckets.set(ruleId, created); return created; }; @@ -103,6 +106,7 @@ export function buildCalibrationTrend( const bucket = bucketsFor(row.ruleId)[offset]!; bucket.confirmed += row.confirmed; bucket.reversed += row.reversed; + bucket.unrecognized += row.unrecognized; } const runBuckets = Array.from({ length: weeks }, () => ({ regressed: 0, improved: 0, unchanged: 0 })); @@ -128,6 +132,7 @@ export function buildCalibrationTrend( confirmed: publishable ? bucket.confirmed : null, reversed: publishable ? bucket.reversed : null, precisionPct: publishable ? roundPct(bucket.confirmed / decided) : null, + unrecognized: bucket.unrecognized, }; }), })); @@ -161,19 +166,28 @@ async function loadFiredDayRows(env: Env, sinceIso: string): Promise { - const rows = await safeAll<{ rule_id: string; day: string; confirmed: number; reversed: number }>( + const rows = await safeAll<{ rule_id: string; day: string; confirmed: number; reversed: number; unrecognized: number }>( env, `SELECT substr(event_type, ${HUMAN_OVERRIDE_EVENT_TYPE_PREFIX.length + 1}) AS rule_id, date(created_at) AS day, - SUM(CASE WHEN json_extract(metadata_json, '$.verdict') = 'reversed' THEN 0 ELSE 1 END) AS confirmed, - SUM(CASE WHEN json_extract(metadata_json, '$.verdict') = 'reversed' THEN 1 ELSE 0 END) AS reversed + SUM(CASE WHEN json_extract(metadata_json, '$.verdict') = 'confirmed' THEN 1 ELSE 0 END) AS confirmed, + SUM(CASE WHEN json_extract(metadata_json, '$.verdict') = 'reversed' THEN 1 ELSE 0 END) AS reversed, + SUM(CASE WHEN json_extract(metadata_json, '$.verdict') NOT IN ('confirmed', 'reversed') OR json_extract(metadata_json, '$.verdict') IS NULL THEN 1 ELSE 0 END) AS unrecognized FROM audit_events WHERE event_type LIKE '${HUMAN_OVERRIDE_EVENT_TYPE_PREFIX}%' AND created_at >= ? GROUP BY rule_id, day`, sinceIso, ); - /* v8 ignore next 2 -- SUM(CASE ...) over a GROUP BY always yields a defined integer, never SQL NULL; the ?? 0 - * fallbacks guard a future query-shape change, mirroring loadOrbDayRows' identical note. */ - return rows.map((row) => ({ ruleId: row.rule_id, day: row.day, confirmed: row.confirmed ?? 0, reversed: row.reversed ?? 0 })); + return rows.map((row) => ({ + ruleId: row.rule_id, + day: row.day, + /* v8 ignore next -- SUM(CASE ...) over a GROUP BY always yields a defined integer, never SQL NULL; the + * ?? 0 fallbacks guard a future query-shape change, mirroring loadOrbDayRows' identical note. */ + confirmed: row.confirmed ?? 0, + /* v8 ignore next */ + reversed: row.reversed ?? 0, + /* v8 ignore next */ + unrecognized: row.unrecognized ?? 0, + })); } /** Day-bucketed backtest runs across BOTH sibling event types, verdict read from the persisted diff --git a/test/unit/eval-score-records.test.ts b/test/unit/eval-score-records.test.ts index 9243212e07..1430bdaa10 100644 --- a/test/unit/eval-score-records.test.ts +++ b/test/unit/eval-score-records.test.ts @@ -21,8 +21,8 @@ const ISSUED_AT = "2026-07-27T12:00:00.000Z"; const PRECISION_WITH_FREEZE_POINT: PublicRulePrecision = { windowDays: 90, rules: [ - { ruleId: "ai_consensus_defect", decided: 25, confirmed: 20, precision: 0.8 }, - { ruleId: "sparse_rule", decided: 9, confirmed: 9, precision: null }, + { ruleId: "ai_consensus_defect", decided: 25, confirmed: 20, precision: 0.8, unrecognized: 0 }, + { ruleId: "sparse_rule", decided: 9, confirmed: 9, precision: null, unrecognized: 0 }, ], reversals: { reopened: 2, reverted: 1, superseded: 3 }, latestBacktestRun: { corpusChecksum: "abc123def456", at: "2026-07-27T10:00:00.000Z" }, @@ -99,7 +99,7 @@ describe("buildEvalScoreRecordsFromRulePrecision (#9266)", () => { // than a hardcoded constant. Previously EVERY record published null, including the 25-decided one above, // so a validator re-deriving decided/(decided+abstained) per #9215 computed 1 and disagreed with the field. const records = await buildEvalScoreRecordsFromRulePrecision( - { ...PRECISION_WITH_FREEZE_POINT, rules: [{ ruleId: "never_fired", decided: 0, confirmed: 0, precision: null }] }, + { ...PRECISION_WITH_FREEZE_POINT, rules: [{ ruleId: "never_fired", decided: 0, confirmed: 0, precision: null, unrecognized: 0 }] }, ISSUED_AT, ); expect(records).toHaveLength(1); @@ -207,8 +207,7 @@ describe("per-rule corpus commitments when no backtest run is persisted (#9805)" reversals: { reopened: 0, reverted: 0, superseded: 0 }, latestBacktestRun, }); - const rule = (ruleId: string): PublicRulePrecision["rules"][number] => - ({ ruleId, decided: 40, confirmed: 25, precision: 0.625 }) as PublicRulePrecision["rules"][number]; + const rule = (ruleId: string): PublicRulePrecision["rules"][number] => ({ ruleId, decided: 40, confirmed: 25, precision: 0.625, unrecognized: 0 }); it("REGRESSION: publishes a record per rule instead of [], committing to that rule's published corpus", async () => { const records = await buildEvalScoreRecordsFromRulePrecision( diff --git a/test/unit/public-rule-precision.test.ts b/test/unit/public-rule-precision.test.ts index bc2cbc39b5..5ea0b8613e 100644 --- a/test/unit/public-rule-precision.test.ts +++ b/test/unit/public-rule-precision.test.ts @@ -38,8 +38,8 @@ describe("loadPublicRulePrecision (#8230)", () => { const block = await loadPublicRulePrecision(env, NOW); expect(block.windowDays).toBe(PUBLIC_PRECISION_WINDOW_DAYS); expect(block.rules).toEqual([ - { ruleId: "ai_consensus_defect", decided: 25, confirmed: 20, precision: 0.8 }, - { ruleId: "linked_issue_scope_mismatch", decided: 12, confirmed: 9, precision: 0.75 }, + { ruleId: "ai_consensus_defect", decided: 25, confirmed: 20, precision: 0.8, unrecognized: 0 }, + { ruleId: "linked_issue_scope_mismatch", decided: 12, confirmed: 9, precision: 0.75, unrecognized: 0 }, ]); }); @@ -55,7 +55,9 @@ describe("loadPublicRulePrecision (#8230)", () => { }); const block = await loadPublicRulePrecision(env, NOW); - expect(block.rules).toEqual([{ ruleId: "sparse_rule", decided: PUBLIC_PRECISION_MIN_DECIDED - 1, confirmed: PUBLIC_PRECISION_MIN_DECIDED - 1, precision: null }]); + expect(block.rules).toEqual([ + { ruleId: "sparse_rule", decided: PUBLIC_PRECISION_MIN_DECIDED - 1, confirmed: PUBLIC_PRECISION_MIN_DECIDED - 1, precision: null, unrecognized: 0 }, + ]); }); it("REGRESSION: excludes counterfactual-replay rows whose label came from a DIFFERENT rule's human verdicts", async () => { @@ -76,7 +78,7 @@ describe("loadPublicRulePrecision (#8230)", () => { } const block = await loadPublicRulePrecision(env, NOW); - expect(block.rules).toEqual([{ ruleId: "ai_consensus_defect", decided: 20, confirmed: 15, precision: 0.75 }]); + expect(block.rules).toEqual([{ ruleId: "ai_consensus_defect", decided: 20, confirmed: 15, precision: 0.75, unrecognized: 0 }]); }); it("keeps synthesized rows whose labels ARE about the rule they are filed under", async () => { @@ -95,7 +97,30 @@ describe("loadPublicRulePrecision (#8230)", () => { } const block = await loadPublicRulePrecision(env, NOW); - expect(block.rules).toEqual([{ ruleId: "ai_consensus_defect", decided: 12, confirmed: 9, precision: 0.75 }]); + expect(block.rules).toEqual([{ ruleId: "ai_consensus_defect", decided: 12, confirmed: 9, precision: 0.75, unrecognized: 0 }]); + }); + + it("REGRESSION (#9640): a missing or unrecognized $.verdict is counted as unrecognized, never folded into confirmed", async () => { + // Before the fix, `decided` was COUNT(*) and `confirmed = decided - reversed`, so any row whose verdict + // was absent or neither 'reversed' nor 'confirmed' inflated confirmed AND cleared the sample floor: this + // would have reported decided: 15, confirmed: 14, precision: 0.933. + const env = createTestEnv(); + await seedVerdicts(env, "linked_issue_scope_mismatch", 9, 1); + for (let i = 0; i < 5; i += 1) { + await recordAuditEvent(env, { + eventType: "signal.human_override:linked_issue_scope_mismatch", + actor: "human", + targetKey: `acme/widgets#${i + 100}`, + outcome: "completed", + metadata: {}, + createdAt: new Date(NOW - 2000 - i).toISOString(), + }); + } + + const block = await loadPublicRulePrecision(env, NOW); + expect(block.rules).toEqual([ + { ruleId: "linked_issue_scope_mismatch", decided: 10, confirmed: 9, precision: 0.9, unrecognized: 5 }, + ]); }); it("counts all three reversal shapes over the window and surfaces the latest backtest run's corpus checksum", async () => { @@ -156,7 +181,7 @@ describe("loadPublicRulePrecision (#8230)", () => { expect(block.latestBacktestRun).toBeNull(); // The scores come from a different dataset (human-override events) and are unaffected -- an empty corpus // means the numbers are uncommitted, never that they are zero. - expect(block.rules).toEqual([{ ruleId: "ai_consensus_defect", decided: 20, confirmed: 15, precision: 0.75 }]); + expect(block.rules).toEqual([{ ruleId: "ai_consensus_defect", decided: 20, confirmed: 15, precision: 0.75, unrecognized: 0 }]); }); it("EMPTY_CORPUS_CHECKSUM is the exporter's own checksum over zero cases", async () => { diff --git a/test/unit/rule-calibration-trend.test.ts b/test/unit/rule-calibration-trend.test.ts index a263c7762a..f7b6878ddc 100644 --- a/test/unit/rule-calibration-trend.test.ts +++ b/test/unit/rule-calibration-trend.test.ts @@ -25,30 +25,57 @@ describe("buildCalibrationTrend (#8113)", () => { // A second day in the SAME week — must accumulate. { ruleId: "linked_issue_scope_mismatch", day: priorMonday, fired: 2 }, ]; - const overrides: OverrideDayRow[] = [{ ruleId: "linked_issue_scope_mismatch", day: priorMonday, confirmed: 3, reversed: 1 }]; + const overrides: OverrideDayRow[] = [ + { ruleId: "linked_issue_scope_mismatch", day: priorMonday, confirmed: 3, reversed: 1, unrecognized: 0 }, + ]; const trend = buildCalibrationTrend(fired, overrides, [], NOW, 2); expect(trend.rules).toHaveLength(1); const [rule] = trend.rules; expect(rule!.ruleId).toBe("linked_issue_scope_mismatch"); expect(rule!.weeks).toEqual([ - { weekStart: priorMonday, fired: 6, confirmed: 3, reversed: 1, precisionPct: 75 }, - { weekStart: currentMonday, fired: 0, confirmed: null, reversed: null, precisionPct: null }, + { weekStart: priorMonday, fired: 6, confirmed: 3, reversed: 1, precisionPct: 75, unrecognized: 0 }, + { weekStart: currentMonday, fired: 0, confirmed: null, reversed: null, precisionPct: null, unrecognized: 0 }, ]); }); it("keeps a week's verdict split null below MIN_CALIBRATION_TREND_SAMPLE decided — unknown never fakes 0 or 100", () => { - const overrides: OverrideDayRow[] = [{ ruleId: "duplicate_pr_risk", day: currentMonday, confirmed: MIN_CALIBRATION_TREND_SAMPLE - 1, reversed: 0 }]; + const overrides: OverrideDayRow[] = [ + { ruleId: "duplicate_pr_risk", day: currentMonday, confirmed: MIN_CALIBRATION_TREND_SAMPLE - 1, reversed: 0, unrecognized: 0 }, + ]; const trend = buildCalibrationTrend([], overrides, [], NOW, 1); - expect(trend.rules[0]!.weeks[0]).toEqual({ weekStart: currentMonday, fired: 0, confirmed: null, reversed: null, precisionPct: null }); + expect(trend.rules[0]!.weeks[0]).toEqual({ + weekStart: currentMonday, + fired: 0, + confirmed: null, + reversed: null, + precisionPct: null, + unrecognized: 0, + }); }); it("creates a rule bucket from an override-only history (no firings recorded in the window)", () => { - const overrides: OverrideDayRow[] = [{ ruleId: "missing_linked_issue", day: currentMonday, confirmed: 2, reversed: 2 }]; + const overrides: OverrideDayRow[] = [{ ruleId: "missing_linked_issue", day: currentMonday, confirmed: 2, reversed: 2, unrecognized: 0 }]; const trend = buildCalibrationTrend([], overrides, [], NOW, 1); expect(trend.rules[0]!.weeks[0]!.precisionPct).toBe(50); expect(trend.rules[0]!.weeks[0]!.fired).toBe(0); }); + it("REGRESSION (#9640): counts a missing/unrecognized $.verdict as unrecognized, excluded from decided/confirmed", () => { + const overrides: OverrideDayRow[] = [ + { ruleId: "linked_issue_scope_mismatch", day: currentMonday, confirmed: 9, reversed: 1, unrecognized: 5 }, + ]; + const trend = buildCalibrationTrend([], overrides, [], NOW, 1); + // Below the fix, decided would have been 15 (confirmed folded the 5 unrecognized rows in) and precision 0.933. + expect(trend.rules[0]!.weeks[0]).toEqual({ + weekStart: currentMonday, + fired: 0, + confirmed: 9, + reversed: 1, + precisionPct: 90, + unrecognized: 5, + }); + }); + it("sorts rules by ruleId for byte-stable output", () => { const fired: FiredDayRow[] = [ { ruleId: "zeta_rule", day: currentMonday, fired: 1 }, @@ -64,7 +91,7 @@ describe("buildCalibrationTrend (#8113)", () => { { ruleId: "r", day: outside, fired: 5 }, { ruleId: "r", day: "not-a-day", fired: 5 }, ]; - const overrides: OverrideDayRow[] = [{ ruleId: "r", day: future, confirmed: 5, reversed: 5 }]; + const overrides: OverrideDayRow[] = [{ ruleId: "r", day: future, confirmed: 5, reversed: 5, unrecognized: 0 }]; const runs: BacktestRunDayRow[] = [{ day: "junk", regressed: 1, improved: 1, unchanged: 1 }]; const trend = buildCalibrationTrend(fired, overrides, runs, NOW, 2); expect(trend.rules).toEqual([]); @@ -154,7 +181,7 @@ describe("loadCalibrationTrend (#8113)", () => { const trend = await loadCalibrationTrend(env, NOW); const rule = trend.rules.find((entry) => entry.ruleId === "linked_issue_scope_mismatch"); const priorWeek = rule!.weeks.find((week) => week.weekStart === priorMonday); - expect(priorWeek).toEqual({ weekStart: priorMonday, fired: 1, confirmed: 3, reversed: 1, precisionPct: 75 }); + expect(priorWeek).toEqual({ weekStart: priorMonday, fired: 1, confirmed: 3, reversed: 1, precisionPct: 75, unrecognized: 0 }); const runWeek = trend.backtestRuns.find((week) => week.weekStart === priorMonday); expect(runWeek).toEqual({ weekStart: priorMonday, runs: 3, regressed: 1, improved: 1, unchanged: 1 }); });