Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions src/services/gate-precision.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@ import { fetchOfficialGittensorMinerLogins } from "../gittensor/api";
import type { GateOutcomeRecord, PullRequestRecord } from "../types";
import { nowIso } from "../utils/json";

// Below this per-gate-type blocked sample the false-positive rate is too noisy to judge.
const MIN_SAMPLE = 5;
// Below this per-gate-type blocked sample the false-positive rate is too noisy to judge. Exported (#9691) so the
// recap's aggregate rates share the one floor instead of each consumer re-deriving it and drifting.
export const MIN_GATE_PRECISION_SAMPLE = 5;

export type GatePrecisionPerType = {
gateType: string;
Expand Down Expand Up @@ -69,7 +70,7 @@ function sameRepo(a: string | null | undefined, b: string): boolean {

/** #4520: the fold core, extracted so buildGatePrecisionReport can run it up to three times (blended, miner,
* human) over disjoint outcome subsets without duplicating the accumulation logic. Pure -- the same
* MIN_SAMPLE floor is applied independently per call, so a small cohort correctly reads null rather than a
* MIN_GATE_PRECISION_SAMPLE floor is applied independently per call, so a small cohort correctly reads null rather than a
* noisy rate. */
function foldGateOutcomes(outcomes: GateOutcomeRecord[], prByNumber: Map<number, PullRequestRecord>): GatePrecisionCohortReport {
const perType = new Map<string, { blocked: number; blockedThenMerged: number; overridden: number }>();
Expand Down Expand Up @@ -97,7 +98,7 @@ function foldGateOutcomes(outcomes: GateOutcomeRecord[], prByNumber: Map<number,
blockedThenMerged: entry.blockedThenMerged,
overridden: entry.overridden,
// Null below the min sample — a 1-of-1 "false positive" is noise, not a precision signal.
falsePositiveRate: entry.blocked >= MIN_SAMPLE ? round(entry.blockedThenMerged / entry.blocked) : null,
falsePositiveRate: entry.blocked >= MIN_GATE_PRECISION_SAMPLE ? round(entry.blockedThenMerged / entry.blocked) : null,
}))
.sort((a, b) => b.blocked - a.blocked || a.gateType.localeCompare(b.gateType));

Expand All @@ -106,7 +107,7 @@ function foldGateOutcomes(outcomes: GateOutcomeRecord[], prByNumber: Map<number,
overall: {
blocked: overallBlocked,
blockedThenMerged: overallMerged,
falsePositiveRate: overallBlocked >= MIN_SAMPLE ? round(overallMerged / overallBlocked) : null,
falsePositiveRate: overallBlocked >= MIN_GATE_PRECISION_SAMPLE ? round(overallMerged / overallBlocked) : null,
},
};
}
Expand All @@ -124,7 +125,7 @@ function isMinerAuthoredOutcome(outcome: GateOutcomeRecord, prByNumber: Map<numb
* PR's terminal outcome; a blocked PR that later MERGED is a false positive. Each blocker `code` on the row
* contributes to that code's bucket (a block citing two codes counts toward both). Overridden-then-merged is
* the strongest signal — `overridden` is counted separately per type. When `options.repoFullName` is given,
* only blocks for that repo are counted. The rate is null below MIN_SAMPLE. When `options.minerLogins` is
* only blocks for that repo are counted. The rate is null below MIN_GATE_PRECISION_SAMPLE. When `options.minerLogins` is
* given (#4520), an additive miner-vs-human `cohorts` split is computed on top of the SAME blended fold;
* omitting it keeps every existing caller byte-identical.
*/
Expand Down Expand Up @@ -165,7 +166,7 @@ export function buildGatePrecisionReport(

export function buildGatePrecisionSignals(perGateType: GatePrecisionPerType[], overallBlocked: number, overallMerged: number): string[] {
const signals: string[] = [];
if (overallBlocked < MIN_SAMPLE) {
if (overallBlocked < MIN_GATE_PRECISION_SAMPLE) {
signals.push(`Not enough recorded gate blocks to judge precision yet (${overallBlocked} blocked).`);
return signals;
}
Expand Down
18 changes: 8 additions & 10 deletions src/services/maintainer-recap-gate-outcomes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,11 @@
// straight from the same GatePrecisionReport totals that services/gate-precision.ts aggregates (the source
// src/review/ops-wire.ts reads). No delivery, no scheduling, no new queries.
//
// The false-positive rate is NULLED below MIN_SAMPLE exactly as gate-precision.ts:103 — a 1-of-1 "false
// The false-positive rate is NULLED below MIN_GATE_PRECISION_SAMPLE exactly as gate-precision.ts:103 — a 1-of-1 "false
// positive" is noise, not a precision signal. Own file (mirroring maintainer-recap-calibration.ts) so it
// stays decoupled from the foundation builder and sibling sections (zero shared-file conflict surface).
import { PUBLIC_LOCAL_PATH_SCRUB_PATTERN } from "../signals/redaction";

// Mirror gate-precision.ts:22 — the rate is noise below this many blocks, so it reports as null (n/a).
const MIN_SAMPLE = 5;
import { MIN_GATE_PRECISION_SAMPLE } from "./gate-precision";

/** Mirror gate-precision.ts:41 round() — three decimal places. */
function round(value: number): number {
Expand All @@ -37,7 +35,7 @@ export type GateOutcomesRecapSection = {
blocked: number;
overridden: number;
falsePositives: number;
/** blockedThenMerged / blocked, 3 dp — NULL below MIN_SAMPLE (gate-precision.ts:103). */
/** blockedThenMerged / blocked, 3 dp — NULL below MIN_GATE_PRECISION_SAMPLE (gate-precision.ts:103). */
falsePositiveRate: number | null;
lines: string[];
};
Expand All @@ -51,19 +49,19 @@ function sanitizeRecapText(value: string): string {
/**
* Pure gate-outcomes section over a RecapReport projection.
*
* - `falsePositiveRate` = gateFalsePositives / blocked, rounded to 3 dp — but **null below MIN_SAMPLE**
* - `falsePositiveRate` = gateFalsePositives / blocked, rounded to 3 dp — but **null below MIN_GATE_PRECISION_SAMPLE**
* (and therefore also when nothing was blocked), exactly as gate-precision.ts nulls a low-sample rate.
* - The rate line reads "n/a" on the null arm so the digest still carries a gate-outcomes section.
*/
export function buildGateOutcomesRecapSection(report: GateOutcomesRecapSource): GateOutcomesRecapSection {
const { blocked, gateFalsePositives, gateOverrides } = report.totals;
// Null the rate below MIN_SAMPLE (gate-precision.ts:103) — a 1-of-1 "false positive" is noise. This also
// covers the divide-by-zero arm (blocked === 0 < MIN_SAMPLE), so the ratio is never evaluated at 0.
const falsePositiveRate = blocked >= MIN_SAMPLE ? round(gateFalsePositives / blocked) : null;
// Null the rate below MIN_GATE_PRECISION_SAMPLE (gate-precision.ts:103) — a 1-of-1 "false positive" is noise. This also
// covers the divide-by-zero arm (blocked === 0 < MIN_GATE_PRECISION_SAMPLE), so the ratio is never evaluated at 0.
const falsePositiveRate = blocked >= MIN_GATE_PRECISION_SAMPLE ? round(gateFalsePositives / blocked) : null;

const rateLine =
falsePositiveRate === null
? `False-positive rate: n/a (fewer than ${MIN_SAMPLE} blocks in the last ${report.windowDays} day(s))`
? `False-positive rate: n/a (fewer than ${MIN_GATE_PRECISION_SAMPLE} blocks in the last ${report.windowDays} day(s))`
: `False-positive rate: ${Math.round(falsePositiveRate * 100)}% (${gateFalsePositives} of ${blocked} blocks merged anyway)`;

const title = "Gate outcomes";
Expand Down
10 changes: 7 additions & 3 deletions src/services/maintainer-recap.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import { PUBLIC_LOCAL_PATH_SCRUB_PATTERN, PUBLIC_UNSAFE_PATTERN } from "../signals/redaction";
import { deliverRecapToDiscord, deliverRecapToSlack } from "./notify-discord";
import type { GatePrecisionReport } from "./gate-precision";
import { MIN_GATE_PRECISION_SAMPLE } from "./gate-precision";
import type { DriftRecapSection } from "./maintainer-recap-drift";
// #8372: these three section builders shipped fully implemented + unit-tested but were never composed into
// the delivered digest -- the same "built, tested, never called from production" shape as #6636.
Expand Down Expand Up @@ -142,20 +143,23 @@ export function buildMaintainerRecap(args: MaintainerRecapInputs): RecapReport {
cohortTotals.human.gateFalsePositives += gatePrecision.cohorts.human.overall.blockedThenMerged;
}
}
// Null below the shared noise floor (#9691), matching gate-precision.ts and the gate-outcomes section — a
// digest must not print a false-positive percentage in its summary/Totals while its own Gate-outcomes section
// says the sample is too small to judge. Subsumes the old `blocked === 0` divide-by-zero guard.
totals.gateFalsePositiveRate =
totals.blocked > 0 ? Math.round((totals.gateFalsePositives / totals.blocked) * 100) / 100 : null;
totals.blocked >= MIN_GATE_PRECISION_SAMPLE ? Math.round((totals.gateFalsePositives / totals.blocked) * 100) / 100 : null;
const cohorts =
cohortBlockedRepos > 0
? {
miner: {
blocked: cohortTotals.miner.blocked,
gateFalsePositives: cohortTotals.miner.gateFalsePositives,
gateFalsePositiveRate: cohortTotals.miner.blocked > 0 ? Math.round((cohortTotals.miner.gateFalsePositives / cohortTotals.miner.blocked) * 100) / 100 : null,
gateFalsePositiveRate: cohortTotals.miner.blocked >= MIN_GATE_PRECISION_SAMPLE ? Math.round((cohortTotals.miner.gateFalsePositives / cohortTotals.miner.blocked) * 100) / 100 : null,
},
human: {
blocked: cohortTotals.human.blocked,
gateFalsePositives: cohortTotals.human.gateFalsePositives,
gateFalsePositiveRate: cohortTotals.human.blocked > 0 ? Math.round((cohortTotals.human.gateFalsePositives / cohortTotals.human.blocked) * 100) / 100 : null,
gateFalsePositiveRate: cohortTotals.human.blocked >= MIN_GATE_PRECISION_SAMPLE ? Math.round((cohortTotals.human.gateFalsePositives / cohortTotals.human.blocked) * 100) / 100 : null,
},
}
: undefined;
Expand Down
22 changes: 22 additions & 0 deletions test/unit/maintainer-recap-format.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,28 @@ describe("formatMaintainerRecap (#2240)", () => {
expect(body).not.toContain("payout");
});

it("renders n/a (not a percentage) for a below-floor report — Totals and Gate-outcomes agree (#9691)", () => {
const report: RecapReport = {
...emptyReport(),
totals: {
...emptyReport().totals,
reviewed: 2,
blocked: 2,
gateFalsePositives: 2,
gateFalsePositiveRate: null,
},
summary: ["Gate false-positive rate: not enough blocked PRs in the window to report."],
};
const body = formatMaintainerRecap(report);
// Totals still reports the raw counts, but the rate reads n/a — no contradictory percentage anywhere.
expect(body).toContain("- Gate false positives: 2/2 (n/a)");
expect(body).not.toMatch(/Gate false positives: 2\/2 \(\d+%\)/);
// The Gate-outcomes section's own below-floor copy is present and consistent.
expect(body).toContain("False-positive rate: n/a (fewer than 5 blocks in the last 7 day(s))");
// The summary's not-enough-blocked arm survives verbatim.
expect(body).toContain("- Gate false-positive rate: not enough blocked PRs in the window to report.");
});

it("omits cohort diagnostics from the public recap even when totals.cohorts is present", () => {
const report: RecapReport = {
...emptyReport(),
Expand Down
80 changes: 72 additions & 8 deletions test/unit/maintainer-recap.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,12 @@ function repoInput(
const bands: OutcomeCalibration["slop"]["bands"] = c.emptyBands
? []
: [{ band: "clean", sampleSize: 0, merged: c.merged ?? 0, closed: c.closed ?? 0, mergeRate: 0 }];
// Mirror gate-precision.ts's real per-cohort floor (#9691): a GatePrecisionReport nulls the rate below
// MIN_GATE_PRECISION_SAMPLE (5) blocks, so a fixture simulating that report must too — otherwise it feeds the
// recap a per-repo rate production would never produce.
const cohortReport = (counts: { blocked: number; blockedThenMerged: number }) => ({
perGateType: [],
overall: { blocked: counts.blocked, blockedThenMerged: counts.blockedThenMerged, falsePositiveRate: counts.blocked > 0 ? Math.round((counts.blockedThenMerged / counts.blocked) * 1000) / 1000 : null },
overall: { blocked: counts.blocked, blockedThenMerged: counts.blockedThenMerged, falsePositiveRate: counts.blocked >= 5 ? Math.round((counts.blockedThenMerged / counts.blocked) * 1000) / 1000 : null },
});
return {
gatePrecision: {
Expand Down Expand Up @@ -145,13 +148,15 @@ describe("buildMaintainerRecap cohort split (#4521)", () => {
}),
],
});
// Both cohorts are below the 5-block floor, so their rate reads null (#9691) — per-repo (fed from the
// already-floored GatePrecisionReport) and the cross-repo aggregate alike. Raw counts are still reported.
expect(report.repos[0]?.cohorts).toMatchObject({
miner: { blocked: 2, gateFalsePositives: 1, gateFalsePositiveRate: 0.5 },
human: { blocked: 4, gateFalsePositives: 1, gateFalsePositiveRate: 0.25 },
miner: { blocked: 2, gateFalsePositives: 1, gateFalsePositiveRate: null },
human: { blocked: 4, gateFalsePositives: 1, gateFalsePositiveRate: null },
});
expect(report.totals.cohorts).toMatchObject({
miner: { blocked: 2, gateFalsePositives: 1, gateFalsePositiveRate: 0.5 },
human: { blocked: 4, gateFalsePositives: 1, gateFalsePositiveRate: 0.25 },
miner: { blocked: 2, gateFalsePositives: 1, gateFalsePositiveRate: null },
human: { blocked: 4, gateFalsePositives: 1, gateFalsePositiveRate: null },
});
expect(report.summary).toHaveLength(3);
expect(report.summary.join("\n")).not.toContain("Miner-originated");
Expand All @@ -166,9 +171,66 @@ describe("buildMaintainerRecap cohort split (#4521)", () => {
repoInput("owner/repo-b", { blocked: 3, blockedThenMerged: 1, cohorts: { miner: { blocked: 1, blockedThenMerged: 0 }, human: { blocked: 2, blockedThenMerged: 1 } } }),
],
});
// Both aggregated cohorts are below the 5-block floor → null rate, not 0 or 0.5 (#9691).
expect(report.totals.cohorts).toMatchObject({
miner: { blocked: 3, gateFalsePositives: 0, gateFalsePositiveRate: 0 },
human: { blocked: 2, gateFalsePositives: 1, gateFalsePositiveRate: 0.5 },
miner: { blocked: 3, gateFalsePositives: 0, gateFalsePositiveRate: null },
human: { blocked: 2, gateFalsePositives: 1, gateFalsePositiveRate: null },
});
});

describe("aggregate gate false-positive rate honors the shared MIN_GATE_PRECISION_SAMPLE floor (#9691)", () => {
it("nulls the blended rate when two repos each block only once (2/2 total, below the 5-floor)", () => {
const report = buildMaintainerRecap({
generatedAt: GEN,
repos: [
repoInput("owner/repo-a", { blocked: 1, blockedThenMerged: 1 }),
repoInput("owner/repo-b", { blocked: 1, blockedThenMerged: 1 }),
],
});
// Raw counts still reported; the rate reads null instead of a contradictory 100%.
expect(report.totals).toMatchObject({ blocked: 2, gateFalsePositives: 2, gateFalsePositiveRate: null });
});

it("reports the rate once the blended sample reaches the floor (blocked 5, fp 1 → 0.2)", () => {
const report = buildMaintainerRecap({
generatedAt: GEN,
repos: [
repoInput("owner/repo-a", { blocked: 3, blockedThenMerged: 1 }),
repoInput("owner/repo-b", { blocked: 2, blockedThenMerged: 0 }),
],
});
expect(report.totals).toMatchObject({ blocked: 5, gateFalsePositives: 1, gateFalsePositiveRate: 0.2 });
});

it("floors each cohort against its OWN blocked denominator, not the blended one", () => {
const report = buildMaintainerRecap({
generatedAt: GEN,
repos: [
// Blended blocked = 8 (≥5), but the human cohort's own blocked is 2 and the miner's is 6.
repoInput("owner/repo-a", { blocked: 6, blockedThenMerged: 3, cohorts: { miner: { blocked: 6, blockedThenMerged: 3 }, human: { blocked: 0, blockedThenMerged: 0 } } }),
repoInput("owner/repo-b", { blocked: 2, blockedThenMerged: 1, cohorts: { miner: { blocked: 0, blockedThenMerged: 0 }, human: { blocked: 2, blockedThenMerged: 1 } } }),
],
});
expect(report.totals.cohorts).toMatchObject({
miner: { blocked: 6, gateFalsePositives: 3, gateFalsePositiveRate: 0.5 }, // ≥5 → reported
human: { blocked: 2, gateFalsePositives: 1, gateFalsePositiveRate: null }, // <5 → null despite blended ≥5
});
// The blended rate itself is reported (total blocked 8 ≥ 5).
expect(report.totals.gateFalsePositiveRate).toBe(0.5);
});

it("reports each cohort rate when BOTH cohorts are at or above their own floor", () => {
const report = buildMaintainerRecap({
generatedAt: GEN,
repos: [
repoInput("owner/repo-a", { blocked: 10, blockedThenMerged: 5, cohorts: { miner: { blocked: 5, blockedThenMerged: 1 }, human: { blocked: 5, blockedThenMerged: 2 } } }),
],
});
// Both cohorts' own blocked ≥ 5 → both report a number (the ≥-floor truthy arm on each).
expect(report.totals.cohorts).toMatchObject({
miner: { blocked: 5, gateFalsePositives: 1, gateFalsePositiveRate: 0.2 },
human: { blocked: 5, gateFalsePositives: 2, gateFalsePositiveRate: 0.4 },
});
});
});

Expand Down Expand Up @@ -351,7 +413,9 @@ describe("runMaintainerRecap (#2252 end-to-end orchestration)", () => {

it("builds, formats, and fans out to BOTH channels when both webhooks are configured", async () => {
const calls = stubRecapChannelFetch();
const result = await runMaintainerRecap(envWithBothWebhooks(), { repos: [repoInput("owner/repo-a", { blocked: 4, blockedThenMerged: 1, totalResolved: 2, merged: 1, closed: 1 })] });
// blocked: 5 keeps this fan-out check at or above the false-positive floor (#9691) so a rendered percentage
// still exists to assert against; the digest content itself is exercised in dedicated builder/format tests.
const result = await runMaintainerRecap(envWithBothWebhooks(), { repos: [repoInput("owner/repo-a", { blocked: 5, blockedThenMerged: 1, totalResolved: 2, merged: 1, closed: 1 })] });
expect(result.skipped).toBe(false);
if (result.skipped) return;
expect(result.formatted).toContain("# Maintainer recap");
Expand Down