Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions apps/loopover-ui/content/docs/verify-this-review.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,33 @@ this site, so fetch it absolutely — a bare `/v1/public/stats` resolves against
curl -s "https://api.loopover.ai/v1/public/stats" | jq '.rulePrecision'
```

### Re-evaluation counts and author-class parity

The same endpoint publishes `reviewParity` (#9743): how often a verdict was re-evaluated and why, and
whether a pull request faced the same scrutiny regardless of who wrote it.

```bash
curl -s "https://api.loopover.ai/v1/public/stats" | jq '.reviewParity'
```

Every figure in that block is computed from `decision_records` alone — the anchored ledger — so you can
recompute all of it from an export without trusting this endpoint. The definitions are published beside
the code that implements them, in `src/review/review-parity-rollups.ts`. The three that are easy to get
wrong, and that are therefore stated explicitly:

- **`verdicts` counts evaluations, not pull requests.** A repeat evaluation of the same head SHA is its
own ledger row, which is exactly what `reviewsPerPr` divides out.
- **`findingsPerPr` excludes verdicts that recorded no finding count**, rather than averaging them in as
zero. `findingsBasis` beside it is the coverage the mean was earned at — a mean over 3 of 400 verdicts
is not the same claim as a mean over 400.
- **Author class is GitHub's own `author_association`**, not a list this project maintains. A PR whose
association was never recorded is reported as `unknown` in its own bucket rather than folded into
either side, because folding it would bias the exact comparison being published.

A rate over an empty window is `null`, never `0` — "nothing was held" and "nothing was measured" are
different claims, and the fairness report renders the second as *measured zero* with its window rather
than as a reassuring number.

The same per-rule numbers are also published as digest-committed
[EvalScoreRecords](/docs/what-you-can-verify), each independently re-derivable without trusting the
transport — recompute `recordDigest` over the record's own remaining fields and compare:
Expand Down
172 changes: 172 additions & 0 deletions apps/loopover-ui/public/openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -828,6 +828,177 @@
"accuracyPct"
]
}
},
"reviewParity": {
"type": "object",
"properties": {
"windowStart": {
"type": "string"
},
"windowEnd": {
"type": "string"
},
"verdicts": {
"type": "number"
},
"reevaluations": {
"type": "number"
},
"reevaluationRatePct": {
"type": "number",
"nullable": true
},
"byReason": {
"type": "array",
"items": {
"type": "object",
"properties": {
"reason": {
"type": "string"
},
"count": {
"type": "number"
},
"shareOfVerdictsPct": {
"type": "number",
"nullable": true
}
},
"required": [
"reason",
"count",
"shareOfVerdictsPct"
]
}
},
"byAuthorClass": {
"type": "array",
"items": {
"type": "object",
"properties": {
"authorClass": {
"type": "string",
"enum": [
"maintainer",
"contributor",
"unknown"
]
},
"verdicts": {
"type": "number"
},
"pullRequests": {
"type": "number"
},
"reviewsPerPr": {
"type": "number",
"nullable": true
},
"findingsPerPr": {
"type": "number",
"nullable": true
},
"findingsBasis": {
"type": "number"
},
"closeRate": {
"type": "number",
"nullable": true
},
"holdRate": {
"type": "number",
"nullable": true
}
},
"required": [
"authorClass",
"verdicts",
"pullRequests",
"reviewsPerPr",
"findingsPerPr",
"findingsBasis",
"closeRate",
"holdRate"
]
}
},
"byProject": {
"type": "array",
"items": {
"type": "object",
"properties": {
"project": {
"type": "string"
},
"byAuthorClass": {
"type": "array",
"items": {
"type": "object",
"properties": {
"authorClass": {
"type": "string",
"enum": [
"maintainer",
"contributor",
"unknown"
]
},
"verdicts": {
"type": "number"
},
"pullRequests": {
"type": "number"
},
"reviewsPerPr": {
"type": "number",
"nullable": true
},
"findingsPerPr": {
"type": "number",
"nullable": true
},
"findingsBasis": {
"type": "number"
},
"closeRate": {
"type": "number",
"nullable": true
},
"holdRate": {
"type": "number",
"nullable": true
}
},
"required": [
"authorClass",
"verdicts",
"pullRequests",
"reviewsPerPr",
"findingsPerPr",
"findingsBasis",
"closeRate",
"holdRate"
]
}
}
},
"required": [
"project",
"byAuthorClass"
]
}
}
},
"required": [
"windowStart",
"windowEnd",
"verdicts",
"reevaluations",
"reevaluationRatePct",
"byReason",
"byAuthorClass",
"byProject"
]
}
},
"required": [
Expand All @@ -836,6 +1007,7 @@
"totals",
"weekly",
"rulePrecision",
"reviewParity",
"byProject",
"fleetAccuracy",
"accuracyTrend",
Expand Down
113 changes: 112 additions & 1 deletion apps/loopover-ui/src/components/site/fairness-report-page.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,16 @@ function renderWithClient(ui: ReactNode) {
const FIXTURE: PublicStats = {
// The wire always carries rulePrecision (#8230/#8231). A fixture without it is not a payload the
// current backend can produce -- the one test that needs that shape strips it explicitly.
reviewParity: {
windowStart: "2026-07-22T00:00:00.000Z",
windowEnd: "2026-07-29T00:00:00.000Z",
verdicts: 0,
reevaluations: 0,
reevaluationRatePct: null,
byReason: [],
byAuthorClass: [],
byProject: [],
},
rulePrecision: {
windowDays: 90,
rules: [],
Expand Down Expand Up @@ -123,7 +133,9 @@ describe("FairnessReportPage (#fairness-analytics)", () => {
expect(screen.getByText(/Reproducibility freeze point/)).toBeTruthy();
expect(screen.getByText(/aaaaaaaaaaaaaaaa…/)).toBeTruthy();
// And the walkthrough link points at the docs page.
expect(screen.getByRole("link", { name: /verify this review/i })).toBeTruthy();
// Both the per-rule precision block and the review-parity block (#9744) invite the reader to
// reproduce their numbers, so there is deliberately more than one of these links.
expect(screen.getAllByRole("link", { name: /verify this review/i }).length).toBeGreaterThan(0);
});

it("hides the per-rule section entirely when the API response predates rulePrecision (deployment skew) or has no rules (#8231)", async () => {
Expand Down Expand Up @@ -398,4 +410,103 @@ describe("FairnessReportPage (#fairness-analytics)", () => {
).toContain("—");
expect(screen.queryByRole("alert")).toBeNull();
});

// #9744: the two series #9743 computes, and the zero-state conventions they must honour.
function parityFixture(over: Record<string, unknown>) {
return {
ok: true,
durationMs: 10,
data: {
...FIXTURE,
reviewParity: {
windowStart: "2026-07-22T00:00:00.000Z",
windowEnd: "2026-07-29T00:00:00.000Z",
verdicts: 0,
reevaluations: 0,
reevaluationRatePct: null,
byReason: [],
byAuthorClass: [],
byProject: [],
...over,
},
},
};
}

it("renders the re-evaluation reason table and the author-class parity table", async () => {
apiFetch.mockResolvedValue(
parityFixture({
verdicts: 10,
reevaluations: 3,
reevaluationRatePct: 30,
byReason: [{ reason: "scheduled_recheck", count: 3, shareOfVerdictsPct: 30 }],
byAuthorClass: [
{
authorClass: "maintainer",
verdicts: 4,
pullRequests: 4,
reviewsPerPr: 1,
findingsPerPr: 2,
findingsBasis: 4,
closeRate: 0,
holdRate: 25,
},
{
authorClass: "contributor",
verdicts: 6,
pullRequests: 3,
reviewsPerPr: 2,
findingsPerPr: null,
findingsBasis: 0,
closeRate: 50,
holdRate: 0,
},
],
}),
);
renderWithClient(<FairnessReportPage />);

expect(await screen.findByText(/Re-evaluation and review parity/i)).toBeTruthy();
expect(screen.getByText("scheduled_recheck")).toBeTruthy();
expect(screen.getByText(/3 of 10 verdicts were re-evaluations/i)).toBeTruthy();
expect(screen.getByText("maintainer")).toBeTruthy();
expect(screen.getByText("contributor")).toBeTruthy();
// The coverage a mean was earned at is published beside it, and an absent mean reads as
// insufficient data rather than as 0.
expect(screen.getByText(/n=4/)).toBeTruthy();
expect(screen.getAllByText(/insufficient data/i).length).toBeGreaterThan(0);
});

it("renders an EMPTY window as a measured zero with its dates, not as missing data", async () => {
apiFetch.mockResolvedValue(parityFixture({}));
renderWithClient(<FairnessReportPage />);

expect(await screen.findByText(/No verdicts recorded/i)).toBeTruthy();
expect(screen.getByText(/measured zero over the dates above, not missing data/i)).toBeTruthy();
});

it("distinguishes 'no re-evaluations' from 'no verdicts' — both are measured zeros, not the same one", async () => {
apiFetch.mockResolvedValue(
parityFixture({
verdicts: 5,
reevaluationRatePct: 0,
byAuthorClass: [
{
authorClass: "contributor",
verdicts: 5,
pullRequests: 5,
reviewsPerPr: 1,
findingsPerPr: 1,
findingsBasis: 5,
closeRate: 20,
holdRate: 0,
},
],
}),
);
renderWithClient(<FairnessReportPage />);

expect(await screen.findByText(/No re-evaluations recorded/i)).toBeTruthy();
expect(screen.queryByText(/No verdicts recorded/i)).toBeNull();
});
});
Loading
Loading