diff --git a/src/services/operator-dashboard.ts b/src/services/operator-dashboard.ts index b80b74ad46..6877d0f707 100644 --- a/src/services/operator-dashboard.ts +++ b/src/services/operator-dashboard.ts @@ -259,7 +259,7 @@ export async function buildOperatorDashboardPayload( { label: "Recommendation quality", value: `${recommendationQuality.totals.positive}/${recommendationQuality.totals.total}`, - delta: recommendationQuality.empty ? "no evaluated outcomes" : `${Math.round(recommendationQuality.totals.positiveRate * 100)}% positive`, + delta: recommendationQuality.totals.positiveRate === null ? "no evaluated outcomes" : `${Math.round(recommendationQuality.totals.positiveRate * 100)}% positive`, }, { label: "Install issues", diff --git a/src/services/outcome-calibration.ts b/src/services/outcome-calibration.ts index 87bff559a4..4f27139b28 100644 --- a/src/services/outcome-calibration.ts +++ b/src/services/outcome-calibration.ts @@ -10,9 +10,17 @@ // agent_recommendation_outcomes ledger (#543's recommendation half) is populated by evaluateRecommendationOutcomes. import { listAgentRecommendationOutcomes, listPullRequests } from "../db/repositories"; import type { SlopBand } from "../signals/slop"; -import type { AgentRecommendationOutcomeRecord, PullRequestRecord } from "../types"; +import type { AgentRecommendationOutcomeRecord, AgentRecommendationOutcomeState, PullRequestRecord } from "../types"; import { nowIso } from "../utils/json"; +// The single place the recommendation-outcome states are grouped into positive/negative/pending. Both +// aggregators over the agent_recommendation_outcomes ledger — this module and recommendation-quality-report.ts, +// which imports these — must classify identically, so a window of unactioned (stale/ignored) recommendations +// can't read as a 100% success rate here and a 50% failure rate there (#9701). +export const RECOMMENDATION_POSITIVE_STATES: readonly AgentRecommendationOutcomeState[] = ["accepted", "merged", "improved"]; +export const RECOMMENDATION_NEGATIVE_STATES: readonly AgentRecommendationOutcomeState[] = ["rejected", "closed"]; +export const RECOMMENDATION_PENDING_STATES: readonly AgentRecommendationOutcomeState[] = ["stale", "ignored"]; + // Severity order — calibration checks that merge rate is non-increasing along it. const SLOP_BAND_ORDER: readonly SlopBand[] = ["clean", "low", "elevated", "high"]; // Below this per-band sample the merge rate is too noisy to judge discrimination. @@ -130,9 +138,9 @@ export function buildRecommendationOutcomeCalibration( ): RecommendationOutcomeCalibration { const repoScoped = repoFullName ? outcomes.filter((o) => sameRepo(o.outcomeRepoFullName ?? o.targetRepoFullName, repoFullName)) : outcomes; const scoped = options.maintainerOnly ? repoScoped.filter((o) => o.maintainerLane) : repoScoped; - const positive = scoped.filter((o) => o.outcomeState === "accepted" || o.outcomeState === "merged" || o.outcomeState === "improved").length; - const negative = scoped.filter((o) => o.outcomeState === "rejected" || o.outcomeState === "closed").length; - const pending = scoped.filter((o) => o.outcomeState === "stale" || o.outcomeState === "ignored").length; + const positive = scoped.filter((o) => RECOMMENDATION_POSITIVE_STATES.includes(o.outcomeState)).length; + const negative = scoped.filter((o) => RECOMMENDATION_NEGATIVE_STATES.includes(o.outcomeState)).length; + const pending = scoped.filter((o) => RECOMMENDATION_PENDING_STATES.includes(o.outcomeState)).length; const resolved = positive + negative; return { total: scoped.length, positive, negative, pending, positiveRate: resolved > 0 ? round(positive / resolved) : null }; } diff --git a/src/services/recommendation-quality-report.ts b/src/services/recommendation-quality-report.ts index 5b3e867ee4..6085d1c316 100644 --- a/src/services/recommendation-quality-report.ts +++ b/src/services/recommendation-quality-report.ts @@ -1,6 +1,7 @@ import { listAgentRecommendationOutcomes } from "../db/repositories"; import type { AgentActionType, AgentRecommendationOutcomeRecord, AgentRecommendationOutcomeState, AgentSurface, JsonValue, ProductUsageRole } from "../types"; import { nowIso } from "../utils/json"; +import { RECOMMENDATION_NEGATIVE_STATES, RECOMMENDATION_PENDING_STATES, RECOMMENDATION_POSITIVE_STATES } from "./outcome-calibration"; export type RecommendationQualityRole = Extract; export type RecommendationQualityLane = "contributor" | "maintainer"; @@ -10,7 +11,8 @@ export type RecommendationQualityTotals = { total: number; positive: number; negative: number; - positiveRate: number; + pending: number; + positiveRate: number | null; maintainerLaneTotal: number; highConfidence: number; mediumConfidence: number; @@ -23,7 +25,7 @@ export type RecommendationQualityTrendBucket = RecommendationQualityTotals & { }; export type RecommendationQualityFailureCategory = { - category: "closed_without_merge" | "rejected" | "stale" | "ignored" | "low_confidence" | "maintainer_lane"; + category: "closed_without_merge" | "rejected" | "low_confidence" | "maintainer_lane"; label: string; count: number; detail: string; @@ -71,8 +73,6 @@ export type RecommendationQualityReport = { }; const ROLE_ORDER: RecommendationQualityRole[] = ["miner", "maintainer", "owner", "operator"]; -const POSITIVE_STATES: AgentRecommendationOutcomeState[] = ["accepted", "merged", "improved"]; -const NEGATIVE_STATES: AgentRecommendationOutcomeState[] = ["closed", "rejected", "stale", "ignored"]; export async function buildRecommendationQualityReport( env: Env, @@ -159,14 +159,16 @@ function roleSurface(role: RecommendationQualityRole, outcomes: AgentRecommendat } function qualityTotals(outcomes: AgentRecommendationOutcomeRecord[]): RecommendationQualityTotals { - const positive = outcomes.filter((outcome) => POSITIVE_STATES.includes(outcome.outcomeState)).length; - const negative = outcomes.filter((outcome) => NEGATIVE_STATES.includes(outcome.outcomeState)).length; + const positive = outcomes.filter((outcome) => RECOMMENDATION_POSITIVE_STATES.includes(outcome.outcomeState)).length; + const negative = outcomes.filter((outcome) => RECOMMENDATION_NEGATIVE_STATES.includes(outcome.outcomeState)).length; + const pending = outcomes.filter((outcome) => RECOMMENDATION_PENDING_STATES.includes(outcome.outcomeState)).length; const total = positive + negative; return { total, positive, negative, - positiveRate: total > 0 ? roundRate(positive / total) : 0, + pending, + positiveRate: total > 0 ? roundRate(positive / total) : null, maintainerLaneTotal: outcomes.filter((outcome) => outcome.maintainerLane).length, highConfidence: outcomes.filter((outcome) => outcome.confidence === "high").length, mediumConfidence: outcomes.filter((outcome) => outcome.confidence === "medium").length, @@ -188,18 +190,6 @@ function failureCategoryRows(outcomes: AgentRecommendationOutcomeRecord[]): Reco count: outcomes.filter((outcome) => outcome.outcomeState === "rejected").length, detail: "Recommended PR work received a changes-requested review signal.", }, - { - category: "stale", - label: "Stale follow-through", - count: outcomes.filter((outcome) => outcome.outcomeState === "stale").length, - detail: "Recommended work remained open without fresh cached activity past the freshness window.", - }, - { - category: "ignored", - label: "No matched activity", - count: outcomes.filter((outcome) => outcome.outcomeState === "ignored").length, - detail: "No cached PR or issue activity matched the recommendation after the action window.", - }, { category: "low_confidence", label: "Low confidence matches", diff --git a/test/unit/operator-dashboard.test.ts b/test/unit/operator-dashboard.test.ts index 40f840b5c2..07a26a9628 100644 --- a/test/unit/operator-dashboard.test.ts +++ b/test/unit/operator-dashboard.test.ts @@ -5,7 +5,8 @@ import { latestUsageRollup, __operatorDashboardInternals, } from "../../src/services/operator-dashboard"; -import type { ProductUsageDailyRollupRecord } from "../../src/types"; +import { createAgentRun, replaceAgentActions, upsertAgentRecommendationOutcome } from "../../src/db/repositories"; +import type { AgentRecommendationOutcomeRecord, ProductUsageDailyRollupRecord } from "../../src/types"; import { createTestEnv } from "../helpers/d1"; const FORBIDDEN_EXPORT_TERMS = @@ -336,8 +337,81 @@ describe("operator dashboard payload", () => { expect(latestUsageRollup(rollups)?.day).toBe("2026-05-30"); expect(latestUsageRollup([])).toBeNull(); }); + + it("renders the recommendation-quality tile with a positive rate when resolved outcomes exist", async () => { + const env = createTestEnv(); + const now = new Date().toISOString(); // keep the outcome inside the dashboard's rolling analytics window + await createAgentRun(env, { + id: "run:dashboard-merged", + objective: "Track quality", + actorLogin: "dashboard-dev", + surface: "api", + mode: "copilot", + status: "completed", + dataQualityStatus: "complete", + payload: {}, + createdAt: now, + updatedAt: now, + }); + await replaceAgentActions(env, "run:dashboard-merged", [ + { + id: "action:dashboard-merged", + runId: "run:dashboard-merged", + actionType: "choose_next_work", + targetRepoFullName: "owner/repo", + targetPullNumber: null, + targetIssueNumber: null, + status: "recommended", + recommendation: "pursue", + why: ["Safe aggregate fixture."], + blockedBy: [], + publicSafeSummary: "Safe aggregate fixture.", + approvalRequired: true, + safetyClass: "private", + payload: {}, + createdAt: now, + }, + ]); + await upsertAgentRecommendationOutcome(env, recommendationOutcome("merged", now)); + + const payload = await buildOperatorDashboardPayload(env); + + expect(payload.metrics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ label: "Recommendation quality", value: "1/1", delta: "100% positive" }), + ]), + ); + }); }); +function recommendationOutcome(outcomeState: AgentRecommendationOutcomeRecord["outcomeState"], now: string): AgentRecommendationOutcomeRecord { + return { + id: `outcome:dashboard-${outcomeState}`, + actionId: `action:dashboard-${outcomeState}`, + runId: `run:dashboard-${outcomeState}`, + actorLogin: "dashboard-dev", + actionType: "choose_next_work", + surface: null, + targetRepoFullName: "owner/repo", + targetPullNumber: null, + targetIssueNumber: null, + source: "inferred", + outcomeState, + outcomeTargetType: "repository", + outcomeRepoFullName: "owner/repo", + outcomePullNumber: null, + outcomeIssueNumber: null, + maintainerLane: false, + confidence: "high", + reason: "safe aggregate fixture", + sourceUpdatedAt: now, + detectedAt: now, + metadata: {}, + createdAt: now, + updatedAt: now, + }; +} + function rollup(day: string): ProductUsageDailyRollupRecord { return { day, diff --git a/test/unit/recommendation-quality-report.test.ts b/test/unit/recommendation-quality-report.test.ts index 5818ba9b0d..eca5dc129a 100644 --- a/test/unit/recommendation-quality-report.test.ts +++ b/test/unit/recommendation-quality-report.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { createAgentRun, replaceAgentActions, upsertAgentRecommendationOutcome } from "../../src/db/repositories"; +import { buildRecommendationOutcomeCalibration } from "../../src/services/outcome-calibration"; import { buildRecommendationQualityReport, buildRecommendationQualityReportFromOutcomes } from "../../src/services/recommendation-quality-report"; import type { AgentRecommendationOutcomeRecord } from "../../src/types"; import { createTestEnv } from "../helpers/d1"; @@ -30,14 +31,21 @@ describe("recommendation quality report", () => { expect(report).toMatchObject({ visibility: "operator_only", empty: false, - sparse: false, - totals: { total: 6, positive: 3, negative: 3, positiveRate: 0.5, maintainerLaneTotal: 1, lowConfidence: 1 }, + // Only 4 outcomes now count toward the resolved total (the 2 stale/ignored are pending), so it reads as sparse. + sparse: true, + totals: { total: 4, positive: 3, negative: 1, pending: 2, positiveRate: 0.75, maintainerLaneTotal: 1, lowConfidence: 1 }, publicExport: { available: false }, }); - expect(report.roleSurfaces.map((surface) => surface.role)).toEqual(["miner", "maintainer", "owner", "operator"]); + // operator's only outcome (operator-ignored) is pending now, so it has no resolved total and drops from + // the role surfaces; its pending count survives in the top-level totals.pending. + expect(report.roleSurfaces.map((surface) => surface.role)).toEqual(["miner", "maintainer", "owner"]); expect(report.roleSurfaces.find((surface) => surface.role === "miner")).toMatchObject({ total: 2, positive: 1, negative: 1 }); + // stale/ignored are pending now, not failures, so they no longer appear as failure categories (#9701). expect(report.failureCategories.map((category) => category.category)).toEqual( - expect.arrayContaining(["closed_without_merge", "stale", "ignored", "low_confidence", "maintainer_lane"]), + expect.arrayContaining(["closed_without_merge", "low_confidence", "maintainer_lane"]), + ); + expect(report.failureCategories.map((category) => category.category)).not.toEqual( + expect.arrayContaining(["stale", "ignored"]), ); expect(report.trends).toHaveLength(6); expect(JSON.stringify(report)).not.toMatch(FORBIDDEN_REPORT_TERMS); @@ -175,7 +183,7 @@ describe("recommendation quality report", () => { expect(empty).toMatchObject({ empty: true, sparse: false, - totals: { total: 0, positive: 0, negative: 0, positiveRate: 0 }, + totals: { total: 0, positive: 0, negative: 0, pending: 0, positiveRate: null }, roleSurfaces: [], failureCategories: [], }); @@ -194,7 +202,7 @@ describe("recommendation quality report", () => { [ outcome("metadata-array", "accepted", { metadata: { roles: ["unknown", "repo-owner"] }, repo: "owner/mixed" }), outcome("metadata-array-negative", "closed", { metadata: { roles: ["owner"] }, repo: "owner/mixed", updatedAt: "2026-05-30T00:00:00.000Z" }), - outcome("metadata-contributor", "ignored", { metadata: { audience: "contributor" }, repo: "owner/miner", confidence: "medium" }), + outcome("metadata-contributor", "closed", { metadata: { audience: "contributor" }, repo: "owner/miner", confidence: "medium" }), outcome("metadata-unknown", "merged", { metadata: { role: "reviewability" }, repo: "owner/fallback", actionType: "check_duplicate_risk" }), outcome("no-repo", "stale", { repo: null, actionType: "explain_repo_fit" }), ], @@ -251,7 +259,7 @@ describe("recommendation quality report", () => { createdAt: "2026-05-26T00:00:00.000Z", }; const missingTimestamp = { - ...outcome("missing-time", "ignored", { + ...outcome("missing-time", "closed", { repo: "owner/no-time", metadata: { role: "operator" }, }), @@ -332,6 +340,38 @@ describe("recommendation quality report", () => { totals: { total: 0 }, }); }); + + it("agrees with buildRecommendationOutcomeCalibration on positiveRate for a stale-heavy ledger (#9701)", () => { + const outcomes = [ + ...Array.from({ length: 5 }, (_unused, index) => outcome(`accepted-${index}`, "accepted", { repo: "owner/repo" })), + ...Array.from({ length: 5 }, (_unused, index) => outcome(`stale-${index}`, "stale", { repo: "owner/repo" })), + ]; + + const calibration = buildRecommendationOutcomeCalibration(outcomes); + const report = buildRecommendationQualityReportFromOutcomes(outcomes, { generatedAt: "2026-06-01T00:00:00.000Z", windowDays: 42 }); + + // Same window, same ledger: both aggregators must report the same positive rate (unactioned stale + // recommendations are pending, not failures, so 5 accepted + 5 stale is 100% positive on both surfaces). + expect(calibration.positiveRate).toBe(1); + expect(report.totals.positiveRate).toBe(1); + expect(report.totals.positiveRate).toBe(calibration.positiveRate); + // The stale bucket survives as pending, mirroring RecommendationOutcomeCalibration.pending. + expect(report.totals.pending).toBe(5); + expect(calibration.pending).toBe(5); + // ...and the stale outcomes are no longer attributed as failures. + expect(report.failureCategories.some((category) => category.category === "closed_without_merge" || category.category === "rejected")).toBe(false); + expect(report.failureCategories.map((category) => category.category)).not.toEqual(expect.arrayContaining(["stale", "ignored"])); + }); + + it("reports positiveRate null on an empty ledger, matching buildRecommendationOutcomeCalibration (#9701)", () => { + const calibration = buildRecommendationOutcomeCalibration([]); + const report = buildRecommendationQualityReportFromOutcomes([], { generatedAt: "2026-06-01T00:00:00.000Z", windowDays: 42 }); + + expect(calibration.positiveRate).toBeNull(); + expect(report.totals.positiveRate).toBeNull(); + expect(report.totals.positiveRate).toBe(calibration.positiveRate); + expect(report.totals.pending).toBe(0); + }); }); function outcome(