fix(services): share one positive/negative/pending split across the recommendation aggregators

src/services/operator-dashboard.ts

@@ -259,7 +259,7 @@ export async function buildOperatorDashboardPayload(
       {
         label: "Recommendation quality",
         value: `${recommendationQuality.totals.positive}/${recommendationQuality.totals.total}`,
-        delta: recommendationQuality.empty ? "no evaluated outcomes" : `${Math.round(recommendationQuality.totals.positiveRate * 100)}% positive`,
+        delta: recommendationQuality.totals.positiveRate === null ? "no evaluated outcomes" : `${Math.round(recommendationQuality.totals.positiveRate * 100)}% positive`,
       },
       {
         label: "Install issues",

src/services/outcome-calibration.ts

@@ -10,9 +10,17 @@
 // agent_recommendation_outcomes ledger (#543's recommendation half) is populated by evaluateRecommendationOutcomes.
 import { listAgentRecommendationOutcomes, listPullRequests } from "../db/repositories";
 import type { SlopBand } from "../signals/slop";
-import type { AgentRecommendationOutcomeRecord, PullRequestRecord } from "../types";
+import type { AgentRecommendationOutcomeRecord, AgentRecommendationOutcomeState, PullRequestRecord } from "../types";
 import { nowIso } from "../utils/json";
 
+// The single place the recommendation-outcome states are grouped into positive/negative/pending. Both
+// aggregators over the agent_recommendation_outcomes ledger — this module and recommendation-quality-report.ts,
+// which imports these — must classify identically, so a window of unactioned (stale/ignored) recommendations
+// can't read as a 100% success rate here and a 50% failure rate there (#9701).
+export const RECOMMENDATION_POSITIVE_STATES: readonly AgentRecommendationOutcomeState[] = ["accepted", "merged", "improved"];
+export const RECOMMENDATION_NEGATIVE_STATES: readonly AgentRecommendationOutcomeState[] = ["rejected", "closed"];
+export const RECOMMENDATION_PENDING_STATES: readonly AgentRecommendationOutcomeState[] = ["stale", "ignored"];
+
 // Severity order — calibration checks that merge rate is non-increasing along it.
 const SLOP_BAND_ORDER: readonly SlopBand[] = ["clean", "low", "elevated", "high"];
 // Below this per-band sample the merge rate is too noisy to judge discrimination.
@@ -130,9 +138,9 @@ export function buildRecommendationOutcomeCalibration(
 ): RecommendationOutcomeCalibration {
   const repoScoped = repoFullName ? outcomes.filter((o) => sameRepo(o.outcomeRepoFullName ?? o.targetRepoFullName, repoFullName)) : outcomes;
   const scoped = options.maintainerOnly ? repoScoped.filter((o) => o.maintainerLane) : repoScoped;
-  const positive = scoped.filter((o) => o.outcomeState === "accepted" || o.outcomeState === "merged" || o.outcomeState === "improved").length;
-  const negative = scoped.filter((o) => o.outcomeState === "rejected" || o.outcomeState === "closed").length;
-  const pending = scoped.filter((o) => o.outcomeState === "stale" || o.outcomeState === "ignored").length;
+  const positive = scoped.filter((o) => RECOMMENDATION_POSITIVE_STATES.includes(o.outcomeState)).length;
+  const negative = scoped.filter((o) => RECOMMENDATION_NEGATIVE_STATES.includes(o.outcomeState)).length;
+  const pending = scoped.filter((o) => RECOMMENDATION_PENDING_STATES.includes(o.outcomeState)).length;
   const resolved = positive + negative;
   return { total: scoped.length, positive, negative, pending, positiveRate: resolved > 0 ? round(positive / resolved) : null };
 }

src/services/recommendation-quality-report.ts

@@ -1,6 +1,7 @@
 import { listAgentRecommendationOutcomes } from "../db/repositories";
 import type { AgentActionType, AgentRecommendationOutcomeRecord, AgentRecommendationOutcomeState, AgentSurface, JsonValue, ProductUsageRole } from "../types";
 import { nowIso } from "../utils/json";
+import { RECOMMENDATION_NEGATIVE_STATES, RECOMMENDATION_PENDING_STATES, RECOMMENDATION_POSITIVE_STATES } from "./outcome-calibration";
 
 export type RecommendationQualityRole = Extract<ProductUsageRole, "miner" | "maintainer" | "owner" | "operator">;
 export type RecommendationQualityLane = "contributor" | "maintainer";
@@ -10,7 +11,8 @@ export type RecommendationQualityTotals = {
   total: number;
   positive: number;
   negative: number;
-  positiveRate: number;
+  pending: number;
+  positiveRate: number | null;
   maintainerLaneTotal: number;
   highConfidence: number;
   mediumConfidence: number;
@@ -23,7 +25,7 @@ export type RecommendationQualityTrendBucket = RecommendationQualityTotals & {
 };
 
 export type RecommendationQualityFailureCategory = {
-  category: "closed_without_merge" | "rejected" | "stale" | "ignored" | "low_confidence" | "maintainer_lane";
+  category: "closed_without_merge" | "rejected" | "low_confidence" | "maintainer_lane";
   label: string;
   count: number;
   detail: string;
@@ -71,8 +73,6 @@ export type RecommendationQualityReport = {
 };
 
 const ROLE_ORDER: RecommendationQualityRole[] = ["miner", "maintainer", "owner", "operator"];
-const POSITIVE_STATES: AgentRecommendationOutcomeState[] = ["accepted", "merged", "improved"];
-const NEGATIVE_STATES: AgentRecommendationOutcomeState[] = ["closed", "rejected", "stale", "ignored"];
 
 export async function buildRecommendationQualityReport(
   env: Env,
@@ -159,14 +159,16 @@ function roleSurface(role: RecommendationQualityRole, outcomes: AgentRecommendat
 }
 
 function qualityTotals(outcomes: AgentRecommendationOutcomeRecord[]): RecommendationQualityTotals {
-  const positive = outcomes.filter((outcome) => POSITIVE_STATES.includes(outcome.outcomeState)).length;
-  const negative = outcomes.filter((outcome) => NEGATIVE_STATES.includes(outcome.outcomeState)).length;
+  const positive = outcomes.filter((outcome) => RECOMMENDATION_POSITIVE_STATES.includes(outcome.outcomeState)).length;
+  const negative = outcomes.filter((outcome) => RECOMMENDATION_NEGATIVE_STATES.includes(outcome.outcomeState)).length;
+  const pending = outcomes.filter((outcome) => RECOMMENDATION_PENDING_STATES.includes(outcome.outcomeState)).length;
   const total = positive + negative;
   return {
     total,
     positive,
     negative,
-    positiveRate: total > 0 ? roundRate(positive / total) : 0,
+    pending,
+    positiveRate: total > 0 ? roundRate(positive / total) : null,
     maintainerLaneTotal: outcomes.filter((outcome) => outcome.maintainerLane).length,
     highConfidence: outcomes.filter((outcome) => outcome.confidence === "high").length,
     mediumConfidence: outcomes.filter((outcome) => outcome.confidence === "medium").length,
@@ -188,18 +190,6 @@ function failureCategoryRows(outcomes: AgentRecommendationOutcomeRecord[]): Reco
       count: outcomes.filter((outcome) => outcome.outcomeState === "rejected").length,
       detail: "Recommended PR work received a changes-requested review signal.",
     },
-    {
-      category: "stale",
-      label: "Stale follow-through",
-      count: outcomes.filter((outcome) => outcome.outcomeState === "stale").length,
-      detail: "Recommended work remained open without fresh cached activity past the freshness window.",
-    },
-    {
-      category: "ignored",
-      label: "No matched activity",
-      count: outcomes.filter((outcome) => outcome.outcomeState === "ignored").length,
-      detail: "No cached PR or issue activity matched the recommendation after the action window.",
-    },
     {
       category: "low_confidence",
       label: "Low confidence matches",

test/unit/operator-dashboard.test.ts

@@ -5,7 +5,8 @@ import {
   latestUsageRollup,
   __operatorDashboardInternals,
 } from "../../src/services/operator-dashboard";
-import type { ProductUsageDailyRollupRecord } from "../../src/types";
+import { createAgentRun, replaceAgentActions, upsertAgentRecommendationOutcome } from "../../src/db/repositories";
+import type { AgentRecommendationOutcomeRecord, ProductUsageDailyRollupRecord } from "../../src/types";
 import { createTestEnv } from "../helpers/d1";
 
 const FORBIDDEN_EXPORT_TERMS =
@@ -336,8 +337,81 @@ describe("operator dashboard payload", () => {
     expect(latestUsageRollup(rollups)?.day).toBe("2026-05-30");
     expect(latestUsageRollup([])).toBeNull();
   });
+
+  it("renders the recommendation-quality tile with a positive rate when resolved outcomes exist", async () => {
+    const env = createTestEnv();
+    const now = new Date().toISOString(); // keep the outcome inside the dashboard's rolling analytics window
+    await createAgentRun(env, {
+      id: "run:dashboard-merged",
+      objective: "Track quality",
+      actorLogin: "dashboard-dev",
+      surface: "api",
+      mode: "copilot",
+      status: "completed",
+      dataQualityStatus: "complete",
+      payload: {},
+      createdAt: now,
+      updatedAt: now,
+    });
+    await replaceAgentActions(env, "run:dashboard-merged", [
+      {
+        id: "action:dashboard-merged",
+        runId: "run:dashboard-merged",
+        actionType: "choose_next_work",
+        targetRepoFullName: "owner/repo",
+        targetPullNumber: null,
+        targetIssueNumber: null,
+        status: "recommended",
+        recommendation: "pursue",
+        why: ["Safe aggregate fixture."],
+        blockedBy: [],
+        publicSafeSummary: "Safe aggregate fixture.",
+        approvalRequired: true,
+        safetyClass: "private",
+        payload: {},
+        createdAt: now,
+      },
+    ]);
+    await upsertAgentRecommendationOutcome(env, recommendationOutcome("merged", now));
+
+    const payload = await buildOperatorDashboardPayload(env);
+
+    expect(payload.metrics).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ label: "Recommendation quality", value: "1/1", delta: "100% positive" }),
+      ]),
+    );
+  });
 });
 
+function recommendationOutcome(outcomeState: AgentRecommendationOutcomeRecord["outcomeState"], now: string): AgentRecommendationOutcomeRecord {
+  return {
+    id: `outcome:dashboard-${outcomeState}`,
+    actionId: `action:dashboard-${outcomeState}`,
+    runId: `run:dashboard-${outcomeState}`,
+    actorLogin: "dashboard-dev",
+    actionType: "choose_next_work",
+    surface: null,
+    targetRepoFullName: "owner/repo",
+    targetPullNumber: null,
+    targetIssueNumber: null,
+    source: "inferred",
+    outcomeState,
+    outcomeTargetType: "repository",
+    outcomeRepoFullName: "owner/repo",
+    outcomePullNumber: null,
+    outcomeIssueNumber: null,
+    maintainerLane: false,
+    confidence: "high",
+    reason: "safe aggregate fixture",
+    sourceUpdatedAt: now,
+    detectedAt: now,
+    metadata: {},
+    createdAt: now,
+    updatedAt: now,
+  };
+}
+
 function rollup(day: string): ProductUsageDailyRollupRecord {
   return {
     day,

test/unit/recommendation-quality-report.test.ts

@@ -1,5 +1,6 @@
 import { describe, expect, it } from "vitest";
 import { createAgentRun, replaceAgentActions, upsertAgentRecommendationOutcome } from "../../src/db/repositories";
+import { buildRecommendationOutcomeCalibration } from "../../src/services/outcome-calibration";
 import { buildRecommendationQualityReport, buildRecommendationQualityReportFromOutcomes } from "../../src/services/recommendation-quality-report";
 import type { AgentRecommendationOutcomeRecord } from "../../src/types";
 import { createTestEnv } from "../helpers/d1";
@@ -30,14 +31,21 @@ describe("recommendation quality report", () => {
     expect(report).toMatchObject({
       visibility: "operator_only",
       empty: false,
-      sparse: false,
-      totals: { total: 6, positive: 3, negative: 3, positiveRate: 0.5, maintainerLaneTotal: 1, lowConfidence: 1 },
+      // Only 4 outcomes now count toward the resolved total (the 2 stale/ignored are pending), so it reads as sparse.
+      sparse: true,
+      totals: { total: 4, positive: 3, negative: 1, pending: 2, positiveRate: 0.75, maintainerLaneTotal: 1, lowConfidence: 1 },
       publicExport: { available: false },
     });
-    expect(report.roleSurfaces.map((surface) => surface.role)).toEqual(["miner", "maintainer", "owner", "operator"]);
+    // operator's only outcome (operator-ignored) is pending now, so it has no resolved total and drops from
+    // the role surfaces; its pending count survives in the top-level totals.pending.
+    expect(report.roleSurfaces.map((surface) => surface.role)).toEqual(["miner", "maintainer", "owner"]);
     expect(report.roleSurfaces.find((surface) => surface.role === "miner")).toMatchObject({ total: 2, positive: 1, negative: 1 });
+    // stale/ignored are pending now, not failures, so they no longer appear as failure categories (#9701).
     expect(report.failureCategories.map((category) => category.category)).toEqual(
-      expect.arrayContaining(["closed_without_merge", "stale", "ignored", "low_confidence", "maintainer_lane"]),
+      expect.arrayContaining(["closed_without_merge", "low_confidence", "maintainer_lane"]),
+    );
+    expect(report.failureCategories.map((category) => category.category)).not.toEqual(
+      expect.arrayContaining(["stale", "ignored"]),
     );
     expect(report.trends).toHaveLength(6);
     expect(JSON.stringify(report)).not.toMatch(FORBIDDEN_REPORT_TERMS);
@@ -175,7 +183,7 @@ describe("recommendation quality report", () => {
     expect(empty).toMatchObject({
       empty: true,
       sparse: false,
-      totals: { total: 0, positive: 0, negative: 0, positiveRate: 0 },
+      totals: { total: 0, positive: 0, negative: 0, pending: 0, positiveRate: null },
       roleSurfaces: [],
       failureCategories: [],
     });
@@ -194,7 +202,7 @@ describe("recommendation quality report", () => {
       [
         outcome("metadata-array", "accepted", { metadata: { roles: ["unknown", "repo-owner"] }, repo: "owner/mixed" }),
         outcome("metadata-array-negative", "closed", { metadata: { roles: ["owner"] }, repo: "owner/mixed", updatedAt: "2026-05-30T00:00:00.000Z" }),
-        outcome("metadata-contributor", "ignored", { metadata: { audience: "contributor" }, repo: "owner/miner", confidence: "medium" }),
+        outcome("metadata-contributor", "closed", { metadata: { audience: "contributor" }, repo: "owner/miner", confidence: "medium" }),
         outcome("metadata-unknown", "merged", { metadata: { role: "reviewability" }, repo: "owner/fallback", actionType: "check_duplicate_risk" }),
         outcome("no-repo", "stale", { repo: null, actionType: "explain_repo_fit" }),
       ],
@@ -251,7 +259,7 @@ describe("recommendation quality report", () => {
       createdAt: "2026-05-26T00:00:00.000Z",
     };
     const missingTimestamp = {
-      ...outcome("missing-time", "ignored", {
+      ...outcome("missing-time", "closed", {
         repo: "owner/no-time",
         metadata: { role: "operator" },
       }),
@@ -332,6 +340,38 @@ describe("recommendation quality report", () => {
       totals: { total: 0 },
     });
   });
+
+  it("agrees with buildRecommendationOutcomeCalibration on positiveRate for a stale-heavy ledger (#9701)", () => {
+    const outcomes = [
+      ...Array.from({ length: 5 }, (_unused, index) => outcome(`accepted-${index}`, "accepted", { repo: "owner/repo" })),
+      ...Array.from({ length: 5 }, (_unused, index) => outcome(`stale-${index}`, "stale", { repo: "owner/repo" })),
+    ];
+
+    const calibration = buildRecommendationOutcomeCalibration(outcomes);
+    const report = buildRecommendationQualityReportFromOutcomes(outcomes, { generatedAt: "2026-06-01T00:00:00.000Z", windowDays: 42 });
+
+    // Same window, same ledger: both aggregators must report the same positive rate (unactioned stale
+    // recommendations are pending, not failures, so 5 accepted + 5 stale is 100% positive on both surfaces).
+    expect(calibration.positiveRate).toBe(1);
+    expect(report.totals.positiveRate).toBe(1);
+    expect(report.totals.positiveRate).toBe(calibration.positiveRate);
+    // The stale bucket survives as pending, mirroring RecommendationOutcomeCalibration.pending.
+    expect(report.totals.pending).toBe(5);
+    expect(calibration.pending).toBe(5);
+    // ...and the stale outcomes are no longer attributed as failures.
+    expect(report.failureCategories.some((category) => category.category === "closed_without_merge" || category.category === "rejected")).toBe(false);
+    expect(report.failureCategories.map((category) => category.category)).not.toEqual(expect.arrayContaining(["stale", "ignored"]));
+  });
+
+  it("reports positiveRate null on an empty ledger, matching buildRecommendationOutcomeCalibration (#9701)", () => {
+    const calibration = buildRecommendationOutcomeCalibration([]);
+    const report = buildRecommendationQualityReportFromOutcomes([], { generatedAt: "2026-06-01T00:00:00.000Z", windowDays: 42 });
+
+    expect(calibration.positiveRate).toBeNull();
+    expect(report.totals.positiveRate).toBeNull();
+    expect(report.totals.positiveRate).toBe(calibration.positiveRate);
+    expect(report.totals.pending).toBe(0);
+  });
 });
 
 function outcome(