From 1b6e60d79dea2da61e2905513827da6915fbd4f2 Mon Sep 17 00:00:00 2001 From: Andriy Polanski Date: Fri, 24 Jul 2026 21:30:35 +0000 Subject: [PATCH 1/2] feat(miner): add calibration backtest-eligibility for ContributionProfile replay --- .../loopover-miner/lib/ams-calibration.ts | 76 ++- .../lib/ams-eligibility-backtest.ts | 267 +++++++++ .../loopover-miner/lib/calibration-cli.ts | 108 +++- .../miner-ams-eligibility-backtest.test.ts | 516 ++++++++++++++++++ test/unit/miner-calibration-cli.test.ts | 233 +++++++- 5 files changed, 1191 insertions(+), 9 deletions(-) create mode 100644 packages/loopover-miner/lib/ams-eligibility-backtest.ts create mode 100644 test/unit/miner-ams-eligibility-backtest.test.ts diff --git a/packages/loopover-miner/lib/ams-calibration.ts b/packages/loopover-miner/lib/ams-calibration.ts index aa75e06b23..4fd47cdccb 100644 --- a/packages/loopover-miner/lib/ams-calibration.ts +++ b/packages/loopover-miner/lib/ams-calibration.ts @@ -30,6 +30,8 @@ import { MINER_PR_OUTCOME_EVENT } from "./pr-outcome.js"; /** Event-ledger vocabulary for one persisted advisory min-rank backtest run (the AMS analog of ORB's * `calibration.threshold_backtest_run` -- the #8185 track record aggregates over these). */ export const MINER_AMS_THRESHOLD_BACKTEST_EVENT = "ams_threshold_backtest_run"; +/** Event-ledger vocabulary for one persisted advisory eligibility-exclusion backtest run (#8545). */ +export const MINER_AMS_ELIGIBILITY_BACKTEST_EVENT = "ams_eligibility_backtest_run"; /** Event-ledger vocabulary for an approved min-rank override apply (#8187). */ export const MINER_AMS_MIN_RANK_APPLIED_EVENT = "ams_min_rank_override_applied"; /** Event-ledger vocabulary for a min-rank override reversion (#8187's one-command revert). */ @@ -146,6 +148,34 @@ export function recordAmsThresholdBacktestRun(result: AmsMinRankBacktestResult, }); } +/** Persist one advisory eligibility-exclusion backtest run (#8545): comparison metadata only, same shape + * discipline as {@link recordAmsThresholdBacktestRun} so #8185's aggregation stays byte-compatible. */ +export function recordAmsEligibilityBacktestRun( + result: { + ruleId: string; + skippedNoContext: number; + visibleCases: number; + heldOutCases: number; + visible: BacktestComparison; + heldOut: BacktestComparison; + }, + options: { eventLedger?: LedgerWriter } = {}, +): LedgerEntry { + const eventLedger = options.eventLedger; + if (!eventLedger || typeof eventLedger.appendEvent !== "function") throw new Error("invalid_event_ledger"); + return eventLedger.appendEvent({ + type: MINER_AMS_ELIGIBILITY_BACKTEST_EVENT, + payload: { + ruleId: result.ruleId, + skippedNoContext: result.skippedNoContext, + visibleCases: result.visibleCases, + heldOutCases: result.heldOutCases, + visible: result.visible as unknown as Record, + heldOut: result.heldOut as unknown as Record, + }, + }); +} + export type PersistedAmsBacktestRun = { createdAt: string | null; currentThreshold: number; @@ -156,6 +186,15 @@ export type PersistedAmsBacktestRun = { heldOut: BacktestComparison; }; +export type PersistedAmsEligibilityBacktestRun = { + createdAt: string | null; + skippedNoContext: number; + visibleCases: number; + heldOutCases: number; + visible: BacktestComparison; + heldOut: BacktestComparison; +}; + function isComparison(value: unknown): value is BacktestComparison { const record = value as Record | null | undefined; return ( @@ -189,11 +228,38 @@ export function readAmsThresholdBacktestRuns(eventLedger: LedgerReader): Persist return runs; } -/** #8185: the REGRESSED-verdict track record over every persisted run's comparisons -- the SAME aggregation - * ORB uses (`computeRegressedVerdictTrackRecord`), zero new math. Both slices count: a held-out REGRESSED - * is exactly as real a verdict as a visible one. */ -export function computeAmsBacktestTrackRecord(runs: readonly PersistedAmsBacktestRun[]): RegressedVerdictTrackRecord { - return computeRegressedVerdictTrackRecord(runs.flatMap((run) => [run.visible, run.heldOut])); +/** Read every persisted eligibility-exclusion backtest run, oldest first; foreign types and malformed payloads + * are skipped. */ +export function readAmsEligibilityBacktestRuns(eventLedger: LedgerReader): PersistedAmsEligibilityBacktestRun[] { + const runs: PersistedAmsEligibilityBacktestRun[] = []; + for (const record of readAll(eventLedger)) { + if (record.type !== MINER_AMS_ELIGIBILITY_BACKTEST_EVENT) continue; + const payload = record.payload as Record | null | undefined; + if (!payload || typeof payload !== "object") continue; + if (!Number.isInteger(payload.skippedNoContext)) continue; + if (!isComparison(payload.visible) || !isComparison(payload.heldOut)) continue; + runs.push({ + createdAt: typeof record.createdAt === "string" ? record.createdAt : null, + skippedNoContext: payload.skippedNoContext as number, + visibleCases: Number.isInteger(payload.visibleCases) ? (payload.visibleCases as number) : 0, + heldOutCases: Number.isInteger(payload.heldOutCases) ? (payload.heldOutCases as number) : 0, + visible: payload.visible, + heldOut: payload.heldOut, + }); + } + return runs; +} + +/** #8185/#8545: the REGRESSED-verdict track record over every persisted threshold AND eligibility backtest + * comparison — the SAME aggregation ORB uses (`computeRegressedVerdictTrackRecord`), zero new math. */ +export function computeAmsBacktestTrackRecord( + runs: readonly PersistedAmsBacktestRun[], + eligibilityRuns: readonly PersistedAmsEligibilityBacktestRun[] = [], +): RegressedVerdictTrackRecord { + return computeRegressedVerdictTrackRecord([ + ...runs.flatMap((run) => [run.visible, run.heldOut]), + ...eligibilityRuns.flatMap((run) => [run.visible, run.heldOut]), + ]); } export type AmsBacktestProposal = { diff --git a/packages/loopover-miner/lib/ams-eligibility-backtest.ts b/packages/loopover-miner/lib/ams-eligibility-backtest.ts new file mode 100644 index 0000000000..3f3965bd42 --- /dev/null +++ b/packages/loopover-miner/lib/ams-eligibility-backtest.ts @@ -0,0 +1,267 @@ +// AMS eligibility-exclusion advisory backtest (#8545, epic #8107) — replays a candidate ContributionProfile +// against the miner's own captured eligibility-exclusion signal history (rule-fired + human-override events +// with bounded metadata from discover-cli's #8544 capture). Mirrors #8184's min-rank backtest shape: pure +// corpus assembly + engine replay math, zero live discovery behavior change. + +import { + AMS_MIN_RANK_HELD_OUT_FRACTION, + AMS_MIN_RANK_MIN_HELD_OUT_CASES, + AMS_MIN_RANK_MIN_VISIBLE_CASES, + buildBacktestCorpus, + compareBacktestScores, + scoreBacktest, + splitBacktestCorpus, + type BacktestCase, + type BacktestComparison, + type HumanOverrideEvent, + type RuleFiredEvent, +} from "@loopover/engine"; +import { buildEligibilityExclusionMetadata } from "./discover-cli.js"; +import type { ContributionProfile } from "./contribution-profile.js"; +import { ELIGIBILITY_EXCLUSION_REASONS, filterCandidatesByProfiles } from "./contribution-profile-filter.js"; +import { SIGNAL_HUMAN_OVERRIDE_EVENT, SIGNAL_RULE_FIRED_EVENT } from "./signal-tracking-store.js"; + +/** Synthetic rule id every eligibility replay case carries — namespaced away from the four live exclusion + * reason strings stored on the original fired events. */ +export const AMS_ELIGIBILITY_RULE_ID = "ams_eligibility_exclusion"; + +/** Fixed-seed held-out split for eligibility replay — same fraction/floors as min-rank (#8184 reads them from + * ams-rank-corpus.ts; this module reuses those constants verbatim, per #8545). */ +export const AMS_ELIGIBILITY_SPLIT_SEED = "ams-eligibility-exclusion-v1"; + +export const AMS_ELIGIBILITY_RULE_IDS = Object.freeze([ + ELIGIBILITY_EXCLUSION_REASONS.EXCLUSION_LABEL, + ELIGIBILITY_EXCLUSION_REASONS.MISSING_ELIGIBILITY_LABEL, + ELIGIBILITY_EXCLUSION_REASONS.CONFLICTING_SIGNALS, + ELIGIBILITY_EXCLUSION_REASONS.EXCLUDED_ASSIGNEE, +] as const); + +type FilterCandidate = { + repoFullName: string; + owner?: string; + labels?: string[]; + assignees?: string[]; +}; + +type RuleFiredPayload = { + ruleId: string; + targetKey: string; + outcome: string; + occurredAt: string; + metadata?: Record; +}; + +type HumanOverridePayload = { + ruleId: string; + targetKey: string; + verdict: "reversed" | "confirmed"; + occurredAt: string; + metadata?: Record; +}; + +function isEligibilityRuleId(ruleId: unknown): ruleId is (typeof AMS_ELIGIBILITY_RULE_IDS)[number] { + return typeof ruleId === "string" && (AMS_ELIGIBILITY_RULE_IDS as readonly string[]).includes(ruleId); +} + +function isRuleFiredPayload(payload: Record, ruleId: string): payload is RuleFiredPayload { + return ( + payload.ruleId === ruleId && + typeof payload.targetKey === "string" && + typeof payload.outcome === "string" && + typeof payload.occurredAt === "string" + ); +} + +function isHumanOverridePayload(payload: Record, ruleId: string): payload is HumanOverridePayload { + return ( + payload.ruleId === ruleId && + typeof payload.targetKey === "string" && + (payload.verdict === "reversed" || payload.verdict === "confirmed") && + typeof payload.occurredAt === "string" + ); +} + +function metadataCapturable(metadata: Record | undefined): boolean { + if (!metadata || typeof metadata !== "object") return false; + const candidate: { + owner?: string; + labels?: string[]; + assignees?: string[]; + } = {}; + if (typeof metadata.owner === "string") candidate.owner = metadata.owner; + if (Array.isArray(metadata.labels)) candidate.labels = metadata.labels as string[]; + if (Array.isArray(metadata.assignees)) candidate.assignees = metadata.assignees as string[]; + return buildEligibilityExclusionMetadata(candidate) !== undefined; +} + +/** Reconstruct a {@link filterCandidatesByProfiles} candidate from a fired event's targetKey + metadata. */ +export function reconstructEligibilityFilterCandidate( + targetKey: string, + metadata: Record, +): FilterCandidate | null { + const match = /^([^/]+\/[^/#]+)#issue-(\d+)$/.exec(targetKey); + if (!match) return null; + const candidate: FilterCandidate = { repoFullName: match[1]! }; + if (typeof metadata.owner === "string" && metadata.owner) candidate.owner = metadata.owner; + const labels = Array.isArray(metadata.labels) + ? metadata.labels.filter((entry): entry is string => typeof entry === "string") + : []; + if (labels.length > 0) candidate.labels = labels; + const assignees = Array.isArray(metadata.assignees) + ? metadata.assignees.filter((entry): entry is string => typeof entry === "string") + : []; + if (assignees.length > 0) candidate.assignees = assignees; + return candidate; +} + +/** Scan raw ledger rows for eligibility rule-fired + override events — same client-side filter discipline + * signal-tracking-store.ts uses, but across all four exclusion ruleIds at once. */ +export function extractEligibilitySignalEvents(events: readonly unknown[]): { + fired: RuleFiredEvent[]; + overrides: HumanOverrideEvent[]; + skippedNoContext: number; +} { + const fired: RuleFiredEvent[] = []; + const overrides: HumanOverrideEvent[] = []; + let skippedNoContext = 0; + for (const event of Array.isArray(events) ? events : []) { + const record = event as Record | null | undefined; + if (!record || typeof record !== "object") continue; + const payload = record.payload as Record | null | undefined; + if (!payload || typeof payload !== "object") continue; + const ruleId = payload.ruleId; + if (!isEligibilityRuleId(ruleId)) continue; + if (record.type === SIGNAL_RULE_FIRED_EVENT) { + if (!isRuleFiredPayload(payload, ruleId)) continue; + if (!metadataCapturable(payload.metadata)) { + skippedNoContext += 1; + continue; + } + fired.push({ + ruleId, + targetKey: payload.targetKey, + outcome: payload.outcome, + occurredAt: payload.occurredAt, + metadata: payload.metadata!, + }); + } else if (record.type === SIGNAL_HUMAN_OVERRIDE_EVENT && isHumanOverridePayload(payload, ruleId)) { + overrides.push({ + ruleId, + targetKey: payload.targetKey, + verdict: payload.verdict, + occurredAt: payload.occurredAt, + ...(payload.metadata ? { metadata: payload.metadata } : {}), + }); + } + } + return { fired, overrides, skippedNoContext }; +} + +/** Repo full names referenced by capturable eligibility fired events — used to hydrate current profiles. */ +export function repoFullNamesInEligibilityEvents(events: readonly unknown[]): Set { + const repos = new Set(); + for (const event of extractEligibilitySignalEvents(events).fired) { + const match = /^([^/]+\/[^/#]+)#issue-\d+$/.exec(event.targetKey); + if (match) repos.add(match[1]!); + } + return repos; +} + +/** Build a labeled replay corpus from eligibility signal history. Fired events without capturable metadata are + * counted in `skippedNoContext` and never guessed at. */ +export function buildEligibilityExclusionCorpus(events: readonly unknown[]): { + cases: BacktestCase[]; + skippedNoContext: number; +} { + const { fired, overrides, skippedNoContext } = extractEligibilitySignalEvents(events); + const cases: BacktestCase[] = []; + for (const ruleId of AMS_ELIGIBILITY_RULE_IDS) { + for (const backtestCase of buildBacktestCorpus(ruleId, fired, overrides)) { + cases.push({ ...backtestCase, ruleId: AMS_ELIGIBILITY_RULE_ID }); + } + } + cases.sort((left, right) => { + const key = left.targetKey.localeCompare(right.targetKey); + return key !== 0 ? key : left.firedAt.localeCompare(right.firedAt); + }); + return { cases, skippedNoContext }; +} + +/** Classifier factory: re-run {@link filterCandidatesByProfiles} under `profilesByRepo`. Excluding the + * reconstructed candidate classifies as the fired outcome (`confirmed`); keeping it classifies as not-fired + * (`reversed`). */ +export function buildEligibilityProfileClassifier( + profilesByRepo: Map, +): (backtestCase: BacktestCase) => "reversed" | "confirmed" { + return (backtestCase: BacktestCase) => { + const metadata = backtestCase.metadata; + if (!metadata) return "reversed"; + const candidate = reconstructEligibilityFilterCandidate(backtestCase.targetKey, metadata); + if (!candidate) return "reversed"; + const { excluded } = filterCandidatesByProfiles([candidate], profilesByRepo); + return excluded.length > 0 ? "confirmed" : "reversed"; + }; +} + +export type AmsEligibilityBacktestResult = { + ruleId: string; + skippedNoContext: number; + visibleCases: number; + heldOutCases: number; + visible: BacktestComparison; + heldOut: BacktestComparison; +}; + +function runEligibilitySlice( + cases: readonly BacktestCase[], + currentProfiles: Map, + candidateProfiles: Map, +): BacktestComparison { + const baseline = scoreBacktest( + AMS_ELIGIBILITY_RULE_ID, + cases, + buildEligibilityProfileClassifier(currentProfiles), + ); + const candidate = scoreBacktest( + AMS_ELIGIBILITY_RULE_ID, + cases, + buildEligibilityProfileClassifier(candidateProfiles), + ); + return compareBacktestScores(baseline, candidate); +} + +/** Advisory replay of a candidate ContributionProfile against the eligibility-exclusion corpus — null when + * either split misses the shared min-rank sample floors (reused verbatim, per #8545). */ +export function runAmsEligibilityBacktest( + cases: readonly BacktestCase[], + currentProfiles: Map, + candidateProfiles: Map, +): Omit | null { + const { visible, heldOut } = splitBacktestCorpus( + cases, + AMS_MIN_RANK_HELD_OUT_FRACTION, + AMS_ELIGIBILITY_SPLIT_SEED, + ); + if (visible.length < AMS_MIN_RANK_MIN_VISIBLE_CASES || heldOut.length < AMS_MIN_RANK_MIN_HELD_OUT_CASES) { + return null; + } + return { + ruleId: AMS_ELIGIBILITY_RULE_ID, + visibleCases: visible.length, + heldOutCases: heldOut.length, + visible: runEligibilitySlice(visible, currentProfiles, candidateProfiles), + heldOut: runEligibilitySlice(heldOut, currentProfiles, candidateProfiles), + }; +} + +/** Convenience composition: ledger events -> labeled corpus -> advisory current-vs-candidate comparison. */ +export function backtestEligibilityCandidate( + events: readonly unknown[], + currentProfiles: Map, + candidateProfiles: Map, +): AmsEligibilityBacktestResult | null { + const { cases, skippedNoContext } = buildEligibilityExclusionCorpus(events); + const replay = runAmsEligibilityBacktest(cases, currentProfiles, candidateProfiles); + if (!replay) return null; + return { skippedNoContext, ...replay }; +} diff --git a/packages/loopover-miner/lib/calibration-cli.ts b/packages/loopover-miner/lib/calibration-cli.ts index e40003c12c..9ed529c339 100644 --- a/packages/loopover-miner/lib/calibration-cli.ts +++ b/packages/loopover-miner/lib/calibration-cli.ts @@ -12,12 +12,17 @@ import { backtestMinRankCandidate, buildAmsBacktestProposals, computeAmsBacktestTrackRecord, + readAmsEligibilityBacktestRuns, readAmsThresholdBacktestRuns, readMinRankAutotuneEnabled, readMinRankOverride, + recordAmsEligibilityBacktestRun, recordAmsThresholdBacktestRun, revertMinRankOverride, } from "./ams-calibration.js"; +import { backtestEligibilityCandidate, repoFullNamesInEligibilityEvents } from "./ams-eligibility-backtest.js"; +import { initContributionProfileCache, resolveContributionProfileCacheDbPath } from "./contribution-profile-cache.js"; +import type { ContributionProfile } from "./contribution-profile.js"; import { buildCalibrationReport } from "./calibration.js"; import { initEventLedger, resolveEventLedgerDbPath } from "./event-ledger.js"; import type { LedgerEntry } from "./event-ledger.js"; @@ -29,7 +34,7 @@ import { reportCliFailure, describeCliError } from "./cli-error.js"; import { runHistoricalReplayCalibrationCycle, type CalibrationSnapshotPayload } from "./calibration-run.js"; const CALIBRATION_USAGE = - "Usage: loopover-miner calibration [--json] | calibration snapshot [--json] | calibration backtest-threshold --candidate [--json] | calibration apply-min-rank --candidate --approve [--json] | calibration revert-min-rank --approve [--json]"; + "Usage: loopover-miner calibration [--json] | calibration snapshot [--json] | calibration backtest-threshold --candidate [--json] | calibration backtest-eligibility --profile [--json] | calibration apply-min-rank --candidate --approve [--json] | calibration revert-min-rank --approve [--json]"; export type CalibrationCliDeps = { readFileSync?: typeof readFileSync; @@ -44,6 +49,49 @@ function parseCandidate(args: string[]): number | null { return Number.isFinite(value) ? value : null; } +function parseProfilePath(args: string[]): string | null { + const index = args.indexOf("--profile"); + if (index === -1 || index + 1 >= args.length) return null; + const value = args[index + 1]; + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +function loadCandidateProfiles( + profilePath: string, + deps: CalibrationCliDeps, +): Map { + const readFileSync = deps.readFileSync; + if (!readFileSync) throw new Error("readFileSync_unavailable"); + let raw: unknown; + try { + raw = JSON.parse(String(readFileSync(profilePath, "utf8"))) as unknown; + } catch { + throw new Error("invalid_profile_file"); + } + if (!raw || typeof raw !== "object" || Array.isArray(raw)) throw new Error("invalid_profile_file"); + const profiles = new Map(); + for (const [repoFullName, profile] of Object.entries(raw as Record)) { + if (!profile || typeof profile !== "object") continue; + const typed = profile as ContributionProfile; + if (typeof typed.repoFullName !== "string" || typed.repoFullName !== repoFullName) continue; + profiles.set(repoFullName, typed); + } + if (profiles.size === 0) throw new Error("invalid_profile_file"); + return profiles; +} + +function readCurrentContributionProfiles( + cache: ReturnType, + repoFullNames: Iterable, +): Map { + const profiles = new Map(); + for (const repoFullName of repoFullNames) { + const cached = cache.get(repoFullName); + if (cached?.profile) profiles.set(repoFullName, cached.profile); + } + return profiles; +} + /** Map prediction-ledger rows to predicted-verdict records: the target id becomes a string key and the recorded * prediction verdict is the `conclusion`. Exported so callers other than this CLI (the MCP calibration-report * tool, #5821) can build the identical join without re-implementing the mapping. */ @@ -165,7 +213,10 @@ function runCalibrationSnapshot(args: string[], env: Record 0 ? trackRecord : null; @@ -253,6 +304,56 @@ function runBacktestThreshold(args: string[], env: Record` (#8545): advisory replay of a candidate + * ContributionProfile against the captured eligibility-exclusion corpus. Exit is nonzero ONLY on operational + * error — never on verdict (the #8138 advisory guarantee). */ +function runBacktestEligibility(args: string[], env: Record, deps: CalibrationCliDeps): number { + const json = args.includes("--json"); + const profilePath = parseProfilePath(args); + if (profilePath === null) return reportCliFailure(json, `Missing or invalid --profile. ${CALIBRATION_USAGE}`, 1); + + let eventLedger; + let profileCache; + try { + eventLedger = initEventLedger(resolveEventLedgerDbPath(env)); + profileCache = initContributionProfileCache(resolveContributionProfileCacheDbPath(env)); + const events = eventLedger.readEvents(); + const candidateProfiles = loadCandidateProfiles(profilePath, deps); + const currentProfiles = readCurrentContributionProfiles(profileCache, [ + ...repoFullNamesInEligibilityEvents(events), + ...candidateProfiles.keys(), + ]); + const result = backtestEligibilityCandidate(events, currentProfiles, candidateProfiles); + if (!result) { + const message = + "backtest-eligibility: not enough labeled eligibility-exclusion cases yet (both splits must clear their sample floors); no verdict, nothing persisted."; + console.log(json ? JSON.stringify({ ran: false, reason: "insufficient_corpus" }) : message); + return 0; + } + recordAmsEligibilityBacktestRun(result, { eventLedger }); + if (json) { + console.log(JSON.stringify({ ran: true, ...result }, null, 2)); + } else { + console.log("eligibility profile backtest: current cache -> candidate profile file"); + console.log(`skipped (no metadata context): ${result.skippedNoContext}`); + console.log(`visible split (${result.visibleCases} case(s)):`); + console.log(renderBacktestComparison(result.visible)); + console.log(`held-out split (${result.heldOutCases} case(s)):`); + console.log(renderBacktestComparison(result.heldOut)); + console.log("advisory only: no profile changed; the run event was persisted for the track record."); + } + return 0; + } catch (error) { + if (error instanceof Error && error.message === "invalid_profile_file") { + return reportCliFailure(json, `Invalid --profile file. ${CALIBRATION_USAGE}`, 1); + } + return reportCliFailure(json, describeCliError(error)); + } finally { + eventLedger?.close(); + profileCache?.close(); + } +} + /** `calibration apply-min-rank --candidate --approve` / `calibration revert-min-rank --approve` * (#8187): the double-gated apply and its one-command revert. Exit 1 when the command did NOT move the * knob (refusal or usage error) so scripts can tell; 0 only on a real apply/revert. */ @@ -302,6 +403,7 @@ function runMinRankMutation(kind: "apply" | "revert", args: string[], env: Recor export function runCalibrationCli(args: string[] = [], env: Record = process.env, deps: CalibrationCliDeps = {}): number { if (args[0] === "snapshot") return runCalibrationSnapshot(args.slice(1), env, deps); if (args[0] === "backtest-threshold") return runBacktestThreshold(args.slice(1), env, deps); + if (args[0] === "backtest-eligibility") return runBacktestEligibility(args.slice(1), env, deps); if (args[0] === "apply-min-rank") return runMinRankMutation("apply", args.slice(1), env, deps); if (args[0] === "revert-min-rank") return runMinRankMutation("revert", args.slice(1), env, deps); const json = args.includes("--json"); @@ -326,7 +428,7 @@ export function runCalibrationCli(args: string[] = [], env: Record { + for (const ledger of ledgers.splice(0)) ledger.close(); + for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +function tempLedger(): EventLedger { + const dir = mkdtempSync(join(tmpdir(), "miner-ams-eligibility-backtest-")); + tempDirs.push(dir); + const ledger = initEventLedger(resolveEventLedgerDbPath({ LOOPOVER_MINER_CONFIG_DIR: dir })); + ledgers.push(ledger); + return ledger; +} + +function trustworthyProfile(over: Partial = {}): ContributionProfile { + return { + repoFullName: "acme/widgets", + schemaVersion: 1, + generatedAt: "2026-07-18T00:00:00.000Z", + eligibilityLabels: { + value: [{ field: "name", contains: "help wanted" }], + confidence: "explicit", + provenance: [{ source: "labels", detail: "help wanted" }], + }, + exclusionLabels: { + value: [{ field: "name", contains: "blocked" }], + confidence: "inferred", + provenance: [{ source: "labels", detail: "blocked" }], + }, + prBody: { value: null, confidence: "absent", provenance: [] }, + completeness: "inferred", + ...over, + }; +} + +function stricterCandidateProfile(): ContributionProfile { + return trustworthyProfile({ + eligibilityLabels: { + value: [{ field: "name", contains: "help wanted" }, { field: "name", contains: "good first issue" }], + confidence: "explicit", + provenance: [ + { source: "labels", detail: "help wanted" }, + { source: "labels", detail: "good first issue" }, + ], + }, + }); +} + +function seedEligibilityFired( + ledger: EventLedger, + issueNumber: number, + ruleId: string, + metadata: Record | undefined, + occurredAt: string, +) { + ledger.appendEvent({ + type: SIGNAL_RULE_FIRED_EVENT, + repoFullName: "acme/widgets", + payload: { + ruleId, + targetKey: `acme/widgets#issue-${issueNumber}`, + outcome: "exclude", + occurredAt, + ...(metadata ? { metadata } : {}), + }, + }); +} + +function seedEligibilityOverride( + ledger: EventLedger, + issueNumber: number, + ruleId: string, + verdict: "reversed" | "confirmed", + occurredAt: string, +) { + ledger.appendEvent({ + type: SIGNAL_HUMAN_OVERRIDE_EVENT, + repoFullName: "acme/widgets", + payload: { + ruleId, + targetKey: `acme/widgets#issue-${issueNumber}`, + verdict, + occurredAt, + }, + }); +} + +function seedLabeledCorpus(ledger: EventLedger, count: number): void { + for (let issueNumber = 1; issueNumber <= count; issueNumber += 1) { + const ruleId = + issueNumber % 4 === 0 + ? ELIGIBILITY_EXCLUSION_REASONS.EXCLUDED_ASSIGNEE + : issueNumber % 3 === 0 + ? ELIGIBILITY_EXCLUSION_REASONS.MISSING_ELIGIBILITY_LABEL + : issueNumber % 2 === 0 + ? ELIGIBILITY_EXCLUSION_REASONS.EXCLUSION_LABEL + : ELIGIBILITY_EXCLUSION_REASONS.CONFLICTING_SIGNALS; + const labels = + ruleId === ELIGIBILITY_EXCLUSION_REASONS.EXCLUSION_LABEL + ? ["blocked"] + : ruleId === ELIGIBILITY_EXCLUSION_REASONS.CONFLICTING_SIGNALS + ? ["help wanted", "blocked"] + : ruleId === ELIGIBILITY_EXCLUSION_REASONS.MISSING_ELIGIBILITY_LABEL + ? ["bug"] + : ["help wanted"]; + const assignees = ruleId === ELIGIBILITY_EXCLUSION_REASONS.EXCLUDED_ASSIGNEE ? ["acme"] : []; + const firedAt = `2026-07-${String((issueNumber % 28) + 1).padStart(2, "0")}T12:00:00.000Z`; + seedEligibilityFired( + ledger, + issueNumber, + ruleId, + { owner: "acme", labels, ...(assignees.length > 0 ? { assignees } : {}) }, + firedAt, + ); + seedEligibilityOverride(ledger, issueNumber, ruleId, issueNumber % 5 === 0 ? "reversed" : "confirmed", `${firedAt.replace("T12:", "T13:")}`); + } +} + +describe("ams-eligibility-backtest pure core (#8545)", () => { + it("counts metadata-less fired events in skippedNoContext and never fabricates corpus rows for them", () => { + const ledger = tempLedger(); + seedEligibilityFired(ledger, 1, ELIGIBILITY_EXCLUSION_REASONS.EXCLUSION_LABEL, undefined, "2026-07-01T12:00:00.000Z"); + seedEligibilityFired(ledger, 2, ELIGIBILITY_EXCLUSION_REASONS.EXCLUSION_LABEL, {}, "2026-07-02T12:00:00.000Z"); + seedEligibilityFired( + ledger, + 3, + ELIGIBILITY_EXCLUSION_REASONS.EXCLUSION_LABEL, + { owner: "acme", labels: ["blocked"] }, + "2026-07-03T12:00:00.000Z", + ); + seedEligibilityOverride( + ledger, + 3, + ELIGIBILITY_EXCLUSION_REASONS.EXCLUSION_LABEL, + "confirmed", + "2026-07-03T13:00:00.000Z", + ); + const events = ledger.readEvents(); + expect(extractEligibilitySignalEvents(events).skippedNoContext).toBe(2); + const { cases, skippedNoContext } = buildEligibilityExclusionCorpus(events); + expect(skippedNoContext).toBe(2); + expect(cases).toHaveLength(1); + expect(cases[0]?.ruleId).toBe(AMS_ELIGIBILITY_RULE_ID); + expect(cases[0]?.metadata).toEqual({ owner: "acme", labels: ["blocked"] }); + }); + + it("reconstructs FilterCandidate fields from targetKey + metadata", () => { + expect( + reconstructEligibilityFilterCandidate("acme/widgets#issue-7", { + owner: "acme", + labels: ["blocked"], + assignees: ["alice"], + }), + ).toEqual({ + repoFullName: "acme/widgets", + owner: "acme", + labels: ["blocked"], + assignees: ["alice"], + }); + expect(reconstructEligibilityFilterCandidate("acme/widgets#issue-7", { labels: ["bug"] })).toEqual({ + repoFullName: "acme/widgets", + labels: ["bug"], + }); + expect(reconstructEligibilityFilterCandidate("bad-target", { labels: ["bug"] })).toBeNull(); + expect( + reconstructEligibilityFilterCandidate("acme/widgets#issue-8", { + owner: "", + labels: "not-an-array" as unknown as string[], + assignees: 42 as unknown as string[], + }), + ).toEqual({ repoFullName: "acme/widgets" }); + }); + + it("extractEligibilitySignalEvents tolerates garbage rows and records overrides with optional metadata", () => { + const extracted = extractEligibilitySignalEvents([ + null, + { type: SIGNAL_RULE_FIRED_EVENT, payload: "not-an-object" }, + { type: SIGNAL_RULE_FIRED_EVENT, payload: { ruleId: "exclusion_label" } }, + { + type: SIGNAL_RULE_FIRED_EVENT, + repoFullName: "acme/widgets", + payload: { + ruleId: ELIGIBILITY_EXCLUSION_REASONS.EXCLUSION_LABEL, + targetKey: "acme/widgets#issue-1", + outcome: "exclude", + occurredAt: "2026-07-01T12:00:00.000Z", + metadata: { labels: ["blocked"] }, + }, + }, + { + type: SIGNAL_HUMAN_OVERRIDE_EVENT, + repoFullName: "acme/widgets", + payload: { + ruleId: ELIGIBILITY_EXCLUSION_REASONS.EXCLUSION_LABEL, + targetKey: "acme/widgets#issue-1", + verdict: "confirmed", + occurredAt: "2026-07-01T13:00:00.000Z", + metadata: { note: "reviewed" }, + }, + }, + { + type: "discovered_issue", + payload: { + ruleId: ELIGIBILITY_EXCLUSION_REASONS.EXCLUSION_LABEL, + targetKey: "acme/widgets#issue-9", + outcome: "exclude", + occurredAt: "2026-07-02T12:00:00.000Z", + metadata: { labels: ["blocked"] }, + }, + }, + ]); + expect(extracted.skippedNoContext).toBe(0); + expect(extracted.fired).toHaveLength(1); + expect(extracted.overrides).toEqual([ + expect.objectContaining({ + targetKey: "acme/widgets#issue-1", + metadata: { note: "reviewed" }, + }), + ]); + expect(extractEligibilitySignalEvents("not-an-array" as unknown as readonly unknown[])).toEqual({ + fired: [], + overrides: [], + skippedNoContext: 0, + }); + }); + + it("repoFullNamesInEligibilityEvents ignores fired rows whose targetKey does not parse", () => { + const repos = repoFullNamesInEligibilityEvents([ + { + type: SIGNAL_RULE_FIRED_EVENT, + payload: { + ruleId: ELIGIBILITY_EXCLUSION_REASONS.EXCLUSION_LABEL, + targetKey: "not-a-valid-target", + outcome: "exclude", + occurredAt: "2026-07-01T12:00:00.000Z", + metadata: { labels: ["blocked"] }, + }, + }, + { + type: SIGNAL_RULE_FIRED_EVENT, + payload: { + ruleId: ELIGIBILITY_EXCLUSION_REASONS.EXCLUSION_LABEL, + targetKey: "acme/widgets#issue-2", + outcome: "exclude", + occurredAt: "2026-07-02T12:00:00.000Z", + metadata: { labels: ["blocked"] }, + }, + }, + ]); + expect([...repos]).toEqual(["acme/widgets"]); + }); + + it("sorts corpus rows with the same targetKey by firedAt", () => { + const targetKey = "acme/widgets#issue-42"; + const metadata = { owner: "acme", labels: ["blocked"] }; + const { cases } = buildEligibilityExclusionCorpus([ + { + type: SIGNAL_RULE_FIRED_EVENT, + payload: { + ruleId: ELIGIBILITY_EXCLUSION_REASONS.EXCLUSION_LABEL, + targetKey, + outcome: "exclude", + occurredAt: "2026-07-02T12:00:00.000Z", + metadata, + }, + }, + { + type: SIGNAL_HUMAN_OVERRIDE_EVENT, + payload: { + ruleId: ELIGIBILITY_EXCLUSION_REASONS.EXCLUSION_LABEL, + targetKey, + verdict: "confirmed", + occurredAt: "2026-07-02T13:00:00.000Z", + }, + }, + { + type: SIGNAL_RULE_FIRED_EVENT, + payload: { + ruleId: ELIGIBILITY_EXCLUSION_REASONS.CONFLICTING_SIGNALS, + targetKey, + outcome: "exclude", + occurredAt: "2026-07-01T12:00:00.000Z", + metadata: { owner: "acme", labels: ["help wanted", "blocked"] }, + }, + }, + { + type: SIGNAL_HUMAN_OVERRIDE_EVENT, + payload: { + ruleId: ELIGIBILITY_EXCLUSION_REASONS.CONFLICTING_SIGNALS, + targetKey, + verdict: "confirmed", + occurredAt: "2026-07-01T13:00:00.000Z", + }, + }, + ]); + const sameTarget = cases.filter((entry) => entry.targetKey === targetKey); + expect(sameTarget.length).toBeGreaterThanOrEqual(2); + expect(sameTarget.map((entry) => entry.firedAt)).toEqual([...sameTarget.map((entry) => entry.firedAt)].sort()); + }); + + it("classifier returns reversed when metadata is missing, targetKey is invalid, or the candidate is kept", () => { + const profiles = new Map([["acme/widgets", trustworthyProfile()]]); + const classify = buildEligibilityProfileClassifier(profiles); + expect( + classify({ + ruleId: AMS_ELIGIBILITY_RULE_ID, + targetKey: "acme/widgets#issue-1", + outcome: "exclude", + firedAt: "2026-07-01T12:00:00.000Z", + decidedAt: "2026-07-01T13:00:00.000Z", + label: "confirmed", + }), + ).toBe("reversed"); + expect( + classify({ + ruleId: AMS_ELIGIBILITY_RULE_ID, + targetKey: "bad-target", + outcome: "exclude", + firedAt: "2026-07-01T12:00:00.000Z", + decidedAt: "2026-07-01T13:00:00.000Z", + label: "confirmed", + metadata: { labels: ["blocked"] }, + }), + ).toBe("reversed"); + expect( + classify({ + ruleId: AMS_ELIGIBILITY_RULE_ID, + targetKey: "acme/widgets#issue-2", + outcome: "exclude", + firedAt: "2026-07-02T12:00:00.000Z", + decidedAt: "2026-07-02T13:00:00.000Z", + label: "confirmed", + metadata: { labels: ["help wanted"] }, + }), + ).toBe("reversed"); + }); + + it("classifier agreement matches filterCandidatesByProfiles on a fixture case", () => { + const ledger = tempLedger(); + seedEligibilityFired( + ledger, + 9, + ELIGIBILITY_EXCLUSION_REASONS.EXCLUSION_LABEL, + { owner: "acme", labels: ["blocked"] }, + "2026-07-09T12:00:00.000Z", + ); + seedEligibilityOverride( + ledger, + 9, + ELIGIBILITY_EXCLUSION_REASONS.EXCLUSION_LABEL, + "confirmed", + "2026-07-09T13:00:00.000Z", + ); + const { cases } = buildEligibilityExclusionCorpus(ledger.readEvents()); + expect(cases).toHaveLength(1); + const backtestCase = cases[0]!; + const profiles = new Map([["acme/widgets", trustworthyProfile()]]); + const candidate = reconstructEligibilityFilterCandidate(backtestCase.targetKey, backtestCase.metadata!); + expect(candidate).not.toBeNull(); + const { excluded } = filterCandidatesByProfiles([candidate!], profiles); + expect(buildEligibilityProfileClassifier(profiles)(backtestCase)).toBe(excluded.length > 0 ? "confirmed" : "reversed"); + }); + + it("returns null when the labeled corpus is under the shared sample floors", () => { + const ledger = tempLedger(); + seedLabeledCorpus(ledger, 10); + const current = new Map([["acme/widgets", trustworthyProfile()]]); + const candidate = new Map([["acme/widgets", stricterCandidateProfile()]]); + expect(backtestEligibilityCandidate(ledger.readEvents(), current, candidate)).toBeNull(); + expect(runAmsEligibilityBacktest(buildEligibilityExclusionCorpus(ledger.readEvents()).cases, current, candidate)).toBeNull(); + }); + + it("composes into a full current-vs-candidate advisory result over a real ledger", () => { + const ledger = tempLedger(); + seedLabeledCorpus(ledger, 40); + const current = new Map([["acme/widgets", trustworthyProfile()]]); + const candidate = new Map([["acme/widgets", stricterCandidateProfile()]]); + const result = backtestEligibilityCandidate(ledger.readEvents(), current, candidate); + expect(result).not.toBeNull(); + expect(result!.ruleId).toBe(AMS_ELIGIBILITY_RULE_ID); + expect(result!.visibleCases).toBeGreaterThanOrEqual(20); + expect(result!.heldOutCases).toBeGreaterThanOrEqual(5); + expect(["improved", "regressed", "unchanged"]).toContain(result!.visible.verdict); + expect(["improved", "regressed", "unchanged"]).toContain(result!.heldOut.verdict); + }); + + it("persists and aggregates eligibility backtest runs into the shared track record (#8545/#8185 parity)", () => { + const ledger = tempLedger(); + seedLabeledCorpus(ledger, 40); + const current = new Map([["acme/widgets", trustworthyProfile()]]); + const candidate = new Map([["acme/widgets", stricterCandidateProfile()]]); + const result = backtestEligibilityCandidate(ledger.readEvents(), current, candidate); + expect(result).not.toBeNull(); + recordAmsEligibilityBacktestRun(result!, { eventLedger: ledger }); + const runs = readAmsEligibilityBacktestRuns(ledger); + expect(runs).toHaveLength(1); + expect(runs[0]).toMatchObject({ + skippedNoContext: result!.skippedNoContext, + visibleCases: result!.visibleCases, + heldOutCases: result!.heldOutCases, + }); + ledger.appendEvent({ type: MINER_AMS_ELIGIBILITY_BACKTEST_EVENT, payload: { skippedNoContext: 1 } }); + expect(readAmsEligibilityBacktestRuns(ledger)).toHaveLength(1); + const trackRecord = computeAmsBacktestTrackRecord([], runs); + expect(trackRecord.totalRuns).toBe(2); + expect(() => recordAmsEligibilityBacktestRun(result!, {})).toThrow("invalid_event_ledger"); + }); + + it("readAmsEligibilityBacktestRuns skips malformed rows and defaults missing case counts", () => { + const ledger = tempLedger(); + seedLabeledCorpus(ledger, 40); + const result = backtestEligibilityCandidate( + ledger.readEvents(), + new Map([["acme/widgets", trustworthyProfile()]]), + new Map([["acme/widgets", stricterCandidateProfile()]]), + ); + recordAmsEligibilityBacktestRun(result!, { eventLedger: ledger }); + ledger.appendEvent({ type: MINER_AMS_ELIGIBILITY_BACKTEST_EVENT, payload: { skippedNoContext: 1.5 } }); + ledger.appendEvent({ + type: MINER_AMS_ELIGIBILITY_BACKTEST_EVENT, + payload: { + skippedNoContext: 2, + visible: { ruleId: "x", verdict: "weird" }, + heldOut: result!.heldOut, + }, + }); + const runs = readAmsEligibilityBacktestRuns(ledger); + expect(runs).toHaveLength(1); + expect(runs[0]?.visibleCases).toBe(result!.visibleCases); + expect(computeAmsBacktestTrackRecord([], runs).totalRuns).toBe(2); + const malformed = readAmsEligibilityBacktestRuns({ + readEvents: () => [ + { type: MINER_AMS_ELIGIBILITY_BACKTEST_EVENT, payload: "bad" }, + { + type: MINER_AMS_ELIGIBILITY_BACKTEST_EVENT, + createdAt: 123, + payload: { + skippedNoContext: 0, + visibleCases: "many", + heldOutCases: null, + visible: result!.visible, + heldOut: result!.heldOut, + }, + }, + ], + }); + expect(malformed).toHaveLength(1); + expect(malformed[0]?.createdAt).toBeNull(); + expect(malformed[0]?.visibleCases).toBe(0); + expect(malformed[0]?.heldOutCases).toBe(0); + }); + + it("aggregates threshold and eligibility backtest runs in one track record", () => { + const ledger = tempLedger(); + for (let i = 1; i <= 60; i += 1) { + ledger.appendEvent({ type: "discovered_issue", repoFullName: "acme/widgets", payload: { issueNumber: i, rankScore: 0.15, title: "t", labels: [] } }); + ledger.appendEvent({ + type: "pr_outcome", + repoFullName: "acme/widgets", + payload: { prNumber: 1000 + i, decision: "closed", closedAt: "2026-07-10T00:00:00Z", reason: null, issueNumber: i }, + }); + } + const threshold = backtestMinRankCandidate(ledger.readEvents(), AMS_MIN_RANK_SHIPPED, 0.2); + expect(threshold).not.toBeNull(); + recordAmsThresholdBacktestRun(threshold!, { eventLedger: ledger }); + seedLabeledCorpus(ledger, 40); + const eligibility = backtestEligibilityCandidate( + ledger.readEvents(), + new Map([["acme/widgets", trustworthyProfile()]]), + new Map([["acme/widgets", stricterCandidateProfile()]]), + ); + recordAmsEligibilityBacktestRun(eligibility!, { eventLedger: ledger }); + const trackRecord = computeAmsBacktestTrackRecord( + readAmsThresholdBacktestRuns(ledger), + readAmsEligibilityBacktestRuns(ledger), + ); + expect(trackRecord.totalRuns).toBe(4); + }); +}); diff --git a/test/unit/miner-calibration-cli.test.ts b/test/unit/miner-calibration-cli.test.ts index 0e561578f9..a3bd1d029d 100644 --- a/test/unit/miner-calibration-cli.test.ts +++ b/test/unit/miner-calibration-cli.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -17,6 +17,16 @@ const { runCalibrationCli, toAmsRealizedOutcomes } = (await import(CALIBRATION_C const { MINER_CALIBRATION_SNAPSHOT_EVENT } = calibrationRun; import { initEventLedger, resolveEventLedgerDbPath } from "../../packages/loopover-miner/lib/event-ledger.js"; +import type { ContributionProfile } from "../../packages/loopover-miner/lib/contribution-profile.js"; +import { + initContributionProfileCache, + resolveContributionProfileCacheDbPath, +} from "../../packages/loopover-miner/lib/contribution-profile-cache.js"; +import { ELIGIBILITY_EXCLUSION_REASONS } from "../../packages/loopover-miner/lib/contribution-profile-filter.js"; +import { + SIGNAL_HUMAN_OVERRIDE_EVENT, + SIGNAL_RULE_FIRED_EVENT, +} from "../../packages/loopover-miner/lib/signal-tracking-store.js"; import { initPredictionLedger, resolvePredictionLedgerDbPath, @@ -202,6 +212,7 @@ describe("loopover-miner calibration CLI (#4849)", () => { import { resolveAmsPolicyConfigPath } from "../../packages/loopover-miner/lib/ams-policy.js"; import { + MINER_AMS_ELIGIBILITY_BACKTEST_EVENT, MINER_AMS_THRESHOLD_BACKTEST_EVENT, readMinRankOverride, } from "../../packages/loopover-miner/lib/ams-calibration.js"; @@ -555,3 +566,223 @@ describe("calibration snapshot (#8317)", () => { expect(output).toContain("calibration snapshot acme/widgets: enabled=true combined=86% delta=0.156 sources=pr_outcome"); }); }); + +function eligibilityProfileJson(): string { + return JSON.stringify({ + "acme/widgets": { + repoFullName: "acme/widgets", + schemaVersion: 1, + generatedAt: "2026-07-18T00:00:00.000Z", + eligibilityLabels: { + value: [ + { field: "name", contains: "help wanted" }, + { field: "name", contains: "good first issue" }, + ], + confidence: "explicit", + provenance: [ + { source: "labels", detail: "help wanted" }, + { source: "labels", detail: "good first issue" }, + ], + }, + exclusionLabels: { + value: [{ field: "name", contains: "blocked" }], + confidence: "inferred", + provenance: [{ source: "labels", detail: "blocked" }], + }, + prBody: { value: null, confidence: "absent", provenance: [] }, + completeness: "inferred", + }, + }); +} + +function seedEligibilityBacktestHistory(env: Record): string { + const ledger = initEventLedger(resolveEventLedgerDbPath(env)); + for (let i = 1; i <= 40; i += 1) { + const ruleId = + i % 2 === 0 ? ELIGIBILITY_EXCLUSION_REASONS.EXCLUSION_LABEL : ELIGIBILITY_EXCLUSION_REASONS.MISSING_ELIGIBILITY_LABEL; + const labels = ruleId === ELIGIBILITY_EXCLUSION_REASONS.EXCLUSION_LABEL ? ["blocked"] : ["bug"]; + const firedAt = `2026-07-${String((i % 28) + 1).padStart(2, "0")}T12:00:00.000Z`; + ledger.appendEvent({ + type: SIGNAL_RULE_FIRED_EVENT, + repoFullName: "acme/widgets", + payload: { + ruleId, + targetKey: `acme/widgets#issue-${i}`, + outcome: "exclude", + occurredAt: firedAt, + metadata: { owner: "acme", labels }, + }, + }); + ledger.appendEvent({ + type: SIGNAL_HUMAN_OVERRIDE_EVENT, + repoFullName: "acme/widgets", + payload: { + ruleId, + targetKey: `acme/widgets#issue-${i}`, + verdict: i % 5 === 0 ? "reversed" : "confirmed", + occurredAt: firedAt.replace("T12:", "T13:"), + }, + }); + } + ledger.close(); + const profilePath = join(env.LOOPOVER_MINER_CONFIG_DIR!, "candidate-profile.json"); + writeFileSync(profilePath, eligibilityProfileJson()); + const cache = initContributionProfileCache(resolveContributionProfileCacheDbPath(env)); + cache.put({ + repoFullName: "acme/widgets", + schemaVersion: 1, + generatedAt: "2026-07-18T00:00:00.000Z", + eligibilityLabels: { + value: [{ field: "name", contains: "help wanted" }], + confidence: "explicit", + provenance: [{ source: "labels", detail: "help wanted" }], + }, + exclusionLabels: { + value: [{ field: "name", contains: "blocked" }], + confidence: "inferred", + provenance: [{ source: "labels", detail: "blocked" }], + }, + prBody: { value: null, confidence: "absent", provenance: [] }, + completeness: "inferred", + }); + cache.close(); + return profilePath; +} + +describe("calibration backtest-eligibility (#8545)", () => { + it("replays, prints skippedNoContext + both split reports, persists the run event, exits 0", () => { + const env = envForTempStores(); + const profilePath = seedEligibilityBacktestHistory(env); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + expect(runCalibrationCli(["backtest-eligibility", "--profile", profilePath], env, { readFileSync })).toBe(0); + const output = logSpy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(output).toContain("skipped (no metadata context):"); + expect(output).toContain("visible split"); + expect(output).toContain("held-out split"); + expect(output).toContain("advisory only"); + const ledger = initEventLedger(resolveEventLedgerDbPath(env)); + expect(ledger.readEvents().filter((event) => event.type === MINER_AMS_ELIGIBILITY_BACKTEST_EVENT)).toHaveLength(1); + ledger.close(); + }); + + it("an under-floored corpus prints the explicit line, persists NOTHING, and still exits 0", () => { + const env = envForTempStores(); + const profilePath = join(env.LOOPOVER_MINER_CONFIG_DIR!, "candidate-profile.json"); + writeFileSync(profilePath, eligibilityProfileJson()); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + expect(runCalibrationCli(["backtest-eligibility", "--profile", profilePath], env, { readFileSync })).toBe(0); + expect(logSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain("not enough labeled eligibility-exclusion cases"); + expect(runCalibrationCli(["backtest-eligibility", "--profile", profilePath, "--json"], env, { readFileSync })).toBe(0); + const jsonLine = logSpy.mock.calls.map((call) => String(call[0])).find((line) => line.includes("insufficient_corpus")); + expect(jsonLine).toBeDefined(); + const ledger = initEventLedger(resolveEventLedgerDbPath(env)); + expect(ledger.readEvents().filter((event) => event.type === MINER_AMS_ELIGIBILITY_BACKTEST_EVENT)).toHaveLength(0); + ledger.close(); + }); + + it("JSON mode dumps the full result; a missing/invalid --profile is a usage error (exit 1)", () => { + const env = envForTempStores(); + const profilePath = seedEligibilityBacktestHistory(env); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + expect(runCalibrationCli(["backtest-eligibility", "--profile", profilePath, "--json"], env, { readFileSync })).toBe(0); + const parsed = JSON.parse(logSpy.mock.calls.map((call) => String(call[0])).find((line) => line.trimStart().startsWith("{"))!) as { + ran: boolean; + visible: { verdict: string }; + }; + expect(parsed.ran).toBe(true); + expect(parsed.visible.verdict).toBeDefined(); + expect(runCalibrationCli(["backtest-eligibility"], env, { readFileSync })).toBe(1); + expect(runCalibrationCli(["backtest-eligibility", "--profile", " "], env, { readFileSync })).toBe(1); + writeFileSync(join(env.LOOPOVER_MINER_CONFIG_DIR!, "bad.json"), "[]"); + expect(runCalibrationCli(["backtest-eligibility", "--profile", join(env.LOOPOVER_MINER_CONFIG_DIR!, "bad.json")], env, { readFileSync })).toBe(1); + }); + + it("fails operationally when readFileSync is unavailable or the ledger store cannot open", () => { + vi.spyOn(console, "error").mockImplementation(() => undefined); + const env = envForTempStores(); + const profilePath = join(env.LOOPOVER_MINER_CONFIG_DIR!, "candidate-profile.json"); + writeFileSync(profilePath, eligibilityProfileJson()); + expect(runCalibrationCli(["backtest-eligibility", "--profile", profilePath], env, {})).toBe(2); + const brokenEnv = { LOOPOVER_MINER_EVENT_LEDGER_DB: "/dev/null/nope/ledger.sqlite" }; + expect(runCalibrationCli(["backtest-eligibility", "--profile", profilePath], brokenEnv, { readFileSync: (() => eligibilityProfileJson()) as never })).toBe(2); + }); + + it("rejects invalid JSON syntax and profile entries whose repoFullName does not match the map key", () => { + const env = envForTempStores(); + vi.spyOn(console, "error").mockImplementation(() => undefined); + const badSyntax = join(env.LOOPOVER_MINER_CONFIG_DIR!, "bad-syntax.json"); + writeFileSync(badSyntax, "{not-json"); + expect(runCalibrationCli(["backtest-eligibility", "--profile", badSyntax], env, { readFileSync })).toBe(1); + const mismatched = join(env.LOOPOVER_MINER_CONFIG_DIR!, "mismatch.json"); + writeFileSync( + mismatched, + JSON.stringify({ + "acme/widgets": { + ...(JSON.parse(eligibilityProfileJson()) as Record)["acme/widgets"], + repoFullName: "other/repo", + }, + }), + ); + expect(runCalibrationCli(["backtest-eligibility", "--profile", mismatched], env, { readFileSync })).toBe(1); + const mixed = join(env.LOOPOVER_MINER_CONFIG_DIR!, "mixed.json"); + writeFileSync( + mixed, + JSON.stringify({ + "acme/widgets": JSON.parse(eligibilityProfileJson())["acme/widgets"], + "other/repo": { repoFullName: "wrong/repo" }, + "skip/null": null, + "skip/string": "not-a-profile", + }), + ); + expect(runCalibrationCli(["backtest-eligibility", "--profile", mixed], env, { readFileSync })).toBe(0); + }); + + it("includes eligibility backtest runs in the main calibration report track record", () => { + const env = envForTempStores(); + const profilePath = seedEligibilityBacktestHistory(env); + vi.spyOn(console, "log").mockImplementation(() => undefined); + expect(runCalibrationCli(["backtest-eligibility", "--profile", profilePath], env, { readFileSync })).toBe(0); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + expect(runCalibrationCli([], env)).toBe(0); + expect(log.mock.calls.map((call) => String(call[0])).join("\n")).toContain("backtest track record:"); + }); + + it("includes eligibility backtest runs in calibration snapshot track record attachment", () => { + const env = envForTempStores(); + seedPrediction(env, 42, "merge"); + seedOutcomeEvent(env, { prNumber: 42, decision: "merged" }); + const profilePath = seedEligibilityBacktestHistory(env); + vi.spyOn(console, "log").mockImplementation(() => undefined); + expect(runCalibrationCli(["backtest-eligibility", "--profile", profilePath], env, { readFileSync })).toBe(0); + vi.spyOn(calibrationRun, "runHistoricalReplayCalibrationCycle").mockReturnValue({ + result: {} as never, + snapshot: { + enabled: false, + combinedAccuracy: null, + baselineAccuracy: null, + deltaFromBaseline: null, + autonomyIncreasePermitted: false, + replayHarnessHold: false, + replayHarnessStatus: "healthy", + replayRunDue: false, + holdReasons: [], + contributingSources: [], + replayRunId: null, + observedAt: null, + replaySampleSize: 0, + backtestTrackRecord: null, + }, + recorded: null, + historicalReplay: null, + compositeScore: null, + sampleSize: 0, + scores: [], + }); + expect(runCalibrationCli(["snapshot"], env)).toBe(0); + expect(calibrationRun.runHistoricalReplayCalibrationCycle).toHaveBeenCalledWith( + expect.objectContaining({ backtestTrackRecord: expect.objectContaining({ totalRuns: 2 }) }), + expect.any(Object), + ); + }); +}); From 381c59e7f1a9d1abb928baabe022981f0890d01f Mon Sep 17 00:00:00 2001 From: Andriy Polanski Date: Fri, 24 Jul 2026 21:40:46 +0000 Subject: [PATCH 2/2] codecov coverage recheck --- .../unit/miner-ams-eligibility-backtest.test.ts | 17 +++++++++-------- test/unit/miner-calibration-cli.test.ts | 6 ++++-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/test/unit/miner-ams-eligibility-backtest.test.ts b/test/unit/miner-ams-eligibility-backtest.test.ts index 8f70f3211d..e6020f257a 100644 --- a/test/unit/miner-ams-eligibility-backtest.test.ts +++ b/test/unit/miner-ams-eligibility-backtest.test.ts @@ -5,7 +5,15 @@ import { afterEach, describe, expect, it } from "vitest"; import type { ContributionProfile } from "../../packages/loopover-miner/lib/contribution-profile.js"; import { ELIGIBILITY_EXCLUSION_REASONS } from "../../packages/loopover-miner/lib/contribution-profile-filter.js"; import { filterCandidatesByProfiles } from "../../packages/loopover-miner/lib/contribution-profile-filter.js"; +import { initEventLedger, resolveEventLedgerDbPath, type EventLedger } from "../../packages/loopover-miner/lib/event-ledger.js"; import { + SIGNAL_HUMAN_OVERRIDE_EVENT, + SIGNAL_RULE_FIRED_EVENT, +} from "../../packages/loopover-miner/lib/signal-tracking-store.js"; + +const AMS_CALIBRATION_MODULE = "../../packages/loopover-miner/lib/ams-calibration.ts"; +const AMS_ELIGIBILITY_MODULE = "../../packages/loopover-miner/lib/ams-eligibility-backtest.ts"; +const { AMS_MIN_RANK_SHIPPED, MINER_AMS_ELIGIBILITY_BACKTEST_EVENT, backtestMinRankCandidate, @@ -14,14 +22,7 @@ import { readAmsThresholdBacktestRuns, recordAmsEligibilityBacktestRun, recordAmsThresholdBacktestRun, -} from "../../packages/loopover-miner/lib/ams-calibration.js"; -import { initEventLedger, resolveEventLedgerDbPath, type EventLedger } from "../../packages/loopover-miner/lib/event-ledger.js"; -import { - SIGNAL_HUMAN_OVERRIDE_EVENT, - SIGNAL_RULE_FIRED_EVENT, -} from "../../packages/loopover-miner/lib/signal-tracking-store.js"; - -const AMS_ELIGIBILITY_MODULE = "../../packages/loopover-miner/lib/ams-eligibility-backtest.ts"; +} = (await import(AMS_CALIBRATION_MODULE)) as typeof import("../../packages/loopover-miner/lib/ams-calibration.js"); const { AMS_ELIGIBILITY_RULE_ID, backtestEligibilityCandidate, diff --git a/test/unit/miner-calibration-cli.test.ts b/test/unit/miner-calibration-cli.test.ts index a3bd1d029d..50e1945a80 100644 --- a/test/unit/miner-calibration-cli.test.ts +++ b/test/unit/miner-calibration-cli.test.ts @@ -211,11 +211,13 @@ describe("loopover-miner calibration CLI (#4849)", () => { // ── #8184/#8185/#8186/#8187: the calibration subcommands + report sections ───────────────────────────────── import { resolveAmsPolicyConfigPath } from "../../packages/loopover-miner/lib/ams-policy.js"; -import { + +const AMS_CALIBRATION_MODULE = "../../packages/loopover-miner/lib/ams-calibration.ts"; +const { MINER_AMS_ELIGIBILITY_BACKTEST_EVENT, MINER_AMS_THRESHOLD_BACKTEST_EVENT, readMinRankOverride, -} from "../../packages/loopover-miner/lib/ams-calibration.js"; +} = (await import(AMS_CALIBRATION_MODULE)) as typeof import("../../packages/loopover-miner/lib/ams-calibration.js"); function seedTakenHistory(env: Record): void { const ledger = initEventLedger(resolveEventLedgerDbPath(env));