diff --git a/docs/statement-schema.md b/docs/statement-schema.md index e48c2f8..dd55933 100644 --- a/docs/statement-schema.md +++ b/docs/statement-schema.md @@ -66,12 +66,38 @@ legitimate producer could ever have emitted it — are *not* breaking and do | `period_end` | ISO date `YYYY-MM-DD` | Inclusive. Must be ≥ `period_start`. Must be a **real calendar day**. | | `bankroll_usd` | number, > 0 | All `*_pnl_usd` claims reference this single bankroll. | | `alert_count` | non-negative integer | Total alerts emitted in the period. | -| `hit_rate` | number in `[0, 1]` | Fraction. `0.58` means 58 %. | +| `hit_rate` | number in `[0, 1]` | Fraction. `0.58` means 58 %. Not interpretable without `resolved_count` — see below. | +| `resolved_count` | non-negative integer, **optional** | How many of `alert_count` had resolved by publication: the denominator behind `hit_rate`. | | `hypothetical_pnl_usd` | number (signed) | Hypothetical PnL on `bankroll_usd`. | | `categories` | non-empty array of category enum, **no duplicates** | See `Category` below. | | `top_wallets` | array of `{ truncated_id, category }` | See `TopWallets` below. | | `draft` | boolean, optional, defaults `false` | When `true`, the entry is excluded from feeds, the homepage teaser, and both dual-format routes (HTML + `.md`). Used for skeletons before publication. | +### `hit_rate` and `resolved_count` + +`hit_rate` is `in_favor / resolved`, over alerts **emitted** in the period. A +market that fires an alert on Monday and settles in December contributes to +`alert_count` immediately and to `hit_rate` months later. + +That makes `hit_rate: 0` ambiguous on its own between two very different weeks: + +| | `hit_rate` | `resolved_count` | meaning | +|---|---:|---:|---| +| genuinely bad week | `0` | `240` | 240 outcomes settled, none went our way | +| nothing settled yet | `0` | `0` | the rate is vacuous — no outcomes to score | + +Macro-only statements are almost always the second case, because those markets +resolve months after the alert. Publishing a bare `0.00` for one of them reads as +a 0 % success rate, which is why `2026-07-28-weekly.md` was unpublished +(auditmos/ogsfrompoly#236). + +`resolved_count` is optional because every statement published before it existed +omits it — adding an optional field is deliberately **not** a version bump, per +the rules above. Absent means *unknown*, never zero: the sanity lint +(`statement-sanity.ts`) keeps flagging `hit_rate: 0` with positive PnL when the +field is missing, so adding it does not retire that check for the back catalogue. +Producers should emit it on every new statement. + ## Weekly-only Weekly statements add no extra top-level fields beyond the shared set. diff --git a/src/content/statement-sanity.test.ts b/src/content/statement-sanity.test.ts index a7593b8..eb4da7a 100644 --- a/src/content/statement-sanity.test.ts +++ b/src/content/statement-sanity.test.ts @@ -41,6 +41,76 @@ describe("findDataInconsistencies", () => { ); }); + // --- resolved_count: telling "0 of 0" apart from "0 of 240" --------------- + // + // Without a denominator the 0/0 collapse and a genuine all-miss week are the + // same two numbers, so the guard above had to treat both as suspect. That made + // a truthful statement for an unresolved window unpublishable — see + // auditmos/ogsfrompoly#236, and ogsfrompoly-lp#30 before it. + + it("does not flag hit_rate 0 with positive PnL when resolved_count is explicitly 0", () => { + // Macro markets resolve months out, so a 7-day window routinely resolves + // nothing. hit_rate is then vacuous rather than a 0% success rate, and the + // PnL is real: hypothetical PnL marks to price, not to resolution. + expect( + findDataInconsistencies({ + ...clean, + alert_count: 263, + hit_rate: 0, + hypothetical_pnl_usd: 199.11, + resolved_count: 0, + }), + ).toEqual([]); + }); + + it("still flags hit_rate 0 with positive PnL when outcomes DID resolve", () => { + // The original 0/0 collapse must stay caught: 240 alerts resolved, not one + // went in favour, yet PnL is positive. That is the upstream join bug. + expect( + findDataInconsistencies({ + ...clean, + alert_count: 263, + hit_rate: 0, + hypothetical_pnl_usd: 199.11, + resolved_count: 240, + }), + ).not.toHaveLength(0); + }); + + it("keeps flagging hit_rate 0 with positive PnL when resolved_count is absent", () => { + // Every statement published before this field existed omits it. Absent must + // stay suspect, or adding the field would silently retire the guard for the + // whole back catalogue. + expect( + findDataInconsistencies({ + ...clean, + alert_count: 263, + hit_rate: 0, + hypothetical_pnl_usd: 199.11, + }), + ).not.toHaveLength(0); + }); + + it("flags a non-zero hit_rate that claims zero resolved outcomes", () => { + // A rate needs a denominator: 0 resolved cannot yield 0.47. + expect( + findDataInconsistencies({ ...clean, hit_rate: 0.47, resolved_count: 0 }), + ).not.toHaveLength(0); + }); + + it("flags resolved_count exceeding alert_count", () => { + // More outcomes than alerts means the join fanned out. + expect( + findDataInconsistencies({ ...clean, alert_count: 263, resolved_count: 264 }), + ).not.toHaveLength(0); + }); + + it("passes a healthy statement carrying its denominator", () => { + expect(findDataInconsistencies({ ...clean, alert_count: 2453, resolved_count: 1200 })).toEqual( + [], + ); + }); + it("flags a positive hit_rate reported with zero alerts", () => { expect(findDataInconsistencies({ ...clean, alert_count: 0, hit_rate: 0.5 })).not.toHaveLength( 0, diff --git a/src/content/statement-sanity.ts b/src/content/statement-sanity.ts index e06b2f8..10c08cd 100644 --- a/src/content/statement-sanity.ts +++ b/src/content/statement-sanity.ts @@ -38,6 +38,12 @@ export interface StatementSanityInput { period_start?: string; alert_count: number; hit_rate: number; + /** + * How many of `alert_count` had a resolved outcome — the denominator behind + * `hit_rate`. Optional: every statement published before the field existed + * omits it, and absent is treated as "unknown", never as zero. + */ + resolved_count?: number; hypothetical_pnl_usd: number; bankroll_usd: number; top_wallets?: ReadonlyArray<{ hypothetical_pnl_usd?: number }>; @@ -51,18 +57,8 @@ export interface StatementSanityInput { export function findDataInconsistencies(data: StatementSanityInput): string[] { const issues: string[] = []; - // A 0 hit rate over resolved alerts cannot coexist with positive PnL under the - // mirror-every-alert model — the tell of a `0/0 → 0` resolution collapse. - if (data.alert_count > 0 && data.hit_rate === 0 && data.hypothetical_pnl_usd > 0) { - issues.push( - `alert_count=${data.alert_count} with hit_rate=0 but hypothetical_pnl_usd=${data.hypothetical_pnl_usd} > 0 (0 resolved alerts collapsing 0/0 → 0?)`, - ); - } - - // A non-zero hit rate is undefined without alerts to have hit. - if (data.hit_rate > 0 && data.alert_count === 0) { - issues.push(`hit_rate=${data.hit_rate} with alert_count=0 (a rate over no alerts)`); - } + // Hit-rate cross-field checks (the 0/0 collapse and its mirrors). + issues.push(...findHitRateInconsistencies(data)); // Monthly P&L cross-field checks (balance, recurring-opex floor, runway). if (data.pnl) { @@ -86,6 +82,62 @@ export function findDataInconsistencies(data: StatementSanityInput): string[] { return issues; } +/** + * Cross-field checks tying `hit_rate` to its denominator. Empty array = consistent. + * + * `hit_rate` is `in_favor / resolved` over alerts **emitted** in the period, so a + * market alerted on Monday that settles in December counts toward `alert_count` + * now and toward `hit_rate` months later. `hit_rate: 0` is therefore ambiguous + * on its own between "240 settled, none in favour" and "nothing has settled yet". + * `resolved_count` is what separates them. + */ +function findHitRateInconsistencies(data: StatementSanityInput): string[] { + const issues: string[] = []; + const { alert_count, hit_rate, hypothetical_pnl_usd, resolved_count } = data; + const resolvedKnown = resolved_count !== undefined; + + // A 0 hit rate over resolved alerts cannot coexist with positive PnL under the + // mirror-every-alert model — the tell of a `0/0 → 0` resolution collapse. + // + // An explicit `resolved_count === 0` exempts it: the rate is vacuous rather + // than a 0% success rate, while the PnL is still real, because hypothetical + // PnL marks to price, not to resolution. That is the normal shape of a + // Macro-only statement, whose markets resolve months after the alert fires + // (auditmos/ogsfrompoly#236). + // + // Absent stays suspect. Every statement published before the field existed + // omits it, and treating absent as zero would silently retire this guard for + // the whole back catalogue — including the ogsfrompoly-lp#30 case it exists for. + if (alert_count > 0 && hit_rate === 0 && hypothetical_pnl_usd > 0 && resolved_count !== 0) { + const denominator = resolvedKnown ? ` over ${resolved_count} resolved` : ""; + issues.push( + `alert_count=${alert_count} with hit_rate=0${denominator} but hypothetical_pnl_usd=${hypothetical_pnl_usd} > 0 (0 resolved alerts collapsing 0/0 → 0?)`, + ); + } + + // A non-zero hit rate is undefined without alerts to have hit. + if (hit_rate > 0 && alert_count === 0) { + issues.push(`hit_rate=${hit_rate} with alert_count=0 (a rate over no alerts)`); + } + + // ...and equally undefined without outcomes to have hit. The mirror of the + // exemption above: claiming a rate while declaring an empty denominator is the + // same collapse pointing the other way. + if (resolved_count === 0 && hit_rate > 0) { + issues.push(`hit_rate=${hit_rate} with resolved_count=0 (a rate over no resolved outcomes)`); + } + + // More resolved outcomes than alerts means the resolution join fanned out — + // one alert matching several markets, or duplicate rows counted twice. + if (resolvedKnown && resolved_count > alert_count) { + issues.push( + `resolved_count=${resolved_count} exceeds alert_count=${alert_count} (resolution join fanned out?)`, + ); + } + + return issues; +} + type PnlInput = NonNullable; /** Cross-field checks for the monthly `pnl` block. Empty array = consistent. */ diff --git a/src/content/statement-schema.ts b/src/content/statement-schema.ts index 57c2d0d..eaae3e0 100644 --- a/src/content/statement-schema.ts +++ b/src/content/statement-schema.ts @@ -56,6 +56,22 @@ const sharedFields = { bankroll_usd: z.number().positive(), alert_count: z.number().int().nonnegative(), hit_rate: z.number().min(0).max(1), + /** + * How many of `alert_count` had a resolved outcome by publication — the + * denominator behind `hit_rate`. + * + * OPTIONAL, which per `docs/statement-schema.md` is explicitly not a version + * bump. Statements published before this field existed omit it and stay + * valid; the consumer-side lint treats absent as "unknown", never as zero. + * + * Without it, `hit_rate: 0` is ambiguous between "0 of 240 went our way" and + * "0 of 0 have settled" — and for Macro-only statements the second is the + * norm, because those markets resolve months after the alert fires. That + * ambiguity made a truthful statement for an unresolved window unpublishable: + * the sanity lint could not clear it without also retiring the check that + * catches a genuine `0/0 → 0` collapse. See auditmos/ogsfrompoly#236. + */ + resolved_count: z.number().int().nonnegative().optional(), hypothetical_pnl_usd: z.number(), categories: z .array(category)