From 686de7fad6d50c2bcaa63a80632118c3f1815218 Mon Sep 17 00:00:00 2001 From: Tomasz Kowalczyk Date: Wed, 29 Jul 2026 12:50:49 +0200 Subject: [PATCH] feat(schema): add optional resolved_count so hit_rate 0 can be read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `hit_rate` is `in_favor / resolved` over alerts EMITTED in the period, so a market alerted on Monday that settles in December counts toward `alert_count` now and toward `hit_rate` months later. On its own `hit_rate: 0` is therefore ambiguous between two very different weeks: hit_rate 0, resolved 240 → 240 outcomes settled and none went our way hit_rate 0, resolved 0 → nothing has settled; the rate is vacuous Macro-only statements are almost always the second — those markets resolve months after the alert. Without a denominator the sanity lint had to treat both as suspect, which made a truthful statement for an unresolved window unpublishable: `2026-07-28-weekly.md` could not clear CI without either asserting a 0% success rate that never happened, or falsifying its PnL back to 0 to match. It was unpublished instead. `resolved_count` is OPTIONAL, which per docs/statement-schema.md is explicitly not a version bump — `schema_version` stays 1 and all ten existing statements keep validating. Absent means UNKNOWN, never zero: the 0/0-collapse guard still fires when the field is missing, so this does not retire that check for the back catalogue, including the ogsfrompoly-lp#30 case it was written for. Two mirror checks come with it, since a denominator is only useful if it is also constrained: - hit_rate > 0 with resolved_count === 0 — a rate over no resolved outcomes - resolved_count > alert_count — the resolution join fanned out The hit-rate checks move into `findHitRateInconsistencies`, mirroring the existing `findPnlInconsistencies` split; inlining the new branches tripped noExcessiveCognitiveComplexity, and the helper is the shape this file already uses. Producer side (emitting the field) lands in auditmos/ogsfrompoly. Ordered consumer-first so the value is understood before it is sent. Refs auditmos/ogsfrompoly#236, auditmos/ogsfrompoly#234 --- docs/statement-schema.md | 28 +++++++++- src/content/statement-sanity.test.ts | 70 +++++++++++++++++++++++++ src/content/statement-sanity.ts | 76 +++++++++++++++++++++++----- src/content/statement-schema.ts | 16 ++++++ 4 files changed, 177 insertions(+), 13 deletions(-) diff --git a/docs/statement-schema.md b/docs/statement-schema.md index e48c2f8..dd55933 100644 --- a/docs/statement-schema.md +++ b/docs/statement-schema.md @@ -66,12 +66,38 @@ legitimate producer could ever have emitted it — are *not* breaking and do | `period_end` | ISO date `YYYY-MM-DD` | Inclusive. Must be ≥ `period_start`. Must be a **real calendar day**. | | `bankroll_usd` | number, > 0 | All `*_pnl_usd` claims reference this single bankroll. | | `alert_count` | non-negative integer | Total alerts emitted in the period. | -| `hit_rate` | number in `[0, 1]` | Fraction. `0.58` means 58 %. | +| `hit_rate` | number in `[0, 1]` | Fraction. `0.58` means 58 %. Not interpretable without `resolved_count` — see below. | +| `resolved_count` | non-negative integer, **optional** | How many of `alert_count` had resolved by publication: the denominator behind `hit_rate`. | | `hypothetical_pnl_usd` | number (signed) | Hypothetical PnL on `bankroll_usd`. | | `categories` | non-empty array of category enum, **no duplicates** | See `Category` below. | | `top_wallets` | array of `{ truncated_id, category }` | See `TopWallets` below. | | `draft` | boolean, optional, defaults `false` | When `true`, the entry is excluded from feeds, the homepage teaser, and both dual-format routes (HTML + `.md`). Used for skeletons before publication. | +### `hit_rate` and `resolved_count` + +`hit_rate` is `in_favor / resolved`, over alerts **emitted** in the period. A +market that fires an alert on Monday and settles in December contributes to +`alert_count` immediately and to `hit_rate` months later. + +That makes `hit_rate: 0` ambiguous on its own between two very different weeks: + +| | `hit_rate` | `resolved_count` | meaning | +|---|---:|---:|---| +| genuinely bad week | `0` | `240` | 240 outcomes settled, none went our way | +| nothing settled yet | `0` | `0` | the rate is vacuous — no outcomes to score | + +Macro-only statements are almost always the second case, because those markets +resolve months after the alert. Publishing a bare `0.00` for one of them reads as +a 0 % success rate, which is why `2026-07-28-weekly.md` was unpublished +(auditmos/ogsfrompoly#236). + +`resolved_count` is optional because every statement published before it existed +omits it — adding an optional field is deliberately **not** a version bump, per +the rules above. Absent means *unknown*, never zero: the sanity lint +(`statement-sanity.ts`) keeps flagging `hit_rate: 0` with positive PnL when the +field is missing, so adding it does not retire that check for the back catalogue. +Producers should emit it on every new statement. + ## Weekly-only Weekly statements add no extra top-level fields beyond the shared set. diff --git a/src/content/statement-sanity.test.ts b/src/content/statement-sanity.test.ts index a7593b8..eb4da7a 100644 --- a/src/content/statement-sanity.test.ts +++ b/src/content/statement-sanity.test.ts @@ -41,6 +41,76 @@ describe("findDataInconsistencies", () => { ); }); + // --- resolved_count: telling "0 of 0" apart from "0 of 240" --------------- + // + // Without a denominator the 0/0 collapse and a genuine all-miss week are the + // same two numbers, so the guard above had to treat both as suspect. That made + // a truthful statement for an unresolved window unpublishable — see + // auditmos/ogsfrompoly#236, and ogsfrompoly-lp#30 before it. + + it("does not flag hit_rate 0 with positive PnL when resolved_count is explicitly 0", () => { + // Macro markets resolve months out, so a 7-day window routinely resolves + // nothing. hit_rate is then vacuous rather than a 0% success rate, and the + // PnL is real: hypothetical PnL marks to price, not to resolution. + expect( + findDataInconsistencies({ + ...clean, + alert_count: 263, + hit_rate: 0, + hypothetical_pnl_usd: 199.11, + resolved_count: 0, + }), + ).toEqual([]); + }); + + it("still flags hit_rate 0 with positive PnL when outcomes DID resolve", () => { + // The original 0/0 collapse must stay caught: 240 alerts resolved, not one + // went in favour, yet PnL is positive. That is the upstream join bug. + expect( + findDataInconsistencies({ + ...clean, + alert_count: 263, + hit_rate: 0, + hypothetical_pnl_usd: 199.11, + resolved_count: 240, + }), + ).not.toHaveLength(0); + }); + + it("keeps flagging hit_rate 0 with positive PnL when resolved_count is absent", () => { + // Every statement published before this field existed omits it. Absent must + // stay suspect, or adding the field would silently retire the guard for the + // whole back catalogue. + expect( + findDataInconsistencies({ + ...clean, + alert_count: 263, + hit_rate: 0, + hypothetical_pnl_usd: 199.11, + }), + ).not.toHaveLength(0); + }); + + it("flags a non-zero hit_rate that claims zero resolved outcomes", () => { + // A rate needs a denominator: 0 resolved cannot yield 0.47. + expect( + findDataInconsistencies({ ...clean, hit_rate: 0.47, resolved_count: 0 }), + ).not.toHaveLength(0); + }); + + it("flags resolved_count exceeding alert_count", () => { + // More outcomes than alerts means the join fanned out. + expect( + findDataInconsistencies({ ...clean, alert_count: 263, resolved_count: 264 }), + ).not.toHaveLength(0); + }); + + it("passes a healthy statement carrying its denominator", () => { + expect(findDataInconsistencies({ ...clean, alert_count: 2453, resolved_count: 1200 })).toEqual( + [], + ); + }); + it("flags a positive hit_rate reported with zero alerts", () => { expect(findDataInconsistencies({ ...clean, alert_count: 0, hit_rate: 0.5 })).not.toHaveLength( 0, diff --git a/src/content/statement-sanity.ts b/src/content/statement-sanity.ts index e06b2f8..10c08cd 100644 --- a/src/content/statement-sanity.ts +++ b/src/content/statement-sanity.ts @@ -38,6 +38,12 @@ export interface StatementSanityInput { period_start?: string; alert_count: number; hit_rate: number; + /** + * How many of `alert_count` had a resolved outcome — the denominator behind + * `hit_rate`. Optional: every statement published before the field existed + * omits it, and absent is treated as "unknown", never as zero. + */ + resolved_count?: number; hypothetical_pnl_usd: number; bankroll_usd: number; top_wallets?: ReadonlyArray<{ hypothetical_pnl_usd?: number }>; @@ -51,18 +57,8 @@ export interface StatementSanityInput { export function findDataInconsistencies(data: StatementSanityInput): string[] { const issues: string[] = []; - // A 0 hit rate over resolved alerts cannot coexist with positive PnL under the - // mirror-every-alert model — the tell of a `0/0 → 0` resolution collapse. - if (data.alert_count > 0 && data.hit_rate === 0 && data.hypothetical_pnl_usd > 0) { - issues.push( - `alert_count=${data.alert_count} with hit_rate=0 but hypothetical_pnl_usd=${data.hypothetical_pnl_usd} > 0 (0 resolved alerts collapsing 0/0 → 0?)`, - ); - } - - // A non-zero hit rate is undefined without alerts to have hit. - if (data.hit_rate > 0 && data.alert_count === 0) { - issues.push(`hit_rate=${data.hit_rate} with alert_count=0 (a rate over no alerts)`); - } + // Hit-rate cross-field checks (the 0/0 collapse and its mirrors). + issues.push(...findHitRateInconsistencies(data)); // Monthly P&L cross-field checks (balance, recurring-opex floor, runway). if (data.pnl) { @@ -86,6 +82,62 @@ export function findDataInconsistencies(data: StatementSanityInput): string[] { return issues; } +/** + * Cross-field checks tying `hit_rate` to its denominator. Empty array = consistent. + * + * `hit_rate` is `in_favor / resolved` over alerts **emitted** in the period, so a + * market alerted on Monday that settles in December counts toward `alert_count` + * now and toward `hit_rate` months later. `hit_rate: 0` is therefore ambiguous + * on its own between "240 settled, none in favour" and "nothing has settled yet". + * `resolved_count` is what separates them. + */ +function findHitRateInconsistencies(data: StatementSanityInput): string[] { + const issues: string[] = []; + const { alert_count, hit_rate, hypothetical_pnl_usd, resolved_count } = data; + const resolvedKnown = resolved_count !== undefined; + + // A 0 hit rate over resolved alerts cannot coexist with positive PnL under the + // mirror-every-alert model — the tell of a `0/0 → 0` resolution collapse. + // + // An explicit `resolved_count === 0` exempts it: the rate is vacuous rather + // than a 0% success rate, while the PnL is still real, because hypothetical + // PnL marks to price, not to resolution. That is the normal shape of a + // Macro-only statement, whose markets resolve months after the alert fires + // (auditmos/ogsfrompoly#236). + // + // Absent stays suspect. Every statement published before the field existed + // omits it, and treating absent as zero would silently retire this guard for + // the whole back catalogue — including the ogsfrompoly-lp#30 case it exists for. + if (alert_count > 0 && hit_rate === 0 && hypothetical_pnl_usd > 0 && resolved_count !== 0) { + const denominator = resolvedKnown ? ` over ${resolved_count} resolved` : ""; + issues.push( + `alert_count=${alert_count} with hit_rate=0${denominator} but hypothetical_pnl_usd=${hypothetical_pnl_usd} > 0 (0 resolved alerts collapsing 0/0 → 0?)`, + ); + } + + // A non-zero hit rate is undefined without alerts to have hit. + if (hit_rate > 0 && alert_count === 0) { + issues.push(`hit_rate=${hit_rate} with alert_count=0 (a rate over no alerts)`); + } + + // ...and equally undefined without outcomes to have hit. The mirror of the + // exemption above: claiming a rate while declaring an empty denominator is the + // same collapse pointing the other way. + if (resolved_count === 0 && hit_rate > 0) { + issues.push(`hit_rate=${hit_rate} with resolved_count=0 (a rate over no resolved outcomes)`); + } + + // More resolved outcomes than alerts means the resolution join fanned out — + // one alert matching several markets, or duplicate rows counted twice. + if (resolvedKnown && resolved_count > alert_count) { + issues.push( + `resolved_count=${resolved_count} exceeds alert_count=${alert_count} (resolution join fanned out?)`, + ); + } + + return issues; +} + type PnlInput = NonNullable; /** Cross-field checks for the monthly `pnl` block. Empty array = consistent. */ diff --git a/src/content/statement-schema.ts b/src/content/statement-schema.ts index 57c2d0d..eaae3e0 100644 --- a/src/content/statement-schema.ts +++ b/src/content/statement-schema.ts @@ -56,6 +56,22 @@ const sharedFields = { bankroll_usd: z.number().positive(), alert_count: z.number().int().nonnegative(), hit_rate: z.number().min(0).max(1), + /** + * How many of `alert_count` had a resolved outcome by publication — the + * denominator behind `hit_rate`. + * + * OPTIONAL, which per `docs/statement-schema.md` is explicitly not a version + * bump. Statements published before this field existed omit it and stay + * valid; the consumer-side lint treats absent as "unknown", never as zero. + * + * Without it, `hit_rate: 0` is ambiguous between "0 of 240 went our way" and + * "0 of 0 have settled" — and for Macro-only statements the second is the + * norm, because those markets resolve months after the alert fires. That + * ambiguity made a truthful statement for an unresolved window unpublishable: + * the sanity lint could not clear it without also retiring the check that + * catches a genuine `0/0 → 0` collapse. See auditmos/ogsfrompoly#236. + */ + resolved_count: z.number().int().nonnegative().optional(), hypothetical_pnl_usd: z.number(), categories: z .array(category)