diff --git a/.github/workflows/canon-quality.yml b/.github/workflows/canon-quality.yml index b7377a64..82275432 100644 --- a/.github/workflows/canon-quality.yml +++ b/.github/workflows/canon-quality.yml @@ -557,3 +557,70 @@ jobs: print(f"- **Result**: audit did not produce output ({e})") PY } >> "$GITHUB_STEP_SUMMARY" + + surfacing: + name: Homepage surfacing report (soft) + runs-on: ubuntu-latest + timeout-minutes: 3 + # Soft-only. Reports where each writings/ essay surfaces (homepage feed / + # nav / hidden) and flags anything not on the homepage. The HARD field gate + # lives in the `frontmatter` job; this job exists to make the legitimate- + # but-easy-to-miss `exposure: nav` state LOUD instead of silent. Never fails. + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install dependencies + run: pip install --quiet pyyaml + + - name: Run surfacing report + run: | + python3 scripts/surfacing-report.py --json writings/ > /tmp/surface.json || true + python3 scripts/surfacing-report.py writings/ || true + + - name: Render PR comment + if: github.event_name == 'pull_request' + run: | + python3 - <<'PY' + import json + d = json.load(open('/tmp/surface.json')) + essays = d['essays'] + off = d['not_on_homepage'] + lines = [] + icon = '✅' if not off else 'ℹ️' + lines.append(f"### Canon Quality — Homepage Surfacing {icon}") + lines.append('') + lines.append(f"{len(essays)} essay(s) scanned. **Soft report** — never blocks; " + f"the hard field gate is the Frontmatter Schema job.") + lines.append('') + if off: + lines.append(f"**{len(off)} essay(s) are NOT on the homepage feed** " + f"(confirm this is intentional):") + lines.append('') + lines.append('| Essay | Surface |') + lines.append('|---|---|') + by = {e['path']: e for e in essays} + for p in off: + e = by[p] + lines.append(f"| `{p}` | {e['surface']} — {e['explanation']} |") + lines.append('') + lines.append('> To promote to the homepage: set `public: true` and `exposure: public`.') + else: + lines.append('All published essays resolve to the homepage feed.') + lines.append('') + lines.append('Report: `scripts/surfacing-report.py` · Canon: ' + '`klappy://canon/constraints/frontmatter-validation-before-merge`') + open('/tmp/surface-comment.md', 'w').write('\n'.join(lines)) + PY + + - name: Sticky comment + if: github.event_name == 'pull_request' + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: canon-quality-surfacing + path: /tmp/surface-comment.md diff --git a/.husky/pre-commit b/.husky/pre-commit index d6e51bec..b1f161c1 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,5 +1,39 @@ #!/usr/bin/env sh +# +# Catch frontmatter problems at WRITE time, before they ever reach CI. +# +# 1. validate-frontmatter.py (HARD): blocks the commit if any staged +# writings/ essay has missing/malformed renderer-critical fields. Same +# gate CI runs — running it here gives you the failure in seconds instead +# of after a push. +# 2. surfacing-report.py (SOFT): prints where each staged essay will surface +# (homepage / nav / hidden) so an essay quietly on `nav` instead of the +# homepage is visible immediately. Never blocks. +# +# Requires python3 + pyyaml. Degrades to a warning if python3 is unavailable. -# E0005.1: Pre-commit hooks for defunct pipeline removed. -# sync-content, export-book, and build-docs-index scripts -# were part of the lane-era pipeline (see D0016). +staged_writings=$(git diff --cached --name-only --diff-filter=ACM | grep -E '^writings/.*\.md$' | grep -vE '/_[^/]*\.md$' || true) + +if [ -z "$staged_writings" ]; then + exit 0 +fi + +if ! command -v python3 >/dev/null 2>&1; then + echo "pre-commit: python3 not found — skipping frontmatter validation." >&2 + exit 0 +fi + +echo "pre-commit: validating frontmatter for staged essays…" +# shellcheck disable=SC2086 +if ! python3 scripts/validate-frontmatter.py $staged_writings; then + echo "" >&2 + echo "Commit blocked: frontmatter validation failed. Fix the fields above" >&2 + echo "(or copy writings/_TEMPLATE.md for the full required set), then re-commit." >&2 + exit 1 +fi + +# Soft surfacing heads-up — informational only. +# shellcheck disable=SC2086 +python3 scripts/surfacing-report.py $staged_writings || true + +exit 0 diff --git a/canon/constraints/frontmatter-validation-before-merge.md b/canon/constraints/frontmatter-validation-before-merge.md index ff34f443..520ad9a6 100644 --- a/canon/constraints/frontmatter-validation-before-merge.md +++ b/canon/constraints/frontmatter-validation-before-merge.md @@ -53,6 +53,8 @@ These specific combinations have caused renderer crashes in production: | Missing `type` on public documents | Renderer cannot select template | | Quoted booleans (`"true"` instead of `true`) | YAML parses as string, renderer expects boolean | | Missing `hook` or `description` | Social card generation fails silently | +| Missing `public` field entirely | Essay is treated as unpublished — it merges, CI is green, and it is silently absent from the homepage | +| `exposure: nav` + missing `type` / `slug` / `public` | The silent-drop bug. Passed the OLD gate (which only checked `exposure: public`), then never surfaced anywhere | --- @@ -79,7 +81,7 @@ The validator emits findings under five rule_ids, each mapped directly to a |---------|---------| | `frontmatter-missing-block` | File has no `---`-delimited frontmatter at all | | `frontmatter-parse-error` | Frontmatter block exists but YAML is malformed | -| `frontmatter-missing-required` | One of the eight universal fields, or one of `type` / `slug` / `hook` / `description` on a public essay in writings/, is missing or empty | +| `frontmatter-missing-required` | One of the eight universal fields, or one of `public` / `type` / `slug` / `hook` / `description` on **any** essay in writings/ (regardless of exposure), is missing or empty | | `frontmatter-invalid-enum` | `exposure`, `voice`, `tier`, or `audience` has a value not in the canonical allowed set | | `frontmatter-type-mismatch` | Quoted boolean (`public: "true"`) or quoted integer (`tier: "3"`) | | `frontmatter-contradictory` | `public: false` combined with `exposure: public` | @@ -93,6 +95,55 @@ schema doc wins and the validator's enum mirror must be updated to match. --- +## Homepage Surfacing — Where Essays Appear, and Why They Vanish + +Two distinct frontmatter signals decide where a writings/ essay shows up. Per +`canon/meta/frontmatter-schema.md` (the source of truth): + +- **Homepage feed** — `public: true` **and** `exposure: public`. This is the + default published surface. +- **Curated reading path** — `start_here: true` (ordered by `start_here_order`). + An additional, editorial "start here" path on the homepage. Independent of + the feed; set it deliberately, not by default. +- **Navigation only** — `public: true` **and** `exposure: nav`. Reachable + through site navigation but **not** promoted on the homepage. This is a + legitimate, intentional state for some essays — it is NOT an error. +- **Hidden / draft** — `public: false`/absent, or `exposure` in + `draft` / `hidden` / `internal`. + +### The recurring failure this prevents + +An essay authored with `exposure: nav` and missing `public` / `type` / `slug` +**merges clean and then never appears on the homepage.** The original gate only +required the renderer-critical fields when `exposure: public`, so a `nav` essay +sailed through with a green check and vanished silently. "Be more careful" does +not fix a blind spot in the gate; widening the gate does. + +### The strengthened rule (no exceptions) + +`public`, `type`, `slug`, `hook`, and `description` are required on **every** +essay in writings/, **regardless of exposure**. An essay cannot exist in +writings/ without declaring `public` explicitly. This is enforced +unconditionally by `scripts/validate-frontmatter.py`. + +Because `public: true` + `exposure: nav` is legitimate, the gate cannot +hard-fail it. Instead, `scripts/surfacing-report.py` makes the state **loud**: +it reports, per essay, exactly which surface it lands on and flags anything not +on the homepage feed — so "I meant to publish and it landed on nav" is visible +at write time, not discovered in production. + +### Caught at every layer + +| Layer | Mechanism | +|-------|-----------| +| Writing (scaffold) | `writings/_TEMPLATE.md` ships the complete required field set; copy it to start a new essay correct | +| Writing (commit) | `.husky/pre-commit` runs the validator (hard) + surfacing report (soft) on staged writings/ essays | +| Validating | `scripts/validate-frontmatter.py` — fields required for all essays unconditionally | +| Challenging | This constraint; oddkit `preflight`/`challenge`/`validate` surface it when a deliverable includes writings/ | +| CI/CD | `.github/workflows/canon-quality.yml` `frontmatter` job hard-blocks; the surfacing report runs as a soft PR comment | + +--- + ## Enforcement This constraint is part of the Definition of Done for any writing. A writing that exists but has broken frontmatter is not complete — it is a liability that will crash the renderer. diff --git a/scripts/surfacing-report.py b/scripts/surfacing-report.py new file mode 100644 index 00000000..be38288d --- /dev/null +++ b/scripts/surfacing-report.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +""" +surfacing-report.py — report where each writings/ essay will surface. + +The frontmatter validator (validate-frontmatter.py) is a HARD gate: it fails +the build when renderer-critical fields are missing or malformed. But there is +a second, quieter failure mode it deliberately does NOT block, because the +state is sometimes intentional: + + A complete, valid essay set to `exposure: nav` is reachable through site + navigation but is NOT promoted on the homepage feed. + +`public: true` + `exposure: nav` is a legitimate, established pattern in this +repo (several essays use it on purpose). So we cannot hard-fail it. But the +recurring pain is that an author MEANT to publish to the homepage and the essay +silently landed on nav — and nothing said so. + +This script makes that state LOUD instead of silent. It classifies every essay +by the surface it will appear on, using the schema's own definitions +(canon/meta/frontmatter-schema.md): + + homepage = public: true AND exposure: public (homepage feed) + start_here = start_here: true (curated reading path) + nav = public: true AND exposure: nav (navigable, NOT promoted) + hidden = exposure in {draft, hidden, internal} OR public is false/absent + +It NEVER fails the build (exit 0 always). It is a report. Wire it into CI as a +soft PR comment and into the pre-commit hook so the author sees, at write time, +exactly where each essay they touched will (and will not) show up. + +Usage: + python3 scripts/surfacing-report.py [path ...] # human-readable + python3 scripts/surfacing-report.py --json [path ...] +""" +from __future__ import annotations +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Any + +try: + import yaml +except ImportError: + sys.stderr.write("This script requires PyYAML. Install with: pip install pyyaml\n") + sys.exit(0) # report-only: never break the build on a missing dep + +FRONTMATTER_BLOCK_RE = re.compile(r"^---\n(.*?)\n---\n", re.DOTALL) + + +def parse_fm(path: str) -> dict[str, Any] | None: + try: + text = Path(path).read_text(encoding="utf-8") + except OSError: + return None + m = FRONTMATTER_BLOCK_RE.match(text) + if not m: + return None + try: + fm = yaml.safe_load(m.group(1)) + except yaml.YAMLError: + return None + return fm if isinstance(fm, dict) else None + + +def classify(fm: dict[str, Any]) -> tuple[str, str]: + """Return (surface, human_explanation).""" + public = fm.get("public") + exposure = fm.get("exposure") + start_here = bool(fm.get("start_here")) + + if public is not True or public is None: + return ("hidden", f"public={public!r} — NOT a published essay; will not surface publicly") + if exposure == "public": + if start_here: + return ("homepage+start_here", "homepage feed AND curated start_here reading path") + return ("homepage", "homepage feed (exposure: public)") + if exposure == "nav": + return ("nav", "navigation only — reachable but NOT promoted on the homepage") + return ("hidden", f"exposure={exposure!r} — not listed on public surfaces") + + +def discover(paths: list[str]) -> list[str]: + def keep(p: Path) -> bool: + return (p.suffix == ".md" + and p.name != "README.md" + and not p.name.startswith("_")) + if paths: + out: list[str] = [] + for p in paths: + pp = Path(p) + if pp.is_dir(): + out.extend(str(x) for x in sorted(pp.rglob("*.md")) if keep(x)) + elif pp.is_file() and keep(pp): + out.append(str(pp)) + return out + base = Path("writings") + return [str(p) for p in sorted(base.rglob("*.md")) if keep(p)] if base.is_dir() else [] + + +def main() -> int: + ap = argparse.ArgumentParser(description="Report where each writings/ essay surfaces.") + ap.add_argument("paths", nargs="*", help="Files/dirs. Default: writings/.") + ap.add_argument("--json", action="store_true") + args = ap.parse_args() + + rows = [] + for path in discover(args.paths): + fm = parse_fm(path) + if fm is None: + rows.append({"path": path, "surface": "unknown", + "explanation": "no parseable frontmatter"}) + continue + surface, explanation = classify(fm) + rows.append({"path": path, "surface": surface, "explanation": explanation}) + + not_promoted = [r for r in rows if r["surface"] in ("nav", "hidden", "unknown")] + + if args.json: + json.dump({"essays": rows, + "not_on_homepage": [r["path"] for r in not_promoted]}, + sys.stdout, indent=2) + sys.stdout.write("\n") + return 0 # report-only + + name_w = max((len(Path(r["path"]).name) for r in rows), default=4) + print(f"Surfacing report — {len(rows)} essay(s)\n") + for r in rows: + print(f" {Path(r['path']).name.ljust(name_w)} {r['surface']:18} {r['explanation']}") + if not_promoted: + print("\n⚠ NOT on the homepage feed (confirm this is intentional):") + for r in not_promoted: + print(f" - {Path(r['path']).name}: {r['explanation']}") + print("\n To promote to the homepage: set `public: true` and `exposure: public`.") + print(" To keep it intentionally off the homepage: leave as-is (this is just a heads-up).") + return 0 # ALWAYS report-only — never fails the build + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/fixtures/writings/broken-nav-missing-discovery.md b/scripts/tests/fixtures/writings/broken-nav-missing-discovery.md new file mode 100644 index 00000000..78e09f8b --- /dev/null +++ b/scripts/tests/fixtures/writings/broken-nav-missing-discovery.md @@ -0,0 +1,15 @@ +--- +uri: klappy://writings/broken-nav-missing-discovery +title: "A nav essay that forgot to be publishable" +audience: public +exposure: nav +tier: 2 +voice: first_person +stability: draft +tags: ["test"] +date: 2026-06-05 +hook: "Has a hook and description, but no public flag, type, or slug." +description: "This reproduces the recurring 'merged but invisible' bug: exposure=nav with missing public/type/slug used to pass the old conditional gate." +--- + +Body text. diff --git a/scripts/tests/test_validator.py b/scripts/tests/test_validator.py index 32ea799e..82ce2168 100755 --- a/scripts/tests/test_validator.py +++ b/scripts/tests/test_validator.py @@ -69,6 +69,25 @@ def main() -> None: "broken-no-frontmatter: missing-block fires", ) + # 4b. REGRESSION: the recurring "merged but invisible" bug. A writings + # essay on exposure=nav with missing public/type/slug used to pass the + # old conditional gate (which only checked exposure=public). It must + # now fail. This fixture must produce a `public` finding AND `type`/ + # `slug` discovery findings regardless of its nav exposure. + d, rc = run(str(FIXTURES / "writings" / "broken-nav-missing-discovery.md")) + assert rc == 1, f"expected exit 1 for nav-missing-discovery, got {rc}" + occurrences = {f["occurrence"] for f in d["findings"]} + for required in ("public", "type", "slug"): + assert required in occurrences, ( + f"nav-missing-discovery should flag missing {required!r}; " + f"got occurrences {occurrences}" + ) + expect( + {"frontmatter-missing-required"}, + d["findings"], + "broken-nav-missing-discovery: nav essay missing public/type/slug fails", + ) + # 5. Real writings/ directory must be clean (this enforces that we never # ship the validator with existing breakage) d, rc = run("writings/") diff --git a/scripts/validate-frontmatter.py b/scripts/validate-frontmatter.py index f71754e6..2998e848 100755 --- a/scripts/validate-frontmatter.py +++ b/scripts/validate-frontmatter.py @@ -16,9 +16,12 @@ parses these as strings, which the renderer rejects - Contradictory flags (`public: false` + `exposure: public`) — renderer builds a route with no content - - Public essays in writings/ missing renderer-critical discovery fields - (type, slug, hook, description) — homepage card renders empty without - them; the May 10 incident that motivated this gate + - ANY essay in writings/ missing renderer-critical fields (public, type, + slug, hook, description) — regardless of exposure. The card renders empty + (or the essay silently never surfaces) without them. This is unconditional: + the old gate only checked exposure=public essays, which let nav essays slip + through with missing fields and then vanish from the homepage — the + recurring "merged but invisible" bug. What this does NOT catch (deferred — separate concerns): - Terminological drift, projection staleness, epoch gaps @@ -217,17 +220,48 @@ def validate_file(path: str) -> list[dict[str, Any]]: f"Set both consistently. Canon: {CONSTRAINT_REF}", )) - # 6. Essay-critical discovery fields (only for writings/ with exposure=public) + # 6. Renderer-critical fields for ALL essays in writings/ (regardless of + # exposure). The OLD gate only fired for exposure=public, which let an + # essay sit on exposure=nav with missing type/slug/public and pass + # clean — then silently never appear on the homepage. That conditional + # blind spot is the recurring "merged but invisible" bug. Every essay in + # writings/ needs these fields to render a card on ANY surface (the + # homepage feed at exposure=public, or the nav list at exposure=nav), so + # require them unconditionally. is_writing = path.startswith("writings/") or "/writings/" in path - if is_writing and fm.get("exposure") == "public": + if is_writing: + # 6a. `public` must be PRESENT and a real boolean. Its absence is the + # exact shape of the silent-drop bug: the essay merges, CI is + # green, and it never surfaces. Every published essay in the corpus + # already carries `public: true`; the only files that omit it are + # the ones that vanished. + if "public" not in fm or fm.get("public") is None: + findings.append(finding( + "frontmatter-missing-required", "error", path, "public", + f'Essay in writings/ is missing the "public" field. Every ' + f"essay must declare `public: true` (a real published essay) " + f"or `public: false` (draft/internal). Its absence is the " + f"silent-drop pattern — the essay merges but never surfaces. " + f"Canon: {CONSTRAINT_REF}", + )) + elif not isinstance(fm.get("public"), bool): + findings.append(finding( + "frontmatter-type-mismatch", "error", path, + f"public: {fm.get('public')!r}", + f'Field "public" must be an unquoted boolean (true/false); ' + f"got a {type(fm.get('public')).__name__}. Canon: {CANON_REF}", + )) + + # 6b. Discovery fields required for EVERY essay, not just public ones. for field in ESSAY_DISCOVERY_REQUIRED: v = fm.get(field) if v is None or v == "" or v == []: findings.append(finding( "frontmatter-missing-required", "error", path, field, - f'Public essay in writings/ is missing renderer-critical ' - f'field "{field}". Without it the homepage card renders ' - f"empty. Required for exposure=public writings: " + f'Essay in writings/ is missing renderer-critical field ' + f'"{field}". Without it the card renders empty on whatever ' + f"surface lists it (homepage feed or nav). Required for " + f"ALL writings essays: " f"{', '.join(ESSAY_DISCOVERY_REQUIRED)}. " f"Canon: {CONSTRAINT_REF}", )) @@ -239,7 +273,11 @@ def discover_targets(args_paths: list[str]) -> list[str]: """Resolve CLI args to a list of .md files to scan. README.md files are skipped as they are section indexes with a different shape from articles.""" def keep(p: Path) -> bool: - return p.suffix == ".md" and p.name != "README.md" + # Skip README indexes and underscore-prefixed files (templates, + # partials, scaffolds like writings/_TEMPLATE.md). + return (p.suffix == ".md" + and p.name != "README.md" + and not p.name.startswith("_")) if args_paths: out: list[str] = [] diff --git a/writings/_TEMPLATE.md b/writings/_TEMPLATE.md new file mode 100644 index 00000000..e62471d4 --- /dev/null +++ b/writings/_TEMPLATE.md @@ -0,0 +1,42 @@ +--- +# ─── writings/ essay template ────────────────────────────────────────────── +# Copy this file to writings/.md and fill it in. Every field below +# the divider is REQUIRED by the frontmatter validator +# (scripts/validate-frontmatter.py) for ALL essays in writings/, regardless of +# where they surface. Leaving any of them out fails CI — and, historically, +# silently dropped the essay from the homepage. +# +# Underscore-prefixed files (this one) are skipped by the validator. + +# ── Universal (every document) ── +uri: "klappy://writings/REPLACE-WITH-SLUG" +title: "REPLACE — the essay title" +audience: public # public | canon | docs | odd | operators | apocrypha +exposure: public # public = ON THE HOMEPAGE | nav = navigable, NOT promoted | draft | hidden | internal +tier: 2 # 1 foundational | 2 governance | 3 operational | 4 ephemeral +voice: first_person # first_person | neutral | direct | narrative | conversational | authoritative +stability: draft # stable | semi_stable | evolving | draft | experimental +tags: ["REPLACE", "tags"] + +# ── Renderer-critical for EVERY essay (missing = empty card / silent drop) ── +public: true # true = real published essay; false = draft/internal. MUST be an unquoted boolean. +type: "essay" # essay | article +slug: "REPLACE-WITH-SLUG" # must match the filename and the uri tail +hook: "REPLACE — one or two sentences that open with the reader's pain." +description: "REPLACE — 1-3 sentence summary used for the card and social preview." + +# ── Recommended ── +date: "2026-01-01" +epoch: "E0009" +og_description: "REPLACE — social/OG description (can mirror description)." + +# ── Optional: curated homepage reading path ── +# Set BOTH of these only if this essay belongs on the ordered 'start here' +# path on the homepage. Omit them otherwise — exposure: public alone already +# puts the essay in the homepage feed. +# start_here: true +# start_here_order: 99 +# ───────────────────────────────────────────────────────────────────────────── +--- + +Write the essay here. diff --git a/writings/great-minds-think-alike.md b/writings/great-minds-think-alike.md index 2a319090..24a38422 100644 --- a/writings/great-minds-think-alike.md +++ b/writings/great-minds-think-alike.md @@ -1,8 +1,11 @@ --- uri: klappy://writings/great-minds-think-alike title: "Great Minds Think Alike. Here's What That's Actually Telling You." +public: true +type: "essay" +slug: "great-minds-think-alike" audience: public -exposure: nav +exposure: public tier: 2 voice: first_person stability: draft diff --git a/writings/own-your-vertical.md b/writings/own-your-vertical.md index 1f8dfcc1..bfdf384d 100644 --- a/writings/own-your-vertical.md +++ b/writings/own-your-vertical.md @@ -1,8 +1,11 @@ --- uri: klappy://writings/own-your-vertical title: "Own Your Vertical. Let Me Carry the Layer Beneath It." +public: true +type: "essay" +slug: "own-your-vertical" audience: public -exposure: nav +exposure: public tier: 2 voice: first_person stability: draft diff --git a/writings/the-broken-wall-and-the-buried-talent.md b/writings/the-broken-wall-and-the-buried-talent.md index 28b0693f..29931232 100644 --- a/writings/the-broken-wall-and-the-buried-talent.md +++ b/writings/the-broken-wall-and-the-buried-talent.md @@ -1,6 +1,7 @@ --- uri: klappy://writings/the-broken-wall-and-the-buried-talent title: "The Broken Wall and the Buried Talent" +public: true subtitle: "Two ancient stories collided in a conversation about AI — and I haven't been the same since" author: "Klappy" type: article