Notify demo maintainers when their demo breaks on main

.github/workflows/ci.yml

@@ -16,6 +16,7 @@ permissions:
   contents: read
   pages: write
   id-token: write
+  issues: write # broken-demos rollup issue (bin/notify-broken-demos.mjs)
 
 jobs:
   build-job:
@@ -31,6 +32,9 @@ jobs:
     container:
       image: mcr.microsoft.com/playwright:v1.62.0-jammy
 
+    outputs:
+      e2e-outcome: ${{ steps.e2e.outcome }}
+
     steps:
       #
       # Setup
@@ -51,14 +55,32 @@ jobs:
       #
       - run: pnpm lint:metadata
       - run: pnpm lint:versions
+      - run: pnpm test:unit
 
       #
       # Test
       #
+      # PRs: fail fast — a broken demo blocks the PR
       - run: pnpm test
-        if: ${{ !inputs.update_snapshots }}
+        if: ${{ github.event_name == 'pull_request' && !inputs.update_snapshots }}
         env:
           BASE_PATH: ${{ steps.configurepages.outputs.base_path }}
+      # main (prod monitoring): play ALL tests; one broken demo must not hold
+      # the other demos hostage, so the job carries on (build+deploy still run)
+      # and the e2e-status-job turns the run red instead
+      - id: e2e
+        run: pnpm exec turbo test --continue=dependencies-successful --summarize
+        if: ${{ github.event_name != 'pull_request' && !inputs.update_snapshots }}
+        continue-on-error: true
+        env:
+          BASE_PATH: ${{ steps.configurepages.outputs.base_path }}
+      # continue-on-error: a GitHub API hiccup must not block the deploy either
+      - name: Notify maintainers of broken demos
+        run: node bin/notify-broken-demos.mjs
+        if: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/main' && !inputs.update_snapshots }}
+        continue-on-error: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       # Baselines regen: gh workflow run CI --ref <branch> -f update_snapshots=true
       # --force: a regen must never replay cached test results (their cached
       # outputs would restore the stale baselines we're regenerating)
@@ -85,6 +107,17 @@ jobs:
         with:
           path: ./out${{ steps.configurepages.outputs.base_path }}
 
+  # Mirrors the outcome of the (continue-on-error) e2e step: keeps the run red
+  # when a demo is broken on main, without holding back build+deploy
+  e2e-status-job:
+    if: ${{ github.event_name != 'pull_request' && !inputs.update_snapshots }}
+    runs-on: ubuntu-latest
+    needs: build-job
+    steps:
+      - run: |
+          echo "e2e outcome: ${{ needs.build-job.outputs.e2e-outcome }}"
+          [ "${{ needs.build-job.outputs.e2e-outcome }}" != "failure" ]
+
   deploy-job:
     # only for pushes on main
     if: ${{ github.event_name != 'pull_request' && !inputs.update_snapshots && github.ref == 'refs/heads/main' }}

AGENTS.md

@@ -10,6 +10,10 @@ Issues are tracked on GitHub (pmndrs/examples) via the `gh` CLI; external PRs ar
 
 Canonical vocabulary: `needs-triage`, `needs-info`, `ready-for-agent`, `ready-for-human`, `wontfix`. See `docs/agents/triage-labels.md`.
 
+### Broken demos & maintainers
+
+Each demo's `pmndrs.json` declares `maintainers` (GitHub logins). On main, CI plays all e2e tests and maintains one `broken-demos` issue per broken demo, @mentioning its maintainers on green→broken. See `docs/agents/broken-demos.md`.
+
 ### Domain docs
 
 Single-context: one `CONTEXT.md` + `docs/adr/` at the repo root (created lazily by `/domain-modeling`). See `docs/agents/domain.md`.

apps/website/lib/helper.ts

@@ -31,6 +31,7 @@ export type DemoMetadata = {
   description: string;
   tags: string[];
   authors: string[];
+  maintainers: string[];
   publishedAt?: string;
   source: string;
   libraries: string[];

bin/lib/broken-demos.mjs

@@ -0,0 +1,66 @@
+// Pure logic behind bin/notify-broken-demos.mjs: which demos a turbo run
+// summary says are broken, and how each broken demo maps to / renders into
+// its own GitHub issue. Kept IO-free so it can be tested with node --test.
+
+export const ISSUE_LABEL = "broken-demos";
+
+const DEMO_TASKS = new Set(["build2", "test"]);
+// Hidden marker tying an issue to its demo: sturdier than parsing titles.
+const DEMO_MARKER = /<!-- broken-demo:([a-z0-9-]+) -->/;
+
+export function brokenDemosFromSummary(summary, packageToDir) {
+  const broken = new Set();
+  for (const task of summary.tasks ?? []) {
+    const dir = packageToDir.get(task.package);
+    if (!dir || !DEMO_TASKS.has(task.task)) continue;
+    // A task skipped by --continue=dependencies-successful has no (or a null)
+    // exitCode: only an actually-executed, non-zero task marks the demo broken.
+    const exitCode = task.execution?.exitCode;
+    if (Number.isInteger(exitCode) && exitCode !== 0) broken.add(dir);
+  }
+  return [...broken].sort();
+}
+
+export function issueTitle(demo) {
+  return `🔴 \`${demo}\` is broken on main`;
+}
+
+export function demoFromIssueBody(body) {
+  const match = (body ?? "").match(DEMO_MARKER);
+  return match ? match[1] : null;
+}
+
+function mentions(logins) {
+  return logins.map((login) => `@${login}`).join(" ");
+}
+
+export function renderIssueBody({
+  demo,
+  maintainers,
+  runUrl,
+  sha,
+  siteBaseUrl,
+}) {
+  const who = maintainers.length
+    ? `Maintainers: ${mentions(maintainers)}`
+    : `_No maintainer_ — add your GitHub login to \`demos/${demo}/pmndrs.json\` → \`maintainers\` to get pinged about this demo.`;
+  return [
+    `<!-- broken-demo:${demo} -->`,
+    "",
+    `[\`${demo}\`](${siteBaseUrl}/demos/${demo})'s e2e test (or build) fails on \`main\` — see [the run](${runUrl}) (\`${sha.slice(0, 7)}\`).`,
+    "",
+    who,
+    "",
+    "Managed by `bin/notify-broken-demos.mjs`: the body tracks the latest",
+    "failing run, and the issue closes itself once the demo is green again.",
+  ].join("\n");
+}
+
+export function renderReopenComment({ demo, maintainers, runUrl }) {
+  const cc = maintainers.length ? ` cc ${mentions(maintainers)}` : "";
+  return `🔴 \`${demo}\` is broken again on \`main\` ([run](${runUrl})).${cc}`;
+}
+
+export function renderFixedComment({ demo, runUrl }) {
+  return `✅ \`${demo}\` is green again on \`main\` ([run](${runUrl})). Closing.`;
+}

bin/lib/broken-demos.test.mjs

@@ -0,0 +1,147 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+
+import {
+  brokenDemosFromSummary,
+  issueTitle,
+  demoFromIssueBody,
+  renderIssueBody,
+  renderReopenComment,
+  renderFixedComment,
+} from "./broken-demos.mjs";
+
+const packageToDir = new Map([
+  ["@demo/aquarium", "aquarium"],
+  ["@demo/baking-soft-shadows", "baking-soft-shadows"],
+  ["@demo/backdrop-and-cables", "backdrop-and-cables"],
+]);
+
+function summaryTask(pkg, task, exitCode) {
+  return {
+    taskId: `${pkg}#${task}`,
+    package: pkg,
+    task,
+    execution: exitCode === undefined ? undefined : { exitCode },
+  };
+}
+
+test("brokenDemosFromSummary flags demos whose test failed", () => {
+  const summary = {
+    tasks: [
+      summaryTask("@demo/aquarium", "build2", 0),
+      summaryTask("@demo/aquarium", "test", 1),
+      summaryTask("@demo/baking-soft-shadows", "build2", 0),
+      summaryTask("@demo/baking-soft-shadows", "test", 0),
+    ],
+  };
+  assert.deepEqual(brokenDemosFromSummary(summary, packageToDir), ["aquarium"]);
+});
+
+test("brokenDemosFromSummary flags demos whose build failed (test skipped)", () => {
+  const summary = {
+    tasks: [
+      summaryTask("@demo/aquarium", "build2", 1),
+      // test task skipped by --continue=dependencies-successful: whether turbo
+      // reports it without execution or with a null exitCode, it is NOT an
+      // extra failure (build2 already marks the demo broken)
+      summaryTask("@demo/aquarium", "test", undefined),
+      summaryTask("@demo/baking-soft-shadows", "build2", 0),
+      summaryTask("@demo/baking-soft-shadows", "test", null),
+    ],
+  };
+  assert.deepEqual(brokenDemosFromSummary(summary, packageToDir), ["aquarium"]);
+});
+
+test("brokenDemosFromSummary dedupes, sorts, and ignores non-demo packages", () => {
+  const summary = {
+    tasks: [
+      summaryTask("website", "build3", 1),
+      summaryTask("@demo/backdrop-and-cables", "test", 1),
+      summaryTask("@demo/backdrop-and-cables", "build2", 1),
+      summaryTask("@demo/aquarium", "test", 1),
+    ],
+  };
+  assert.deepEqual(brokenDemosFromSummary(summary, packageToDir), [
+    "aquarium",
+    "backdrop-and-cables",
+  ]);
+});
+
+test("demoFromIssueBody roundtrips through renderIssueBody", () => {
+  const body = renderIssueBody({
+    demo: "re-using-geometry-and-level-of-detail",
+    maintainers: ["abernier"],
+    runUrl: "https://github.com/pmndrs/examples/actions/runs/1",
+    sha: "abc1234def",
+    siteBaseUrl: "https://pmndrs.github.io/examples",
+  });
+  assert.equal(
+    demoFromIssueBody(body),
+    "re-using-geometry-and-level-of-detail",
+  );
+});
+
+test("demoFromIssueBody returns null without a marker", () => {
+  assert.equal(demoFromIssueBody("no marker here"), null);
+  assert.equal(demoFromIssueBody(undefined), null);
+});
+
+test("renderIssueBody links the demo and mentions its maintainers", () => {
+  const body = renderIssueBody({
+    demo: "aquarium",
+    maintainers: ["abernier", "drcmda"],
+    runUrl: "https://github.com/pmndrs/examples/actions/runs/1",
+    sha: "abc1234def",
+    siteBaseUrl: "https://pmndrs.github.io/examples",
+  });
+  assert.match(body, /https:\/\/pmndrs\.github\.io\/examples\/demos\/aquarium/);
+  assert.match(body, /@abernier @drcmda/);
+  assert.match(body, /actions\/runs\/1/);
+  assert.match(body, /abc1234/);
+});
+
+test("renderIssueBody without maintainers points at pmndrs.json instead", () => {
+  const body = renderIssueBody({
+    demo: "aquarium",
+    maintainers: [],
+    runUrl: "https://github.com/pmndrs/examples/actions/runs/1",
+    sha: "abc1234def",
+    siteBaseUrl: "https://pmndrs.github.io/examples",
+  });
+  assert.match(body, /No maintainer/);
+  assert.match(body, /demos\/aquarium\/pmndrs\.json/);
+  assert.doesNotMatch(body, /@\w/);
+});
+
+test("renderReopenComment mentions maintainers", () => {
+  const comment = renderReopenComment({
+    demo: "aquarium",
+    maintainers: ["abernier"],
+    runUrl: "https://github.com/pmndrs/examples/actions/runs/2",
+  });
+  assert.match(comment, /broken again/);
+  assert.match(comment, /cc @abernier/);
+});
+
+test("renderReopenComment without maintainers has no mentions", () => {
+  const comment = renderReopenComment({
+    demo: "aquarium",
+    maintainers: [],
+    runUrl: "https://github.com/pmndrs/examples/actions/runs/2",
+  });
+  assert.doesNotMatch(comment, /@\w/);
+});
+
+test("renderFixedComment has no mentions", () => {
+  const comment = renderFixedComment({
+    demo: "aquarium",
+    runUrl: "https://github.com/pmndrs/examples/actions/runs/2",
+  });
+  assert.match(comment, /green again/);
+  assert.doesNotMatch(comment, /@\w/);
+});
+
+test("issueTitle is stable per demo", () => {
+  assert.equal(issueTitle("aquarium"), issueTitle("aquarium"));
+  assert.match(issueTitle("aquarium"), /aquarium/);
+});