Move conformance off the CI hot path

.github/workflows/chart-library-benchmarks.yml

@@ -119,59 +119,6 @@ jobs:
           path: .benchmark-output/results
           if-no-files-found: error
 
-  conformance:
-    name: Conformance (${{ matrix.shard }}/8)
-    runs-on: ubuntu-24.04
-    timeout-minutes: 25
-    strategy:
-      fail-fast: false
-      matrix:
-        shard:
-          - 1
-          - 2
-          - 3
-          - 4
-          - 5
-          - 6
-          - 7
-          - 8
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-        with:
-          persist-credentials: false
-
-      - name: Setup
-        uses: ./.github/actions/setup
-        with:
-          playwright: 'true'
-
-      - name: Run pull-request conformance shard
-        if: github.event_name == 'pull_request'
-        run: pnpm conformance:quick -- --shard=${{ matrix.shard }}/8
-
-      - name: Run production conformance shard
-        if: github.event_name != 'pull_request'
-        run: pnpm conformance -- --shard=${{ matrix.shard }}/8
-
-      - name: Publish conformance summary
-        if: success()
-        run: |
-          summaries=(.benchmark-output/conformance/results/*.md)
-          test -e "${summaries[0]}"
-          cat "${summaries[@]}" >> "$GITHUB_STEP_SUMMARY"
-
-      - name: Upload conformance
-        if: always()
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
-        with:
-          name: chart-library-conformance-${{ matrix.shard }}-${{ github.run_id }}
-          path: |
-            .benchmark-output/conformance/results
-            .benchmark-output/conformance/screenshots
-          if-no-files-found: error
-
   stress:
     name: Stress (${{ matrix.name }})
     runs-on: ubuntu-24.04
@@ -233,7 +180,6 @@ jobs:
       - static
       - bundle-baseline
       - compare
-      - conformance
       - stress
     runs-on: ubuntu-24.04
     timeout-minutes: 5
@@ -244,14 +190,12 @@ jobs:
           STATIC_RESULT: ${{ needs.static.result }}
           BUNDLE_RESULT: ${{ needs.bundle-baseline.result }}
           COMPARISON_RESULT: ${{ needs.compare.result }}
-          CONFORMANCE_RESULT: ${{ needs.conformance.result }}
           STRESS_RESULT: ${{ needs.stress.result }}
         run: |
           for result in \
             "$STATIC_RESULT" \
             "$BUNDLE_RESULT" \
             "$COMPARISON_RESULT" \
-            "$CONFORMANCE_RESULT" \
             "$STRESS_RESULT"
           do
             test "$result" = success

.github/workflows/conformance.yml

@@ -0,0 +1,124 @@
+name: Conformance monitoring
+
+on:
+  pull_request:
+    types:
+      - opened
+      - labeled
+      - synchronize
+      - reopened
+      - ready_for_review
+  schedule:
+    - cron: '43 7 * * *'
+    - cron: '13 9 * * 1'
+  workflow_dispatch:
+    inputs:
+      shard:
+        description: Standard conformance scope
+        required: true
+        type: choice
+        default: full
+        options:
+          - full
+          - '1'
+          - '2'
+          - '3'
+          - '4'
+          - '5'
+          - '6'
+          - '7'
+          - '8'
+
+permissions:
+  contents: read
+
+jobs:
+  select:
+    name: Select conformance shards
+    if: >-
+      github.event_name == 'schedule' ||
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'pull_request' &&
+       contains(github.event.pull_request.labels.*.name, 'full-conformance') &&
+       (github.event.action != 'labeled' || github.event.label.name == 'full-conformance'))
+    runs-on: ubuntu-24.04
+    timeout-minutes: 2
+    outputs:
+      mode: ${{ steps.selection.outputs.mode }}
+      shards: ${{ steps.selection.outputs.shards }}
+
+    steps:
+      - id: selection
+        name: Resolve scope
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          EVENT_SCHEDULE: ${{ github.event.schedule }}
+          REQUESTED_SHARD: ${{ inputs.shard }}
+        run: |
+          if [ "$EVENT_NAME" = schedule ] && [ "$EVENT_SCHEDULE" = '43 7 * * *' ]; then
+            epoch_day=$(( $(date -u +%s) / 86400 ))
+            shard=$(( epoch_day % 8 + 1 ))
+            mode=nightly
+            shards="[$shard]"
+          elif [ "$EVENT_NAME" = workflow_dispatch ] && [ "$REQUESTED_SHARD" != full ]; then
+            mode=manual
+            shards="[$REQUESTED_SHARD]"
+          elif [ "$EVENT_NAME" = schedule ]; then
+            mode=weekly
+            shards='[1,2,3,4,5,6,7,8]'
+          elif [ "$EVENT_NAME" = pull_request ]; then
+            mode=label
+            shards='[1,2,3,4,5,6,7,8]'
+          else
+            mode=manual
+            shards='[1,2,3,4,5,6,7,8]'
+          fi
+
+          echo "mode=$mode" >> "$GITHUB_OUTPUT"
+          echo "shards=$shards" >> "$GITHUB_OUTPUT"
+          echo "Mode: $mode; shards: $shards; revision: $GITHUB_SHA" >> "$GITHUB_STEP_SUMMARY"
+
+  conformance:
+    name: Conformance (${{ matrix.shard }}/8, ${{ needs.select.outputs.mode }})
+    needs: select
+    runs-on: ubuntu-24.04
+    timeout-minutes: 25
+    concurrency:
+      group: charts-conformance-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }}-${{ matrix.shard }}
+      cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: ${{ fromJSON(needs.select.outputs.shards) }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+
+      - name: Setup
+        uses: ./.github/actions/setup
+        with:
+          playwright: 'true'
+
+      - name: Run standard conformance shard
+        run: pnpm conformance -- --shard=${{ matrix.shard }}/8
+
+      - name: Publish conformance summary
+        if: success()
+        run: |
+          summaries=(.benchmark-output/conformance/results/*.md)
+          test -e "${summaries[0]}"
+          cat "${summaries[@]}" >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload conformance
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
+        with:
+          name: chart-library-conformance-${{ needs.select.outputs.mode }}-${{ matrix.shard }}-${{ github.run_id }}
+          path: |
+            .benchmark-output/conformance/results
+            .benchmark-output/conformance/screenshots
+          if-no-files-found: error
+          retention-days: 14

API-FRICTION.md

@@ -190,6 +190,7 @@ Each entry records:
 | F-152 | Version bumps invalidated workspace bundle evidence      | Tooling/Release | resolved   |
 | F-153 | Changesets left release-facing version claims behind     | Tooling/Release | resolved   |
 | F-154 | Root barrels crossed the browser host boundary           | API/Tooling     | resolved   |
+| F-155 | Conformance monitoring blocked unrelated changes         | Tooling         | resolved   |
 
 ## Findings
 
@@ -2816,13 +2817,15 @@ Each entry records:
   headers, cache policy, and content delivery behavior.
 - Decision: treat the catalog as generated structured content. Charts CI builds
   schema-v4 `catalog.json` plus only the recursively allowlisted implementation
-  modules, then replaces the generated `catalog-dist` branch after validation
-  and the unfiltered conformance matrix. TanStack.com's existing content
-  pipeline reads that branch, verifies hashes and limits, renders native routes
-  and embeds, and serves modules below an artifact-commit namespace. Charts
-  source and dependencies remain out of the site repository and default site
-  bundle. The previous Worker, staging tree, deployment scripts, credentials,
-  and route ownership are removed from the Charts workflow.
+  modules, then replaces the generated `catalog-dist` branch after the static,
+  package, bundle, comparison, and stress gates pass. Conformance is independent
+  regression monitoring rather than an artifact-integrity gate. TanStack.com's
+  existing content pipeline reads that branch, verifies hashes and limits,
+  renders native routes and embeds, and serves modules below an artifact-commit
+  namespace. Charts source and dependencies remain out of the site repository
+  and default site bundle. The previous Worker, staging tree, deployment
+  scripts, credentials, and route ownership are removed from the Charts
+  workflow.
 - Verification: the artifact generator records an exact Charts revision,
   deterministic SHA-256 allowlist, safe repository source paths, recursive
   imports, debug-only comparison roots, and role-aware authored-source
@@ -3681,3 +3684,28 @@ Each entry records:
   export, reconciliation, renderer, and SVG surface modules. That full
   portable barrel measures 55.26 kB minified and 17.04 kB gzip; granular
   subpaths remain the bundle-sensitive option.
+
+### F-155 — Conformance monitoring blocked unrelated changes
+
+- Status: resolved
+- Severity: high
+- Owner: Tooling
+- Observed in: the first pull-request and main runs after release automation
+  was simplified
+- Friction: every pull request ran all 100 paired cases in the quick profile,
+  then every main commit reran all 100 in the standard profile. Release-only PR
+  `#17` therefore consumed 15.9 conformance runner-minutes before merge and
+  21.6 after merge despite changing no chart source. The eight shards also
+  repeated the complete TypeScript program and 27 type-protection probes. npm
+  publication did not wait on either run, so the cost added no release gate.
+- Decision: move conformance into a read-only monitoring workflow. Run one
+  deterministic standard shard nightly, all eight standard shards weekly, all
+  eight on manual request by default, and all eight for a pull request carrying
+  the `full-conformance` label. Manual dispatch may select one exact shard for
+  reproduction. Normal pull-request, main, catalog, and release paths do not
+  wait on conformance.
+- Verification: workflow contracts require deterministic eight-day rotation,
+  complete weekly and labeled-PR matrices, exact manual shard selection,
+  read-only permissions, immutable action pins, and standard-profile browser
+  execution. The main CI contract rejects any conformance dependency while
+  retaining every exact-revision catalog publication guard.

CONTRIBUTING.md

@@ -36,9 +36,15 @@ Pull-request CI runs the cached Nx target graph, the locked comparison bundle
 gate, and these uncached browser matrices in parallel:
 
 - four chart-library comparison shards;
-- eight conformance shards;
 - four stress-workload shards.
 
+Conformance runs as regression monitoring outside normal validation. One
+deterministically rotated standard shard runs nightly, all eight standard
+shards run weekly, and a manual run can select all shards or reproduce one
+exact shard. Add the `full-conformance` label to a risky pull request to run
+the complete standard matrix against that pull request; later commits rerun
+it while the label remains.
+
 Browser measurements and the revision-stamped catalog artifact are not cached
 because their results depend on the browser environment or exact Git commit.
 Install Chromium before running a browser suite locally:
@@ -87,9 +93,10 @@ Every push to `main` starts the release workflow:
    attestations, the workflow creates one annotated `vX.Y.Z` tag and GitHub
    release from the root changelog.
 
-The complete chart, browser, and catalog workflow still runs on `main`, but it
-is independent from npm publication. User-visible package work must pass that
-workflow in its feature pull request before merging.
+The chart comparison, stress, and catalog workflow still runs on `main`, but
+it is independent from npm publication. Scheduled conformance monitoring is
+also independent from release and catalog publication. User-visible package
+work must pass normal validation in its feature pull request before merging.
 
 Never move or reuse a release tag. If publishing succeeds but tag or GitHub
 release creation fails, rerun the failed `Release` workflow; its registry

README.md

@@ -315,10 +315,11 @@ pnpm conformance:quick
 These measurements are development evidence, not release claims. See each
 suite's README for its protocol, output, and limitations.
 
-Pull requests first gate formatting, types, tests, packed exports and
-declarations, locked bundles, comparison bundles, and catalog metadata.
-Browser comparison, all 100 conformance cases, and the quick five-library stress
-matrix then run as separate artifact-producing jobs.
+Pull requests gate formatting, types, tests, packed exports and declarations,
+locked bundles, comparison bundles, catalog metadata, browser comparison, and
+the quick five-library stress matrix. Conformance is monitored separately with
+a rotating nightly shard, a complete weekly matrix, manual runs, and an
+opt-in `full-conformance` pull-request label.
 
 ## License
 

benchmarks/conformance/README.md

@@ -187,13 +187,15 @@ pnpm catalog:build
 pnpm catalog:loading:check
 ```
 
-Main-branch CI publishes a new generated commit only after validation and the
-unfiltered standard conformance matrix pass. TanStack.com's existing content
-pipeline reads that branch and verifies the schema, revision, module allowlist,
-sizes, and hashes before serving it. It composes `site.assetBasePath`, the
-resolved `catalog-dist` commit SHA, and each relative module path into the
-immutable asset URL. A rollback points `catalog-dist` back to a prior generated
-commit; the catalog has no mutable runtime state.
+Main-branch CI publishes a new generated commit only after the static,
+package, bundle, comparison, and stress gates pass. Conformance runs
+independently as nightly rotating, weekly complete, manual, and labeled-PR
+monitoring. TanStack.com's existing content pipeline reads the generated branch
+and verifies the schema, revision, module allowlist, sizes, and hashes before
+serving it. It composes `site.assetBasePath`, the resolved `catalog-dist` commit
+SHA, and each relative module path into the immutable asset URL. A rollback
+points `catalog-dist` back to a prior generated commit; the catalog has no
+mutable runtime state.
 
 ## What is and is not equivalent