perf(ci): parallelize the repo audits and guard env-dependent tests

The 21 independent audits ran as 21 sequential CI steps, each a single-threaded
read-only walk of the tree. scripts/run-audits.ts runs them concurrently:
28s serial -> 5.0s wall locally at 13-way. It buffers each audit's output and
replays only failures, so a green run stays quiet and a red one still names the
audit and shows why. Audits needing a git base ref (block registry, migration
safety) or that write files (drizzle generate) stay as their own steps.

Also fixes 5 tests that fail for every macOS dev and are invisible in CI. They
shell out to python3 using `match` statements and 3.12 f-string nesting, which
need >= 3.10; stock macOS ships 3.9.6, so `bun run test` produced raw Python
SyntaxErrors with no guard and nothing tying them to a missing tool. One also
needs ripgrep, which CI installs and a Mac usually does not.

@sim/testing/environment detects both and the tests skip with a reason via
vitest's ctx.skip(). Under CI it throws instead: these suites deliberately run
the real helper rather than a mock -- the cloud-review path/read-size bounds and
the placeholder compiler's generated Python are only observable that way -- so a
missing tool in CI means a security boundary silently stopped being covered,
which is worse than a red build.

Drops the Codecov upload. The workflow already documented it as a dead path:
nothing generates apps/sim/coverage, vitest runs without --coverage, and
fail_ci_if_error hides it, so it reported green having uploaded nothing.

Author: waleedlatif1

.github/workflows/test-build.yml

@@ -117,70 +117,22 @@ jobs:
       - name: Lint code
         run: bun run lint:check
 
-      - name: Enforce monorepo boundaries
-        run: bun run check:boundaries
-
-      - name: API contract boundary audit
-        run: bun run check:api-validation:strict
-
-      - name: Desktop bridge contract audit
-        run: bun run check:desktop-bridge
-
-      # Complements the bridge audit above, which compares against a snapshot
-      # this same PR is allowed to regenerate. This one derives every fact from
-      # the source both sides execute, so it has no such blind spot.
-      - name: Desktop IPC contract audit
-        run: bun run check:desktop-ipc
-
-      - name: Shared utils enforcement audit
-        run: bun run check:utils
-
-      - name: Zustand v5 selector audit
-        run: bun run check:zustand-v5
-
-      - name: React Query pattern audit
-        run: bun run check:react-query
-
-      - name: Client boundary import audit
-        run: bun run check:client-boundary
-
-      - name: Bare-icon theme-safety audit
-        run: bun run check:bare-icons
-
-      - name: Icon SVG path validity audit
-        run: bun run check:icon-paths
-
-      - name: Verify realtime prune graph
-        run: bun run check:realtime-prune
-
-      - name: Tool registry client-boundary audit
-        run: bun run check:tool-registry-boundary
-
-      - name: Tool request transport boundary audit
-        run: bun run check:tool-request-boundary
-
-      - name: Trigger/block initialization cycle audit
-        run: bun run check:trigger-block-cycle
-
-      # This job builds with webpack; devs run Turbopack. A specifier only webpack
-      # resolves passes here and breaks every dev server.
-      - name: Import specifier hygiene audit
-        run: bun run check:import-specifiers
-
-      - name: SQL Date binding audit
-        run: bun run check:sql-date-binding
-
-      - name: Verify generated tool metadata is in sync
-        run: bun run tool-metadata:check
-
-      - name: Verify integration deployment metadata is in sync
-        run: bun run integration-catalog:check
-
-      - name: Verify skill projections are in sync
-        run: bun run skills:check
-
-      - name: Verify agent stream capability docs are in sync
-        run: bun run agent-stream-docs:check
+      # These 21 audits are independent read-only passes over the tree, so running
+      # them as 21 sequential steps spent the whole time waiting on single-threaded
+      # file walks (~28s serial, ~5s at 13-way locally). scripts/run-audits.ts owns
+      # the list; it buffers each audit's output and replays only the failures, so a
+      # red run still names the audit and shows why. Audits that need a git base ref
+      # or write files stay as their own steps below.
+      #
+      # Notable members: the desktop IPC audit complements the bridge audit by
+      # deriving every fact from the source both sides execute, so it has no
+      # snapshot blind spot; the import-specifier audit catches specifiers that only
+      # webpack (which this job uses) resolves and that break every dev server on
+      # Turbopack; and the native-typecheck audit catches a bare `tsc` falling back
+      # to the ~10x slower JavaScript TypeScript 6 compiler, which otherwise still
+      # passes and just burns minutes.
+      - name: Repo audits
+        run: bun run check:audits
 
       - name: Migration safety (zero-downtime) audit
         run: |
@@ -192,12 +144,6 @@ jobs:
           fi
           bun run check:migrations "$BASE_REF"
 
-      # Runs before the type-check itself: if a bare `tsc` has fallen back to the
-      # JavaScript TypeScript 6 compiler that `@typescript/typescript6` drags in
-      # transitively, the type-check below still passes — it just takes ~10x longer.
-      - name: Native type-check audit
-        run: bun run check:native-typecheck
-
       # Every workspace, not just realtime. packages/emcn, packages/utils,
       # apps/desktop and apps/docs had no type check in CI at all; apps/sim's
       # source was covered only as a side effect of `next build` in the separate
@@ -236,21 +182,6 @@ jobs:
           fi
           echo "✅ Schema and migrations are in sync"
 
-      # DEAD PATH: nothing generates `apps/sim/coverage`. The test step runs
-      # `vitest run` without `--coverage`, and vitest.config.ts declares no
-      # coverage provider, so this uploads nothing and still reports success in
-      # ~1s (`fail_ci_if_error: false` hides it). `@vitest/coverage-v8` IS
-      # installed, so wiring it up is possible — but coverage instrumentation
-      # costs test time and nothing gates on the result today. Left in place
-      # pending a decision to either enable coverage or drop this step; do not
-      # read its green tick as "coverage was published".
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5
-        with:
-          directory: ./apps/sim/coverage
-          fail_ci_if_error: false
-          verbose: true
-
   # Next.js production build, in parallel with lint + tests. Sticky disks are
   # cloned from the last committed snapshot per job and committed last-writer-
   # wins, so concurrent mounts are safe. The bun/node_modules disks are shared

apps/sim/executor/handlers/pi/cloud-review-tools.test.ts

@@ -7,6 +7,12 @@ import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { promisify } from 'node:util'
 import * as sdk from '@earendil-works/pi-coding-agent'
+import {
+  hasPython3,
+  hasRipgrep,
+  PYTHON_SKIP_REASON,
+  RIPGREP_SKIP_REASON,
+} from '@sim/testing/environment'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 import type { PiSandboxRunner } from '@/lib/execution/remote-sandbox'
 import {
@@ -61,7 +67,9 @@ describe('cloud review tools', () => {
     expect(source).not.toContain('--unified=20')
   })
 
-  it('enforces read-size and canonical path bounds in the actual helper', async () => {
+  it('enforces read-size and canonical path bounds in the actual helper', async (ctx) => {
+    if (!hasPython3()) ctx.skip(PYTHON_SKIP_REASON)
+    if (!hasRipgrep()) ctx.skip(RIPGREP_SKIP_REASON)
     await installCloudReviewTools(runner)
     const source = writeFile.mock.calls[0][1] as string
     const testDir = await mkdtemp(join(tmpdir(), 'sim-review-tools-'))

apps/sim/lib/execution/code-placeholders/compiler.test.ts

@@ -2,6 +2,7 @@
  * @vitest-environment node
  */
 import { spawnSync } from 'node:child_process'
+import { hasPython3, PYTHON_SKIP_REASON } from '@sim/testing/environment'
 import { afterEach, describe, expect, it } from 'vitest'
 import {
   analyzeCodePlaceholders,
@@ -580,7 +581,8 @@ describe('code placeholder compiler', () => {
     )
   })
 
-  it('distinguishes Python lambda identifiers, match guards, and soft keywords from names', async () => {
+  it('distinguishes Python lambda identifiers, match guards, and soft keywords from names', async (ctx) => {
+    if (!hasPython3()) ctx.skip(PYTHON_SKIP_REASON)
     const compiled = await compileCodePlaceholders({
       code: [
         'lambda_value = "prefix-"',
@@ -688,7 +690,8 @@ describe('code placeholder compiler', () => {
     expect(executePython(compiled.code, compiled.bindings)).toBe('  7\n')
   })
 
-  it('supports quoted and embedded strings inside Python f-string expressions', async () => {
+  it('supports quoted and embedded strings inside Python f-string expressions', async (ctx) => {
+    if (!hasPython3()) ctx.skip(PYTHON_SKIP_REASON)
     const value = 'quote"\\\n{{OTHER}}'
     const compiled = await compileCodePlaceholders({
       code: [
@@ -734,7 +737,8 @@ describe('code placeholder compiler', () => {
     }
   })
 
-  it('ignores Python f-string comments and does not group adjacent strings across dedents', async () => {
+  it('ignores Python f-string comments and does not group adjacent strings across dedents', async (ctx) => {
+    if (!hasPython3()) ctx.skip(PYTHON_SKIP_REASON)
     const code = [
       'def build():',
       '    value = "{{KEY}}"',

apps/sim/lib/execution/remote-sandbox/fellows-digest.acceptance.test.ts

@@ -7,6 +7,7 @@ import { createHash } from 'node:crypto'
 import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join, resolve } from 'node:path'
+import { hasPython3, PYTHON_SKIP_REASON } from '@sim/testing/environment'
 import { afterEach, describe, expect, it, vi } from 'vitest'
 import { CodeLanguage } from '@/lib/execution/languages'
 import {
@@ -159,7 +160,8 @@ describe('Fellows Council Weekly sandbox acceptance fixture', () => {
     )
   })
 
-  it('stubs every external service and exports deterministic HTML and JSON in a ZIP', () => {
+  it('stubs every external service and exports deterministic HTML and JSON in a ZIP', (ctx) => {
+    if (!hasPython3()) ctx.skip(PYTHON_SKIP_REASON)
     const firstRoot = makeTempRoot()
     const secondRoot = makeTempRoot()
     const first = runFixture(firstRoot)

package.json

@@ -44,6 +44,7 @@
     "check:icon-paths": "bun run scripts/check-icon-paths.ts",
     "check:migrations": "bun run scripts/check-migrations-safety.ts",
     "check:native-typecheck": "bun run scripts/check-native-typecheck.ts",
+    "check:audits": "bun run scripts/run-audits.ts",
     "check:desktop-bridge": "bun run scripts/check-desktop-bridge-contract.ts --check",
     "check:desktop-ipc": "bun run scripts/check-desktop-ipc-contract.ts",
     "desktop-bridge-contract:update": "bun run scripts/check-desktop-bridge-contract.ts --update",

packages/testing/package.json

@@ -39,6 +39,10 @@
     "./setup": {
       "types": "./src/setup/vitest.setup.ts",
       "default": "./src/setup/vitest.setup.ts"
+    },
+    "./environment": {
+      "types": "./src/environment/index.ts",
+      "default": "./src/environment/index.ts"
     }
   },
   "scripts": {

packages/testing/src/environment/index.ts

@@ -0,0 +1,68 @@
+/**
+ * Detects external command-line tools that a handful of tests shell out to.
+ *
+ * A few suites deliberately execute the real thing rather than a mock — the cloud-review
+ * helper's path and read-size bounds, and the code-placeholder compiler's generated Python.
+ * That is the point of those tests, but it makes them depend on tools the repo does not
+ * vendor, and the failure mode is a raw `SyntaxError` or `ENOENT` from a subprocess with
+ * nothing tying it back to a missing tool.
+ *
+ * Locally these report `false` and print one actionable line, so the affected tests skip.
+ * Under `CI` they throw instead: a missing tool there means the gate silently stopped
+ * covering a security boundary, which is strictly worse than a red build.
+ */
+import { spawnSync } from 'node:child_process'
+
+/** Python 3.10 is the floor: the compiler suite generates `match` statements. */
+export const MIN_PYTHON: readonly [number, number] = [3, 10]
+
+/** Reasons to pass to vitest's `ctx.skip(...)` so the report says why, not just that. */
+export const PYTHON_SKIP_REASON = 'needs python3 >= 3.10; macOS ships 3.9'
+export const RIPGREP_SKIP_REASON = 'needs ripgrep (`rg`) on PATH'
+
+const warned = new Set<string>()
+
+function unavailable(tool: string, hint: string): false {
+  if (process.env.CI) {
+    throw new Error(
+      `${tool} is required to run this suite and was not found. CI must never skip these tests — they cover behavior that is only observable by running the real tool. ${hint}`
+    )
+  }
+  if (!warned.has(tool)) {
+    warned.add(tool)
+    console.warn(`[@sim/testing] Skipping tests that require ${tool}. ${hint}`)
+  }
+  return false
+}
+
+/** Parses `python3 --version`, returning null when the interpreter is missing or unreadable. */
+export function detectPython3(): { major: number; minor: number } | null {
+  const result = spawnSync('python3', ['--version'], { encoding: 'utf8' })
+  const match = /Python (\d+)\.(\d+)/.exec(`${result.stdout ?? ''}${result.stderr ?? ''}`)
+  if (!match) return null
+  return { major: Number(match[1]), minor: Number(match[2]) }
+}
+
+/**
+ * True when `python3` resolves to at least {@link MIN_PYTHON}.
+ *
+ * macOS ships 3.9 as the system `python3` and Homebrew's newer builds are not linked as
+ * `python3`, so this is false on a stock Mac even when a modern Python is installed.
+ */
+export function hasPython3(): boolean {
+  const version = detectPython3()
+  const [minMajor, minMinor] = MIN_PYTHON
+  const label = `python3 >= ${minMajor}.${minMinor}`
+  const hint = `Found ${version ? `${version.major}.${version.minor}` : 'no python3 on PATH'}. Install a newer Python (e.g. \`brew install python@3.13\`) and put it on PATH ahead of /usr/bin.`
+  if (!version) return unavailable(label, hint)
+  if (version.major > minMajor) return true
+  if (version.major === minMajor && version.minor >= minMinor) return true
+  return unavailable(label, hint)
+}
+
+/** True when `rg` is an executable on PATH. */
+export function hasRipgrep(): boolean {
+  const result = spawnSync('rg', ['--version'], { encoding: 'utf8' })
+  if (result.status === 0) return true
+  return unavailable('ripgrep (`rg`)', 'Install it with `brew install ripgrep`.')
+}

packages/testing/src/index.ts

@@ -43,6 +43,7 @@
 
 export * from './assertions'
 export * from './builders'
+export * from './environment'
 export * from './factories'
 export * from './mocks'
 export * from './types'

scripts/run-audits.ts

@@ -0,0 +1,96 @@
+#!/usr/bin/env bun
+/**
+ * Runs the independent repo audits concurrently.
+ *
+ * Each audit is a self-contained read-only pass over the tree, so running them as 20-odd
+ * sequential CI steps spent most of its wall clock waiting on single-threaded file walks.
+ * Only audits that need no extra arguments, working directory, or git base ref belong here —
+ * the ones that diff against a base ref (block registry, migration safety) or write files
+ * (drizzle generate) stay as their own steps.
+ *
+ * Output is buffered per audit and replayed only for failures, so a green run stays quiet
+ * and a red one still shows exactly which audit failed and why.
+ */
+const AUDITS = [
+  'check:boundaries',
+  'check:api-validation:strict',
+  'check:desktop-bridge',
+  'check:desktop-ipc',
+  'check:utils',
+  'check:zustand-v5',
+  'check:react-query',
+  'check:client-boundary',
+  'check:bare-icons',
+  'check:icon-paths',
+  'check:realtime-prune',
+  'check:tool-registry-boundary',
+  'check:tool-request-boundary',
+  'check:trigger-block-cycle',
+  'check:import-specifiers',
+  'check:sql-date-binding',
+  'check:native-typecheck',
+  'tool-metadata:check',
+  'integration-catalog:check',
+  'skills:check',
+  'agent-stream-docs:check',
+] as const
+
+interface AuditResult {
+  script: string
+  ok: boolean
+  durationMs: number
+  output: string
+}
+
+const CONCURRENCY = Math.max(2, navigator.hardwareConcurrency - 1)
+
+async function runAudit(script: string): Promise<AuditResult> {
+  const startedAt = performance.now()
+  const proc = Bun.spawn(['bun', 'run', script], {
+    stdout: 'pipe',
+    stderr: 'pipe',
+    env: { ...process.env, FORCE_COLOR: '0' },
+  })
+  const [stdout, stderr, exitCode] = await Promise.all([
+    new Response(proc.stdout).text(),
+    new Response(proc.stderr).text(),
+    proc.exited,
+  ])
+  return {
+    script,
+    ok: exitCode === 0,
+    durationMs: performance.now() - startedAt,
+    output: `${stdout}${stderr}`.trimEnd(),
+  }
+}
+
+const queue = [...AUDITS]
+const results: AuditResult[] = []
+
+async function worker(): Promise<void> {
+  for (let script = queue.shift(); script; script = queue.shift()) {
+    const result = await runAudit(script)
+    results.push(result)
+    console.log(`${result.ok ? '✓' : '✗'} ${result.script} (${Math.round(result.durationMs)}ms)`)
+  }
+}
+
+const startedAt = performance.now()
+await Promise.all(Array.from({ length: Math.min(CONCURRENCY, AUDITS.length) }, worker))
+const wallMs = performance.now() - startedAt
+
+const failures = results.filter((result) => !result.ok)
+const serialMs = results.reduce((total, result) => total + result.durationMs, 0)
+
+console.log(
+  `\n${results.length} audits in ${(wallMs / 1000).toFixed(1)}s wall (${(serialMs / 1000).toFixed(1)}s serial, ${CONCURRENCY}-way)`
+)
+
+if (failures.length > 0) {
+  for (const failure of failures) {
+    console.error(`\n${'─'.repeat(72)}\n✗ ${failure.script}\n${'─'.repeat(72)}`)
+    console.error(failure.output || '(no output)')
+  }
+  console.error(`\n${failures.length} audit(s) failed: ${failures.map((f) => f.script).join(', ')}`)
+  process.exit(1)
+}