From 3ad2a6095166e0947b9d0fee706eba6f8d87cfba Mon Sep 17 00:00:00 2001 From: krandder Date: Tue, 30 Jun 2026 12:38:17 +0100 Subject: [PATCH] Add auto-qa CI tiers --- .github/workflows/auto-qa-fork.yml | 80 + .github/workflows/auto-qa-interactions.yml | 85 + .github/workflows/auto-qa.yml | 15 +- auto-qa/README.md | 76 +- auto-qa/harness/.gitignore | 29 + auto-qa/harness/ARCHITECTURE.md | 157 ++ auto-qa/harness/CAPABILITIES.md | 360 +++ auto-qa/harness/README.md | 82 + .../docs/ADR-001-synpress-vs-custom-stub.md | 144 ++ .../harness/docs/ADR-002-scenario-format.md | 168 ++ .../docs/spike-002-eth-subscribe-shim.md | 250 ++ auto-qa/harness/fixtures/a11y-heuristics.mjs | 190 ++ auto-qa/harness/fixtures/api-mocks.mjs | 1063 ++++++++ auto-qa/harness/fixtures/cascading-css.mjs | 168 ++ .../harness/fixtures/eth-call-inspector.mjs | 266 ++ auto-qa/harness/fixtures/fork-state-setup.mjs | 250 ++ auto-qa/harness/fixtures/fork-state.mjs | 1125 ++++++++ auto-qa/harness/fixtures/keyboard-nav.mjs | 213 ++ .../fixtures/page-error-exclusions.mjs | 97 + auto-qa/harness/fixtures/scenario-tiers.mjs | 64 + auto-qa/harness/fixtures/text-selection.mjs | 115 + auto-qa/harness/fixtures/wallet-stub.mjs | 682 +++++ auto-qa/harness/flows/INVARIANTS.md | 82 + auto-qa/harness/flows/app-discovery.spec.mjs | 116 + .../harness/flows/dom-api-invariant.spec.mjs | 2286 +++++++++++++++++ auto-qa/harness/flows/scenarios.spec.mjs | 356 +++ .../harness/flows/wallet-injection.spec.mjs | 222 ++ auto-qa/harness/flows/wallet-signing.spec.mjs | 249 ++ auto-qa/harness/package-lock.json | 380 +++ auto-qa/harness/package.json | 41 + auto-qa/harness/playwright.config.mjs | 156 ++ auto-qa/harness/playwright.prod.config.mjs | 120 + .../01-stale-price-shape.scenario.mjs | 75 + .../scenarios/02-registry-down.scenario.mjs | 65 + .../scenarios/03-candles-down.scenario.mjs | 89 + .../scenarios/04-candles-partial.scenario.mjs | 128 + .../05-registry-empty-orgs.scenario.mjs | 94 + .../06-both-endpoints-down.scenario.mjs | 82 + .../07-registry-malformed-body.scenario.mjs | 90 + .../08-candles-malformed-body.scenario.mjs | 104 + .../09-registry-corrupt-org.scenario.mjs | 133 + .../10-market-page-happy.scenario.mjs | 137 + .../11-market-page-trading.scenario.mjs | 159 ++ .../12-market-page-allowances.scenario.mjs | 130 + .../13-market-page-charts.scenario.mjs | 122 + .../14-market-page-positions.scenario.mjs | 164 ++ ...15-market-page-balance-update.scenario.mjs | 164 ++ ...-market-page-isolation-canary.scenario.mjs | 103 + ...7-market-page-position-update.scenario.mjs | 151 ++ ...arket-page-isolation-canary-2.scenario.mjs | 90 + .../scenarios/19-registry-slow.scenario.mjs | 106 + .../scenarios/20-candles-slow.scenario.mjs | 130 + .../scenarios/21-candles-empty.scenario.mjs | 108 + .../22-registry-partial.scenario.mjs | 165 ++ .../23-candles-corrupt-pool.scenario.mjs | 202 ++ .../24-market-page-registry-down.scenario.mjs | 118 + .../25-market-page-candles-down.scenario.mjs | 113 + ...26-market-page-registry-empty.scenario.mjs | 130 + .../27-market-page-candles-empty.scenario.mjs | 150 ++ ...arket-page-registry-malformed.scenario.mjs | 122 + ...market-page-candles-malformed.scenario.mjs | 109 + .../30-market-page-registry-slow.scenario.mjs | 121 + .../31-market-page-candles-slow.scenario.mjs | 128 + ...-market-page-registry-partial.scenario.mjs | 144 ++ ...3-market-page-candles-partial.scenario.mjs | 142 + ...ket-page-registry-corrupt-row.scenario.mjs | 155 ++ ...ket-page-candles-corrupt-pool.scenario.mjs | 144 ++ .../36-registry-rate-limited.scenario.mjs | 109 + .../37-candles-rate-limited.scenario.mjs | 107 + ...et-page-registry-rate-limited.scenario.mjs | 103 + ...ket-page-candles-rate-limited.scenario.mjs | 111 + ...-registry-504-gateway-timeout.scenario.mjs | 153 ++ ...1-candles-504-gateway-timeout.scenario.mjs | 160 ++ ...-registry-504-gateway-timeout.scenario.mjs | 135 + ...e-candles-504-gateway-timeout.scenario.mjs | 151 ++ .../scenarios/44-pr64-real.scenario.mjs | 235 ++ .../45-pr64-prefixed-shape.scenario.mjs | 199 ++ .../46-pr51-liquidity-magnitude.scenario.mjs | 289 +++ ...-checkpoint-schema-strictness.scenario.mjs | 117 + .../48-no-page-errors-companies.scenario.mjs | 132 + .../49-pr52-url-hash-rewrite.scenario.mjs | 126 + .../50-network-shape-companies.scenario.mjs | 167 ++ .../51-pr59-text-selection.scenario.mjs | 107 + .../52-a11y-heuristics-companies.scenario.mjs | 135 + ...-checkpoint-schema-strictness.scenario.mjs | 161 ++ ...pr55-market-singular-redirect.scenario.mjs | 133 + ...r46-filter-archived-proposals.scenario.mjs | 132 + .../56-pr61-filter-archived-orgs.scenario.mjs | 156 ++ ...8-no-supabase-snapshot-lookup.scenario.mjs | 135 + .../58-time-evolution-foundation.scenario.mjs | 140 + .../59-pr54-twap-ended-window.scenario.mjs | 271 ++ ...prediction-market-badge-gated.scenario.mjs | 156 ++ ...pr49-no-supabase-pool-fetcher.scenario.mjs | 159 ++ ...r59-text-selection-pagelayout.scenario.mjs | 150 ++ ...lestones-legacy-id-to-address.scenario.mjs | 173 ++ .../64-pr53-quoter-gas-cap.scenario.mjs | 300 +++ .../65-pr57-hide-max-approval.scenario.mjs | 322 +++ .../66-network-shape-market-page.scenario.mjs | 152 ++ ...7-a11y-heuristics-market-page.scenario.mjs | 142 + ...68-text-selection-market-page.scenario.mjs | 120 + .../69-no-page-errors-milestones.scenario.mjs | 142 + ...70-a11y-heuristics-milestones.scenario.mjs | 137 + .../71-network-shape-milestones.scenario.mjs | 132 + .../72-text-selection-milestones.scenario.mjs | 115 + .../73-keyboard-nav-companies.scenario.mjs | 130 + .../74-keyboard-nav-market-page.scenario.mjs | 130 + .../75-keyboard-nav-milestones.scenario.mjs | 105 + ...6-modal-focus-trap-rainbowkit.scenario.mjs | 148 ++ ...modal-focus-trap-confirm-swap.scenario.mjs | 300 +++ .../78-aria-state-outcome-tabs.scenario.mjs | 192 ++ ...-aria-current-active-nav-link.scenario.mjs | 189 ++ ...nter-events-cascade-companies.scenario.mjs | 145 ++ .../81-cursor-cascade-companies.scenario.mjs | 155 ++ ...t-transform-cascade-companies.scenario.mjs | 146 ++ ...er-events-cascade-market-page.scenario.mjs | 154 ++ ...ter-events-cascade-milestones.scenario.mjs | 167 ++ ...85-cursor-cascade-market-page.scenario.mjs | 137 + .../86-cursor-cascade-milestones.scenario.mjs | 156 ++ ...rection-rtl-cascade-companies.scenario.mjs | 164 ++ ...ent-card-href-shape-companies.scenario.mjs | 187 ++ ...lity-hidden-cascade-companies.scenario.mjs | 157 ++ ...er-back-nav-anchor-milestones.scenario.mjs | 187 ++ ...r-back-nav-anchor-market-page.scenario.mjs | 145 ++ ...pacity-zero-cascade-companies.scenario.mjs | 184 ++ ...transform-cascade-market-page.scenario.mjs | 147 ++ ...-transform-cascade-milestones.scenario.mjs | 158 ++ ...er-select-cascade-market-page.scenario.mjs | 174 ++ ...ser-select-cascade-milestones.scenario.mjs | 168 ++ auto-qa/harness/scenarios/README.md | 83 + auto-qa/harness/scenarios/SCENARIOS.md | 112 + auto-qa/harness/scripts/contracts.mjs | 274 ++ .../harness/scripts/debug-balance-quirk.mjs | 185 ++ .../harness/scripts/invariants-catalog.mjs | 209 ++ .../harness/scripts/scenarios-by-route.mjs | 75 + auto-qa/harness/scripts/scenarios-by-tier.mjs | 56 + auto-qa/harness/scripts/scenarios-catalog.mjs | 89 + .../scripts/scenarios-chaos-matrix.mjs | 240 ++ .../harness/tests/architecture-sync.test.mjs | 56 + auto-qa/harness/tests/contract-calls.test.mjs | 211 ++ .../harness/tests/eth-call-inspector.test.mjs | 177 ++ auto-qa/harness/tests/fork-state.test.mjs | 1013 ++++++++ .../harness/tests/invariants-catalog.test.mjs | 96 + .../harness/tests/live-time-control.test.mjs | 174 ++ .../tests/market-page-fixture.test.mjs | 680 +++++ .../harness/tests/scenarios-by-route.test.mjs | 35 + .../harness/tests/scenarios-by-tier.test.mjs | 28 + .../harness/tests/scenarios-catalog.test.mjs | 75 + .../tests/scenarios-chaos-matrix.test.mjs | 72 + auto-qa/harness/tests/wallet-stub.test.mjs | 354 +++ package.json | 7 +- 150 files changed, 28128 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/auto-qa-fork.yml create mode 100644 .github/workflows/auto-qa-interactions.yml create mode 100644 auto-qa/harness/.gitignore create mode 100644 auto-qa/harness/ARCHITECTURE.md create mode 100644 auto-qa/harness/CAPABILITIES.md create mode 100644 auto-qa/harness/README.md create mode 100644 auto-qa/harness/docs/ADR-001-synpress-vs-custom-stub.md create mode 100644 auto-qa/harness/docs/ADR-002-scenario-format.md create mode 100644 auto-qa/harness/docs/spike-002-eth-subscribe-shim.md create mode 100644 auto-qa/harness/fixtures/a11y-heuristics.mjs create mode 100644 auto-qa/harness/fixtures/api-mocks.mjs create mode 100644 auto-qa/harness/fixtures/cascading-css.mjs create mode 100644 auto-qa/harness/fixtures/eth-call-inspector.mjs create mode 100644 auto-qa/harness/fixtures/fork-state-setup.mjs create mode 100644 auto-qa/harness/fixtures/fork-state.mjs create mode 100644 auto-qa/harness/fixtures/keyboard-nav.mjs create mode 100644 auto-qa/harness/fixtures/page-error-exclusions.mjs create mode 100644 auto-qa/harness/fixtures/scenario-tiers.mjs create mode 100644 auto-qa/harness/fixtures/text-selection.mjs create mode 100644 auto-qa/harness/fixtures/wallet-stub.mjs create mode 100644 auto-qa/harness/flows/INVARIANTS.md create mode 100644 auto-qa/harness/flows/app-discovery.spec.mjs create mode 100644 auto-qa/harness/flows/dom-api-invariant.spec.mjs create mode 100644 auto-qa/harness/flows/scenarios.spec.mjs create mode 100644 auto-qa/harness/flows/wallet-injection.spec.mjs create mode 100644 auto-qa/harness/flows/wallet-signing.spec.mjs create mode 100644 auto-qa/harness/package-lock.json create mode 100644 auto-qa/harness/package.json create mode 100644 auto-qa/harness/playwright.config.mjs create mode 100644 auto-qa/harness/playwright.prod.config.mjs create mode 100644 auto-qa/harness/scenarios/01-stale-price-shape.scenario.mjs create mode 100644 auto-qa/harness/scenarios/02-registry-down.scenario.mjs create mode 100644 auto-qa/harness/scenarios/03-candles-down.scenario.mjs create mode 100644 auto-qa/harness/scenarios/04-candles-partial.scenario.mjs create mode 100644 auto-qa/harness/scenarios/05-registry-empty-orgs.scenario.mjs create mode 100644 auto-qa/harness/scenarios/06-both-endpoints-down.scenario.mjs create mode 100644 auto-qa/harness/scenarios/07-registry-malformed-body.scenario.mjs create mode 100644 auto-qa/harness/scenarios/08-candles-malformed-body.scenario.mjs create mode 100644 auto-qa/harness/scenarios/09-registry-corrupt-org.scenario.mjs create mode 100644 auto-qa/harness/scenarios/10-market-page-happy.scenario.mjs create mode 100644 auto-qa/harness/scenarios/11-market-page-trading.scenario.mjs create mode 100644 auto-qa/harness/scenarios/12-market-page-allowances.scenario.mjs create mode 100644 auto-qa/harness/scenarios/13-market-page-charts.scenario.mjs create mode 100644 auto-qa/harness/scenarios/14-market-page-positions.scenario.mjs create mode 100644 auto-qa/harness/scenarios/15-market-page-balance-update.scenario.mjs create mode 100644 auto-qa/harness/scenarios/16-market-page-isolation-canary.scenario.mjs create mode 100644 auto-qa/harness/scenarios/17-market-page-position-update.scenario.mjs create mode 100644 auto-qa/harness/scenarios/18-market-page-isolation-canary-2.scenario.mjs create mode 100644 auto-qa/harness/scenarios/19-registry-slow.scenario.mjs create mode 100644 auto-qa/harness/scenarios/20-candles-slow.scenario.mjs create mode 100644 auto-qa/harness/scenarios/21-candles-empty.scenario.mjs create mode 100644 auto-qa/harness/scenarios/22-registry-partial.scenario.mjs create mode 100644 auto-qa/harness/scenarios/23-candles-corrupt-pool.scenario.mjs create mode 100644 auto-qa/harness/scenarios/24-market-page-registry-down.scenario.mjs create mode 100644 auto-qa/harness/scenarios/25-market-page-candles-down.scenario.mjs create mode 100644 auto-qa/harness/scenarios/26-market-page-registry-empty.scenario.mjs create mode 100644 auto-qa/harness/scenarios/27-market-page-candles-empty.scenario.mjs create mode 100644 auto-qa/harness/scenarios/28-market-page-registry-malformed.scenario.mjs create mode 100644 auto-qa/harness/scenarios/29-market-page-candles-malformed.scenario.mjs create mode 100644 auto-qa/harness/scenarios/30-market-page-registry-slow.scenario.mjs create mode 100644 auto-qa/harness/scenarios/31-market-page-candles-slow.scenario.mjs create mode 100644 auto-qa/harness/scenarios/32-market-page-registry-partial.scenario.mjs create mode 100644 auto-qa/harness/scenarios/33-market-page-candles-partial.scenario.mjs create mode 100644 auto-qa/harness/scenarios/34-market-page-registry-corrupt-row.scenario.mjs create mode 100644 auto-qa/harness/scenarios/35-market-page-candles-corrupt-pool.scenario.mjs create mode 100644 auto-qa/harness/scenarios/36-registry-rate-limited.scenario.mjs create mode 100644 auto-qa/harness/scenarios/37-candles-rate-limited.scenario.mjs create mode 100644 auto-qa/harness/scenarios/38-market-page-registry-rate-limited.scenario.mjs create mode 100644 auto-qa/harness/scenarios/39-market-page-candles-rate-limited.scenario.mjs create mode 100644 auto-qa/harness/scenarios/40-registry-504-gateway-timeout.scenario.mjs create mode 100644 auto-qa/harness/scenarios/41-candles-504-gateway-timeout.scenario.mjs create mode 100644 auto-qa/harness/scenarios/42-market-page-registry-504-gateway-timeout.scenario.mjs create mode 100644 auto-qa/harness/scenarios/43-market-page-candles-504-gateway-timeout.scenario.mjs create mode 100644 auto-qa/harness/scenarios/44-pr64-real.scenario.mjs create mode 100644 auto-qa/harness/scenarios/45-pr64-prefixed-shape.scenario.mjs create mode 100644 auto-qa/harness/scenarios/46-pr51-liquidity-magnitude.scenario.mjs create mode 100644 auto-qa/harness/scenarios/47-checkpoint-schema-strictness.scenario.mjs create mode 100644 auto-qa/harness/scenarios/48-no-page-errors-companies.scenario.mjs create mode 100644 auto-qa/harness/scenarios/49-pr52-url-hash-rewrite.scenario.mjs create mode 100644 auto-qa/harness/scenarios/50-network-shape-companies.scenario.mjs create mode 100644 auto-qa/harness/scenarios/51-pr59-text-selection.scenario.mjs create mode 100644 auto-qa/harness/scenarios/52-a11y-heuristics-companies.scenario.mjs create mode 100644 auto-qa/harness/scenarios/53-market-page-checkpoint-schema-strictness.scenario.mjs create mode 100644 auto-qa/harness/scenarios/54-pr55-market-singular-redirect.scenario.mjs create mode 100644 auto-qa/harness/scenarios/55-pr46-filter-archived-proposals.scenario.mjs create mode 100644 auto-qa/harness/scenarios/56-pr61-filter-archived-orgs.scenario.mjs create mode 100644 auto-qa/harness/scenarios/57-pr48-no-supabase-snapshot-lookup.scenario.mjs create mode 100644 auto-qa/harness/scenarios/58-time-evolution-foundation.scenario.mjs create mode 100644 auto-qa/harness/scenarios/59-pr54-twap-ended-window.scenario.mjs create mode 100644 auto-qa/harness/scenarios/60-pr56-prediction-market-badge-gated.scenario.mjs create mode 100644 auto-qa/harness/scenarios/61-pr49-no-supabase-pool-fetcher.scenario.mjs create mode 100644 auto-qa/harness/scenarios/62-pr59-text-selection-pagelayout.scenario.mjs create mode 100644 auto-qa/harness/scenarios/63-pr50-milestones-legacy-id-to-address.scenario.mjs create mode 100644 auto-qa/harness/scenarios/64-pr53-quoter-gas-cap.scenario.mjs create mode 100644 auto-qa/harness/scenarios/65-pr57-hide-max-approval.scenario.mjs create mode 100644 auto-qa/harness/scenarios/66-network-shape-market-page.scenario.mjs create mode 100644 auto-qa/harness/scenarios/67-a11y-heuristics-market-page.scenario.mjs create mode 100644 auto-qa/harness/scenarios/68-text-selection-market-page.scenario.mjs create mode 100644 auto-qa/harness/scenarios/69-no-page-errors-milestones.scenario.mjs create mode 100644 auto-qa/harness/scenarios/70-a11y-heuristics-milestones.scenario.mjs create mode 100644 auto-qa/harness/scenarios/71-network-shape-milestones.scenario.mjs create mode 100644 auto-qa/harness/scenarios/72-text-selection-milestones.scenario.mjs create mode 100644 auto-qa/harness/scenarios/73-keyboard-nav-companies.scenario.mjs create mode 100644 auto-qa/harness/scenarios/74-keyboard-nav-market-page.scenario.mjs create mode 100644 auto-qa/harness/scenarios/75-keyboard-nav-milestones.scenario.mjs create mode 100644 auto-qa/harness/scenarios/76-modal-focus-trap-rainbowkit.scenario.mjs create mode 100644 auto-qa/harness/scenarios/77-modal-focus-trap-confirm-swap.scenario.mjs create mode 100644 auto-qa/harness/scenarios/78-aria-state-outcome-tabs.scenario.mjs create mode 100644 auto-qa/harness/scenarios/79-aria-current-active-nav-link.scenario.mjs create mode 100644 auto-qa/harness/scenarios/80-pointer-events-cascade-companies.scenario.mjs create mode 100644 auto-qa/harness/scenarios/81-cursor-cascade-companies.scenario.mjs create mode 100644 auto-qa/harness/scenarios/82-text-transform-cascade-companies.scenario.mjs create mode 100644 auto-qa/harness/scenarios/83-pointer-events-cascade-market-page.scenario.mjs create mode 100644 auto-qa/harness/scenarios/84-pointer-events-cascade-milestones.scenario.mjs create mode 100644 auto-qa/harness/scenarios/85-cursor-cascade-market-page.scenario.mjs create mode 100644 auto-qa/harness/scenarios/86-cursor-cascade-milestones.scenario.mjs create mode 100644 auto-qa/harness/scenarios/87-direction-rtl-cascade-companies.scenario.mjs create mode 100644 auto-qa/harness/scenarios/88-event-card-href-shape-companies.scenario.mjs create mode 100644 auto-qa/harness/scenarios/89-visibility-hidden-cascade-companies.scenario.mjs create mode 100644 auto-qa/harness/scenarios/90-header-back-nav-anchor-milestones.scenario.mjs create mode 100644 auto-qa/harness/scenarios/91-header-back-nav-anchor-market-page.scenario.mjs create mode 100644 auto-qa/harness/scenarios/92-opacity-zero-cascade-companies.scenario.mjs create mode 100644 auto-qa/harness/scenarios/93-text-transform-cascade-market-page.scenario.mjs create mode 100644 auto-qa/harness/scenarios/94-text-transform-cascade-milestones.scenario.mjs create mode 100644 auto-qa/harness/scenarios/95-user-select-cascade-market-page.scenario.mjs create mode 100644 auto-qa/harness/scenarios/96-user-select-cascade-milestones.scenario.mjs create mode 100644 auto-qa/harness/scenarios/README.md create mode 100644 auto-qa/harness/scenarios/SCENARIOS.md create mode 100644 auto-qa/harness/scripts/contracts.mjs create mode 100644 auto-qa/harness/scripts/debug-balance-quirk.mjs create mode 100644 auto-qa/harness/scripts/invariants-catalog.mjs create mode 100644 auto-qa/harness/scripts/scenarios-by-route.mjs create mode 100644 auto-qa/harness/scripts/scenarios-by-tier.mjs create mode 100644 auto-qa/harness/scripts/scenarios-catalog.mjs create mode 100644 auto-qa/harness/scripts/scenarios-chaos-matrix.mjs create mode 100644 auto-qa/harness/tests/architecture-sync.test.mjs create mode 100644 auto-qa/harness/tests/contract-calls.test.mjs create mode 100644 auto-qa/harness/tests/eth-call-inspector.test.mjs create mode 100644 auto-qa/harness/tests/fork-state.test.mjs create mode 100644 auto-qa/harness/tests/invariants-catalog.test.mjs create mode 100644 auto-qa/harness/tests/live-time-control.test.mjs create mode 100644 auto-qa/harness/tests/market-page-fixture.test.mjs create mode 100644 auto-qa/harness/tests/scenarios-by-route.test.mjs create mode 100644 auto-qa/harness/tests/scenarios-by-tier.test.mjs create mode 100644 auto-qa/harness/tests/scenarios-catalog.test.mjs create mode 100644 auto-qa/harness/tests/scenarios-chaos-matrix.test.mjs create mode 100644 auto-qa/harness/tests/wallet-stub.test.mjs diff --git a/.github/workflows/auto-qa-fork.yml b/.github/workflows/auto-qa-fork.yml new file mode 100644 index 0000000..3e7fb2a --- /dev/null +++ b/.github/workflows/auto-qa-fork.yml @@ -0,0 +1,80 @@ +name: auto-qa fork + +on: + workflow_dispatch: + inputs: + fork_url: + description: 'Override Gnosis fork RPC URL' + required: false + default: '' + schedule: + - cron: '37 4 * * 2' + +jobs: + fork: + name: Forked-chain auto-qa + runs-on: ubuntu-latest + timeout-minutes: 25 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: 'npm' + cache-dependency-path: | + package-lock.json + auto-qa/harness/package-lock.json + + - name: Install app dependencies + run: npm ci + + - name: Install harness dependencies + working-directory: auto-qa/harness + run: npm ci + + - name: Install Foundry + uses: foundry-rs/foundry-toolchain@v1 + with: + version: stable + + - name: Cache Playwright browsers + id: playwright-cache + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: ${{ runner.os }}-playwright-${{ hashFiles('auto-qa/harness/package-lock.json') }} + + - name: Install Chromium + if: steps.playwright-cache.outputs.cache-hit != 'true' + working-directory: auto-qa/harness + run: npx playwright install --with-deps chromium + + - name: Install Chromium system dependencies + if: steps.playwright-cache.outputs.cache-hit == 'true' + working-directory: auto-qa/harness + run: npx playwright install-deps chromium + + - name: Run fork tier + env: + FORK_URL_INPUT: ${{ github.event.inputs.fork_url || '' }} + FORK_URL_SECRET: ${{ secrets.GNOSIS_FORK_URL }} + HARNESS_ANVIL_LOG: '1' + run: | + export FORK_URL="${FORK_URL_INPUT:-${FORK_URL_SECRET:-https://rpc.gnosis.gateway.fm}}" + npm run auto-qa:harness:fork + + - name: Upload fork artifacts on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: fork-auto-qa-results-${{ github.run_attempt }} + path: | + auto-qa/harness/playwright-report/ + auto-qa/harness/test-results/ + /tmp/anvil-harness.log + retention-days: 14 + if-no-files-found: ignore diff --git a/.github/workflows/auto-qa-interactions.yml b/.github/workflows/auto-qa-interactions.yml new file mode 100644 index 0000000..706d4de --- /dev/null +++ b/.github/workflows/auto-qa-interactions.yml @@ -0,0 +1,85 @@ +name: auto-qa interactions + +on: + pull_request: + paths: + - '.github/workflows/auto-qa-interactions.yml' + - 'auto-qa/harness/**' + - 'app/**' + - 'src/**' + - 'public/**' + - 'package.json' + - 'package-lock.json' + - 'next.config.mjs' + - 'tailwind.config.js' + push: + branches: + - main + paths: + - '.github/workflows/auto-qa-interactions.yml' + - 'auto-qa/harness/**' + - 'app/**' + - 'src/**' + - 'public/**' + - 'package.json' + - 'package-lock.json' + - 'next.config.mjs' + - 'tailwind.config.js' + workflow_dispatch: + +jobs: + interaction: + name: Browser interaction auto-qa + runs-on: ubuntu-latest + timeout-minutes: 12 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: 'npm' + cache-dependency-path: | + package-lock.json + auto-qa/harness/package-lock.json + + - name: Install app dependencies + run: npm ci + + - name: Install harness dependencies + working-directory: auto-qa/harness + run: npm ci + + - name: Cache Playwright browsers + id: playwright-cache + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: ${{ runner.os }}-playwright-${{ hashFiles('auto-qa/harness/package-lock.json') }} + + - name: Install Chromium + if: steps.playwright-cache.outputs.cache-hit != 'true' + working-directory: auto-qa/harness + run: npx playwright install --with-deps chromium + + - name: Install Chromium system dependencies + if: steps.playwright-cache.outputs.cache-hit == 'true' + working-directory: auto-qa/harness + run: npx playwright install-deps chromium + + - name: Run browser interaction tier + run: npm run auto-qa:harness:interaction + + - name: Upload Playwright artifacts on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: playwright-interaction-results-${{ github.run_attempt }} + path: | + auto-qa/harness/playwright-report/ + auto-qa/harness/test-results/ + retention-days: 14 + if-no-files-found: ignore diff --git a/.github/workflows/auto-qa.yml b/.github/workflows/auto-qa.yml index a75cfc1..6d37192 100644 --- a/.github/workflows/auto-qa.yml +++ b/.github/workflows/auto-qa.yml @@ -5,6 +5,8 @@ on: paths: - '.github/workflows/auto-qa.yml' - '.github/workflows/auto-qa-live.yml' + - '.github/workflows/auto-qa-interactions.yml' + - '.github/workflows/auto-qa-fork.yml' - 'auto-qa/**' - 'package.json' - 'src/utils/proposalLifecycle.js' @@ -14,6 +16,8 @@ on: paths: - '.github/workflows/auto-qa.yml' - '.github/workflows/auto-qa-live.yml' + - '.github/workflows/auto-qa-interactions.yml' + - '.github/workflows/auto-qa-fork.yml' - 'auto-qa/**' - 'package.json' - 'src/utils/proposalLifecycle.js' @@ -23,7 +27,7 @@ jobs: unit: name: Deterministic auto-qa runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 8 steps: - name: Checkout @@ -33,6 +37,15 @@ jobs: uses: actions/setup-node@v4 with: node-version: '22' + cache: 'npm' + cache-dependency-path: auto-qa/harness/package-lock.json + + - name: Install harness dependencies + working-directory: auto-qa/harness + run: npm ci - name: Run deterministic auto-qa tests run: npm run auto-qa:test:unit + + - name: Run deterministic harness checks + run: npm run auto-qa:harness:unit diff --git a/auto-qa/README.md b/auto-qa/README.md index bc7ba00..1d43263 100644 --- a/auto-qa/README.md +++ b/auto-qa/README.md @@ -1,6 +1,7 @@ # Auto-QA -This directory is split into CI tiers by flakiness and external dependencies. +This directory is split into explicit CI tiers so each check has a clear +runtime and dependency profile. ## Deterministic Unit Tier @@ -8,12 +9,33 @@ Run: ```sh npm run auto-qa:test:unit +npm run auto-qa:harness:unit ``` -Files live under `auto-qa/tests/`. These tests must not call live services, -start browsers, start an Anvil fork, or submit transactions. They are allowed -to inspect local source files and run local Node helper scripts. This tier is -safe for pull-request CI and is also what `npm run auto-qa:test` runs. +Files live under `auto-qa/tests/` and `auto-qa/harness/tests/`. These tests must +not call live services, start browsers, start an Anvil fork, or submit +transactions. They inspect local source files, local fixtures, and generated +catalogs only. + +GitHub runs this tier on pull requests and pushes to `main` in +`.github/workflows/auto-qa.yml`. + +## Browser Interaction Tier + +Run: + +```sh +npm run auto-qa:harness:interaction +``` + +This tier runs Playwright against the local Next app with mocked network data +and the wallet stub. It is for real browser interactions such as keyboard +navigation, focus handling, and modal behavior. It does not start Anvil and does +not submit transactions. + +Scenarios opt into this tier with `ciTiers: ['interaction']`. GitHub runs it on +pull requests and pushes to `main` in +`.github/workflows/auto-qa-interactions.yml`. ## Live Network Tier @@ -24,14 +46,40 @@ npm run auto-qa:test:live ``` Files live under `auto-qa/live/`. These tests may call public Futarchy -endpoints and should skip when the network is unavailable. They are useful for -catching endpoint drift, but they are intentionally excluded from pull-request -CI. GitHub runs them only by manual dispatch or on the nightly schedule in -`.github/workflows/auto-qa-live.yml`. +endpoints and should skip when the network is unavailable. They catch endpoint +drift, but they are intentionally excluded from pull-request CI. + +GitHub runs this tier only by manual dispatch or on the weekly Monday schedule +in `.github/workflows/auto-qa-live.yml`. + +## Forked-Chain Tier -## Forked Browser Harness +Run: + +```sh +npm run auto-qa:harness:fork +``` + +This tier runs Playwright scenarios against a local Anvil fork. It may submit +transactions to that local fork, but it must not submit real network writes. +Scenarios join this tier with `ciTiers: ['fork']` and must also declare +`requiresAnvil` or `useAnvilRpcProxy`. + +GitHub runs this tier only by manual dispatch or on the weekly Tuesday schedule +in `.github/workflows/auto-qa-fork.yml`. The workflow uses `FORK_URL` from the +manual input, `GNOSIS_FORK_URL`, or the public Gnosis RPC fallback. + +## Promoting Scenarios + +New scenarios should start unassigned unless they are intentionally part of a +CI tier. Before promotion, run: + +```sh +npm run auto-qa:harness:scenarios:by-tier +npm run auto-qa:harness:scenarios:catalog +``` -The larger Playwright and Anvil replay harness from the experimental -`auto-qa` branch is not part of this tier. Forked-chain reads, writes, browser -scenarios, and catalog drift checks need their own explicit workflow once they -are cleaned up and separated from live-write tests. +Promote stable browser-only scenarios with `ciTiers: ['interaction']`. Promote +stable forked scenarios with both `ciTiers: ['fork']` and an Anvil requirement +on the scenario itself. Keep live endpoint checks in `auto-qa/live/`; they +remain weekly/manual unless we explicitly decide to make a live check blocking. diff --git a/auto-qa/harness/.gitignore b/auto-qa/harness/.gitignore new file mode 100644 index 0000000..51215dd --- /dev/null +++ b/auto-qa/harness/.gitignore @@ -0,0 +1,29 @@ +# Local node deps (we keep harness deps separate from the interface repo deps) +node_modules/ + +# Playwright artifacts +.playwright/ +playwright-report/ +playwright-prod-report/ +test-results/ +*.video +*.trace.zip +*.har + +# Browser binaries (Playwright pulls them on install) +.browsers/ + +# Local Next.js dev server state when run from harness +.next-harness/ + +# Local wallet keys for the harness (anvil dev keys; never real) +.wallet-state/ + +# Step 7: cross-process channel for the active anvil snapshot ID +# (globalSetup writes; per-scenario beforeEach reads + rewrites). +# Ephemeral; regenerated on every test run. +.fork-snapshot-id + +# Editor / OS +.DS_Store +*.log diff --git a/auto-qa/harness/ARCHITECTURE.md b/auto-qa/harness/ARCHITECTURE.md new file mode 100644 index 0000000..946f7a4 --- /dev/null +++ b/auto-qa/harness/ARCHITECTURE.md @@ -0,0 +1,157 @@ +# Forked Replay Harness — Architecture + +This is the **shared architecture spec** for the Forked Replay Harness. +The same file is mirrored in both repos: + +- `futarchy-fi/futarchy-api` hosts the **server side**: anvil fork, + indexer, futarchy-api, orchestrator, scenarios, chaos injection. +- `futarchy-fi/interface` hosts the **UI side**: Playwright config, + wallet stub, page-object models, flows, DOM↔API consistency + assertions. + +The two halves communicate over docker-compose's internal network and +share state via the orchestrator. Neither side modifies production code. + +For phasing, status, and effort breakdown see [`PROGRESS.md`](./PROGRESS.md). +For repo-local how-to see [`README.md`](./README.md). + +## Service topology + +``` + ┌──────────────────────────────────────────────────────────┐ + │ Orchestrator (futarchy-api/auto-qa/harness/orchestrator)│ + │ - owns the block clock │ + │ - drives scenario script │ + │ - emits cross-layer assertion failures │ + └─┬───────────────┬───────────────┬─────────────────┬──────┘ + │ JSON-RPC │ JSON-RPC │ HTTP │ HTTP/CDP + ▼ ▼ ▼ ▼ + ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌────────────────┐ + │ anvil │ │ Local │ │ futarchy │ │ Playwright │ + │ (fork │◄──┤Checkpoint│ │ -api │ │ + headless │ + │ Gnosis) │ │ indexer │◄──┤(Express) │◄───┤ Chromium │ + │ port │ │ port │ │ port 3000│ │ (interface │ + │ 8545 │ │ 3001 │ │ │ │ /auto-qa │ + └──────────┘ └──────────┘ └──────────┘ │ /harness) │ + └───────┬────────┘ + │ + ┌────────────▼─────────┐ + │ Next.js dev server │ + │ (futarchy-fi/ │ + │ interface, mounted │ + │ sibling clone) │ + │ port 3010 (host) │ + └──────────────────────┘ +``` + +All five services run inside one docker-compose stack on a private +`harness-net` bridge. The orchestrator exposes no ports; it talks to +everything else via service names. + +## Repo split + +| Concern | Lives in | Path | +|---|---|---| +| anvil fork launcher | futarchy-api | `auto-qa/harness/scripts/start-fork.mjs` | +| Block clock + tx replay | futarchy-api | `auto-qa/harness/scripts/block-clock.mjs` | +| Local Checkpoint launcher | futarchy-api | `auto-qa/harness/scripts/start-indexer.mjs` | +| Local futarchy-api (this repo's `src/`) | futarchy-api | docker-compose service | +| Cross-layer assertion library | futarchy-api | `auto-qa/harness/orchestrator/invariants.mjs` | +| Scenario fixtures | futarchy-api | `auto-qa/harness/scenarios/` | +| Chaos injection | futarchy-api | `auto-qa/harness/scripts/chaos.mjs` | +| docker-compose | futarchy-api | `auto-qa/harness/docker-compose.yml` | +| Playwright config | interface | `auto-qa/harness/playwright.config.mjs` | +| Wallet stub (EIP-1193) | interface | `auto-qa/harness/fixtures/wallet-stub.mjs` | +| Page-object models | interface | `auto-qa/harness/pages/` | +| End-to-end flows | interface | `auto-qa/harness/flows/` | +| DOM↔API invariants | interface | `auto-qa/harness/invariants/` | + +## Boot sequence + +1. `docker compose -f futarchy-api/auto-qa/harness/docker-compose.yml up -d` +2. anvil starts, forks Gnosis at the configured block, healthcheck + passes via `cast block-number` +3. Checkpoint indexer starts, points at `http://anvil:8545`, ingests + from `START_BLOCK` +4. futarchy-api starts, points at `http://anvil:8545` for RPC and + `http://indexer:3001/graphql` for indexed data +5. (Optional, only when running UI tests) interface-dev starts, + points at `http://anvil:8545` and `http://api:3000` +6. Orchestrator runs the chosen scenario script +7. Per-block: orchestrator advances anvil clock, mines, then runs + the cross-layer invariant battery +8. Per-scenario: snapshot/revert anvil between scenarios so they don't + share state + +## Cross-layer invariant catalogue + +| Layer A | vs | Layer B | Invariant | +|---|---|---|---| +| chain (anvil) | vs | indexer | every Swap event present, same token amounts, same sqrtPrice | +| indexer | vs | api `/candles/graphql` | candle aggregates match raw swaps | +| api `/spot-candles` | vs | api `/candles/graphql` | rate-applied prices reconcile | +| api `/v2/.../chart` | vs | indexer raw | unified-chart shape consistent | +| frontend DOM | vs | api response | every visible price/volume/TVL matches the API call that produced it | +| Playwright wallet swap | vs | chain receipt | tx mined, balance delta correct, conditional tokens minted | + +## Economic invariants (always-on, evaluated every block) + +- **Conservation**: ∑(YES + NO conditional tokens) = ∑(sDAI deposited) +- **Monotonicity**: TWAP window endpoints respect contract's `min(now, twapEnd)` clamp +- **Probability**: price ∈ [0, 1] for PREDICTION pools +- **No phantom mints**: `balanceOfBatch` sums match historical + synthetic deposits +- **Rate sanity**: sDAI rate from on-chain `getRate()` ≥ 1, monotonically increasing + +## Cloning + running together + +The harness needs both repos checked out as siblings: + +```bash +mkdir -p ~/code/futarchy-fi +cd ~/code/futarchy-fi +git clone -b auto-qa https://github.com/futarchy-fi/futarchy-api.git +git clone -b auto-qa https://github.com/futarchy-fi/interface.git + +# From futarchy-api (the harness host): +cd futarchy-api +docker compose -f auto-qa/harness/docker-compose.yml up -d + +# Wait for healthchecks, then run the orchestrator: +npm run auto-qa:e2e:replay -- --tier interaction +``` + +The compose file references `${INTERFACE_PATH}` (default +`../../../../interface`) for the sibling-clone mount. Override that env +var if your layout differs. + +## Decision records + +ADRs live in `docs/` on each side: + +- `futarchy-api/auto-qa/harness/docs/ADR-001-foundry-vs-hardhat.md` +- `interface/auto-qa/harness/docs/ADR-001-synpress-vs-custom-stub.md` + +Each ADR is one page, opinionated, and dated. New ADRs increment the +number (ADR-002, etc.) — do not amend old ADRs in place; supersede them. + +## Open questions + +These are deferred to the relevant phase but logged here so they don't +get lost: + +1. **Indexer schema migration story** (Phase 3) — Checkpoint's cold-start + warm-up time is the biggest CI risk; do we ship a pre-warmed snapshot + or accept the boot cost? +2. **Multi-user wallet provisioning** (Phase 4) — anvil dev mnemonic + gives us 10 deterministic addresses; do we need more for arbitrage + scenarios? +3. **Frontend test-id discipline** (Phase 5) — DOM-equivalent price + extraction is brittle without `data-testid` attributes in the + production code. Do we add them as part of the harness work, or + accept brittle selectors? +4. **Scenario capture format** (Phase 6) — JSON snapshot of `(block range, + tx list, expected end-state)` vs full state-dump replay. Tradeoff + between fidelity and snapshot size. +5. **CI execution model** (Phase 7) — nightly cron vs manually-triggered + workflow vs PR-gated. Decision deferred until we know the runtime + per scenario. diff --git a/auto-qa/harness/CAPABILITIES.md b/auto-qa/harness/CAPABILITIES.md new file mode 100644 index 0000000..3d8184c --- /dev/null +++ b/auto-qa/harness/CAPABILITIES.md @@ -0,0 +1,360 @@ +# Harness Capabilities — Discovery Index + +Quick lookup for what the harness can already catch and what fixtures +already exist. **Read this before authoring a new scenario** — odds +are good that the capability you need is already here. + +Full slice-by-slice history lives in `PROGRESS.md`. This file is the +flat reference. + +--- + +## The 12 KINDs of bugs the harness catches + +A "KIND" is a class of regression with its own assertion mechanism. +The catalog of 96 scenarios (3 pinned-skipped) is a composition of these. + +| # | KIND | What it pins | Example scenarios | +|---|---|---|---| +| 1 | **DOM text** | rendered text (presence, absence, exact value) | 01 (stale price), 10-15 (market-page happy), 55 (PR #46 archived-proposal absence), 60 (PR #56 Prediction Market badge gated) | +| 2 | **GraphQL shape** | request body + response classification (strict schema, body-level field match) | 47 + 53 (PRs #45/#60/#61/#62/#63/#65 schema strictness), 63 (PR #50 `organization(id:)` body match) | +| 3 | **Page errors** | `pageerror` + console.error capture | 48 (/companies), 10-23 (markets opt-in), 69 (/milestones) — 3-surface grid closed | +| 4 | **URL state** | history mutations, query param transitions, nav-infrastructure shape | 5 scenarios across 4 sub-shapes: 49 (post-mount hash → query rewrite via useEffect on /milestones), 54 (`/market` singular redirect via routing config), 88 (event-card outbound href on /companies), 90 + 91 (Header back-nav anchor on /milestones + /markets — 2/2 natural ceiling closed slice 322; /companies n/a as it's the destination) | +| 5 | **Network requests** | URL pattern presence/absence, request count, request body | 50 (/companies), 66 (/markets), 71 (/milestones) — 3-surface grid closed; 57+61 (negative Supabase), 63 (positive + body match) | +| 6 | **Visual / Computed CSS** | `getComputedStyle` queries on cascading CSS properties | 15 scenarios across 3 surfaces × 7 properties via slice-310 helper. **4 properties at full 3-surface coverage** (pointer-events: 80+83+84; cursor: 81+85+86; text-transform: 82+93+94; user-select: 62+95+96). Other 3 at 1-surface (87 direction:rtl, 89 visibility:hidden, 92 opacity:0 — all /companies). Matrix table below; `assertPageLayoutCascadeStyleIsNot` makes any new cascade catch a 1-liner | +| 7 | **a11y heuristics (static DOM)** | img alt, button accessible-name, input labels via inline DOM walk | 52 (/companies), 67 (/markets), 70 (/milestones) — 3-surface grid closed; refinement landed slice 289 (skip aria-hidden, accept title); fixture extracted slice 293 | +| 8 | **Build-mode runtime** | minified-bundle catches (TDZ, dead-code elim) | 54 (PR #55, `prodModeOnly` flag) | +| 9 | **TIME-EVOLUTION** | chain-side mutations + RPC interception with decoded params | 59 (PR #54 TWAP window, function args), 64 (PR #53 gas param), 65 (PR #57 modal-walkthrough with multicall3) | +| 10 | **Keyboard navigation** | Tab walk + `document.activeElement` chain inspection | 73 (/companies), 74 (/markets), 75 (/milestones) — 3-surface grid closed; fixture extracted slice 300 | +| 11 | **Modal focus-trap** | Tab walk after modal opens; assert focus does NOT escape modal subtree | 76 (RainbowKit modal, active), 77 (ConfirmSwapModal, pinned-latent: 4th latent bug); fixture extended slice 306 with inverted-direction helper | +| 12 | **ARIA-state inspection** | runtime ARIA attributes (aria-selected, aria-current, etc.) on interactive widgets | 78 (outcome tabs, pinned-latent: 4th latent bug), 79 (aria-current, pinned-latent: 5th latent bug + systemic gap confirmed) | + +**Sister-pattern KINDs at 3-surface coverage**: **6 of 12** (page-error, network shape, a11y heuristics, user-CSS interactive, keyboard-nav, **Visual/Computed CSS** — joined slice 313 via the `pointer-events` sub-grid 80+83+84). Per-surface chaos matrix structurally complete for these. + +### KIND 6 sub-grid matrix (slice 319 snapshot) + +KIND 6 introduced a **sub-grid framework** at slice 316 — a single KIND can host multiple properties each independently at 3-surface coverage via the slice-310 helper. The matrix grows on two independent axes (property + surface), each combination a 1-liner. + +``` + /companies /markets /milestones +user-select 62 95 96 (3/3 ✓) +pointer-events 80 83 84 (3/3 ✓) +cursor 81 85 86 (3/3 ✓) +text-transform 82 93 94 (3/3 ✓) +direction 87 — — (1/3) +visibility 89 — — (1/3) +opacity 92 — — (1/3) +``` + +Snapshot evolution: +- Slice 313: 1 sub-grid (pointer-events) at 3-surface. +- Slice 316: 2 sub-grids (+cursor). +- Slice 326: 3 sub-grids (+text-transform). +- **Slice 328: 4 sub-grids (+user-select).** +- 3 properties remain single-surface (direction, visibility, opacity); ~6 more slices to fill 7×3 matrix at the established 2-slice-per-property cadence. + +Filling a cell is a 1-liner: `assertPageLayoutCascadeStyleIsNot(page, { propertyName, expectedNot, scenarioLabel })`. Helper validated route-agnostic AND property-agnostic across 10 catches with ZERO helper edits. + +--- + +## Fixture capabilities — what already exists + +### `fixtures/api-mocks.mjs` + +GraphQL endpoint mocks + probe data. + +| Export | Use when | +|---|---| +| `REGISTRY_GRAPHQL_URL`, `CANDLES_GRAPHQL_URL` | Routing mocks | +| `PROBE_*` (org/agg/pool/proposal addresses) | /companies-side probe data | +| `MARKET_PROBE_*` (address/title/currency/company/pool) | /markets-side probe data | +| `makeGraphqlMockHandler({ proposals, orgMetadata, orgName, organizations, onCall })` | Registry GraphQL with operation dispatch. **`onCall(query, body)`** signature — pass a 2-arg callback if you need to inspect `variables` (the address is in `body.variables.id`, not in the query string — see scenario 63). | +| `makeStrictCheckpointGraphqlMockHandler` / `makeStrictCandlesGraphqlMockHandler` | Registry/candles mocks that REJECT legacy schema shapes (matches `CHECKPOINT_SCHEMA_VIOLATIONS` list). Use when catching schema regressions (scenarios 47, 53). | +| `CHECKPOINT_SCHEMA_VIOLATIONS` | Catalog of 7 legacy GraphQL patterns Checkpoint indexer rejects. Add a new pattern + verbatim error string here to light up every consumer at once. | +| `fakeProposal`, `fakePoolBearingProposal({ idSuffix, poolYes, poolNo, metadataExtra, organizationId })` | /companies-side proposal stubs. Use `metadataExtra` for `archived: true`, `chain: '10'`, etc. | +| `fakeMarketProposalEntity({ proposalAddress, title, metadataExtra })` | /markets-side proposalentity stub in the format `fetchProposalMetadataFromRegistry` expects. | +| `makeCandlesMockHandler({ prices, onCall })` | /companies-side candles handler (handles `pools(where: id_in:...)` bulk-fetch query). Pass `prices: {}` (empty) to disable bulk prefetch and force per-card fetches — used by scenario 61 for the negative-network catch. | +| `makeMarketCandlesMockHandler(opts)` | /markets-side candles handler (handles `proposal(...) + whitelistedtokens(...)` discovery + per-pool detail + swaps + candles + pool-batch + token-list refresh). | +| `makeSubgraphAwareCandlesHandler({ marketName })` | **OPT-IN** wrapper that enriches the discovery response with `pools[]` + role-tagged whitelistedtokens (`YES_COMPANY`, `NO_CURRENCY`, etc.) so the subgraph adapter populates POOL_CONFIG_YES/NO + MERGE_CONFIG + BASE_TOKENS_CONFIG. **Required** for any scenario that reads from `config.MERGE_CONFIG` or `POOL_CONFIG_YES.address`. Used by 59, 60, 64, 65. | +| `PUBLIC_GNOSIS_RPC_URLS` (6 URLs) | wagmi rotates through these. Any RPC interceptor should route ALL of them + `http://localhost:8546/` to ensure capture. | +| `makeAnvilRpcProxyHandler` + `installAnvilRpcProxy` | Proxy public Gnosis RPCs to local anvil fork so balance reads see fork-funded state. Opt in via `useAnvilRpcProxy: true` on the scenario. | + +### `fixtures/eth-call-inspector.mjs` + +eth_call request/response parsing + Multicall3 helpers. + +| Export | Use when | +|---|---| +| `parseEthCallParams(body)` | Extract `{ to, data }` from a JSON-RPC eth_call body. Returns null for non-eth_call or batched requests. | +| `decodeEthCallData(data, abi)` | Generic: decode any function call's args. Returns `{ functionName, args }` or null. | +| `sameAddress(a, b)` | Case-insensitive address compare (handles checksum vs lowercase). | +| `ALGEBRA_POOL_TIMEPOINTS_ABI` + `ALGEBRA_POOL_TIMEPOINTS_SELECTOR` + `decodeGetTimepointsArgs(data)` | Decode the `getTimepoints(uint32[])` args. Used by scenario 59 to catch PR #54's TWAP window. | +| `MULTICALL3_ADDRESS` + `MULTICALL3_AGGREGATE3_SELECTOR` | wagmi auto-batches reads through multicall3. Any RPC interceptor for `readContract` calls MUST handle this — see slice 103 for the cautionary tale. | +| `decodeMulticall3Aggregate3(data)` | Parse the inner `Call3[]` array from an aggregate3 call. Returns `[{ target, allowFailure, callData }]`. | +| `encodeMulticall3Returns(results)` | Encode a `Result[]` aggregate3 return. Each result is `{ success: bool, returnData: bytes }`. **The returnData length matters** — return 32 bytes of zero for unknown selectors (NOT shorter, viem decodes the outer array and the position assertion trips). | + +### `fixtures/fork-state.mjs` + +Chain-side mutations + time control via anvil RPCs. + +| Export | Use when | +|---|---| +| `anvilRpc(rpcUrl, method, params, timeoutMs)` | Raw anvil RPC client with retry. Use this if your custom mutation isn't covered by a higher-level helper. | +| `setEthBalance` / `getEthBalance` | ETH balance state. | +| `setErc20Balance(rpcUrl, token, holder, amount, slot)` | ERC20 balance via `setStorageAt`. Known flake on cold anvil — wrap with `withProxyPaused` + drain windows (see scenarios 15/17/18 for the documented pattern). | +| `getErc20Balance` | Read ERC20 balance. | +| `fundWalletWithSDAI(rpcUrl, holder, amountWei)` | Convenience: set sDAI balance to a specific wei amount. Defaults to 1000 sDAI. | +| `setErc1155Balance` / `getErc1155Balance` | ERC1155 (Conditional Tokens) state. Uses nested mapping storage layout. | +| `setConditionalPosition` / `getConditionalPosition` | Higher-level helper for ConditionalTokens position state. | +| `setNextBlockTimestamp` / `mineBlock` / `advanceTime(rpcUrl, seconds)` | TIME-EVOLUTION primitives (slice 90). Anvil's `evm_setNextBlockTimestamp` pins exactly — no wall-clock slop. | +| `getBlockTimestamp(rpcUrl)` | Read current chain time. | +| `warmContractCache(rpcUrl, addresses)` / `warmErc20Balances(rpcUrl, tokens, holder)` | Pre-warm anvil's contract/balance cache to reduce cold-call latency. | +| `impersonateAndSend(rpcUrl, fromAddress, tx)` | Send a tx as another address (no signing required). | +| `ctGetCollectionId` / `ctGetPositionId` / `ctDerivePositionId` | ConditionalTokens collection/position ID derivation. | +| `mappingStorageKey` / `nestedMappingStorageKey` | Solidity storage slot computation for mappings. | +| `PAGE_CONTRACT_ADDRESSES`, `PAGE_ERC20_ADDRESSES` | Lists of contracts the page typically touches. Useful for cache-warming or RPC traffic budgeting. | + +### `fixtures/wallet-stub.mjs` + +EIP-1193 wallet stub for Playwright via EIP-6963. + +| Export | Use when | +|---|---| +| `installWalletStub(config)` | Install the synthetic wallet (auto-applied by `scenarios.spec.mjs` beforeEach). | +| `nStubWallets(n, mnemonic)` | Derive N wallets from a mnemonic for multi-account scenarios. | +| `setupSigningTunnel(context, cfg)` | Wire up `__harnessSign` exposeBinding so SIGNING_METHODS route to viem in node (privateKey never enters the page). | +| `WALLET_LOCAL_METHODS` / `RPC_PASSTHROUGH_METHODS` / `SUBSCRIPTION_METHODS` | EIP-1193 method classification (which the stub handles vs forwards). | +| `ANVIL_DEV_MNEMONIC` | Standard anvil dev mnemonic. | + +### `fixtures/a11y-heuristics.mjs` + +Static-DOM a11y heuristics evaluated inside the browser via `page.evaluate(A11Y_HEURISTICS)`. Slice 293 extraction; back-ports slice 289 refinements (skip aria-hidden ancestors, accept `title` as accessible-name) to scenario 52 retroactively. + +| Export | Use when | +|---|---| +| `A11Y_HEURISTICS` | The heuristic function to pass into `page.evaluate(...)`. Returns `Array<{kind, html, ...}>`. Three rule classes: `img-no-alt`, `button-no-name`, `input-no-label`. Skips elements whose ancestor chain has `aria-hidden="true"` or that aren't visible. | +| `isKnownViolation(violation, knownBaseline)` | Filter helper for KNOWN_BASELINE matching. Each rule is `{kind, match}` where `match` is a substring or RegExp tested against the violation's `html` field. | + +Used by scenarios 52 (/companies), 67 (/markets), 70 (/milestones). + +### `fixtures/text-selection.mjs` + +User-CSS interactive KIND helper — triple-click + `getSelection` for catching CSS regressions that block `user-select` cascade. Slice 296 extraction (after 3 inline copies in scenarios 51, 68, 72). + +| Export | Use when | +|---|---| +| `assertTripleClickSelects(page, locator, expectedSubstring)` | Triple-click the locator, read `window.getSelection().toString()`, assert it contains `expectedSubstring`. Clears any inherited selection first. Choose a heading/span/paragraph (NOT a button — buttons commonly have `user-select: none` by default in CSS resets/Tailwind, which would mask the catch direction). | + +### `fixtures/keyboard-nav.mjs` + +Keyboard-navigation simulation — Tab walk + focus-chain inspection. Slice 300 extraction (after 3 inline copies in scenarios 73-75); slice 306 added the inverted-direction sister for modal focus-trap catches. + +| Export | Use when | +|---|---| +| `walkTabOrder(page, { depth })` | Press Tab `depth` times from `document.body`, return the ordered chain `Array<{tag, text, ariaLabel, href}>`. Build custom catches against the chain. | +| `assertTabReachesAnyOf(page, { depth, anchors, minDistinctTags })` | Two-catch shape: (1) ≥ `minDistinctTags` distinct tag names appear (rules out collapsed tab order); (2) at least one of `anchors` (RegExp[]) matches the text or aria-label of some focused element (proves an expected element is keyboard-reachable). | +| `assertTabDoesNotReachAnyOf(page, { depth, anchors })` | Inverted-direction catch (slice 306). After a modal opens, Tab must cycle INSIDE the modal subtree — none of `anchors` (typically background-Header anchors like `/chain selector/i`) may appear in the focus chain. | + +Used by scenarios 73 (/companies), 74 (/markets), 75 (/milestones), 76 (RainbowKit modal — passes), 77 (ConfirmSwapModal — pinned-latent: focus trap absent). + +### `fixtures/cascading-css.mjs` + +Visual / Computed-CSS cascade catches at the PageLayout `
` element. Slice 310 extraction (after 3 inline copies in scenarios 62, 80, 81). Validates that any cascading CSS property regressing on a layout-level wrapper is surfaced via `getComputedStyle`. + +| Export | Use when | +|---|---| +| `assertPageLayoutCascadeStyleIsNot(page, { propertyName, expectedNot, scenarioLabel? })` | Read a cascading CSS property at PageLayout's `
` element and assert it has NOT regressed to the marker value. `propertyName` is camelCase (e.g., `'userSelect'`, `'pointerEvents'`, `'cursor'`, `'textTransform'`). `expectedNot` is the regression marker (e.g., `'none'`, `'not-allowed'`, `'uppercase'`). On failure, dumps the ancestor chain so the cascade source is obvious. Selection logic (Tailwind `mt-20 bg-white` signature with fallback to last `
` in DOM order) lives ONCE in this fixture; battle-tested across /companies, /markets, /milestones (slice 313). | + +Used by 15 scenarios across 3 surfaces × 7 properties — see KIND 6 matrix table above. Sub-grid framework introduced slice 316: a single KIND scales independently on property + surface axes; each combination is a 1-liner. Helper is route-agnostic AND property-agnostic for any PageLayout consumer (battle-tested across /companies, /markets, /milestones with 7 distinct cascading properties — userSelect, pointerEvents, cursor, textTransform, direction, visibility, opacity). Non-PageLayout pages would need a `targetSelector` parameter (deferred until that scenario lands). + +--- + +## The scenario ctx — what assertions can access + +Every scenario assertion is called with `(page, ctx)`. The ctx contains: + +| Field | Source | Use | +|---|---|---| +| `wallet` | beforeEach | The connected wallet (address, signer) | +| `anvilUrl` | beforeEach | Local anvil URL — pass to fork-state helpers | +| `pageErrors[]` | slice 80 monitor | `{ kind: 'pageerror' \| 'console.error', message }` | +| `networkRequests[]` | slice 82 monitor | `{ url, method, resourceType, timestamp }` for every fetch | +| `callsTo(pattern)` | slice 82 helper | Filter networkRequests by URL regex or substring | +| `withProxyPaused(fn, { drainMs })` | slice 17 helper | Pause the page proxy + drain before a mutation. Required for `setStorageAt` on cold anvil. | +| **Scenario-scoped state** | `mocks` factory form | If `mocks` is a function `(ctx) => ({ ... })`, the factory can attach arbitrary state to ctx and assertions read it. Used by scenarios 53 (`onViolation`), 59 (`timepointsCalls`), 60, 63, 64, 65. | + +--- + +## Scenario flags (declarative opt-ins) + +| Flag | What it does | Example | +|---|---|---| +| `requiresAnvil: true` | Skip scenario when `HARNESS_NO_ANVIL=1` (no chain side). Use when reading chain state or calling fork-state helpers. | 58, 59, 64, 65 | +| `prodModeOnly: true` | Skip when `HARNESS_PROD_MODE!=1`. Run via `npm run ui:prod` — uses the minified production bundle. Catches build-mode runtime regressions. | 54 (PR #55 + PR #58 family) | +| `useAnvilRpcProxy: true` | Install the Public Gnosis RPC → local anvil proxy. Required when wallet balance reads should see fork-funded state. | 11, 12 | +| `mocks: (ctx) => ({...})` | Factory form — ctx is shared with assertions. Use when interceptors need to push state for later assertion. | 53, 59, 60, 63, 64, 65 | +| `pinnedLatentBug: ''` | Runner skips the scenario cleanly with the description as the skip reason. Use when a scenario was authored with the correct catch direction, runs, and exposes a real latent bug — preserve it as a future-catch (regression detector once the bug is fixed). Remove the flag when the underlying bug is fixed; the scenario will then catch regressions. | 77 (ConfirmSwapModal focus-trap absent), 78 (outcome tabs missing ARIA state), 79 (app-wide nav missing aria-current) | + +--- + +## Common assertion shapes + +### Positive DOM text +```js +await expect(page.getByText('HARNESS-PROBE-EVENT-001').first()) + .toBeVisible({ timeout: 30_000 }); +``` + +### Negative DOM text (regression catch via absence) +```js +await expect(page.getByText('Max Approval')).toHaveCount(0); +``` + +### Computed CSS property +```js +const userSelect = await page.evaluate(() => { + const main = document.querySelector('main.bg-white'); // pick the right one + return getComputedStyle(main).userSelect; +}); +expect(userSelect).not.toBe('none'); +``` + +### Negative network shape +```js +const deprecated = ctx.callsTo(/pool_candles/); +if (deprecated.length > 0) { + throw new Error(`Found ${deprecated.length} calls to the deprecated table: ${deprecated.slice(0,3).map(r => r.url).join(' | ')}`); +} +``` + +### GraphQL request-body match (variables-aware) +```js +makeGraphqlMockHandler({ + onCall: (query, body) => { + if (query.includes('organization(id:') && + body?.variables?.id?.toLowerCase() === GNOSIS_DAO_ORG_ADDR) { + ctx.orgQueries.push({ query, variables: body.variables }); + } + }, +}); +``` + +### eth_call function-arg inspection (TIME-EVOLUTION) +```js +const call = parseEthCallParams(body); +if (call && sameAddress(call.to, MARKET_PROBE_YES_POOL)) { + const decoded = decodeGetTimepointsArgs(call.data); + if (decoded) ctx.timepointsCalls.push({ secondsAgos: decoded }); +} +``` + +### eth_call gas-field inspection (slice 100 shape) +```js +const gasHex = body?.params?.[0]?.gas; +const gasDec = parseInt(gasHex, 16); +if (gasDec > GNOSIS_BLOCK_CAP) throw new Error(`gas=${gasDec} > cap`); +``` + +### Multicall3-aware response synthesis (slice 103 shape) +```js +if (sameAddress(call.to, MULTICALL3_ADDRESS) && + data.startsWith(MULTICALL3_AGGREGATE3_SELECTOR)) { + const innerCalls = decodeMulticall3Aggregate3(call.data); + const results = innerCalls.map(c => ({ + success: true, + returnData: c.callData.startsWith(ALLOWANCE_SELECTOR) + ? MAX_UINT256_BYTES32 + : ZERO_BYTES32, + })); + return route.fulfill({ ..., result: encodeMulticall3Returns(results) }); +} +``` + +### Modal walkthrough (slice 103 click-through pattern) +```js +// 1. Wait for button to enable (quote completed) +await expect(page.getByRole('button', { name: /^Confirm Swap$/ })) + .toBeEnabled({ timeout: 30_000 }); +// 2. Click to open modal +await page.getByRole('button', { name: /^Confirm Swap$/ }).click(); +// 3. Wait for modal heading +await expect(page.getByRole('heading', { name: /^Confirm Buy$/ }).first()) + .toBeVisible({ timeout: 15_000 }); +// 4. Now assert against modal-internal state +``` + +--- + +## "Where do I start if my scenario needs to..." + +| ...catch a... | start by reading | +|---|---| +| Text presence/absence regression | scenarios 10, 55, 60 | +| URL/route regression | scenarios 49, 54 | +| GraphQL schema regression | scenarios 47, 53 (strict-schema catalog) | +| Network-URL regression (positive or negative) | scenarios 50, 57, 61, 63 | +| Computed-CSS cascade regression (user-select, pointer-events, cursor, text-transform, direction, visibility, opacity, etc.) | 15 scenarios — see KIND 6 matrix in section above + `fixtures/cascading-css.mjs` (`assertPageLayoutCascadeStyleIsNot`). Property axis × surface axis = N×M cells, each a 1-liner. 4 of 7 properties at full 3-surface coverage (slice 328 milestone) | +| URL state regression (history mutation, redirect, nav-infrastructure shape) | 5 scenarios across 4 sub-shapes: 49 (post-mount hash → query rewrite), 54 (`/market` redirect), 88 (outbound event-card href on /companies), 90 + 91 (Header back-nav anchor on /milestones + /markets — 2/2 natural ceiling). Click-mediated nav still uncovered — slice 318 attempt failed in harness setup; slice 321 milestone-card-rendering attempt also failed; both need investigation | +| Build-minification regression | scenario 54 + `playwright.prod.config.mjs` | +| eth_call function-arg regression | scenarios 58, 59 + slice 94 | +| eth_call parameter (gas, etc.) regression | scenario 64 | +| Modal-state regression (Confirm/etc.) | scenario 65 | +| Multicall3-batched regression | scenario 65 + `eth-call-inspector.mjs` helpers | +| Chain-state mutation flow | scenarios 15, 17, 18 + `fork-state.mjs` | +| Time-evolution (block timestamp) | scenarios 58, 59 + slice 90 primitives | +| a11y heuristic regression (img alt, button name, input label) | scenarios 52, 67, 70 + `fixtures/a11y-heuristics.mjs` | +| User-CSS interactive regression (`user-select`, etc.) | scenarios 51, 68, 72 + `fixtures/text-selection.mjs` | +| Keyboard-navigation regression (Tab order reaches/skips an anchor) | scenarios 73, 74, 75 + `fixtures/keyboard-nav.mjs` (`assertTabReachesAnyOf`) | +| Modal focus-trap regression (Tab escapes back to background) | scenarios 76, 77 + `fixtures/keyboard-nav.mjs` (`assertTabDoesNotReachAnyOf`) | +| ARIA-state regression (aria-selected, aria-current, etc.) on interactive widgets | scenarios 78 (tabs), 79 (active nav link) — both pinned-latent; pattern is `page.evaluate` reading the runtime ARIA attribute | +| Latent-bug pinning (scenario passes today AND exposes a bug) | scenarios 77, 78, 79 + `pinnedLatentBug` flag | + +--- + +## Pitfalls documented across slices (avoid re-encountering) + +- **Cold-anvil setStorageAt 30s timeout** — wrap mutations in `withProxyPaused({ drainMs: 5000 })`. Even then, ~3% of full-catalog runs hit it on scenarios 15/18 (slice 21+ remediation). +- **HMR cache during regression-verification** — `lsof -i :3000 -t | xargs kill` before flipping the regression direction (slice 88 lesson). +- **`document.querySelector('main')` ambiguity** — multiple `
` elements on the page (`_app.js`, `RootLayout`, `PageLayout`). For computed-style queries pin via class signature (slice 98). +- **wagmi multicall batching** — `publicClient.readContract` auto-batches through Multicall3. An interceptor matching INNER selectors misses the OUTER aggregate3 call. See slice 103 — use `decodeMulticall3Aggregate3` + `encodeMulticall3Returns`. +- **Bulk-prefetched prices short-circuit per-card fetches** — to surface a regression in `useLatestPoolPrices`, set `makeCandlesMockHandler({ prices: {} })` so `attachPrefetchedPrices` skips and the per-card fetch actually runs (slice 97). +- **Subgraph adapter requires `YES_COMPANY`/`NO_CURRENCY` role tagging** — bare `YES`/`NO` token roles in the candles mock leave pool/token configs null. Use `makeSubgraphAwareCandlesHandler` (slice 95) for anything that reads pool config. +- **Real 20-byte addresses required** when addresses flow to viem-backed `readContract` — 1-byte dummies (`0x10`) fail pre-flight (slice 102/103 lesson). Slice 103 fixed the shared fixture to use full 20-byte forms. + +--- + +## Where the boundary is + +The harness covers **the interface app's UI surface**. As of slice 329: +- **96 scenarios authored**, 3 pinned-skipped via `pinnedLatentBug` (each pinned scenario corresponds to a real latent bug in the app — see ledger below). +- **12 KINDs of bugs** realized — **6 of them at 3-surface coverage** (page-error, network shape, a11y heuristics, user-CSS interactive, keyboard-nav, Visual/Computed CSS via 4 sub-grids). The per-surface chaos matrix is structurally complete for those 6 KINDs. KIND 6 sub-grid framework (slice 316): single KIND scales on property × surface axes, each combination a 1-liner via slice-310 helper. KIND 6 currently has **4 sub-grids at 3-surface** (pointer-events, cursor, text-transform, user-select) and **3 properties at 1-surface** (direction, visibility, opacity). Surface-fill phase (initiated slice 325) closes one sub-grid per ~2 slices. +- **8 shared fixture modules** (`api-mocks`, `eth-call-inspector`, `fork-state`, `wallet-stub`, `a11y-heuristics`, `text-selection`, `keyboard-nav`, `cascading-css`). All extracted at the N=3-inline-copies threshold (slice 289 doctrine), or N=2 for symmetric pairs (slice 306). +- **KIND 4 (URL state)** grew 2 → 5 scenarios across slices 318-322, covering 4 distinct sub-shapes (post-mount mutation, routing redirect, outbound static-href, back-nav static-href). Back-nav sub-shape structurally complete at 2/2 surfaces (/companies is the destination — back-nav from it is n/a). +- **Backup-catch synergy fully realized for `user-select`** (slice 328): both KIND 6 (computed-CSS read via 62+95+96) AND text-selection KIND (interactive triple-click via 51+68+72) cover the same regression at full 3-surface coverage. Maximum robustness against either mechanism path failing. +- **20 of 22 recent PRs (91%)** are mechanically caught. **PR #44** (SDK-only `getLinkableProposals`) has no UI consumer in src/, so it's out of scope without an SDK-side test harness. **PR #47** (dead-code removal) has nothing in src/ to assert against; partially guarded by proxy via scenarios 57+61 (catch any Supabase re-introduction on `market_event_proposal_links` / `pool_candles`). + +If the next round of PRs needs catches that don't fit the existing 12 KINDs, the slice should propose a 13th KIND with its mechanism and minimum-viable assertion shape before writing the scenario. + +--- + +## Latent-bug ledger + +Real bugs the harness discovered while authoring scenarios. Each entry pairs a `pinnedLatentBug` scenario (or, for older finds, a documented commit) with the symptom. When a bug is fixed, remove the `pinnedLatentBug` flag — the scenario becomes a regression catch. + +| # | Bug | Surfaced by | Symptom | Status | +|---|---|---|---|---| +| 1 | `fallback-company.png` 404 | slice 80 (page-error monitor) | Missing static asset triggers `console.error`. Initial baseline added it to the no-page-errors exclusion list so it doesn't blanket-fail page-error scenarios. | Open (latent, excluded from baseline) | +| 2 | React update-in-render warning | slice 80 + 48 (page-error scenarios) | One component calls `setState` during render — surfaces as a React warning the page-error monitor would otherwise blanket-catch. Excluded as a known artifact. | Open (latent, excluded from baseline) | +| 3 | ConfirmSwapModal lacks focus trap | scenario 77 (slice 302) | Tab from inside the open modal walks back to background within ~5 presses (lands on "Companies" link, then "Chain Selector"). Total a11y break for keyboard users while the modal is visible. | Pinned-latent in scenario 77 | +| 4 | Outcome tabs missing ARIA state | scenario 78 (slice 304) | Tab `