[codex] Add tiered auto-qa tests

.github/workflows/auto-qa-live.yml

@@ -0,0 +1,31 @@
+name: auto-qa live
+
+on:
+  workflow_dispatch:
+    inputs:
+      api_base:
+        description: 'Override api.futarchy.fi base URL for live checks'
+        required: false
+        default: 'https://api.futarchy.fi'
+  schedule:
+    - cron: '17 4 * * *'
+
+jobs:
+  live:
+    name: Live endpoint auto-qa
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+
+      - name: Run live auto-qa tests
+        env:
+          AUTO_QA_API_BASE: ${{ inputs.api_base || 'https://api.futarchy.fi' }}
+        run: npm run auto-qa:test:live

.github/workflows/auto-qa.yml

@@ -18,8 +18,8 @@ on:
   workflow_dispatch:
 
 jobs:
-  proposal-lifecycle:
-    name: Proposal lifecycle auto-qa
+  unit:
+    name: Deterministic auto-qa
     runs-on: ubuntu-latest
     timeout-minutes: 5
 
@@ -32,5 +32,5 @@ jobs:
         with:
           node-version: '22'
 
-      - name: Run auto-qa tests
-        run: npm run auto-qa:test
+      - name: Run deterministic auto-qa tests
+        run: npm run auto-qa:test:unit

auto-qa/README.md

@@ -0,0 +1,37 @@
+# Auto-QA
+
+This directory is split into CI tiers by flakiness and external dependencies.
+
+## Deterministic Unit Tier
+
+Run:
+
+```sh
+npm run auto-qa:test:unit
+```
+
+Files live under `auto-qa/tests/`. These tests must not call live services,
+start browsers, start an Anvil fork, or submit transactions. They are allowed
+to inspect local source files and run local Node helper scripts. This tier is
+safe for pull-request CI and is also what `npm run auto-qa:test` runs.
+
+## Live Network Tier
+
+Run:
+
+```sh
+npm run auto-qa:test:live
+```
+
+Files live under `auto-qa/live/`. These tests may call public Futarchy
+endpoints and should skip when the network is unavailable. They are useful for
+catching endpoint drift, but they are intentionally excluded from pull-request
+CI. GitHub runs them only by manual dispatch or on the nightly schedule in
+`.github/workflows/auto-qa-live.yml`.
+
+## Forked Browser Harness
+
+The larger Playwright and Anvil replay harness from the experimental
+`auto-qa` branch is not part of this tier. Forked-chain reads, writes, browser
+scenarios, and catalog drift checks need their own explicit workflow once they
+are cleaned up and separated from live-write tests.

auto-qa/fixtures/known-graphql-failures.json

@@ -0,0 +1,86 @@
+{
+  "knownFailureCount": 16,
+  "notes": "Baseline of GraphQL compat failures captured by auto-qa/tools/probe-graphql.mjs. These are production queries that fail validation against the live Checkpoint schema. Per the /loop directive: failures are documented but NOT fixed in this branch. When a production fix lands on main, regenerate with: npm run auto-qa:probe-graphql -- --baseline > auto-qa/fixtures/known-graphql-failures.json",
+  "failures": [
+    {
+      "file": "src/components/debug/EditCompanyModal.jsx",
+      "line": 23,
+      "errorSignature": "Cannot query field \"organizationEntity\" on type \"Query\"."
+    },
+    {
+      "file": "src/components/debug/EditProposalModal.jsx",
+      "line": 24,
+      "errorSignature": "Cannot query field \"proposalEntity\" on type \"Query\". Did you mean \"proposalentity\" or \"proposalentities\"?"
+    },
+    {
+      "file": "src/components/debug/OrganizationManagerModal.jsx",
+      "line": 64,
+      "errorSignature": "Cannot query field \"organizations\" on type \"Aggregator\"."
+    },
+    {
+      "file": "src/components/debug/OrganizationManagerModal.jsx",
+      "line": 65,
+      "errorSignature": "Cannot query field \"proposals\" on type \"Organization\"."
+    },
+    {
+      "file": "src/components/futarchyFi/marketPage/MarketPageShowcase.jsx",
+      "line": 2028,
+      "errorSignature": "Syntax Error: Unexpected \"}\"."
+    },
+    {
+      "file": "src/components/futarchyFi/proposalsList/page/proposalsPage/ProposalsPage.jsx",
+      "line": 111,
+      "errorSignature": "Cannot query field \"pools\" on type \"Proposal\"."
+    },
+    {
+      "file": "src/hooks/useOrganization.js",
+      "line": 20,
+      "errorSignature": "Cannot query field \"proposals\" on type \"Organization\"."
+    },
+    {
+      "file": "src/hooks/useSearchProposals.js",
+      "line": 20,
+      "errorSignature": "Cannot query field \"pools\" on type \"Proposal\"."
+    },
+    {
+      "file": "src/hooks/useSearchProposals.js",
+      "line": 40,
+      "errorSignature": "Cannot query field \"pools\" on type \"Proposal\"."
+    },
+    {
+      "file": "src/services/subgraphClient.js",
+      "line": 26,
+      "errorSignature": "Field \"proposal\" must not have a selection since type \"String\" has no subfields."
+    },
+    {
+      "file": "src/services/subgraphClient.js",
+      "line": 46,
+      "errorSignature": "Field \"proposal\" must not have a selection since type \"String\" has no subfields."
+    },
+    {
+      "file": "src/services/subgraphClient.js",
+      "line": 67,
+      "errorSignature": "Unknown type \"BigInt\". Did you mean \"Int\"?"
+    },
+    {
+      "file": "src/services/subgraphClient.js",
+      "line": 89,
+      "errorSignature": "Cannot query field \"pools\" on type \"Proposal\"."
+    },
+    {
+      "file": "src/utils/SubgraphBulkPriceFetcher.js",
+      "line": 36,
+      "errorSignature": "Syntax Error: Expected Name, found \"{\"."
+    },
+    {
+      "file": "src/utils/SubgraphPoolFetcher.js",
+      "line": 132,
+      "errorSignature": "Syntax Error: Expected Name, found \"{\"."
+    },
+    {
+      "file": "src/utils/subgraphTradesClient.js",
+      "line": 133,
+      "errorSignature": "String cannot represent a non string value: 0"
+    }
+  ]
+}

auto-qa/live/endpoint-liveness.test.mjs

@@ -0,0 +1,144 @@
+/**
+ * Endpoint-liveness invariant (auto-qa).
+ *
+ * For every subgraph endpoint URL hardcoded in
+ * `src/config/subgraphEndpoints.js`, send a trivial GraphQL introspection
+ * query and assert:
+ *   - HTTP 2xx
+ *   - response is valid JSON with a `data.__schema` envelope (i.e. it's
+ *     actually a working GraphQL endpoint, not a 200-OK landing page)
+ *
+ * Why: PRs #47, #49, #50, #60 all stemmed from the AWS → GCP migration
+ * leaving the frontend pointing at dead URLs. The dead endpoint returned
+ * a `database unavailable` body (so the page rendered empty silently).
+ * If any future endpoint drift, dead deploy, or URL typo lands on main,
+ * this test fails immediately with a clear message.
+ *
+ * The test reads the URLs out of the live source file (no copy-paste),
+ * so adding a new endpoint to subgraphEndpoints.js automatically extends
+ * the coverage.
+ */
+
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { dirname, resolve } from 'node:path';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ENDPOINTS_FILE = resolve(__dirname, '../../src/config/subgraphEndpoints.js');
+// PR #42: futarchy-api base URL referenced from usePoolData.js — extracted
+// here so a regression in the env-var default URL is caught.
+const API_BASE_URL_FILE = resolve(__dirname, '../../src/hooks/usePoolData.js');
+
+/**
+ * Pull every `https://…/graphql` URL out of the endpoints config.
+ * Deliberately permissive: any string literal that ends in `/graphql`
+ * counts. Keeps the test resilient to refactors that rename the
+ * exported constants.
+ */
+function loadEndpointUrls() {
+    const text = readFileSync(ENDPOINTS_FILE, 'utf8');
+    const matches = text.matchAll(/['"`](https?:\/\/[^'"`]+\/graphql)['"`]/g);
+    return [...new Set([...matches].map(m => m[1]))];
+}
+
+/**
+ * Pull the futarchy-api base URL out of usePoolData.js. The string
+ * literal we look for is the `||` fallback after the env-var read:
+ *   process.env.NEXT_PUBLIC_POOL_API_URL || 'https://api.futarchy.fi'
+ * If the file is refactored to use a different default URL we want to
+ * notice. Returns null if not found (test then skips that case).
+ */
+function loadApiBaseUrl() {
+    let text;
+    try { text = readFileSync(API_BASE_URL_FILE, 'utf8'); }
+    catch { return null; }
+    const m = text.match(/NEXT_PUBLIC_POOL_API_URL\s*\|\|\s*['"`](https?:\/\/[^'"`]+)['"`]/);
+    return m ? m[1] : null;
+}
+
+const INTROSPECTION = `{ __schema { queryType { name } } }`;
+
+async function probeEndpoint(url) {
+    const res = await fetch(url, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ query: INTROSPECTION }),
+        signal: AbortSignal.timeout(10000),
+    });
+    let body = null;
+    try { body = await res.json(); } catch { /* non-JSON */ }
+    return { status: res.status, body };
+}
+
+const urls = loadEndpointUrls();
+
+test('subgraphEndpoints.js contains at least one URL', () => {
+    assert.ok(urls.length > 0,
+        `expected to find at least one /graphql URL in ${ENDPOINTS_FILE}`);
+});
+
+for (const url of urls) {
+    test(`endpoint is live: ${url}`, async (t) => {
+        // Quick probe — if the URL is unreachable at all, skip rather than fail
+        // so we don't fail when the user is offline. But once we GET a response,
+        // demand it satisfies the schema-introspection invariant.
+        let result;
+        try {
+            result = await probeEndpoint(url);
+        } catch (err) {
+            t.skip(`network unreachable for ${url}: ${err.message}`);
+            return;
+        }
+        assert.ok(result.status >= 200 && result.status < 300,
+            `${url} returned HTTP ${result.status} (expected 2xx)`);
+        assert.ok(result.body && typeof result.body === 'object',
+            `${url} did not return JSON`);
+        assert.ok(
+            result.body.data?.__schema?.queryType?.name,
+            `${url} did not return a valid GraphQL introspection envelope. ` +
+            `Got: ${JSON.stringify(result.body).slice(0, 200)}…`
+        );
+    });
+}
+
+// ────────────────────────────────────────────────────────────────────────
+// PR #42 — futarchy-api base URL (Express, not GraphQL)
+// ────────────────────────────────────────────────────────────────────────
+const apiBaseUrl = loadApiBaseUrl();
+
+test('PR #42 — usePoolData.js declares a futarchy-api base URL', () => {
+    assert.ok(
+        apiBaseUrl && /^https?:\/\//.test(apiBaseUrl),
+        `Expected to find a NEXT_PUBLIC_POOL_API_URL fallback URL in ` +
+        `${API_BASE_URL_FILE}. Was the constant renamed or removed?`
+    );
+});
+
+test(`PR #42 — futarchy-api base URL is live: ${apiBaseUrl || '(not found)'}`,
+async (t) => {
+    if (!apiBaseUrl) { t.skip('no api base URL discovered'); return; }
+    let res;
+    try {
+        res = await fetch(`${apiBaseUrl}/health`, {
+            signal: AbortSignal.timeout(10000),
+        });
+    } catch (err) {
+        t.skip(`network unreachable for ${apiBaseUrl}: ${err.message}`);
+        return;
+    }
+    assert.ok(res.status >= 200 && res.status < 300,
+        `${apiBaseUrl}/health returned HTTP ${res.status} (expected 2xx)`);
+
+    let body = null;
+    try { body = await res.json(); } catch { /* not JSON */ }
+    // /health on futarchy-api returns { status: 'ok', timestamp }; assert
+    // either that or simply that the body is non-empty so this test still
+    // passes if the health endpoint changes its shape harmlessly.
+    if (body && typeof body === 'object') {
+        assert.ok(body.status === 'ok' || body.timestamp || Object.keys(body).length > 0,
+            `${apiBaseUrl}/health body looks empty: ${JSON.stringify(body)}`);
+    }
+});
+