feat: nightly schema audit against the live API

.changeset/wild-geese-report.md

@@ -0,0 +1,13 @@
+---
+'confluence.js': patch
+---
+
+Adds a schema audit that reports the keys Confluence sends but the schemas do not describe.
+
+Nothing changes for callers. Response validation stays loose, so undocumented fields keep passing through untouched,
+and the published types are byte-identical. The audit is off unless `AUDIT_SCHEMAS=true` is set, which only this
+package's own nightly run does; `src/core/schemaAudit.ts` is internal and is not exported.
+
+The first run found 161 undocumented fields across 93 endpoints — Atlassian's published spec having fallen behind its
+own API rather than anything broken. Each one is a field consumers receive at runtime but cannot see in the types, and
+fixing them is what makes the types truer.

.github/workflows/schema-audit.yml

@@ -0,0 +1,50 @@
+name: 🔬 Schema audit
+
+# Asks a different question from the live suite, which is why it is a separate run.
+#
+# The live suite asks whether the library still works: a field that vanished, a type that changed, an enum that grew.
+# This asks whether the schemas still describe what Confluence actually sends — the keys it returns that the spec does
+# not document. Loose validation cannot answer that by construction, since letting undocumented keys through is exactly
+# what it is for, so the run flips every response schema to strict.
+#
+# Kept apart on purpose: undocumented fields are usually documentation debt rather than breakage, and mixing them into
+# the live run would bury a real failure under a pile of harmless additions.
+on:
+  schedule:
+    # 04:00 UTC, an hour behind the live suite. Both take about ten minutes and
+    # share one tenant, so the gap keeps them from queueing behind each other.
+    - cron: '0 4 * * *'
+  workflow_dispatch:
+
+# Deliberately the same group as live-tests: one Confluence site serves both, and
+# two runs creating and deleting content at once would fight over it.
+concurrency:
+  group: live-tests
+  cancel-in-progress: false
+
+jobs:
+  audit:
+    name: 🔬 Schemas against the live API
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: 🔄 Checkout sources
+        uses: actions/checkout@v7
+      - name: ⚙️ Use Node.js 22.x
+        uses: actions/setup-node@v7
+        with:
+          node-version: 22.x
+      - name: 📦 Install pnpm
+        uses: pnpm/action-setup@v6
+      - name: 📌 Install dependencies
+        run: pnpm install --frozen-lockfile
+      - name: 📝 Creating `.env` file
+        run: |
+          touch .env
+          echo HOST=${{ secrets.HOST }} >> .env
+          echo EMAIL=${{ secrets.EMAIL }} >> .env
+          echo API_TOKEN=${{ secrets.API_TOKEN }} >> .env
+      # Writes the findings to the run summary before deciding the exit code, so a
+      # red run still says which fields it is red about.
+      - name: 🔬 Audit schemas against the live API
+        run: pnpm run audit:schemas

package.json

@@ -108,7 +108,8 @@
     "changeset:status": "changeset status",
     "version:dev": "node scripts/devVersion.ts",
     "release:publish": "pnpm publish --no-git-checks --provenance",
-    "prepare": "husky"
+    "prepare": "husky",
+    "audit:schemas": "node scripts/schemaAudit.ts"
   },
   "publishConfig": {
     "access": "public",

scripts/schemaAudit.ts

@@ -0,0 +1,140 @@
+/**
+ * Runs the schema audit and reports what the live API sends that the schemas do not describe.
+ *
+ * Two things can go wrong and they are not the same thing:
+ *
+ * - a test fails, which means the library is actually broken against the live API;
+ * - drift is found, which means Atlassian's spec has fallen behind its own API.
+ *
+ * Both fail the run, but the summary keeps them apart — a red run for the second reason is a documentation debt, not
+ * an outage, and reading it as one wastes the signal.
+ *
+ * Runs on bare `node` — keep the types here erasable, as in scripts/checkBrowserSafe.ts.
+ */
+import { spawnSync } from 'node:child_process';
+import { appendFileSync, existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const root = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const findings = join(root, 'node_modules', '.cache', 'schema-audit.jsonl');
+
+interface SchemaDrift {
+  endpoint: string;
+  path: string;
+  keys: string[];
+}
+
+// `--report-only` rebuilds the summary from the last run's findings. The suite takes
+// several minutes against a live site, and reformatting the report should not cost that.
+const reportOnly = process.argv.includes('--report-only');
+
+mkdirSync(dirname(findings), { recursive: true });
+
+if (!reportOnly) rmSync(findings, { force: true });
+
+const run = reportOnly
+  ? { status: 0 }
+  : spawnSync('npx', ['vitest', 'run', '--config', 'vitest.config.audit.ts'], {
+    cwd: root,
+    stdio: 'inherit',
+    env: { ...process.env, AUDIT_SCHEMAS_OUTPUT: findings },
+  });
+
+/**
+ * Array indices carry no information here.
+ *
+ * `results.0._links.self` and `results.1._links.self` are one gap in one schema, and left as they are they would fill
+ * the report with the same finding once per element the API happened to return.
+ */
+function normalize(path: string): string {
+  return path.replace(/(^|\.)\d+(\.|$)/g, '$1[]$2');
+}
+
+/**
+ * Collapses the resource ids the suite happened to create.
+ *
+ * Without this the report names `GET /wiki/api/v2/attachments/att17105305` — an id that existed for the length of one
+ * run — and counts every such URL as its own endpoint, so the same gap in one route reads as dozens.
+ *
+ * The digit run has to be long: `v2` and `api` are route, not identity.
+ */
+function normalizeEndpoint(endpoint: string): string {
+  return endpoint
+    .split('/')
+    .map(segment => {
+      if (/^\d{4,}$/.test(segment)) return '{id}';
+      if (/^[a-zA-Z]+\d{4,}$/.test(segment)) return '{id}';
+      if (/^[0-9a-f]{16,}$/i.test(segment)) return '{id}';
+
+      return segment;
+    })
+    .join('/');
+}
+
+const byField = new Map<string, Set<string>>();
+
+if (existsSync(findings)) {
+  const lines = readFileSync(findings, 'utf8').trim();
+
+  for (const line of lines ? lines.split('\n') : []) {
+    const entry = JSON.parse(line) as SchemaDrift;
+
+    for (const key of entry.keys) {
+      const field = normalize(entry.path ? `${entry.path}.${key}` : key);
+
+      if (!byField.has(field)) byField.set(field, new Set());
+
+      byField.get(field)!.add(normalizeEndpoint(entry.endpoint));
+    }
+  }
+}
+
+const ranked = [...byField.entries()].sort(
+  (a, b) => b[1].size - a[1].size || a[0].localeCompare(b[0]),
+);
+const endpoints = new Set([...byField.values()].flatMap(set => [...set]));
+
+const summary: string[] = ['## Schema audit', ''];
+
+if (ranked.length === 0) {
+  summary.push('No drift: every response matched the schema that describes it.');
+} else {
+  summary.push(
+    `**${ranked.length} undocumented fields** across **${endpoints.size} endpoints**.`,
+    '',
+    'These are keys Confluence sends that the schemas do not describe. Usually not breakage —',
+    'the published spec has fallen behind the API — but each one is a field consumers cannot see',
+    'in the types.',
+    '',
+    '| Field | Endpoints | Seen at |',
+    '| --- | ---: | --- |',
+  );
+
+  for (const [field, seen] of ranked) {
+    const sample = [...seen].sort().slice(0, 3).join('<br>');
+    const more = seen.size > 3 ? `<br>…and ${seen.size - 3} more` : '';
+
+    summary.push(`| \`${field}\` | ${seen.size} | ${sample}${more} |`);
+  }
+}
+
+const report = `${summary.join('\n')}\n`;
+
+if (process.env.GITHUB_STEP_SUMMARY) {
+  appendFileSync(process.env.GITHUB_STEP_SUMMARY, report, 'utf8');
+} else {
+  console.log(`\n${report}`);
+}
+
+if (run.status !== 0) {
+  console.error('\nThe audit suite itself failed — that is real breakage, not drift. See the run output above.');
+  process.exit(run.status ?? 1);
+}
+
+if (ranked.length > 0) {
+  console.error(`\nSchema audit: ${ranked.length} undocumented fields across ${endpoints.size} endpoints.`);
+  process.exit(1);
+}
+
+console.log('\nSchema audit: no drift.');

src/core/apiObject.ts

@@ -1,5 +1,27 @@
 import { z, type ZodRawShape } from 'zod';
+import { isSchemaAuditEnabled } from './schemaAudit.js';
 
+/**
+ * Builds an object schema for an API response.
+ *
+ * Loose by default: keys the API sends but the spec does not document pass straight through, so a field added upstream
+ * never breaks a consumer between releases.
+ *
+ * Under `AUDIT_SCHEMAS=true` it builds strict objects instead, which is how the audit run learns *where* an
+ * undocumented key sits — zod reports `unrecognized_keys` with a path, and nothing else in the pipeline knows the
+ * shape well enough to say. Those failures do not reach the caller: `createClient` records them and hands back the
+ * response anyway, so one stale schema cannot cut the audit short. Loose validation cannot report drift at all, since
+ * accepting anything extra is precisely what it is for.
+ *
+ * The declared return type stays the loose one in both modes, deliberately. The switch is read at runtime, so the
+ * compiler cannot follow it, and the published declarations must not shift with an environment variable.
+ */
 export function apiObject<Shape extends ZodRawShape>(shape: Shape) {
-  return z.object(shape).loose();
+  const loose = z.object(shape).loose();
+
+  if (isSchemaAuditEnabled()) {
+    return z.strictObject(shape) as unknown as typeof loose;
+  }
+
+  return loose;
 }

src/core/createClient.ts

@@ -1,3 +1,4 @@
+import type { core as zodCore } from 'zod';
 import { bodyToFetchBody, requiresDuplex, shouldSetJsonContentType } from './bodyToFetchBody.js';
 import type { Auth, ClientConfig, SendRequestOptions } from './schemas/index.js';
 import type { Client } from './interfaces/index.js';
@@ -10,6 +11,7 @@ import {
   TRANSIENT_HTTP_STATUSES,
 } from './errors/index.js';
 import { BufferSchema } from './formData/index.js';
+import { isSchemaAuditEnabled, recordSchemaDrift } from './schemaAudit.js';
 import { buildUrlWithSearchParams } from './serializeSearchParams.js';
 import { clientConfigSchema } from './schemas/index.js';
 import { createOAuth2Manager } from './oauth/index.js';
@@ -27,6 +29,70 @@ async function isScopeMismatchResponse(response: Response): Promise<boolean> {
   }
 }
 
+/**
+ * Removes the keys an audit run found undocumented, so the response can be parsed a second time.
+ *
+ * Audit-only. `path` is a zod issue path, so every segment is an object key or an array index, and anything that is
+ * not there any more is simply skipped — the walk describes a body that was just parsed, not an arbitrary structure.
+ */
+function dropKeys(body: unknown, path: readonly PropertyKey[], keys: readonly PropertyKey[]): void {
+  let target = body;
+
+  for (const segment of path) {
+    if (target === null || typeof target !== 'object') return;
+
+    target = (target as Record<PropertyKey, unknown>)[segment];
+  }
+
+  if (target === null || typeof target !== 'object') return;
+
+  for (const key of keys) {
+    delete (target as Record<PropertyKey, unknown>)[key];
+  }
+}
+
+interface DriftFinding {
+  path: PropertyKey[];
+  keys: PropertyKey[];
+}
+
+/**
+ * Reads a validation failure as pure schema drift, or decides it is not.
+ *
+ * Audit-only. Returns the undocumented keys when *every* complaint is one, and `undefined` the moment anything else
+ * appears — a missing field or a changed type is real breakage, and the audit must not quietly absorb it.
+ *
+ * Unions need the recursion. Zod reports each branch it tried, and branches that failed for their own reasons are
+ * simply the wrong branch; what identifies the right one is a branch whose only complaint is undocumented keys. Without
+ * this, every union-typed response throws instead of being recorded, and the drift inside it accumulates unseen behind
+ * a report that looks complete.
+ */
+function readDrift(issues: readonly zodCore.$ZodIssue[], base: PropertyKey[] = []): DriftFinding[] | undefined {
+  const findings: DriftFinding[] = [];
+
+  for (const issue of issues) {
+    if (issue.code === 'unrecognized_keys') {
+      findings.push({ path: [...base, ...issue.path], keys: [...issue.keys] });
+      continue;
+    }
+
+    if (issue.code === 'invalid_union') {
+      const branch = issue.errors
+        .map(branchIssues => readDrift(branchIssues, [...base, ...issue.path]))
+        .find(result => result !== undefined);
+
+      if (branch === undefined) return undefined;
+
+      findings.push(...branch);
+      continue;
+    }
+
+    return undefined;
+  }
+
+  return findings;
+}
+
 /** A `Client` is anything with `sendRequest`; a `ClientConfig` never has one. */
 function isClient(value: ClientConfig | Client): value is Client {
   return typeof (value as Client).sendRequest === 'function';
@@ -273,6 +339,34 @@ export function createClient(config: ClientConfig | Client): Client {
         const parsed = requestConfig.schema.safeParse(data);
 
         if (!parsed.success) {
+          const endpoint = `${requestConfig.method ?? 'GET'} ${requestConfig.url}`;
+
+          // Under audit, undocumented keys are the finding — not a failure. They
+          // are recorded and the response is handed back unvalidated, so a single
+          // stale schema cannot end the run before the rest has been looked at.
+          // Anything else in the issue list is real breakage and still throws,
+          // which is why this checks that *every* issue is an extra key.
+          const drift = isSchemaAuditEnabled() ? readDrift(parsed.error.issues) : undefined;
+
+          if (drift) {
+            for (const finding of drift) {
+              recordSchemaDrift({
+                endpoint,
+                path: finding.path.join('.'),
+                keys: finding.keys.map(key => String(key)),
+              });
+              dropKeys(data, finding.path, finding.keys);
+            }
+
+            // Re-parsed rather than returned raw, so the caller still gets what
+            // the schema promises: `z.coerce.date()` and friends only run on a
+            // successful parse, and handing back the untouched body would turn
+            // every Date into a string — failures that say nothing about drift.
+            const cleaned = requestConfig.schema.safeParse(data);
+
+            if (cleaned.success) return cleaned.data as T;
+          }
+
           // The response parsed as JSON but is not the shape the endpoint
           // promises. Callers should not have to know zod is the validator to
           // catch this, so it arrives as the library's own error with the