diff --git a/.changeset/wild-geese-report.md b/.changeset/wild-geese-report.md new file mode 100644 index 00000000..f11a8bd4 --- /dev/null +++ b/.changeset/wild-geese-report.md @@ -0,0 +1,13 @@ +--- +'confluence.js': patch +--- + +Adds a schema audit that reports the keys Confluence sends but the schemas do not describe. + +Nothing changes for callers. Response validation stays loose, so undocumented fields keep passing through untouched, +and the published types are byte-identical. The audit is off unless `AUDIT_SCHEMAS=true` is set, which only this +package's own nightly run does; `src/core/schemaAudit.ts` is internal and is not exported. + +The first run found 161 undocumented fields across 93 endpoints β€” Atlassian's published spec having fallen behind its +own API rather than anything broken. Each one is a field consumers receive at runtime but cannot see in the types, and +fixing them is what makes the types truer. diff --git a/.github/workflows/schema-audit.yml b/.github/workflows/schema-audit.yml new file mode 100644 index 00000000..32f96839 --- /dev/null +++ b/.github/workflows/schema-audit.yml @@ -0,0 +1,50 @@ +name: πŸ”¬ Schema audit + +# Asks a different question from the live suite, which is why it is a separate run. +# +# The live suite asks whether the library still works: a field that vanished, a type that changed, an enum that grew. +# This asks whether the schemas still describe what Confluence actually sends β€” the keys it returns that the spec does +# not document. Loose validation cannot answer that by construction, since letting undocumented keys through is exactly +# what it is for, so the run flips every response schema to strict. +# +# Kept apart on purpose: undocumented fields are usually documentation debt rather than breakage, and mixing them into +# the live run would bury a real failure under a pile of harmless additions. +on: + schedule: + # 04:00 UTC, an hour behind the live suite. Both take about ten minutes and + # share one tenant, so the gap keeps them from queueing behind each other. + - cron: '0 4 * * *' + workflow_dispatch: + +# Deliberately the same group as live-tests: one Confluence site serves both, and +# two runs creating and deleting content at once would fight over it. +concurrency: + group: live-tests + cancel-in-progress: false + +jobs: + audit: + name: πŸ”¬ Schemas against the live API + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: πŸ”„ Checkout sources + uses: actions/checkout@v7 + - name: βš™οΈ Use Node.js 22.x + uses: actions/setup-node@v7 + with: + node-version: 22.x + - name: πŸ“¦ Install pnpm + uses: pnpm/action-setup@v6 + - name: πŸ“Œ Install dependencies + run: pnpm install --frozen-lockfile + - name: πŸ“ Creating `.env` file + run: | + touch .env + echo HOST=${{ secrets.HOST }} >> .env + echo EMAIL=${{ secrets.EMAIL }} >> .env + echo API_TOKEN=${{ secrets.API_TOKEN }} >> .env + # Writes the findings to the run summary before deciding the exit code, so a + # red run still says which fields it is red about. + - name: πŸ”¬ Audit schemas against the live API + run: pnpm run audit:schemas diff --git a/package.json b/package.json index 5c04ec72..ed897d49 100644 --- a/package.json +++ b/package.json @@ -108,7 +108,8 @@ "changeset:status": "changeset status", "version:dev": "node scripts/devVersion.ts", "release:publish": "pnpm publish --no-git-checks --provenance", - "prepare": "husky" + "prepare": "husky", + "audit:schemas": "node scripts/schemaAudit.ts" }, "publishConfig": { "access": "public", diff --git a/scripts/schemaAudit.ts b/scripts/schemaAudit.ts new file mode 100644 index 00000000..e3a2e52f --- /dev/null +++ b/scripts/schemaAudit.ts @@ -0,0 +1,140 @@ +/** + * Runs the schema audit and reports what the live API sends that the schemas do not describe. + * + * Two things can go wrong and they are not the same thing: + * + * - a test fails, which means the library is actually broken against the live API; + * - drift is found, which means Atlassian's spec has fallen behind its own API. + * + * Both fail the run, but the summary keeps them apart β€” a red run for the second reason is a documentation debt, not + * an outage, and reading it as one wastes the signal. + * + * Runs on bare `node` β€” keep the types here erasable, as in scripts/checkBrowserSafe.ts. + */ +import { spawnSync } from 'node:child_process'; +import { appendFileSync, existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const findings = join(root, 'node_modules', '.cache', 'schema-audit.jsonl'); + +interface SchemaDrift { + endpoint: string; + path: string; + keys: string[]; +} + +// `--report-only` rebuilds the summary from the last run's findings. The suite takes +// several minutes against a live site, and reformatting the report should not cost that. +const reportOnly = process.argv.includes('--report-only'); + +mkdirSync(dirname(findings), { recursive: true }); + +if (!reportOnly) rmSync(findings, { force: true }); + +const run = reportOnly + ? { status: 0 } + : spawnSync('npx', ['vitest', 'run', '--config', 'vitest.config.audit.ts'], { + cwd: root, + stdio: 'inherit', + env: { ...process.env, AUDIT_SCHEMAS_OUTPUT: findings }, + }); + +/** + * Array indices carry no information here. + * + * `results.0._links.self` and `results.1._links.self` are one gap in one schema, and left as they are they would fill + * the report with the same finding once per element the API happened to return. + */ +function normalize(path: string): string { + return path.replace(/(^|\.)\d+(\.|$)/g, '$1[]$2'); +} + +/** + * Collapses the resource ids the suite happened to create. + * + * Without this the report names `GET /wiki/api/v2/attachments/att17105305` β€” an id that existed for the length of one + * run β€” and counts every such URL as its own endpoint, so the same gap in one route reads as dozens. + * + * The digit run has to be long: `v2` and `api` are route, not identity. + */ +function normalizeEndpoint(endpoint: string): string { + return endpoint + .split('/') + .map(segment => { + if (/^\d{4,}$/.test(segment)) return '{id}'; + if (/^[a-zA-Z]+\d{4,}$/.test(segment)) return '{id}'; + if (/^[0-9a-f]{16,}$/i.test(segment)) return '{id}'; + + return segment; + }) + .join('/'); +} + +const byField = new Map>(); + +if (existsSync(findings)) { + const lines = readFileSync(findings, 'utf8').trim(); + + for (const line of lines ? lines.split('\n') : []) { + const entry = JSON.parse(line) as SchemaDrift; + + for (const key of entry.keys) { + const field = normalize(entry.path ? `${entry.path}.${key}` : key); + + if (!byField.has(field)) byField.set(field, new Set()); + + byField.get(field)!.add(normalizeEndpoint(entry.endpoint)); + } + } +} + +const ranked = [...byField.entries()].sort( + (a, b) => b[1].size - a[1].size || a[0].localeCompare(b[0]), +); +const endpoints = new Set([...byField.values()].flatMap(set => [...set])); + +const summary: string[] = ['## Schema audit', '']; + +if (ranked.length === 0) { + summary.push('No drift: every response matched the schema that describes it.'); +} else { + summary.push( + `**${ranked.length} undocumented fields** across **${endpoints.size} endpoints**.`, + '', + 'These are keys Confluence sends that the schemas do not describe. Usually not breakage β€”', + 'the published spec has fallen behind the API β€” but each one is a field consumers cannot see', + 'in the types.', + '', + '| Field | Endpoints | Seen at |', + '| --- | ---: | --- |', + ); + + for (const [field, seen] of ranked) { + const sample = [...seen].sort().slice(0, 3).join('
'); + const more = seen.size > 3 ? `
…and ${seen.size - 3} more` : ''; + + summary.push(`| \`${field}\` | ${seen.size} | ${sample}${more} |`); + } +} + +const report = `${summary.join('\n')}\n`; + +if (process.env.GITHUB_STEP_SUMMARY) { + appendFileSync(process.env.GITHUB_STEP_SUMMARY, report, 'utf8'); +} else { + console.log(`\n${report}`); +} + +if (run.status !== 0) { + console.error('\nThe audit suite itself failed β€” that is real breakage, not drift. See the run output above.'); + process.exit(run.status ?? 1); +} + +if (ranked.length > 0) { + console.error(`\nSchema audit: ${ranked.length} undocumented fields across ${endpoints.size} endpoints.`); + process.exit(1); +} + +console.log('\nSchema audit: no drift.'); diff --git a/src/core/apiObject.ts b/src/core/apiObject.ts index 556552cf..4c0d3a9b 100644 --- a/src/core/apiObject.ts +++ b/src/core/apiObject.ts @@ -1,5 +1,27 @@ import { z, type ZodRawShape } from 'zod'; +import { isSchemaAuditEnabled } from './schemaAudit.js'; +/** + * Builds an object schema for an API response. + * + * Loose by default: keys the API sends but the spec does not document pass straight through, so a field added upstream + * never breaks a consumer between releases. + * + * Under `AUDIT_SCHEMAS=true` it builds strict objects instead, which is how the audit run learns *where* an + * undocumented key sits β€” zod reports `unrecognized_keys` with a path, and nothing else in the pipeline knows the + * shape well enough to say. Those failures do not reach the caller: `createClient` records them and hands back the + * response anyway, so one stale schema cannot cut the audit short. Loose validation cannot report drift at all, since + * accepting anything extra is precisely what it is for. + * + * The declared return type stays the loose one in both modes, deliberately. The switch is read at runtime, so the + * compiler cannot follow it, and the published declarations must not shift with an environment variable. + */ export function apiObject(shape: Shape) { - return z.object(shape).loose(); + const loose = z.object(shape).loose(); + + if (isSchemaAuditEnabled()) { + return z.strictObject(shape) as unknown as typeof loose; + } + + return loose; } diff --git a/src/core/createClient.ts b/src/core/createClient.ts index 0d426d7c..f8380b94 100644 --- a/src/core/createClient.ts +++ b/src/core/createClient.ts @@ -1,3 +1,4 @@ +import type { core as zodCore } from 'zod'; import { bodyToFetchBody, requiresDuplex, shouldSetJsonContentType } from './bodyToFetchBody.js'; import type { Auth, ClientConfig, SendRequestOptions } from './schemas/index.js'; import type { Client } from './interfaces/index.js'; @@ -10,6 +11,7 @@ import { TRANSIENT_HTTP_STATUSES, } from './errors/index.js'; import { BufferSchema } from './formData/index.js'; +import { isSchemaAuditEnabled, recordSchemaDrift } from './schemaAudit.js'; import { buildUrlWithSearchParams } from './serializeSearchParams.js'; import { clientConfigSchema } from './schemas/index.js'; import { createOAuth2Manager } from './oauth/index.js'; @@ -27,6 +29,70 @@ async function isScopeMismatchResponse(response: Response): Promise { } } +/** + * Removes the keys an audit run found undocumented, so the response can be parsed a second time. + * + * Audit-only. `path` is a zod issue path, so every segment is an object key or an array index, and anything that is + * not there any more is simply skipped β€” the walk describes a body that was just parsed, not an arbitrary structure. + */ +function dropKeys(body: unknown, path: readonly PropertyKey[], keys: readonly PropertyKey[]): void { + let target = body; + + for (const segment of path) { + if (target === null || typeof target !== 'object') return; + + target = (target as Record)[segment]; + } + + if (target === null || typeof target !== 'object') return; + + for (const key of keys) { + delete (target as Record)[key]; + } +} + +interface DriftFinding { + path: PropertyKey[]; + keys: PropertyKey[]; +} + +/** + * Reads a validation failure as pure schema drift, or decides it is not. + * + * Audit-only. Returns the undocumented keys when *every* complaint is one, and `undefined` the moment anything else + * appears β€” a missing field or a changed type is real breakage, and the audit must not quietly absorb it. + * + * Unions need the recursion. Zod reports each branch it tried, and branches that failed for their own reasons are + * simply the wrong branch; what identifies the right one is a branch whose only complaint is undocumented keys. Without + * this, every union-typed response throws instead of being recorded, and the drift inside it accumulates unseen behind + * a report that looks complete. + */ +function readDrift(issues: readonly zodCore.$ZodIssue[], base: PropertyKey[] = []): DriftFinding[] | undefined { + const findings: DriftFinding[] = []; + + for (const issue of issues) { + if (issue.code === 'unrecognized_keys') { + findings.push({ path: [...base, ...issue.path], keys: [...issue.keys] }); + continue; + } + + if (issue.code === 'invalid_union') { + const branch = issue.errors + .map(branchIssues => readDrift(branchIssues, [...base, ...issue.path])) + .find(result => result !== undefined); + + if (branch === undefined) return undefined; + + findings.push(...branch); + continue; + } + + return undefined; + } + + return findings; +} + /** A `Client` is anything with `sendRequest`; a `ClientConfig` never has one. */ function isClient(value: ClientConfig | Client): value is Client { return typeof (value as Client).sendRequest === 'function'; @@ -273,6 +339,34 @@ export function createClient(config: ClientConfig | Client): Client { const parsed = requestConfig.schema.safeParse(data); if (!parsed.success) { + const endpoint = `${requestConfig.method ?? 'GET'} ${requestConfig.url}`; + + // Under audit, undocumented keys are the finding β€” not a failure. They + // are recorded and the response is handed back unvalidated, so a single + // stale schema cannot end the run before the rest has been looked at. + // Anything else in the issue list is real breakage and still throws, + // which is why this checks that *every* issue is an extra key. + const drift = isSchemaAuditEnabled() ? readDrift(parsed.error.issues) : undefined; + + if (drift) { + for (const finding of drift) { + recordSchemaDrift({ + endpoint, + path: finding.path.join('.'), + keys: finding.keys.map(key => String(key)), + }); + dropKeys(data, finding.path, finding.keys); + } + + // Re-parsed rather than returned raw, so the caller still gets what + // the schema promises: `z.coerce.date()` and friends only run on a + // successful parse, and handing back the untouched body would turn + // every Date into a string β€” failures that say nothing about drift. + const cleaned = requestConfig.schema.safeParse(data); + + if (cleaned.success) return cleaned.data as T; + } + // The response parsed as JSON but is not the shape the endpoint // promises. Callers should not have to know zod is the validator to // catch this, so it arrives as the library's own error with the diff --git a/src/core/schemaAudit.ts b/src/core/schemaAudit.ts new file mode 100644 index 00000000..39a5cc77 --- /dev/null +++ b/src/core/schemaAudit.ts @@ -0,0 +1,39 @@ +/** + * Collects the gaps between the schemas and what the API actually sends. + * + * Only active when `AUDIT_SCHEMAS=true`, which the package's own audit run sets and nothing else does. In every other + * process this module holds an empty array and is never written to. + * + * It deliberately keeps findings in memory rather than writing them anywhere: `core/` ships to browsers, so it cannot + * reach for the filesystem. The audit runner reads the collection at the end of the run and reports it. + */ + +/** Symbol.for, not a local const: the collection has to survive two copies of the package in one process. */ +const STORE = Symbol.for('apis-code-gen.schemaAudit'); + +export interface SchemaDrift { + /** The request that produced the response, e.g. `GET /wiki/api/v2/spaces`. */ + endpoint: string; + /** Where in the response body the keys turned up. Empty string for the top level. */ + path: string; + /** The keys the API sent that the schema does not describe. */ + keys: string[]; +} + +type Store = { [STORE]?: SchemaDrift[] }; + +export function isSchemaAuditEnabled(): boolean { + const env = (globalThis as { process?: { env?: Record } }).process?.env; + + return env?.AUDIT_SCHEMAS === 'true'; +} + +export function recordSchemaDrift(entry: SchemaDrift): void { + const store = globalThis as Store; + + (store[STORE] ??= []).push(entry); +} + +export function collectedSchemaDrift(): readonly SchemaDrift[] { + return (globalThis as Store)[STORE] ?? []; +} diff --git a/tests/live/setup/auditCollector.ts b/tests/live/setup/auditCollector.ts new file mode 100644 index 00000000..668f41ea --- /dev/null +++ b/tests/live/setup/auditCollector.ts @@ -0,0 +1,25 @@ +import { appendFileSync } from 'node:fs'; +import { afterAll } from 'vitest'; +import { collectedSchemaDrift } from '#/core/schemaAudit'; + +/** + * Carries the audit's findings out of the worker. + * + * `core/` ships to browsers and so cannot touch the filesystem β€” it only accumulates findings in memory. This runs + * test-side, where `node:fs` is fine, and appends whatever the worker has gathered after each file. + * + * Everything is written every time and deduplicated when the report is built. Vitest may re-evaluate a setup file per + * test file, which would reset any "already written" cursor kept here and silently drop findings; duplicates in a + * JSONL file are free to remove, dropped findings are not recoverable. + */ +const OUTPUT = process.env.AUDIT_SCHEMAS_OUTPUT; + +afterAll(() => { + if (!OUTPUT) return; + + const drift = collectedSchemaDrift(); + + if (drift.length === 0) return; + + appendFileSync(OUTPUT, `${drift.map(entry => JSON.stringify(entry)).join('\n')}\n`, 'utf8'); +}); diff --git a/vitest.config.audit.ts b/vitest.config.audit.ts new file mode 100644 index 00000000..760556f7 --- /dev/null +++ b/vitest.config.audit.ts @@ -0,0 +1,36 @@ +import { resolve } from 'node:path'; +import { loadEnv } from 'vite'; +import { defineConfig } from 'vitest/config'; + +const repoRoot = import.meta.dirname; + +/** + * The schema audit: the live suite again, but with every response schema switched to strict. + * + * Separate from `vitest.config.live.ts` because the two answer different questions. The live run asks whether the + * library still works β€” a field that vanished, a type that changed, an enum that grew. The audit asks whether the + * schemas still describe what Confluence actually sends, which loose validation cannot report by construction: letting + * undocumented keys through is exactly what it is for. + * + * Failures here are usually not breakage. They mean Atlassian's published spec has fallen behind its own API. + */ +export default defineConfig(({ mode }) => ({ + test: { + include: ['tests/live/**/*.test.ts'], + environment: 'node', + reporters: ['default'], + env: { + ...loadEnv(mode, repoRoot, ''), + AUDIT_SCHEMAS: 'true', + AUDIT_SCHEMAS_OUTPUT: resolve(repoRoot, 'node_modules/.cache/schema-audit.jsonl'), + }, + fileParallelism: false, + globalSetup: ['./tests/live/setup/globalSetup.ts'], + setupFiles: ['./tests/live/setup/auditCollector.ts'], + hookTimeout: 100_000, + testTimeout: 100_000, + }, + resolve: { + alias: [{ find: /^#\/(.*)/, replacement: resolve(repoRoot, 'src/$1') }], + }, +}));